From b7018896268690c6c7e1076de1b0ed28dcf9e04b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Sat, 10 Jan 2026 00:46:00 +0800 Subject: [PATCH 01/61] add the first version franz implementation --- docs/design/2026-01-09-franz-go-kafka-sink.md | 279 +++++++++++++++++ downstreamadapter/sink/kafka/helper.go | 2 +- go.mod | 23 +- go.sum | 26 ++ pkg/sink/kafka/factory_selector.go | 39 +++ pkg/sink/kafka/franz_admin_client.go | 281 +++++++++++++++++ pkg/sink/kafka/franz_async_producer.go | 166 ++++++++++ pkg/sink/kafka/franz_factory.go | 294 ++++++++++++++++++ pkg/sink/kafka/franz_sync_producer.go | 160 ++++++++++ pkg/sink/kafka/options.go | 13 + 10 files changed, 1272 insertions(+), 11 deletions(-) create mode 100644 docs/design/2026-01-09-franz-go-kafka-sink.md create mode 100644 pkg/sink/kafka/factory_selector.go create mode 100644 pkg/sink/kafka/franz_admin_client.go create mode 100644 pkg/sink/kafka/franz_async_producer.go create mode 100644 pkg/sink/kafka/franz_factory.go create mode 100644 pkg/sink/kafka/franz_sync_producer.go diff --git a/docs/design/2026-01-09-franz-go-kafka-sink.md b/docs/design/2026-01-09-franz-go-kafka-sink.md new file mode 100644 index 0000000000..6fc01aa0bf --- /dev/null +++ b/docs/design/2026-01-09-franz-go-kafka-sink.md @@ -0,0 +1,279 @@ +# Kafka Sink 基于 franz-go 的实现设计与实施计划 + +## Status + +- Status: Proposed +- Date: 2026-01-09 +- Owner: TiCDC Team + +## Background / Context + +TiCDC 的 Kafka sink 负责把上游产生的 DML/DDL/checkpoint 事件编码并写入 Kafka，写入成功后通过 callback 将“已落盘”信息回传给上游推进进度。当前实现分为两层： + +- 使用层：`downstreamadapter/sink/kafka`（事件路由、编码流水线、发送调度） +- 客户端抽象层：`pkg/sink/kafka`（Factory、AdminClient、AsyncProducer、SyncProducer 以及 metrics 采集） + +`pkg/sink/kafka` 当前以 Sarama 为主要实现（`sarama_factory.go` / `sarama_*_producer.go` / `admin.go` / `sarama_config.go`）。本设计的目标是引入并基于 `github.com/twmb/franz-go`（`kgo` + `kadm`）实现等价的 Kafka 客户端层，使 `downstreamadapter/sink/kafka` 的使用方式尽可能不变，并支持渐进式切换与回滚。 + +## Problem Statement + +在不破坏既有 Kafka sink 行为与配置的前提下，引入 franz-go 作为 Kafka client 实现，满足： + +- 兼容现有 sink-uri 参数（topic、partition-num、required-acks、compression、TLS、SASL 等） +- 兼容现有模块边界与接口：`pkg/sink/kafka/factory.go`、`pkg/sink/kafka/cluster_admin_client.go` +- 可灰度、可回滚（支持 Sarama 与 franz-go 并存，按配置选择） +- 性能与稳定性不劣于现有实现，并为后续优化留出空间 + +## Goals / Non-Goals + +### Goals + +- 在 `pkg/sink/kafka` 内新增 franz-go 实现：Factory、AdminClient、AsyncProducer、SyncProducer。 +- `downstreamadapter/sink/kafka` 仅做最小化改动（最好只改 factory 选择逻辑）。 +- 覆盖安全连接能力：TLS、SASL PLAIN / SCRAM / OAuth（与现有选项对齐）。 +- 错误语义与诊断信息对齐：保留 `pkg/sink/kafka/logutil.go: AnnotateEventError(...)` 的日志上下文能力。 +- 支持渐进式验证：单元测试 + 复用现有集成测试（通过切换参数跑两套）。 + +### Non-Goals + +- 不修改事件编码协议与路由语义（`downstreamadapter/sink/eventrouter`、`pkg/sink/codec` 不在本设计范围）。 +- 不实现 Kafka consumer 能力（仅生产端与 admin 能力）。 +- 不在第一阶段追求 metrics 完全等价（可先保证功能正确，再补齐指标采集）。 + +## Current State (as-is) + +### 关键接口与调用路径 + +- `pkg/sink/kafka/factory.go: type Factory`：为上层提供 + - `AdminClient(ctx)` + - `AsyncProducer(ctx)`（DML） + - `SyncProducer(ctx)`（DDL/checkpoint） + - `MetricsCollector(adminClient)` +- `downstreamadapter/sink/kafka/helper.go: newKafkaSinkComponent(...)`：默认使用 `kafka.NewSaramaFactory` +- `downstreamadapter/sink/kafka/sink.go`： + - DML：编码后调用 `AsyncProducer.AsyncSend(ctx, topic, partition, message)`，并在 `AsyncRunCallback` 中消费 ack/error + - DDL/checkpoint：调用 `SyncProducer.SendMessage/SendMessages` + - 心跳：每 5s 调用一次 `Producer.Heartbeat()`（DML 与 DDL 分别一个 ticker） +- Topic 管理依赖 admin：`downstreamadapter/sink/topicmanager/kafka_topic_manager.go` +- 配置自适应：`pkg/sink/kafka/options.go: adjustOptions(...)` 通过 `ClusterAdminClient` 读取 topic/broker 配置并调整 `MaxMessageBytes`、`PartitionNum`、`KeepConnAliveInterval` 等 + +## Proposed Design (to-be) + +### 总体架构 + +保持 `downstreamadapter/sink/kafka` 逻辑基本不变，仅将 `pkg/sink/kafka` 的 Sarama 实现扩展为“多实现可选”： + +``` +downstreamadapter/sink/kafka + └─ uses pkg/sink/kafka.Factory + ├─ Sarama (existing): saramaFactory / saramaAdminClient / sarama{Async,Sync}Producer + └─ Franz (new): franzFactory / franzAdminClient / franz{Async,Sync}Producer +``` + +### 组件与职责 + +#### 1) `franzFactory`（新增） + +- 文件建议：`pkg/sink/kafka/franz_factory.go` +- 责任： + - 从 `options` 构造 `kgo.Opt` 集合（seed brokers、TLS、SASL、超时、ack、压缩、producer 行为等） + - 复用现有自适应逻辑：创建临时 admin client → 调用 `pkg/sink/kafka/options.go: adjustOptions(...)` → 关闭临时 admin → 保存调整后的 `options` + - 提供 `AdminClient/AsyncProducer/SyncProducer/MetricsCollector` 的 franz-go 实现 + +#### 2) `franzAdminClient`（新增） + +- 文件建议：`pkg/sink/kafka/franz_admin_client.go` +- 内部使用： + - `kgo.Client`（底层连接与请求） + - `kadm.Client`（admin API 封装） +- 需要实现 `pkg/sink/kafka/cluster_admin_client.go: ClusterAdminClient`： + - `GetAllBrokers()`：`kadm.Client.ListBrokers(ctx)` 或 `BrokerMetadata(ctx)` 解析 broker id + - `GetTopicsMeta(...)` / `GetTopicsPartitionsNum(...)`：`kadm.Client.Metadata(ctx, topics...)` + - `CreateTopic(...)`：`kadm.Client.CreateTopics(ctx, partitions, rf, configs, topic)`；对 “topic already exists” 做兼容性忽略 + - `GetBrokerConfig(...)`：`kadm.Client.BrokerMetadata(ctx)` 获取 controller id，再 `DescribeBrokerConfigs(ctx, controllerID)` + - `GetTopicConfig(...)`：`kadm.Client.DescribeTopicConfigs(ctx, topic)` + - `Heartbeat()`：可实现为 no-op，依赖 `kgo` 的自动重连与 producer 的重试能力；必要时再引入 `Ping`（短超时）的实现以辅助排障 + +#### 3) `franzAsyncProducer`（新增，DML） + +- 文件建议：`pkg/sink/kafka/franz_async_producer.go` +- 对齐上层语义： + - `AsyncSend(ctx, topic, partition, message)`：调用 `kgo.Client.Produce`，record 的 `Topic/Partition/Key/Value` 来自现有路由与编码结果 + - `AsyncRunCallback(ctx)`：阻塞等待第一条 produce error 或 ctx.Done；对齐 Sarama 行为（发生错误导致 sink 退出重建） + - `message.Callback`：在 produce 回调成功时执行（与 Sarama 成功通道消费一致） + - 错误：立刻在边界处包装为带 stack 的错误，并通过 `AnnotateEventError(...)` 附带 message 的 `LogInfo` + - `Heartbeat()`：可实现为 no-op；通过 `kgo.RecordRetries` 在网络抖动、连接被 broker 关闭等场景下提升鲁棒性 + +#### 4) `franzSyncProducer`（新增，DDL/checkpoint） + +- 文件建议：`pkg/sink/kafka/franz_sync_producer.go` +- 对齐上层语义： + - `SendMessage`：构造 1 条 record，`ProduceSync` 并返回错误 + - `SendMessages`：按 partitionNum 构造 N 条 record（与当前逻辑一致），`ProduceSync` 等待全部返回，聚合错误 + - `Heartbeat()`：可实现为 no-op + +#### 5) 选择机制（灰度） + +建议增加一个可选 sink-uri 参数来选择 Kafka client 实现，默认保持 Sarama： + +- 新增参数：`kafka-client=sarama|franz`（默认 sarama） +- 影响范围： + - `pkg/sink/kafka/options.go: urlConfig` 增加字段 + - `downstreamadapter/sink/kafka/helper.go: newKafkaSinkComponent(...)` 根据 options 选择 `kafka.NewSaramaFactory` 或 `kafka.NewFranzFactory` + +该机制允许： + +- CI/集成测试中在不改代码的情况下切换实现 +- 线上灰度（按 changefeed 配置逐个切换） +- 快速回滚（改回 sarama） + +## Detailed Design + +### 配置映射（options → franz-go） + +建议以“对齐现有行为”为优先原则，主要映射如下（示例为概念性描述，具体以实现为准）： + +- Brokers：`options.BrokerEndpoints` → `kgo.SeedBrokers(...)` +- ClientID：`options.ClientID` → `kgo.ClientID(...)` +- Dial timeout：`options.DialTimeout` → `kgo.DialTimeout(...)` +- TLS： + - `options.EnableTLS` / `options.Credential` / `options.InsecureSkipVerify` + - → 构造 `tls.Config` 后 `kgo.DialTLSConfig(tlsConf)` +- SASL（注意能力差异）： + - PLAIN / SCRAM：`kgo.SASL(...)`（基于 `github.com/twmb/franz-go/pkg/sasl/plain`、`.../scram`） + - OAuth：基于 `github.com/twmb/franz-go/pkg/sasl/oauth`，把现有 token provider 适配为 franz-go 的 oauth provider + - GSSAPI：franz-go 默认包不提供现成实现（当前 `pkg/sink/kafka/sarama_config.go: completeSaramaSASLConfig(...)` 支持）。第一阶段建议：若检测到 `sasl-mechanism=GSSAPI` 则强制走 Sarama，或直接返回“暂不支持”的显式错误。 +- RequiredAcks：`options.RequiredAcks` → + - `WaitForAll` → `kgo.RequiredAcks(kgo.AllISRAcks())` + - `WaitForLocal` → `kgo.RequiredAcks(kgo.LeaderAck())` + - `NoResponse` → `kgo.RequiredAcks(kgo.NoAck())` +- Compression：`options.Compression` → `kgo.ProducerBatchCompression(...)`（`SnappyCompression/GzipCompression/Lz4Compression/ZstdCompression/NoCompression`） +- MaxMessageBytes：`options.MaxMessageBytes` → `kgo.ProducerBatchMaxBytes(int32(...))` + +### Producer 行为对齐（重试、顺序、幂等） + +Sarama 现状（见 `pkg/sink/kafka/sarama_config.go`）： + +- DML async：`Producer.Retry.Max = 0`，`Net.MaxOpenRequests = 1`（偏向“顺序安全 + fail fast”） +- DDL/checkpoint sync：`Producer.Retry.Max = 3`（偏向“关键控制面更稳健”） + +franz-go 默认行为差异较大（默认 recordRetries 近似无限、默认开启幂等写），因此需要显式对齐： + +- DML async（建议第一阶段）： + - `kgo.DisableIdempotentWrite()`：避免引入 `IDEMPOTENT_WRITE` ACL 依赖，保持与 Sarama 默认一致 + - `kgo.MaxProduceRequestsInflightPerBroker(1)`：对齐顺序与可预期性 + - `kgo.RecordRetries(N)`：设置一个合理重试次数以提升鲁棒性（例如 N=3 或 5），并依赖 franz-go 的“gapless ordering”语义避免在单分区内越过失败记录继续成功写入 + - `kgo.ProduceRequestTimeout(...)`：与现有 `options.WriteTimeout/ReadTimeout` 对齐，避免重试导致长时间阻塞 + - `kgo.ProducerLinger(0)`：对齐“尽快 flush” +- DDL/checkpoint sync： + - 可采用更保守的重试策略（例如 `RecordRetries(5)`），以提升控制面事件（DDL/checkpoint）的成功率 + - 需要用内部超时兜底，避免在 `SyncProducer` 接口缺少 ctx 的情况下无限阻塞 + +后续如需提升吞吐，可在不影响语义的前提下评估： + +- 允许更大的 in-flight（可能导致乱序） +- 打开幂等写（需评估权限、配额与 broker 版本） +- 适度增加 linger（吞吐上升，延迟增加） + +### 错误处理与诊断信息 + +边界层（franz-go → TiCDC）要做到： + +- franz-go / kadm 返回的错误属于第三方错误：在最接近发生点立即用 TiCDC 的 errors 包装以获得 stack trace +- 附带事件上下文：使用 `pkg/sink/kafka/logutil.go: AnnotateEventError(...)` 把 `MessageLogInfo` 拼入错误，便于定位是哪类事件（dml/ddl/checkpoint）以及表信息、ts 等 +- 上层 caller 对已包装错误不再重复 wrap（减少噪音与重复堆栈） + +### Close 语义与资源管理 + +Sarama 版本中每个 producer/admin 都持有独立 client；close 顺序也写入了注释（先关 client 再关 producer，避免阻塞 flush）。franz-go 可以选择两种实现方式： + +1) **与现状一致：每个组件一个 kgo.Client**（实现简单、行为可控，代价是连接数略多） +2) **同一个 factory 共享一个 kgo.Client**（连接更少、资源更省，但需要引用计数与更严格的 close 协议） + +第一阶段建议采用方案 (1)，降低引入风险；后续可在确认稳定后再做共享优化。 + +## Performance Considerations + +franz-go 的优势通常来自： + +- 更紧凑的编码与更少的反射/分配 +- 统一 client 能力（produce/admin/consume 一套基础设施） +- 可通过 hooks/telemetry 获取更丰富的请求级信息 + +但在 TiCDC Kafka sink 场景，真正的性能瓶颈往往在“上层编码与调度”，并非单纯 client 库。引入 franz-go 后仍需重点关注： + +- `downstreamadapter/sink/kafka` 的无限队列与 per-row 分配（不在本设计范围，但可在后续优化） +- Producer 参数对吞吐/延迟/乱序的权衡（linger、batch、in-flight、retries） +- 若 franz 实现的 `Heartbeat()` 为 no-op，可考虑后续把上层 5s ticker 变为按需或配置化，减少无效调用 + +## Testing Strategy + +### Unit Tests + +- 配置映射测试：给定 `options`，断言构造出的 franz-go 配置与预期一致（acks/compression/TLS/SASL 等）。 +- admin wrapper 行为测试：对 `GetTopicsMeta/GetTopicConfig/GetBrokerConfig/CreateTopic` 的错误处理、已存在 topic 的兼容性处理等。 + +### Integration / E2E + +复用现有 Kafka 集成测试，通过 sink-uri 参数切换实现： + +- 现有测试用例（示例）： + - `tests/integration_tests/kafka_log_info/run.sh`（依赖 failpoint 注入错误与日志上下文） + - `tests/integration_tests/mq_sink_error_resume/run.sh`（错误恢复） +- 新增运行方式： + - 在 sink-uri 增加 `kafka-client=franz`，并确保 failpoint 名称在 franz 实现中兼容（或新增等价 failpoint） + +### 性能回归 + +- A/B 对比：同一 workload 下对比 Sarama 与 franz-go 的吞吐、端到端延迟、CPU、内存、Kafka 请求数量。 +- 关注场景：高并发 DML、批量 DDL、checkpoint 广播、topic 自动创建/metadata 刷新。 + +## Observability / Operations + +- 日志：错误日志必须包含 changefeed 维度（keyspace/changefeed）和事件上下文（eventType/table/ts），但避免在日志文本中拼接函数名与多余格式噪音。 +- Metrics（阶段性计划）： + - 第一阶段：可先保持 `MetricsCollector` 为 no-op（功能优先） + - 第二阶段：基于 franz-go hooks 或 client telemetry 把关键指标接入现有 Prometheus 指标体系（例如 request latency、in-flight、吞吐等） + +## Rollout Plan + +1) **实现与编译通过** + - 新增 `kafka-client=franz` 选项，默认仍为 sarama + - 引入 `NewFranzFactory` 与相关实现文件 +2) **功能验证** + - 单元测试覆盖关键映射与错误处理 + - 本地/CI 跑现有 Kafka 集成测试，分别用 sarama 与 franz-go 跑一遍 +3) **灰度** + - 选取少量 changefeed 开启 franz-go + - 对比关键指标与故障率 +4) **扩大与默认切换** + - 确认稳定后逐步扩大覆盖面 + - 视情况将默认实现切换为 franz-go，并保留 sarama 回滚窗口 + +## Alternatives Considered + +- 继续使用 Sarama：稳定但维护与性能空间受限，且部分行为（如 metadata/连接管理）需要更多定制补丁。 +- 其他 Go Kafka client（如 kafka-go）：API/语义与现有实现差异较大，迁移成本与回归风险更高。 + +## Open Questions / Future Work + +- SASL GSSAPI（Kerberos）在 franz-go 体系下的实现方案（自定义 sasl.Mechanism vs 继续走 Sarama）。 +- franz-go metrics / hooks 与现有 `pkg/sink/kafka/metrics_collector.go` 指标体系的对齐方案与成本评估。 +- 是否要在 factory 内共享 `kgo.Client`（资源更省）以及如何保证 close 语义与并发安全。 + +## References + +- franz-go：`github.com/twmb/franz-go`（核心 `pkg/kgo`） +- kadm：admin 封装 `github.com/twmb/franz-go/pkg/kadm` +- 现有 TiCDC Kafka sink： + - `downstreamadapter/sink/kafka/helper.go` + - `downstreamadapter/sink/kafka/sink.go` + - `pkg/sink/kafka/factory.go` + - `pkg/sink/kafka/cluster_admin_client.go` + - `pkg/sink/kafka/options.go` + - `pkg/sink/kafka/sarama_factory.go` + - `pkg/sink/kafka/sarama_config.go` + - `pkg/sink/kafka/admin.go` + - `pkg/sink/kafka/sarama_async_producer.go` + - `pkg/sink/kafka/sarama_sync_producer.go` + - `pkg/sink/kafka/logutil.go` diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index c45b7eb10f..b6738a2a00 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -135,7 +135,7 @@ func newKafkaSinkComponent( sinkURI *url.URL, sinkConfig *config.SinkConfig, ) (components, config.Protocol, error) { - return newKafkaSinkComponentWithFactory(ctx, changefeedID, sinkURI, sinkConfig, kafka.NewSaramaFactory) + return newKafkaSinkComponentWithFactory(ctx, changefeedID, sinkURI, sinkConfig, kafka.NewFactory) } func newKafkaSinkComponentForTest( diff --git a/go.mod b/go.mod index 79d0b4938c..35f59e4144 100644 --- a/go.mod +++ b/go.mod @@ -42,11 +42,11 @@ require ( github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc github.com/jarcoal/httpmock v1.2.0 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.18.0 + github.com/klauspost/compress v1.18.2 github.com/linkedin/goavro/v2 v2.14.0 github.com/mailru/easyjson v0.7.7 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 - github.com/pierrec/lz4/v4 v4.1.18 + github.com/pierrec/lz4/v4 v4.1.22 github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 github.com/pingcap/errors v0.11.5-0.20250523034308-74f78ae071ee github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 @@ -84,12 +84,12 @@ require ( go.uber.org/mock v0.5.2 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/net v0.43.0 + golang.org/x/net v0.47.0 golang.org/x/oauth2 v0.30.0 - golang.org/x/sync v0.17.0 - golang.org/x/sys v0.35.0 - golang.org/x/term v0.34.0 - golang.org/x/text v0.29.0 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.38.0 + golang.org/x/term v0.37.0 + golang.org/x/text v0.31.0 golang.org/x/time v0.12.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.36.6 @@ -329,6 +329,9 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/twmb/franz-go v1.20.6 // indirect + github.com/twmb/franz-go/pkg/kadm v1.17.1 // indirect + github.com/twmb/franz-go/pkg/kmsg v1.12.0 // indirect github.com/twmb/murmur3 v1.1.6 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect @@ -358,10 +361,10 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.41.0 // indirect + golang.org/x/crypto v0.45.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/mod v0.27.0 // indirect - golang.org/x/tools v0.36.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/tools v0.38.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.170.0 // indirect google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect diff --git a/go.sum b/go.sum index 0e0270cd56..6eee5491be 100644 --- a/go.sum +++ b/go.sum @@ -705,6 +705,8 @@ github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHU github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -850,6 +852,8 @@ github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d h1:eHcokyHxm7HVM+7+Qy1zZwC7NhX9wVNX8oQDcSZw1qI= github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d/go.mod h1:KiO2zumBCWx7yoVYoFRpb+DNrwEPk1pR1LF7NvOACMQ= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= @@ -1087,6 +1091,12 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/twmb/franz-go v1.20.6 h1:TpQTt4QcixJ1cHEmQGPOERvTzo99s8jAutmS7rbSD6w= +github.com/twmb/franz-go v1.20.6/go.mod h1:u+FzH2sInp7b9HNVv2cZN8AxdXy6y/AQ1Bkptu4c0FM= +github.com/twmb/franz-go/pkg/kadm v1.17.1 h1:Bt02Y/RLgnFO2NP2HVP1kd2TFtGRiJZx+fSArjZDtpw= +github.com/twmb/franz-go/pkg/kadm v1.17.1/go.mod h1:s4duQmrDbloVW9QTMXhs6mViTepze7JLG43xwPcAeTg= +github.com/twmb/franz-go/pkg/kmsg v1.12.0 h1:CbatD7ers1KzDNgJqPbKOq0Bz/WLBdsTH75wgzeVaPc= +github.com/twmb/franz-go/pkg/kmsg v1.12.0/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= @@ -1262,6 +1272,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1319,6 +1331,8 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1368,6 +1382,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1391,6 +1407,8 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1456,6 +1474,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1472,6 +1492,8 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1491,6 +1513,8 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1552,6 +1576,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/sink/kafka/factory_selector.go b/pkg/sink/kafka/factory_selector.go new file mode 100644 index 0000000000..402454a507 --- /dev/null +++ b/pkg/sink/kafka/factory_selector.go @@ -0,0 +1,39 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strings" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" +) + +// NewFactory selects a Kafka client implementation based on options. +func NewFactory( + ctx context.Context, + o *options, + changefeedID common.ChangeFeedID, +) (Factory, error) { + switch strings.ToLower(strings.TrimSpace(o.KafkaClient)) { + case "", "sarama": + return NewSaramaFactory(ctx, o, changefeedID) + case "franz": + return NewFranzFactory(ctx, o, changefeedID) + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported kafka client %s", o.KafkaClient) + } +} + diff --git a/pkg/sink/kafka/franz_admin_client.go b/pkg/sink/kafka/franz_admin_client.go new file mode 100644 index 0000000000..c6ca0e2bb9 --- /dev/null +++ b/pkg/sink/kafka/franz_admin_client.go @@ -0,0 +1,281 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strconv" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" +) + +type franzAdminClient struct { + changefeed common.ChangeFeedID + + client *kgo.Client + admin *kadm.Client + timeout time.Duration +} + +func newFranzAdminClient( + ctx context.Context, + changefeedID common.ChangeFeedID, + o *options, +) (ClusterAdminClient, error) { + baseOpts, err := buildFranzBaseOptions(ctx, o) + if err != nil { + return nil, errors.Trace(err) + } + + client, err := kgo.NewClient(baseOpts...) + if err != nil { + return nil, errors.Trace(err) + } + + timeout := o.ReadTimeout + if o.WriteTimeout > timeout { + timeout = o.WriteTimeout + } + if timeout <= 0 { + timeout = 10 * time.Second + } + + return &franzAdminClient{ + changefeed: changefeedID, + client: client, + admin: kadm.NewClient(client), + timeout: timeout, + }, nil +} + +func (a *franzAdminClient) newRequestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(a.client.Context(), a.timeout) +} + +func (a *franzAdminClient) GetAllBrokers() []Broker { + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + log.Warn("Kafka admin client fetch broker metadata failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.Error(err)) + return nil + } + + result := make([]Broker, 0, len(meta.Brokers)) + for _, broker := range meta.Brokers { + if broker.NodeID < 0 { + continue + } + result = append(result, Broker{ID: broker.NodeID}) + } + return result +} + +func (a *franzAdminClient) GetBrokerConfig(configName string) (string, error) { + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + return "", errors.Trace(err) + } + if meta.Controller < 0 { + return "", errors.ErrKafkaInvalidConfig.GenWithStack("kafka controller is not available") + } + + configs, err := a.admin.DescribeBrokerConfigs(ctx, meta.Controller) + if err != nil { + return "", errors.Trace(err) + } + + controllerName := strconv.Itoa(int(meta.Controller)) + resource, err := configs.On(controllerName, nil) + if err != nil { + return "", errors.Trace(err) + } + if resource.Err != nil { + return "", errors.Trace(resource.Err) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + return entry.MaybeValue(), nil + } + } + + log.Warn("Kafka config item not found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName)) + return "", errors.ErrKafkaConfigNotFound.GenWithStack( + "cannot find the `%s` from the broker's configuration", configName) +} + +func (a *franzAdminClient) GetTopicConfig(topicName string, configName string) (string, error) { + ctx, cancel := a.newRequestContext() + defer cancel() + + configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) + if err != nil { + return "", errors.Trace(err) + } + + resource, err := configs.On(topicName, nil) + if err != nil { + return "", errors.Trace(err) + } + if resource.Err != nil { + return "", errors.Trace(resource.Err) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + log.Info("Kafka config item found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName), + zap.String("configValue", entry.MaybeValue())) + return entry.MaybeValue(), nil + } + } + + log.Warn("Kafka config item not found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName)) + return "", errors.ErrKafkaConfigNotFound.GenWithStack( + "cannot find the `%s` from the topic's configuration", configName) +} + +func (a *franzAdminClient) GetTopicsMeta( + topics []string, + ignoreTopicError bool, +) (map[string]TopicDetail, error) { + if len(topics) == 0 { + return make(map[string]TopicDetail), nil + } + + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.Metadata(ctx, topics...) + if err != nil { + return nil, errors.Trace(err) + } + + result := make(map[string]TopicDetail, len(topics)) + for _, topic := range topics { + detail, ok := meta.Topics[topic] + if !ok { + continue + } + if detail.Err != nil { + if errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { + continue + } + if !ignoreTopicError { + return nil, errors.Trace(detail.Err) + } + log.Warn("fetch topic meta failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("topic", topic), + zap.Error(detail.Err)) + continue + } + + result[topic] = TopicDetail{ + Name: topic, + NumPartitions: int32(len(detail.Partitions)), + } + } + return result, nil +} + +func (a *franzAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + if len(topics) == 0 { + return make(map[string]int32), nil + } + + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.Metadata(ctx, topics...) + if err != nil { + return nil, errors.Trace(err) + } + + result := make(map[string]int32, len(topics)) + for _, topic := range topics { + detail, ok := meta.Topics[topic] + if !ok { + return nil, errors.Trace(kerr.UnknownTopicOrPartition) + } + if detail.Err != nil { + return nil, errors.Trace(detail.Err) + } + result[topic] = int32(len(detail.Partitions)) + } + return result, nil +} + +func (a *franzAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { + ctx, cancel := a.newRequestContext() + defer cancel() + + var ( + responses kadm.CreateTopicResponses + err error + ) + if validateOnly { + responses, err = a.admin.ValidateCreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) + } else { + responses, err = a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) + } + if err != nil { + return errors.Trace(err) + } + + resp, ok := responses[detail.Name] + if !ok { + return errors.ErrKafkaCreateTopic.GenWithStack("kafka topic create response is missing") + } + if resp.Err != nil { + if errors.Is(resp.Err, kerr.TopicAlreadyExists) { + return nil + } + return errors.Trace(resp.Err) + } + return nil +} + +func (a *franzAdminClient) Heartbeat() {} + +func (a *franzAdminClient) Close() { + if a.admin != nil { + a.admin.Close() + } +} + diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go new file mode 100644 index 0000000000..c94582a443 --- /dev/null +++ b/pkg/sink/kafka/franz_async_producer.go @@ -0,0 +1,166 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/log" + commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +const franzAsyncRecordRetries = 3 + +type franzAsyncProducer struct { + client *kgo.Client + changefeedID commonType.ChangeFeedID + + closed *atomic.Bool + errCh chan error +} + +func newFranzAsyncProducer( + ctx context.Context, + changefeedID commonType.ChangeFeedID, + o *options, +) (AsyncProducer, error) { + baseOpts, err := buildFranzBaseOptions(ctx, o) + if err != nil { + return nil, errors.Trace(err) + } + producerOpts, err := buildFranzProducerOptions(o, franzAsyncRecordRetries) + if err != nil { + return nil, errors.Trace(err) + } + + client, err := kgo.NewClient(append(baseOpts, producerOpts...)...) + if err != nil { + return nil, errors.Trace(err) + } + + return &franzAsyncProducer{ + client: client, + changefeedID: changefeedID, + closed: atomic.NewBool(false), + errCh: make(chan error, 1), + }, nil +} + +func (p *franzAsyncProducer) Close() { + if !p.closed.CompareAndSwap(false, true) { + return + } + + go func() { + start := time.Now() + p.client.Close() + log.Info("Close kafka async producer success", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.Duration("duration", time.Since(start))) + }() +} + +func (p *franzAsyncProducer) AsyncSend( + ctx context.Context, + topic string, + partition int32, + message *common.Message, +) error { + if p.closed.Load() { + return errors.ErrKafkaProducerClosed.GenWithStackByArgs() + } + + select { + case <-ctx.Done(): + return errors.Trace(ctx.Err()) + default: + } + + failpoint.Inject("KafkaSinkAsyncSendError", func() { + log.Info("KafkaSinkAsyncSendError error injected", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name())) + errWithInfo := AnnotateEventError( + p.changefeedID.Keyspace(), + p.changefeedID.Name(), + message.LogInfo, + errors.New("kafka sink injected error"), + ) + select { + case p.errCh <- errors.WrapError(errors.ErrKafkaAsyncSendMessage, errWithInfo): + default: + } + failpoint.Return(nil) + }) + + callback := message.Callback + logInfo := message.LogInfo + + record := &kgo.Record{ + Topic: topic, + Partition: partition, + Key: message.Key, + Value: message.Value, + } + + p.client.Produce(ctx, record, func(_ *kgo.Record, err error) { + if err != nil { + errWithInfo := AnnotateEventError( + p.changefeedID.Keyspace(), + p.changefeedID.Name(), + logInfo, + err, + ) + select { + case p.errCh <- errors.WrapError(errors.ErrKafkaAsyncSendMessage, errWithInfo): + default: + } + return + } + if callback != nil { + callback() + } + }) + + return nil +} + +func (p *franzAsyncProducer) Heartbeat() {} + +func (p *franzAsyncProducer) AsyncRunCallback(ctx context.Context) error { + defer p.closed.Store(true) + for { + select { + case <-ctx.Done(): + log.Info("async producer exit since context is done", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name())) + return errors.Trace(ctx.Err()) + case err := <-p.errCh: + if err == nil { + return nil + } + return err + } + } +} + diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go new file mode 100644 index 0000000000..d0dcb52793 --- /dev/null +++ b/pkg/sink/kafka/franz_factory.go @@ -0,0 +1,294 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "crypto/tls" + "net/url" + "strings" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/kversion" + "github.com/twmb/franz-go/pkg/sasl" + "github.com/twmb/franz-go/pkg/sasl/oauth" + "github.com/twmb/franz-go/pkg/sasl/plain" + "github.com/twmb/franz-go/pkg/sasl/scram" + "go.uber.org/zap" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +type franzFactory struct { + changefeedID common.ChangeFeedID + option *options +} + +// NewFranzFactory constructs a Factory with franz-go implementation. +func NewFranzFactory( + ctx context.Context, + o *options, + changefeedID common.ChangeFeedID, +) (Factory, error) { + admin, err := newFranzAdminClient(ctx, changefeedID, o) + if err != nil { + return nil, errors.Trace(err) + } + defer admin.Close() + + if err = adjustOptions(ctx, admin, o, o.Topic); err != nil { + return nil, errors.Trace(err) + } + + return &franzFactory{ + changefeedID: changefeedID, + option: o, + }, nil +} + +func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { + admin, err := newFranzAdminClient(ctx, f.changefeedID, f.option) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + } + return admin, nil +} + +func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { + producer, err := newFranzSyncProducer(ctx, f.changefeedID, f.option) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + } + return producer, nil +} + +func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { + producer, err := newFranzAsyncProducer(ctx, f.changefeedID, f.option) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + } + return producer, nil +} + +func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { + return &noopMetricsCollector{} +} + +type noopMetricsCollector struct{} + +func (m *noopMetricsCollector) Run(_ context.Context) {} + +func buildFranzBaseOptions( + ctx context.Context, + o *options, +) ([]kgo.Opt, error) { + timeoutOverhead := o.ReadTimeout + if o.WriteTimeout > timeoutOverhead { + timeoutOverhead = o.WriteTimeout + } + if timeoutOverhead <= 0 { + timeoutOverhead = 10 * time.Second + } + + opts := []kgo.Opt{ + kgo.WithContext(ctx), + kgo.SeedBrokers(o.BrokerEndpoints...), + kgo.ClientID(o.ClientID), + kgo.DialTimeout(o.DialTimeout), + kgo.RequestTimeoutOverhead(timeoutOverhead), + } + + if o.IsAssignedVersion { + versions := kversion.FromString(o.Version) + if versions == nil { + return nil, errors.ErrKafkaInvalidVersion.GenWithStack("invalid kafka version %s", o.Version) + } + opts = append(opts, kgo.MaxVersions(versions)) + } + + if o.EnableTLS { + tlsConfig, err := buildFranzTLSConfig(o) + if err != nil { + return nil, errors.Trace(err) + } + opts = append(opts, kgo.DialTLSConfig(tlsConfig)) + } + + if o.SASL != nil && o.SASL.SASLMechanism != "" { + mechanism, err := buildFranzSaslMechanism(ctx, o) + if err != nil { + return nil, errors.Trace(err) + } + opts = append(opts, kgo.SASL(mechanism)) + } + + return opts, nil +} + +func buildFranzTLSConfig(o *options) (*tls.Config, error) { + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + } + + if o.Credential != nil && o.Credential.IsTLSEnabled() { + credentialTlsConfig, err := o.Credential.ToTLSConfig() + if err != nil { + return nil, errors.Trace(err) + } + tlsConfig = credentialTlsConfig + if tlsConfig.MinVersion == 0 { + tlsConfig.MinVersion = tls.VersionTLS12 + } + if len(tlsConfig.NextProtos) == 0 { + tlsConfig.NextProtos = []string{"h2", "http/1.1"} + } + } + + tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify + return tlsConfig, nil +} + +func buildFranzSaslMechanism(ctx context.Context, o *options) (sasl.Mechanism, error) { + if o.SASL == nil { + return nil, nil + } + + switch security.SASLMechanism(strings.ToUpper(string(o.SASL.SASLMechanism))) { + case security.PlainMechanism: + auth := plain.Auth{ + User: o.SASL.SASLUser, + Pass: o.SASL.SASLPassword, + } + return auth.AsMechanism(), nil + case security.SCRAM256Mechanism: + auth := scram.Auth{ + User: o.SASL.SASLUser, + Pass: o.SASL.SASLPassword, + } + return auth.AsSha256Mechanism(), nil + case security.SCRAM512Mechanism: + auth := scram.Auth{ + User: o.SASL.SASLUser, + Pass: o.SASL.SASLPassword, + } + return auth.AsSha512Mechanism(), nil + case security.OAuthMechanism: + tokenSource, err := buildFranzOauthTokenSource(ctx, o) + if err != nil { + return nil, errors.Trace(err) + } + return oauth.Oauth(func(context.Context) (oauth.Auth, error) { + token, err := tokenSource.Token() + if err != nil { + return oauth.Auth{}, errors.Trace(err) + } + return oauth.Auth{Token: token.AccessToken}, nil + }), nil + case security.GSSAPIMechanism: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("sasl gssapi is not supported by franz client") + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) + } +} + +func buildFranzOauthTokenSource(ctx context.Context, o *options) (oauth2.TokenSource, error) { + endpointParams := url.Values{} + if o.SASL.OAuth2.GrantType != "" { + endpointParams.Set("grant_type", o.SASL.OAuth2.GrantType) + } + if o.SASL.OAuth2.Audience != "" { + endpointParams.Set("audience", o.SASL.OAuth2.Audience) + } + + tokenURL, err := url.Parse(o.SASL.OAuth2.TokenURL) + if err != nil { + return nil, errors.Trace(err) + } + + cfg := &clientcredentials.Config{ + ClientID: o.SASL.OAuth2.ClientID, + ClientSecret: o.SASL.OAuth2.ClientSecret, + TokenURL: tokenURL.String(), + EndpointParams: endpointParams, + Scopes: o.SASL.OAuth2.Scopes, + } + return cfg.TokenSource(ctx), nil +} + +func buildFranzProducerOptions( + o *options, + recordRetries int, +) ([]kgo.Opt, error) { + var acks kgo.Acks + switch o.RequiredAcks { + case WaitForAll: + acks = kgo.AllISRAcks() + case WaitForLocal: + acks = kgo.LeaderAck() + case NoResponse: + acks = kgo.NoAck() + default: + acks = kgo.AllISRAcks() + log.Warn("unknown required acks, use all isr acks", zap.Int16("requiredAcks", int16(o.RequiredAcks))) + } + + compressionOpt, err := buildFranzCompressionOption(o) + if err != nil { + return nil, errors.Trace(err) + } + + produceTimeout := o.ReadTimeout + if produceTimeout < 100*time.Millisecond { + produceTimeout = 10 * time.Second + } + + return []kgo.Opt{ + kgo.RecordPartitioner(kgo.ManualPartitioner()), + kgo.RequiredAcks(acks), + kgo.DisableIdempotentWrite(), + kgo.MaxProduceRequestsInflightPerBroker(1), + kgo.RecordRetries(recordRetries), + kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), + kgo.ProduceRequestTimeout(produceTimeout), + kgo.ProducerLinger(0), + compressionOpt, + }, nil +} + +func buildFranzCompressionOption(o *options) (kgo.Opt, error) { + compression := strings.ToLower(strings.TrimSpace(o.Compression)) + switch compression { + case "none": + return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + case "gzip": + return kgo.ProducerBatchCompression(kgo.GzipCompression()), nil + case "snappy": + return kgo.ProducerBatchCompression(kgo.SnappyCompression()), nil + case "lz4": + return kgo.ProducerBatchCompression(kgo.Lz4Compression()), nil + case "zstd": + return kgo.ProducerBatchCompression(kgo.ZstdCompression()), nil + case "": + return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + default: + log.Warn("unsupported compression algorithm", zap.String("compression", o.Compression)) + return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + } +} diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go new file mode 100644 index 0000000000..21928c8579 --- /dev/null +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -0,0 +1,160 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/log" + commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +const franzSyncRecordRetries = 5 + +type franzSyncProducer struct { + id commonType.ChangeFeedID + + client *kgo.Client + closed *atomic.Bool + timeout time.Duration +} + +func newFranzSyncProducer( + ctx context.Context, + changefeedID commonType.ChangeFeedID, + o *options, +) (SyncProducer, error) { + baseOpts, err := buildFranzBaseOptions(ctx, o) + if err != nil { + return nil, errors.Trace(err) + } + producerOpts, err := buildFranzProducerOptions(o, franzSyncRecordRetries) + if err != nil { + return nil, errors.Trace(err) + } + + client, err := kgo.NewClient(append(baseOpts, producerOpts...)...) + if err != nil { + return nil, errors.Trace(err) + } + + produceTimeout := o.ReadTimeout + if produceTimeout <= 0 { + produceTimeout = 10 * time.Second + } + timeout := time.Duration(franzSyncRecordRetries+1) * produceTimeout + + return &franzSyncProducer{ + id: changefeedID, + client: client, + closed: atomic.NewBool(false), + timeout: timeout, + }, nil +} + +func (p *franzSyncProducer) newRequestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(p.client.Context(), p.timeout) +} + +func (p *franzSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { + if p.closed.Load() { + return errors.ErrKafkaProducerClosed.GenWithStackByArgs() + } + + ctx, cancel := p.newRequestContext() + defer cancel() + + record := &kgo.Record{ + Topic: topic, + Partition: partitionNum, + Key: message.Key, + Value: message.Value, + } + err := p.client.ProduceSync(ctx, record).FirstErr() + + failpoint.Inject("KafkaSinkSyncSendMessageError", func() { + err = errors.New("kafka sink sync send message injected error") + }) + + if err != nil { + err = AnnotateEventError( + p.id.Keyspace(), + p.id.Name(), + message.LogInfo, + err, + ) + } + return errors.WrapError(errors.ErrKafkaSendMessage, err) +} + +func (p *franzSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { + if p.closed.Load() { + return errors.ErrKafkaProducerClosed.GenWithStackByArgs() + } + + records := make([]*kgo.Record, 0, partitionNum) + for i := 0; i < int(partitionNum); i++ { + records = append(records, &kgo.Record{ + Topic: topic, + Partition: int32(i), + Key: message.Key, + Value: message.Value, + }) + } + + ctx, cancel := p.newRequestContext() + defer cancel() + + err := p.client.ProduceSync(ctx, records...).FirstErr() + + failpoint.Inject("KafkaSinkSyncSendMessagesError", func() { + err = errors.New("kafka sink sync send messages injected error") + }) + + if err != nil { + err = AnnotateEventError( + p.id.Keyspace(), + p.id.Name(), + message.LogInfo, + err, + ) + } + return errors.WrapError(errors.ErrKafkaSendMessage, err) +} + +func (p *franzSyncProducer) Heartbeat() {} + +func (p *franzSyncProducer) Close() { + if p.closed.Load() { + log.Warn("kafka DDL producer already closed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name())) + return + } + + p.closed.Store(true) + start := time.Now() + p.client.Close() + log.Info("Kafka DDL producer closed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name()), + zap.Duration("duration", time.Since(start))) +} diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 6a5624427f..4d536fdb46 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -120,6 +120,7 @@ type urlConfig struct { KafkaVersion *string `form:"kafka-version"` MaxMessageBytes *int `form:"max-message-bytes"` Compression *string `form:"compression"` + KafkaClient *string `form:"kafka-client"` KafkaClientID *string `form:"kafka-client-id"` AutoCreateTopic *bool `form:"auto-create-topic"` DialTimeout *string `form:"dial-timeout"` @@ -148,6 +149,7 @@ type urlConfig struct { type options struct { Topic string BrokerEndpoints []string + KafkaClient string // control whether to create topic AutoCreate bool @@ -184,6 +186,7 @@ func NewOptions() *options { Version: "2.4.0", // MaxMessageBytes will be used to initialize producer MaxMessageBytes: config.DefaultMaxMessageBytes, + KafkaClient: "sarama", ReplicationFactor: 1, Compression: "none", RequiredAcks: WaitForAll, @@ -265,6 +268,16 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, o.Compression = *urlParameter.Compression } + if urlParameter.KafkaClient != nil && *urlParameter.KafkaClient != "" { + kafkaClient := strings.ToLower(strings.TrimSpace(*urlParameter.KafkaClient)) + switch kafkaClient { + case "sarama", "franz": + o.KafkaClient = kafkaClient + default: + return cerror.ErrKafkaInvalidConfig.GenWithStack("unsupported kafka client %s", kafkaClient) + } + } + var kafkaClientID string if urlParameter.KafkaClientID != nil { kafkaClientID = *urlParameter.KafkaClientID From 2d85ff779197a7129dad9044b935892f019dd018 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Sat, 7 Feb 2026 14:01:11 +0800 Subject: [PATCH 02/61] fix the franz --- pkg/sink/kafka/franz_factory.go | 2 +- pkg/sink/kafka/franz_factory_test.go | 64 ++++++ pkg/sink/kafka/franz_gssapi.go | 292 +++++++++++++++++++++++++++ 3 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 pkg/sink/kafka/franz_factory_test.go create mode 100644 pkg/sink/kafka/franz_gssapi.go diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index d0dcb52793..f04c299d86 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -202,7 +202,7 @@ func buildFranzSaslMechanism(ctx context.Context, o *options) (sasl.Mechanism, e return oauth.Auth{Token: token.AccessToken}, nil }), nil case security.GSSAPIMechanism: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("sasl gssapi is not supported by franz client") + return buildFranzGSSAPIMechanism(o.SASL.GSSAPI) default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) } diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go new file mode 100644 index 0000000000..ba75c6f70f --- /dev/null +++ b/pkg/sink/kafka/franz_factory_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/security" + "github.com/stretchr/testify/require" +) + +func TestBuildFranzSaslMechanismGSSAPIUserAuth(t *testing.T) { + t.Parallel() + + o := NewOptions() + o.SASL = &security.SASL{ + SASLMechanism: security.GSSAPIMechanism, + GSSAPI: security.GSSAPI{ + AuthType: security.UserAuth, + KerberosConfigPath: "/etc/krb5.conf", + ServiceName: "kafka", + Username: "alice", + Password: "pwd", + Realm: "EXAMPLE.COM", + }, + } + + mechanism, err := buildFranzSaslMechanism(context.Background(), o) + require.NoError(t, err) + require.Equal(t, "GSSAPI", mechanism.Name()) +} + +func TestBuildFranzSaslMechanismGSSAPIKeytabAuth(t *testing.T) { + t.Parallel() + + o := NewOptions() + o.SASL = &security.SASL{ + SASLMechanism: security.GSSAPIMechanism, + GSSAPI: security.GSSAPI{ + AuthType: security.KeyTabAuth, + KerberosConfigPath: "/etc/krb5.conf", + ServiceName: "kafka", + Username: "alice", + KeyTabPath: "/tmp/a.keytab", + Realm: "EXAMPLE.COM", + }, + } + + mechanism, err := buildFranzSaslMechanism(context.Background(), o) + require.NoError(t, err) + require.Equal(t, "GSSAPI", mechanism.Name()) +} diff --git a/pkg/sink/kafka/franz_gssapi.go b/pkg/sink/kafka/franz_gssapi.go new file mode 100644 index 0000000000..8ccc3adcec --- /dev/null +++ b/pkg/sink/kafka/franz_gssapi.go @@ -0,0 +1,292 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "encoding/binary" + "fmt" + "strings" + "sync" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + krb5client "github.com/jcmturner/gokrb5/v8/client" + krb5config "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/gssapi" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" + "github.com/twmb/franz-go/pkg/sasl" +) + +const ( + tokIDKrbAPReq = 256 + gssAPIGeneric = 0x60 + gssAPIInitial = 1 + gssAPIVerify = 2 + gssAPIFinished = 3 +) + +type franzKerberosClient interface { + Login() error + GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) + Domain() string + CName() types.PrincipalName + Destroy() +} + +type franzGSSAPIMechanism struct { + config security.GSSAPI +} + +func (m *franzGSSAPIMechanism) Name() string { + return "GSSAPI" +} + +func (m *franzGSSAPIMechanism) Authenticate( + _ context.Context, + host string, +) (sasl.Session, []byte, error) { + client, err := newFranzKerberosClient(m.config) + if err != nil { + return nil, nil, errors.Trace(err) + } + if err = client.Login(); err != nil { + client.Destroy() + return nil, nil, errors.Trace(err) + } + + serverHost := strings.SplitN(host, ":", 2)[0] + spn := fmt.Sprintf("%s/%s", m.config.ServiceName, serverHost) + ticket, encKey, err := client.GetServiceTicket(spn) + if err != nil { + client.Destroy() + return nil, nil, errors.Trace(err) + } + + session := &franzGSSAPISession{ + client: client, + ticket: ticket, + encKey: encKey, + step: gssAPIInitial, + } + firstMessage, err := session.nextMessage(nil) + if err != nil { + session.close() + return nil, nil, errors.Trace(err) + } + return session, firstMessage, nil +} + +type franzGSSAPISession struct { + client franzKerberosClient + ticket messages.Ticket + encKey types.EncryptionKey + step int + + closeOnce sync.Once +} + +func (s *franzGSSAPISession) Challenge(challenge []byte) (bool, []byte, error) { + switch s.step { + case gssAPIVerify: + msg, err := s.nextMessage(challenge) + if err != nil { + s.close() + return false, nil, errors.Trace(err) + } + // Return a final payload while marking done=true. + // franz-go will write this message and finish the auth flow. + s.close() + return true, msg, nil + case gssAPIFinished: + s.close() + return true, nil, nil + default: + s.close() + return false, nil, errors.New("invalid gssapi session state") + } +} + +func (s *franzGSSAPISession) close() { + s.closeOnce.Do(func() { + if s.client != nil { + s.client.Destroy() + } + }) +} + +func (s *franzGSSAPISession) nextMessage(challenge []byte) ([]byte, error) { + switch s.step { + case gssAPIInitial: + token, err := createKrb5Token(s.client.Domain(), s.client.CName(), s.ticket, s.encKey) + if err != nil { + return nil, errors.Trace(err) + } + s.step = gssAPIVerify + return appendGSSAPIHeader(token) + case gssAPIVerify: + wrapTokenReq := gssapi.WrapToken{} + if err := wrapTokenReq.Unmarshal(challenge, true); err != nil { + return nil, errors.Trace(err) + } + isValid, err := wrapTokenReq.Verify(s.encKey, keyusage.GSSAPI_ACCEPTOR_SEAL) + if !isValid { + if err != nil { + return nil, errors.Trace(err) + } + return nil, errors.New("invalid gssapi wrap token") + } + + wrapTokenResp, err := gssapi.NewInitiatorWrapToken(wrapTokenReq.Payload, s.encKey) + if err != nil { + return nil, errors.Trace(err) + } + s.step = gssAPIFinished + return wrapTokenResp.Marshal() + default: + return nil, errors.New("invalid gssapi session state") + } +} + +func buildFranzGSSAPIMechanism(g security.GSSAPI) (sasl.Mechanism, error) { + if g.ServiceName == "" { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") + } + if g.KerberosConfigPath == "" { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") + } + if g.Username == "" { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") + } + if g.Realm == "" { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") + } + + switch g.AuthType { + case security.UserAuth: + if g.Password == "" { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") + } + case security.KeyTabAuth: + if g.KeyTabPath == "" { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") + } + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "unsupported sasl-gssapi-auth-type %d", g.AuthType) + } + + return &franzGSSAPIMechanism{config: g}, nil +} + +type franzGoKrb5Client struct { + krb5client.Client +} + +func (c *franzGoKrb5Client) Domain() string { + return c.Credentials.Domain() +} + +func (c *franzGoKrb5Client) CName() types.PrincipalName { + return c.Credentials.CName() +} + +func newFranzKerberosClient(g security.GSSAPI) (franzKerberosClient, error) { + cfg, err := krb5config.Load(g.KerberosConfigPath) + if err != nil { + return nil, errors.Trace(err) + } + + var client *krb5client.Client + switch g.AuthType { + case security.KeyTabAuth: + kt, err := keytab.Load(g.KeyTabPath) + if err != nil { + return nil, errors.Trace(err) + } + client = krb5client.NewWithKeytab( + g.Username, g.Realm, kt, cfg, krb5client.DisablePAFXFAST(g.DisablePAFXFAST)) + case security.UserAuth: + client = krb5client.NewWithPassword( + g.Username, g.Realm, g.Password, cfg, krb5client.DisablePAFXFAST(g.DisablePAFXFAST)) + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "unsupported sasl-gssapi-auth-type %d", g.AuthType) + } + return &franzGoKrb5Client{*client}, nil +} + +func createKrb5Token( + domain string, + cname types.PrincipalName, + ticket messages.Ticket, + sessionKey types.EncryptionKey, +) ([]byte, error) { + authenticator, err := types.NewAuthenticator(domain, cname) + if err != nil { + return nil, errors.Trace(err) + } + + authenticator.Cksum = types.Checksum{ + CksumType: chksumtype.GSSAPI, + Checksum: newAuthenticatorChecksum(), + } + apReq, err := messages.NewAPReq(ticket, sessionKey, authenticator) + if err != nil { + return nil, errors.Trace(err) + } + + prefix := make([]byte, 2) + binary.BigEndian.PutUint16(prefix, tokIDKrbAPReq) + body, err := apReq.Marshal() + if err != nil { + return nil, errors.Trace(err) + } + return append(prefix, body...), nil +} + +func newAuthenticatorChecksum() []byte { + sum := make([]byte, 24) + flags := []int{gssapi.ContextFlagInteg, gssapi.ContextFlagConf} + binary.LittleEndian.PutUint32(sum[:4], 16) + for _, flag := range flags { + current := binary.LittleEndian.Uint32(sum[20:24]) + current |= uint32(flag) + binary.LittleEndian.PutUint32(sum[20:24], current) + } + return sum +} + +func appendGSSAPIHeader(payload []byte) ([]byte, error) { + oidBytes, err := asn1.Marshal(gssapi.OIDKRB5.OID()) + if err != nil { + return nil, errors.Trace(err) + } + tkoLengthBytes := asn1tools.MarshalLengthBytes(len(oidBytes) + len(payload)) + header := append([]byte{gssAPIGeneric}, tkoLengthBytes...) + header = append(header, oidBytes...) + return append(header, payload...), nil +} From 0945129dbd01703c4543fc232e42335ae684ac1b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Sat, 7 Feb 2026 14:16:12 +0800 Subject: [PATCH 03/61] fix the franz --- pkg/sink/kafka/franz_admin_client.go | 4 +- pkg/sink/kafka/franz_async_producer.go | 4 +- pkg/sink/kafka/franz_factory.go | 21 +- pkg/sink/kafka/franz_factory_test.go | 43 ++++ pkg/sink/kafka/franz_metrics_collector.go | 233 ++++++++++++++++++++++ pkg/sink/kafka/franz_sync_producer.go | 3 +- 6 files changed, 298 insertions(+), 10 deletions(-) create mode 100644 pkg/sink/kafka/franz_metrics_collector.go diff --git a/pkg/sink/kafka/franz_admin_client.go b/pkg/sink/kafka/franz_admin_client.go index c6ca0e2bb9..dda18d2a80 100644 --- a/pkg/sink/kafka/franz_admin_client.go +++ b/pkg/sink/kafka/franz_admin_client.go @@ -39,8 +39,9 @@ func newFranzAdminClient( ctx context.Context, changefeedID common.ChangeFeedID, o *options, + hook kgo.Hook, ) (ClusterAdminClient, error) { - baseOpts, err := buildFranzBaseOptions(ctx, o) + baseOpts, err := buildFranzBaseOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) } @@ -278,4 +279,3 @@ func (a *franzAdminClient) Close() { a.admin.Close() } } - diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index c94582a443..2b3a44e14e 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -41,8 +41,9 @@ func newFranzAsyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, o *options, + hook kgo.Hook, ) (AsyncProducer, error) { - baseOpts, err := buildFranzBaseOptions(ctx, o) + baseOpts, err := buildFranzBaseOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) } @@ -163,4 +164,3 @@ func (p *franzAsyncProducer) AsyncRunCallback(ctx context.Context) error { } } } - diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index f04c299d86..33a85a4346 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -38,6 +38,7 @@ import ( type franzFactory struct { changefeedID common.ChangeFeedID option *options + metricsHook *franzMetricsHook } // NewFranzFactory constructs a Factory with franz-go implementation. @@ -46,7 +47,7 @@ func NewFranzFactory( o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - admin, err := newFranzAdminClient(ctx, changefeedID, o) + admin, err := newFranzAdminClient(ctx, changefeedID, o, nil) if err != nil { return nil, errors.Trace(err) } @@ -56,14 +57,17 @@ func NewFranzFactory( return nil, errors.Trace(err) } + metricsHook := newFranzMetricsHook() + return &franzFactory{ changefeedID: changefeedID, option: o, + metricsHook: metricsHook, }, nil } func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - admin, err := newFranzAdminClient(ctx, f.changefeedID, f.option) + admin, err := newFranzAdminClient(ctx, f.changefeedID, f.option, f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -71,7 +75,7 @@ func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, err } func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := newFranzSyncProducer(ctx, f.changefeedID, f.option) + producer, err := newFranzSyncProducer(ctx, f.changefeedID, f.option, f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -79,7 +83,7 @@ func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { } func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := newFranzAsyncProducer(ctx, f.changefeedID, f.option) + producer, err := newFranzAsyncProducer(ctx, f.changefeedID, f.option, f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -87,7 +91,10 @@ func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) } func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { - return &noopMetricsCollector{} + return &franzMetricsCollector{ + changefeedID: f.changefeedID, + hook: f.metricsHook, + } } type noopMetricsCollector struct{} @@ -97,6 +104,7 @@ func (m *noopMetricsCollector) Run(_ context.Context) {} func buildFranzBaseOptions( ctx context.Context, o *options, + hook kgo.Hook, ) ([]kgo.Opt, error) { timeoutOverhead := o.ReadTimeout if o.WriteTimeout > timeoutOverhead { @@ -113,6 +121,9 @@ func buildFranzBaseOptions( kgo.DialTimeout(o.DialTimeout), kgo.RequestTimeoutOverhead(timeoutOverhead), } + if hook != nil { + opts = append(opts, kgo.WithHooks(hook)) + } if o.IsAssignedVersion { versions := kversion.FromString(o.Version) diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go index ba75c6f70f..83fdd5e397 100644 --- a/pkg/sink/kafka/franz_factory_test.go +++ b/pkg/sink/kafka/franz_factory_test.go @@ -17,8 +17,11 @@ import ( "context" "testing" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/security" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" ) func TestBuildFranzSaslMechanismGSSAPIUserAuth(t *testing.T) { @@ -62,3 +65,43 @@ func TestBuildFranzSaslMechanismGSSAPIKeytabAuth(t *testing.T) { require.NoError(t, err) require.Equal(t, "GSSAPI", mechanism.Name()) } + +func TestFranzFactoryMetricsCollectorIsNotNoop(t *testing.T) { + t.Parallel() + + f := &franzFactory{} + collector := f.MetricsCollector(nil) + + _, isNoop := collector.(*noopMetricsCollector) + require.False(t, isNoop) +} + +func TestFranzMetricsCollectorCollectMetrics(t *testing.T) { + t.Parallel() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceNamme, "franz-metrics") + hook := newFranzMetricsHook() + collector := &franzMetricsCollector{ + changefeedID: changefeedID, + hook: hook, + } + t.Cleanup(func() { + collector.cleanupMetrics() + }) + + meta := kgo.BrokerMetadata{NodeID: 1} + hook.OnBrokerWrite(meta, 0, 128, 0, 0, nil) + hook.OnProduceBatchWritten(meta, "topic", 0, kgo.ProduceBatchMetrics{ + NumRecords: 8, + UncompressedBytes: 400, + CompressedBytes: 200, + }) + + collector.collectMetrics() + + keyspace := changefeedID.Keyspace() + changefeed := changefeedID.Name() + require.Equal(t, float64(1), testutil.ToFloat64(requestsInFlightGauge.WithLabelValues(keyspace, changefeed, "1"))) + require.Greater(t, testutil.ToFloat64(compressionRatioGauge.WithLabelValues(keyspace, changefeed, avg)), 0.0) + require.Greater(t, testutil.ToFloat64(recordsPerRequestGauge.WithLabelValues(keyspace, changefeed, avg)), 0.0) +} diff --git a/pkg/sink/kafka/franz_metrics_collector.go b/pkg/sink/kafka/franz_metrics_collector.go new file mode 100644 index 0000000000..3eaef5ba36 --- /dev/null +++ b/pkg/sink/kafka/franz_metrics_collector.go @@ -0,0 +1,233 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/rcrowley/go-metrics" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" +) + +type franzBrokerMetrics struct { + outgoingByteRate metrics.Meter + requestRate metrics.Meter + requestLatency metrics.Histogram + responseRate metrics.Meter + inFlight int64 +} + +func newFranzBrokerMetrics() *franzBrokerMetrics { + return &franzBrokerMetrics{ + outgoingByteRate: metrics.NewMeter(), + requestRate: metrics.NewMeter(), + requestLatency: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), + responseRate: metrics.NewMeter(), + } +} + +type franzMetricsHook struct { + mu sync.RWMutex + brokers map[int32]*franzBrokerMetrics + + compressionRatio metrics.Histogram + recordsPerReq metrics.Histogram +} + +func newFranzMetricsHook() *franzMetricsHook { + return &franzMetricsHook{ + brokers: make(map[int32]*franzBrokerMetrics), + compressionRatio: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), + recordsPerReq: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), + } +} + +func (h *franzMetricsHook) getBroker(nodeID int32) *franzBrokerMetrics { + if nodeID < 0 { + return nil + } + + h.mu.RLock() + broker := h.brokers[nodeID] + h.mu.RUnlock() + if broker != nil { + return broker + } + + h.mu.Lock() + defer h.mu.Unlock() + broker = h.brokers[nodeID] + if broker != nil { + return broker + } + broker = newFranzBrokerMetrics() + h.brokers[nodeID] = broker + return broker +} + +func (h *franzMetricsHook) snapshotBrokers() map[int32]*franzBrokerMetrics { + h.mu.RLock() + defer h.mu.RUnlock() + result := make(map[int32]*franzBrokerMetrics, len(h.brokers)) + for id, broker := range h.brokers { + result[id] = broker + } + return result +} + +func (h *franzMetricsHook) OnBrokerWrite( + meta kgo.BrokerMetadata, + _ int16, + bytesWritten int, + _ time.Duration, + _ time.Duration, + err error, +) { + broker := h.getBroker(meta.NodeID) + if broker == nil { + return + } + + if bytesWritten > 0 { + broker.outgoingByteRate.Mark(int64(bytesWritten)) + } + broker.requestRate.Mark(1) + if err == nil { + atomic.AddInt64(&broker.inFlight, 1) + } +} + +func (h *franzMetricsHook) OnBrokerE2E( + meta kgo.BrokerMetadata, + _ int16, + e2e kgo.BrokerE2E, +) { + broker := h.getBroker(meta.NodeID) + if broker == nil { + return + } + + if e2e.WriteErr == nil { + if atomic.AddInt64(&broker.inFlight, -1) < 0 { + atomic.StoreInt64(&broker.inFlight, 0) + } + } + if e2e.BytesRead > 0 && e2e.ReadErr == nil { + broker.responseRate.Mark(1) + } + if e2e.Err() == nil { + broker.requestLatency.Update(e2e.DurationE2E().Microseconds()) + } +} + +func (h *franzMetricsHook) OnProduceBatchWritten( + _ kgo.BrokerMetadata, + _ string, + _ int32, + m kgo.ProduceBatchMetrics, +) { + if m.NumRecords > 0 { + h.recordsPerReq.Update(int64(m.NumRecords)) + } + if m.UncompressedBytes > 0 && m.CompressedBytes > 0 { + ratio := int64(float64(m.UncompressedBytes) / float64(m.CompressedBytes) * 100) + h.compressionRatio.Update(ratio) + } +} + +type franzMetricsCollector struct { + changefeedID common.ChangeFeedID + hook *franzMetricsHook +} + +func (m *franzMetricsCollector) Run(ctx context.Context) { + ticker := time.NewTicker(refreshMetricsInterval) + defer func() { + ticker.Stop() + m.cleanupMetrics() + }() + + for { + select { + case <-ctx.Done(): + log.Info("franz kafka metrics collector stopped", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name())) + return + case <-ticker.C: + m.collectMetrics() + } + } +} + +func (m *franzMetricsCollector) collectMetrics() { + keyspace := m.changefeedID.Keyspace() + changefeedID := m.changefeedID.Name() + + compressionSnapshot := m.hook.compressionRatio.Snapshot() + compressionRatioGauge.WithLabelValues(keyspace, changefeedID, avg).Set(compressionSnapshot.Mean()) + compressionRatioGauge.WithLabelValues(keyspace, changefeedID, p99).Set(compressionSnapshot.Percentile(0.99)) + + recordsSnapshot := m.hook.recordsPerReq.Snapshot() + recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, avg).Set(recordsSnapshot.Mean()) + recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, p99).Set(recordsSnapshot.Percentile(0.99)) + + for id, broker := range m.hook.snapshotBrokers() { + brokerID := strconv.Itoa(int(id)) + OutgoingByteRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( + broker.outgoingByteRate.Snapshot().Rate1(), + ) + RequestRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( + broker.requestRate.Snapshot().Rate1(), + ) + RequestLatencyGauge.WithLabelValues(keyspace, changefeedID, brokerID, avg).Set( + broker.requestLatency.Snapshot().Mean() / 1000, + ) + RequestLatencyGauge.WithLabelValues(keyspace, changefeedID, brokerID, p99).Set( + broker.requestLatency.Snapshot().Percentile(0.99) / 1000, + ) + requestsInFlightGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( + float64(atomic.LoadInt64(&broker.inFlight)), + ) + responseRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( + broker.responseRate.Snapshot().Rate1(), + ) + } +} + +func (m *franzMetricsCollector) cleanupMetrics() { + keyspace := m.changefeedID.Keyspace() + changefeedID := m.changefeedID.Name() + compressionRatioGauge.DeleteLabelValues(keyspace, changefeedID, avg) + compressionRatioGauge.DeleteLabelValues(keyspace, changefeedID, p99) + recordsPerRequestGauge.DeleteLabelValues(keyspace, changefeedID, avg) + recordsPerRequestGauge.DeleteLabelValues(keyspace, changefeedID, p99) + + for id := range m.hook.snapshotBrokers() { + brokerID := strconv.Itoa(int(id)) + OutgoingByteRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) + RequestRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) + RequestLatencyGauge.DeleteLabelValues(keyspace, changefeedID, brokerID, avg) + RequestLatencyGauge.DeleteLabelValues(keyspace, changefeedID, brokerID, p99) + requestsInFlightGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) + responseRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) + } +} diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go index 21928c8579..b0541adbc2 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -41,8 +41,9 @@ func newFranzSyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, o *options, + hook kgo.Hook, ) (SyncProducer, error) { - baseOpts, err := buildFranzBaseOptions(ctx, o) + baseOpts, err := buildFranzBaseOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) } From 539dd0c9697350a0ffd7cb945d3a7e37077761a5 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Sat, 7 Feb 2026 14:30:17 +0800 Subject: [PATCH 04/61] kafka: default sink client to franz --- pkg/sink/kafka/factory_selector.go | 7 +++---- pkg/sink/kafka/options.go | 2 +- pkg/sink/kafka/options_test.go | 7 +++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/sink/kafka/factory_selector.go b/pkg/sink/kafka/factory_selector.go index 402454a507..0aa47abe25 100644 --- a/pkg/sink/kafka/factory_selector.go +++ b/pkg/sink/kafka/factory_selector.go @@ -28,12 +28,11 @@ func NewFactory( changefeedID common.ChangeFeedID, ) (Factory, error) { switch strings.ToLower(strings.TrimSpace(o.KafkaClient)) { - case "", "sarama": - return NewSaramaFactory(ctx, o, changefeedID) - case "franz": + case "", "franz": return NewFranzFactory(ctx, o, changefeedID) + case "sarama": + return NewSaramaFactory(ctx, o, changefeedID) default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported kafka client %s", o.KafkaClient) } } - diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 4d536fdb46..5f4d643f14 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -186,7 +186,7 @@ func NewOptions() *options { Version: "2.4.0", // MaxMessageBytes will be used to initialize producer MaxMessageBytes: config.DefaultMaxMessageBytes, - KafkaClient: "sarama", + KafkaClient: "franz", ReplicationFactor: 1, Compression: "none", RequiredAcks: WaitForAll, diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 25721d21e7..7a317844ee 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -111,6 +111,13 @@ func TestCompleteOptions(t *testing.T) { require.True(t, cerror.ErrKafkaInvalidClientID.Equal(err)) } +func TestNewOptionsDefaultKafkaClient(t *testing.T) { + t.Parallel() + + options := NewOptions() + require.Equal(t, "franz", options.KafkaClient) +} + func TestSetPartitionNum(t *testing.T) { options := NewOptions() err := options.setPartitionNum(2) From 2c341dc44b925b82aa9b8764104318322145d101 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 9 Feb 2026 12:30:25 +0000 Subject: [PATCH 05/61] add more code --- pkg/sink/kafka/franz/admin_client.go | 293 ++++++++++++++++++ .../async_producer.go} | 25 +- pkg/sink/kafka/franz/factory.go | 267 ++++++++++++++++ .../{franz_gssapi.go => franz/gssapi.go} | 2 +- pkg/sink/kafka/franz/metrics_hook.go | 206 ++++++++++++ pkg/sink/kafka/franz/sasl_test.go | 66 ++++ .../sync_producer.go} | 27 +- pkg/sink/kafka/franz_admin_client.go | 271 +++------------- pkg/sink/kafka/franz_factory.go | 255 ++------------- pkg/sink/kafka/franz_factory_test.go | 69 +---- pkg/sink/kafka/franz_metrics_collector.go | 170 +--------- pkg/sink/kafka/internal/logutil/logutil.go | 107 +++++++ pkg/sink/kafka/logutil.go | 77 +---- pkg/sink/kafka/logutil_test.go | 5 +- pkg/sink/kafka/main_test.go | 6 +- 15 files changed, 1078 insertions(+), 768 deletions(-) create mode 100644 pkg/sink/kafka/franz/admin_client.go rename pkg/sink/kafka/{franz_async_producer.go => franz/async_producer.go} (88%) create mode 100644 pkg/sink/kafka/franz/factory.go rename pkg/sink/kafka/{franz_gssapi.go => franz/gssapi.go} (99%) create mode 100644 pkg/sink/kafka/franz/metrics_hook.go create mode 100644 pkg/sink/kafka/franz/sasl_test.go rename pkg/sink/kafka/{franz_sync_producer.go => franz/sync_producer.go} (84%) create mode 100644 pkg/sink/kafka/internal/logutil/logutil.go diff --git a/pkg/sink/kafka/franz/admin_client.go b/pkg/sink/kafka/franz/admin_client.go new file mode 100644 index 0000000000..014417ec93 --- /dev/null +++ b/pkg/sink/kafka/franz/admin_client.go @@ -0,0 +1,293 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "strconv" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" +) + +// TopicDetail represent a topic's detail information. +type TopicDetail struct { + Name string + NumPartitions int32 + ReplicationFactor int16 +} + +// Broker represents a Kafka broker. +type Broker struct { + ID int32 +} + +type AdminClient struct { + changefeed common.ChangeFeedID + + client *kgo.Client + admin *kadm.Client + timeout time.Duration +} + +func NewAdminClient( + ctx context.Context, + changefeedID common.ChangeFeedID, + o *Options, + hook kgo.Hook, +) (*AdminClient, error) { + baseOpts, err := buildFranzBaseOptions(ctx, o, hook) + if err != nil { + return nil, errors.Trace(err) + } + + client, err := kgo.NewClient(baseOpts...) + if err != nil { + return nil, errors.Trace(err) + } + + timeout := o.ReadTimeout + if o.WriteTimeout > timeout { + timeout = o.WriteTimeout + } + if timeout <= 0 { + timeout = 10 * time.Second + } + + return &AdminClient{ + changefeed: changefeedID, + client: client, + admin: kadm.NewClient(client), + timeout: timeout, + }, nil +} + +func (a *AdminClient) newRequestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(a.client.Context(), a.timeout) +} + +func (a *AdminClient) GetAllBrokers() []Broker { + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + log.Warn("Kafka admin client fetch broker metadata failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.Error(err)) + return nil + } + + result := make([]Broker, 0, len(meta.Brokers)) + for _, broker := range meta.Brokers { + if broker.NodeID < 0 { + continue + } + result = append(result, Broker{ID: broker.NodeID}) + } + return result +} + +func (a *AdminClient) GetBrokerConfig(configName string) (string, error) { + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + return "", errors.Trace(err) + } + if meta.Controller < 0 { + return "", errors.ErrKafkaInvalidConfig.GenWithStack("kafka controller is not available") + } + + configs, err := a.admin.DescribeBrokerConfigs(ctx, meta.Controller) + if err != nil { + return "", errors.Trace(err) + } + + controllerName := strconv.Itoa(int(meta.Controller)) + resource, err := configs.On(controllerName, nil) + if err != nil { + return "", errors.Trace(err) + } + if resource.Err != nil { + return "", errors.Trace(resource.Err) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + return entry.MaybeValue(), nil + } + } + + log.Warn("Kafka config item not found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName)) + return "", errors.ErrKafkaConfigNotFound.GenWithStack( + "cannot find the `%s` from the broker's configuration", configName) +} + +func (a *AdminClient) GetTopicConfig(topicName string, configName string) (string, error) { + ctx, cancel := a.newRequestContext() + defer cancel() + + configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) + if err != nil { + return "", errors.Trace(err) + } + + resource, err := configs.On(topicName, nil) + if err != nil { + return "", errors.Trace(err) + } + if resource.Err != nil { + return "", errors.Trace(resource.Err) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + log.Info("Kafka config item found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName), + zap.String("configValue", entry.MaybeValue())) + return entry.MaybeValue(), nil + } + } + + log.Warn("Kafka config item not found", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("configName", configName)) + return "", errors.ErrKafkaConfigNotFound.GenWithStack( + "cannot find the `%s` from the topic's configuration", configName) +} + +func (a *AdminClient) GetTopicsMeta( + topics []string, + ignoreTopicError bool, +) (map[string]TopicDetail, error) { + if len(topics) == 0 { + return make(map[string]TopicDetail), nil + } + + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.Metadata(ctx, topics...) + if err != nil { + return nil, errors.Trace(err) + } + + result := make(map[string]TopicDetail, len(topics)) + for _, topic := range topics { + detail, ok := meta.Topics[topic] + if !ok { + continue + } + if detail.Err != nil { + if errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { + continue + } + if !ignoreTopicError { + return nil, errors.Trace(detail.Err) + } + log.Warn("fetch topic meta failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("topic", topic), + zap.Error(detail.Err)) + continue + } + + result[topic] = TopicDetail{ + Name: topic, + NumPartitions: int32(len(detail.Partitions)), + } + } + return result, nil +} + +func (a *AdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + if len(topics) == 0 { + return make(map[string]int32), nil + } + + ctx, cancel := a.newRequestContext() + defer cancel() + + meta, err := a.admin.Metadata(ctx, topics...) + if err != nil { + return nil, errors.Trace(err) + } + + result := make(map[string]int32, len(topics)) + for _, topic := range topics { + detail, ok := meta.Topics[topic] + if !ok { + return nil, errors.Trace(kerr.UnknownTopicOrPartition) + } + if detail.Err != nil { + return nil, errors.Trace(detail.Err) + } + result[topic] = int32(len(detail.Partitions)) + } + return result, nil +} + +func (a *AdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { + ctx, cancel := a.newRequestContext() + defer cancel() + + var ( + responses kadm.CreateTopicResponses + err error + ) + if validateOnly { + responses, err = a.admin.ValidateCreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) + } else { + responses, err = a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) + } + if err != nil { + return errors.Trace(err) + } + + resp, ok := responses[detail.Name] + if !ok { + return errors.ErrKafkaCreateTopic.GenWithStack("kafka topic create response is missing") + } + if resp.Err != nil { + if errors.Is(resp.Err, kerr.TopicAlreadyExists) { + return nil + } + return errors.Trace(resp.Err) + } + return nil +} + +func (a *AdminClient) Heartbeat() {} + +func (a *AdminClient) Close() { + if a.admin != nil { + a.admin.Close() + } +} diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz/async_producer.go similarity index 88% rename from pkg/sink/kafka/franz_async_producer.go rename to pkg/sink/kafka/franz/async_producer.go index 2b3a44e14e..cca0185181 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz/async_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kafka +package franz import ( "context" @@ -22,6 +22,7 @@ import ( commonType "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/kafka/internal/logutil" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/atomic" "go.uber.org/zap" @@ -29,7 +30,7 @@ import ( const franzAsyncRecordRetries = 3 -type franzAsyncProducer struct { +type AsyncProducer struct { client *kgo.Client changefeedID commonType.ChangeFeedID @@ -37,12 +38,12 @@ type franzAsyncProducer struct { errCh chan error } -func newFranzAsyncProducer( +func NewAsyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, - o *options, + o *Options, hook kgo.Hook, -) (AsyncProducer, error) { +) (*AsyncProducer, error) { baseOpts, err := buildFranzBaseOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -57,7 +58,7 @@ func newFranzAsyncProducer( return nil, errors.Trace(err) } - return &franzAsyncProducer{ + return &AsyncProducer{ client: client, changefeedID: changefeedID, closed: atomic.NewBool(false), @@ -65,7 +66,7 @@ func newFranzAsyncProducer( }, nil } -func (p *franzAsyncProducer) Close() { +func (p *AsyncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { return } @@ -80,7 +81,7 @@ func (p *franzAsyncProducer) Close() { }() } -func (p *franzAsyncProducer) AsyncSend( +func (p *AsyncProducer) AsyncSend( ctx context.Context, topic string, partition int32, @@ -100,7 +101,7 @@ func (p *franzAsyncProducer) AsyncSend( log.Info("KafkaSinkAsyncSendError error injected", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name())) - errWithInfo := AnnotateEventError( + errWithInfo := logutil.AnnotateEventError( p.changefeedID.Keyspace(), p.changefeedID.Name(), message.LogInfo, @@ -125,7 +126,7 @@ func (p *franzAsyncProducer) AsyncSend( p.client.Produce(ctx, record, func(_ *kgo.Record, err error) { if err != nil { - errWithInfo := AnnotateEventError( + errWithInfo := logutil.AnnotateEventError( p.changefeedID.Keyspace(), p.changefeedID.Name(), logInfo, @@ -145,9 +146,9 @@ func (p *franzAsyncProducer) AsyncSend( return nil } -func (p *franzAsyncProducer) Heartbeat() {} +func (p *AsyncProducer) Heartbeat() {} -func (p *franzAsyncProducer) AsyncRunCallback(ctx context.Context) error { +func (p *AsyncProducer) AsyncRunCallback(ctx context.Context) error { defer p.closed.Store(true) for { select { diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/franz/factory.go new file mode 100644 index 0000000000..603038a54d --- /dev/null +++ b/pkg/sink/kafka/franz/factory.go @@ -0,0 +1,267 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "crypto/tls" + "net/url" + "strings" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/kversion" + "github.com/twmb/franz-go/pkg/sasl" + "github.com/twmb/franz-go/pkg/sasl/oauth" + "github.com/twmb/franz-go/pkg/sasl/plain" + "github.com/twmb/franz-go/pkg/sasl/scram" + "go.uber.org/zap" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +type RequiredAcks int16 + +const ( + NoResponse RequiredAcks = 0 + WaitForLocal RequiredAcks = 1 + WaitForAll RequiredAcks = -1 +) + +type Options struct { + BrokerEndpoints []string + ClientID string + + Version string + IsAssignedVersion bool + + MaxMessageBytes int + Compression string + RequiredAcks RequiredAcks + + EnableTLS bool + Credential *security.Credential + InsecureSkipVerify bool + SASL *security.SASL + + DialTimeout time.Duration + WriteTimeout time.Duration + ReadTimeout time.Duration +} + +func buildFranzBaseOptions( + ctx context.Context, + o *Options, + hook kgo.Hook, +) ([]kgo.Opt, error) { + timeoutOverhead := o.ReadTimeout + if o.WriteTimeout > timeoutOverhead { + timeoutOverhead = o.WriteTimeout + } + if timeoutOverhead <= 0 { + timeoutOverhead = 10 * time.Second + } + + opts := []kgo.Opt{ + kgo.WithContext(ctx), + kgo.SeedBrokers(o.BrokerEndpoints...), + kgo.ClientID(o.ClientID), + kgo.DialTimeout(o.DialTimeout), + kgo.RequestTimeoutOverhead(timeoutOverhead), + } + if hook != nil { + opts = append(opts, kgo.WithHooks(hook)) + } + + if o.IsAssignedVersion { + versions := kversion.FromString(o.Version) + if versions == nil { + return nil, errors.ErrKafkaInvalidVersion.GenWithStack("invalid kafka version %s", o.Version) + } + opts = append(opts, kgo.MaxVersions(versions)) + } + + if o.EnableTLS { + tlsConfig, err := buildFranzTLSConfig(o) + if err != nil { + return nil, errors.Trace(err) + } + opts = append(opts, kgo.DialTLSConfig(tlsConfig)) + } + + if o.SASL != nil && o.SASL.SASLMechanism != "" { + mechanism, err := buildFranzSaslMechanism(ctx, o) + if err != nil { + return nil, errors.Trace(err) + } + opts = append(opts, kgo.SASL(mechanism)) + } + + return opts, nil +} + +func buildFranzTLSConfig(o *Options) (*tls.Config, error) { + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + } + + if o.Credential != nil && o.Credential.IsTLSEnabled() { + credentialTlsConfig, err := o.Credential.ToTLSConfig() + if err != nil { + return nil, errors.Trace(err) + } + tlsConfig = credentialTlsConfig + if tlsConfig.MinVersion == 0 { + tlsConfig.MinVersion = tls.VersionTLS12 + } + if len(tlsConfig.NextProtos) == 0 { + tlsConfig.NextProtos = []string{"h2", "http/1.1"} + } + } + + tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify + return tlsConfig, nil +} + +func buildFranzSaslMechanism(ctx context.Context, o *Options) (sasl.Mechanism, error) { + if o.SASL == nil { + return nil, nil + } + + switch security.SASLMechanism(strings.ToUpper(string(o.SASL.SASLMechanism))) { + case security.PlainMechanism: + auth := plain.Auth{ + User: o.SASL.SASLUser, + Pass: o.SASL.SASLPassword, + } + return auth.AsMechanism(), nil + case security.SCRAM256Mechanism: + auth := scram.Auth{ + User: o.SASL.SASLUser, + Pass: o.SASL.SASLPassword, + } + return auth.AsSha256Mechanism(), nil + case security.SCRAM512Mechanism: + auth := scram.Auth{ + User: o.SASL.SASLUser, + Pass: o.SASL.SASLPassword, + } + return auth.AsSha512Mechanism(), nil + case security.OAuthMechanism: + tokenSource, err := buildFranzOauthTokenSource(ctx, o) + if err != nil { + return nil, errors.Trace(err) + } + return oauth.Oauth(func(context.Context) (oauth.Auth, error) { + token, err := tokenSource.Token() + if err != nil { + return oauth.Auth{}, errors.Trace(err) + } + return oauth.Auth{Token: token.AccessToken}, nil + }), nil + case security.GSSAPIMechanism: + return buildFranzGSSAPIMechanism(o.SASL.GSSAPI) + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) + } +} + +func buildFranzOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSource, error) { + endpointParams := url.Values{} + if o.SASL.OAuth2.GrantType != "" { + endpointParams.Set("grant_type", o.SASL.OAuth2.GrantType) + } + if o.SASL.OAuth2.Audience != "" { + endpointParams.Set("audience", o.SASL.OAuth2.Audience) + } + + tokenURL, err := url.Parse(o.SASL.OAuth2.TokenURL) + if err != nil { + return nil, errors.Trace(err) + } + + cfg := &clientcredentials.Config{ + ClientID: o.SASL.OAuth2.ClientID, + ClientSecret: o.SASL.OAuth2.ClientSecret, + TokenURL: tokenURL.String(), + EndpointParams: endpointParams, + Scopes: o.SASL.OAuth2.Scopes, + } + return cfg.TokenSource(ctx), nil +} + +func buildFranzProducerOptions( + o *Options, + recordRetries int, +) ([]kgo.Opt, error) { + var acks kgo.Acks + switch o.RequiredAcks { + case WaitForAll: + acks = kgo.AllISRAcks() + case WaitForLocal: + acks = kgo.LeaderAck() + case NoResponse: + acks = kgo.NoAck() + default: + acks = kgo.AllISRAcks() + log.Warn("unknown required acks, use all isr acks", zap.Int16("requiredAcks", int16(o.RequiredAcks))) + } + + compressionOpt, err := buildFranzCompressionOption(o) + if err != nil { + return nil, errors.Trace(err) + } + + produceTimeout := o.ReadTimeout + if produceTimeout < 100*time.Millisecond { + produceTimeout = 10 * time.Second + } + + return []kgo.Opt{ + kgo.RecordPartitioner(kgo.ManualPartitioner()), + kgo.RequiredAcks(acks), + kgo.DisableIdempotentWrite(), + kgo.MaxProduceRequestsInflightPerBroker(1), + kgo.RecordRetries(recordRetries), + kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), + kgo.ProduceRequestTimeout(produceTimeout), + kgo.ProducerLinger(0), + compressionOpt, + }, nil +} + +func buildFranzCompressionOption(o *Options) (kgo.Opt, error) { + compression := strings.ToLower(strings.TrimSpace(o.Compression)) + switch compression { + case "none": + return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + case "gzip": + return kgo.ProducerBatchCompression(kgo.GzipCompression()), nil + case "snappy": + return kgo.ProducerBatchCompression(kgo.SnappyCompression()), nil + case "lz4": + return kgo.ProducerBatchCompression(kgo.Lz4Compression()), nil + case "zstd": + return kgo.ProducerBatchCompression(kgo.ZstdCompression()), nil + case "": + return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + default: + log.Warn("unsupported compression algorithm", zap.String("compression", o.Compression)) + return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + } +} diff --git a/pkg/sink/kafka/franz_gssapi.go b/pkg/sink/kafka/franz/gssapi.go similarity index 99% rename from pkg/sink/kafka/franz_gssapi.go rename to pkg/sink/kafka/franz/gssapi.go index 8ccc3adcec..d1d1d2e4ad 100644 --- a/pkg/sink/kafka/franz_gssapi.go +++ b/pkg/sink/kafka/franz/gssapi.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kafka +package franz import ( "context" diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/franz/metrics_hook.go new file mode 100644 index 0000000000..8e0bd415ea --- /dev/null +++ b/pkg/sink/kafka/franz/metrics_hook.go @@ -0,0 +1,206 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/rcrowley/go-metrics" + "github.com/twmb/franz-go/pkg/kgo" +) + +type brokerMetrics struct { + outgoingByteRate metrics.Meter + requestRate metrics.Meter + requestLatency metrics.Histogram + responseRate metrics.Meter + inFlight int64 +} + +func newBrokerMetrics() *brokerMetrics { + return &brokerMetrics{ + outgoingByteRate: metrics.NewMeter(), + requestRate: metrics.NewMeter(), + requestLatency: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), + responseRate: metrics.NewMeter(), + } +} + +type MetricsHook struct { + mu sync.RWMutex + brokers map[int32]*brokerMetrics + + compressionRatio metrics.Histogram + recordsPerReq metrics.Histogram +} + +func NewMetricsHook() *MetricsHook { + return &MetricsHook{ + brokers: make(map[int32]*brokerMetrics), + compressionRatio: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), + recordsPerReq: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), + } +} + +func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { + broker := h.getBroker(nodeID) + if broker == nil { + return + } + + if bytesWritten > 0 { + broker.outgoingByteRate.Mark(int64(bytesWritten)) + } + broker.requestRate.Mark(1) + if err == nil { + atomic.AddInt64(&broker.inFlight, 1) + } +} + +func (h *MetricsHook) getBroker(nodeID int32) *brokerMetrics { + if nodeID < 0 { + return nil + } + + h.mu.RLock() + broker := h.brokers[nodeID] + h.mu.RUnlock() + if broker != nil { + return broker + } + + h.mu.Lock() + defer h.mu.Unlock() + broker = h.brokers[nodeID] + if broker != nil { + return broker + } + broker = newBrokerMetrics() + h.brokers[nodeID] = broker + return broker +} + +type BrokerMetricsSnapshot struct { + OutgoingByteRate float64 + RequestRate float64 + RequestLatencyMeanMic float64 + RequestLatencyP99Mic float64 + InFlight int64 + ResponseRate float64 +} + +type MetricsSnapshot struct { + CompressionMean float64 + CompressionP99 float64 + RecordsMean float64 + RecordsP99 float64 + Brokers map[int32]BrokerMetricsSnapshot +} + +func (h *MetricsHook) Snapshot() MetricsSnapshot { + h.mu.RLock() + brokers := make(map[int32]*brokerMetrics, len(h.brokers)) + for id, broker := range h.brokers { + brokers[id] = broker + } + compression := h.compressionRatio.Snapshot() + records := h.recordsPerReq.Snapshot() + h.mu.RUnlock() + + result := MetricsSnapshot{ + CompressionMean: compression.Mean(), + CompressionP99: compression.Percentile(0.99), + RecordsMean: records.Mean(), + RecordsP99: records.Percentile(0.99), + Brokers: make(map[int32]BrokerMetricsSnapshot, len(brokers)), + } + + for id, broker := range brokers { + latencySnapshot := broker.requestLatency.Snapshot() + result.Brokers[id] = BrokerMetricsSnapshot{ + OutgoingByteRate: broker.outgoingByteRate.Snapshot().Rate1(), + RequestRate: broker.requestRate.Snapshot().Rate1(), + RequestLatencyMeanMic: latencySnapshot.Mean(), + RequestLatencyP99Mic: latencySnapshot.Percentile(0.99), + InFlight: atomic.LoadInt64(&broker.inFlight), + ResponseRate: broker.responseRate.Snapshot().Rate1(), + } + } + return result +} + +func (h *MetricsHook) snapshotBrokers() map[int32]*brokerMetrics { + h.mu.RLock() + defer h.mu.RUnlock() + result := make(map[int32]*brokerMetrics, len(h.brokers)) + for id, broker := range h.brokers { + result[id] = broker + } + return result +} + +func (h *MetricsHook) OnBrokerWrite( + meta kgo.BrokerMetadata, + _ int16, + bytesWritten int, + _ time.Duration, + _ time.Duration, + err error, +) { + h.RecordBrokerWrite(meta.NodeID, bytesWritten, err) +} + +func (h *MetricsHook) OnBrokerE2E( + meta kgo.BrokerMetadata, + _ int16, + e2e kgo.BrokerE2E, +) { + broker := h.getBroker(meta.NodeID) + if broker == nil { + return + } + + if e2e.WriteErr == nil { + if atomic.AddInt64(&broker.inFlight, -1) < 0 { + atomic.StoreInt64(&broker.inFlight, 0) + } + } + if e2e.BytesRead > 0 && e2e.ReadErr == nil { + broker.responseRate.Mark(1) + } + if e2e.Err() == nil { + broker.requestLatency.Update(e2e.DurationE2E().Microseconds()) + } +} + +func (h *MetricsHook) OnProduceBatchWritten( + _ kgo.BrokerMetadata, + _ string, + _ int32, + m kgo.ProduceBatchMetrics, +) { + h.RecordProduceBatchWritten(m.NumRecords, m.UncompressedBytes, m.CompressedBytes) +} + +func (h *MetricsHook) RecordProduceBatchWritten(numRecords int, uncompressedBytes int, compressedBytes int) { + if numRecords > 0 { + h.recordsPerReq.Update(int64(numRecords)) + } + if uncompressedBytes > 0 && compressedBytes > 0 { + ratio := int64(float64(uncompressedBytes) / float64(compressedBytes) * 100) + h.compressionRatio.Update(ratio) + } +} diff --git a/pkg/sink/kafka/franz/sasl_test.go b/pkg/sink/kafka/franz/sasl_test.go new file mode 100644 index 0000000000..7961125c88 --- /dev/null +++ b/pkg/sink/kafka/franz/sasl_test.go @@ -0,0 +1,66 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/security" + "github.com/stretchr/testify/require" +) + +func TestBuildFranzSaslMechanismGSSAPIUserAuth(t *testing.T) { + t.Parallel() + + o := &Options{ + SASL: &security.SASL{ + SASLMechanism: security.GSSAPIMechanism, + GSSAPI: security.GSSAPI{ + AuthType: security.UserAuth, + KerberosConfigPath: "/etc/krb5.conf", + ServiceName: "kafka", + Username: "alice", + Password: "pwd", + Realm: "EXAMPLE.COM", + }, + }, + } + + mechanism, err := buildFranzSaslMechanism(context.Background(), o) + require.NoError(t, err) + require.Equal(t, "GSSAPI", mechanism.Name()) +} + +func TestBuildFranzSaslMechanismGSSAPIKeytabAuth(t *testing.T) { + t.Parallel() + + o := &Options{ + SASL: &security.SASL{ + SASLMechanism: security.GSSAPIMechanism, + GSSAPI: security.GSSAPI{ + AuthType: security.KeyTabAuth, + KerberosConfigPath: "/etc/krb5.conf", + ServiceName: "kafka", + Username: "alice", + KeyTabPath: "/tmp/a.keytab", + Realm: "EXAMPLE.COM", + }, + }, + } + + mechanism, err := buildFranzSaslMechanism(context.Background(), o) + require.NoError(t, err) + require.Equal(t, "GSSAPI", mechanism.Name()) +} diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz/sync_producer.go similarity index 84% rename from pkg/sink/kafka/franz_sync_producer.go rename to pkg/sink/kafka/franz/sync_producer.go index b0541adbc2..c234e4bf16 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz/sync_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kafka +package franz import ( "context" @@ -22,6 +22,7 @@ import ( commonType "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/kafka/internal/logutil" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/atomic" "go.uber.org/zap" @@ -29,7 +30,7 @@ import ( const franzSyncRecordRetries = 5 -type franzSyncProducer struct { +type SyncProducer struct { id commonType.ChangeFeedID client *kgo.Client @@ -37,12 +38,12 @@ type franzSyncProducer struct { timeout time.Duration } -func newFranzSyncProducer( +func NewSyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, - o *options, + o *Options, hook kgo.Hook, -) (SyncProducer, error) { +) (*SyncProducer, error) { baseOpts, err := buildFranzBaseOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -63,7 +64,7 @@ func newFranzSyncProducer( } timeout := time.Duration(franzSyncRecordRetries+1) * produceTimeout - return &franzSyncProducer{ + return &SyncProducer{ id: changefeedID, client: client, closed: atomic.NewBool(false), @@ -71,11 +72,11 @@ func newFranzSyncProducer( }, nil } -func (p *franzSyncProducer) newRequestContext() (context.Context, context.CancelFunc) { +func (p *SyncProducer) newRequestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(p.client.Context(), p.timeout) } -func (p *franzSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { +func (p *SyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { if p.closed.Load() { return errors.ErrKafkaProducerClosed.GenWithStackByArgs() } @@ -96,7 +97,7 @@ func (p *franzSyncProducer) SendMessage(topic string, partitionNum int32, messag }) if err != nil { - err = AnnotateEventError( + err = logutil.AnnotateEventError( p.id.Keyspace(), p.id.Name(), message.LogInfo, @@ -106,7 +107,7 @@ func (p *franzSyncProducer) SendMessage(topic string, partitionNum int32, messag return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *franzSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { +func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { if p.closed.Load() { return errors.ErrKafkaProducerClosed.GenWithStackByArgs() } @@ -131,7 +132,7 @@ func (p *franzSyncProducer) SendMessages(topic string, partitionNum int32, messa }) if err != nil { - err = AnnotateEventError( + err = logutil.AnnotateEventError( p.id.Keyspace(), p.id.Name(), message.LogInfo, @@ -141,9 +142,9 @@ func (p *franzSyncProducer) SendMessages(topic string, partitionNum int32, messa return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *franzSyncProducer) Heartbeat() {} +func (p *SyncProducer) Heartbeat() {} -func (p *franzSyncProducer) Close() { +func (p *SyncProducer) Close() { if p.closed.Load() { log.Warn("kafka DDL producer already closed", zap.String("keyspace", p.id.Keyspace()), diff --git a/pkg/sink/kafka/franz_admin_client.go b/pkg/sink/kafka/franz_admin_client.go index dda18d2a80..d1e8790063 100644 --- a/pkg/sink/kafka/franz_admin_client.go +++ b/pkg/sink/kafka/franz_admin_client.go @@ -14,268 +14,75 @@ package kafka import ( - "context" - "strconv" - "time" - - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/twmb/franz-go/pkg/kadm" - "github.com/twmb/franz-go/pkg/kerr" - "github.com/twmb/franz-go/pkg/kgo" - "go.uber.org/zap" + kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" ) -type franzAdminClient struct { - changefeed common.ChangeFeedID - - client *kgo.Client - admin *kadm.Client - timeout time.Duration -} - -func newFranzAdminClient( - ctx context.Context, - changefeedID common.ChangeFeedID, - o *options, - hook kgo.Hook, -) (ClusterAdminClient, error) { - baseOpts, err := buildFranzBaseOptions(ctx, o, hook) - if err != nil { - return nil, errors.Trace(err) - } - - client, err := kgo.NewClient(baseOpts...) - if err != nil { - return nil, errors.Trace(err) - } - - timeout := o.ReadTimeout - if o.WriteTimeout > timeout { - timeout = o.WriteTimeout - } - if timeout <= 0 { - timeout = 10 * time.Second - } - - return &franzAdminClient{ - changefeed: changefeedID, - client: client, - admin: kadm.NewClient(client), - timeout: timeout, - }, nil +// franzAdminClientAdapter adapts the franz-go admin client implementation to kafka.ClusterAdminClient. +// It intentionally lives in the kafka package to reuse existing option adjustment logic without +// introducing an import cycle (kafka -> franz -> kafka). +type franzAdminClientAdapter struct { + inner *kafkafranz.AdminClient } -func (a *franzAdminClient) newRequestContext() (context.Context, context.CancelFunc) { - return context.WithTimeout(a.client.Context(), a.timeout) -} - -func (a *franzAdminClient) GetAllBrokers() []Broker { - ctx, cancel := a.newRequestContext() - defer cancel() - - meta, err := a.admin.BrokerMetadata(ctx) - if err != nil { - log.Warn("Kafka admin client fetch broker metadata failed", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.Error(err)) - return nil - } - - result := make([]Broker, 0, len(meta.Brokers)) - for _, broker := range meta.Brokers { - if broker.NodeID < 0 { - continue - } - result = append(result, Broker{ID: broker.NodeID}) +func (a *franzAdminClientAdapter) GetAllBrokers() []Broker { + brokers := a.inner.GetAllBrokers() + result := make([]Broker, 0, len(brokers)) + for _, b := range brokers { + result = append(result, Broker{ID: b.ID}) } return result } -func (a *franzAdminClient) GetBrokerConfig(configName string) (string, error) { - ctx, cancel := a.newRequestContext() - defer cancel() - - meta, err := a.admin.BrokerMetadata(ctx) - if err != nil { - return "", errors.Trace(err) - } - if meta.Controller < 0 { - return "", errors.ErrKafkaInvalidConfig.GenWithStack("kafka controller is not available") - } - - configs, err := a.admin.DescribeBrokerConfigs(ctx, meta.Controller) - if err != nil { - return "", errors.Trace(err) - } - - controllerName := strconv.Itoa(int(meta.Controller)) - resource, err := configs.On(controllerName, nil) - if err != nil { - return "", errors.Trace(err) - } - if resource.Err != nil { - return "", errors.Trace(resource.Err) - } - - for _, entry := range resource.Configs { - if entry.Key == configName { - return entry.MaybeValue(), nil - } - } - - log.Warn("Kafka config item not found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName)) - return "", errors.ErrKafkaConfigNotFound.GenWithStack( - "cannot find the `%s` from the broker's configuration", configName) +func (a *franzAdminClientAdapter) GetBrokerConfig(configName string) (string, error) { + return a.inner.GetBrokerConfig(configName) } -func (a *franzAdminClient) GetTopicConfig(topicName string, configName string) (string, error) { - ctx, cancel := a.newRequestContext() - defer cancel() - - configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) - if err != nil { - return "", errors.Trace(err) - } - - resource, err := configs.On(topicName, nil) - if err != nil { - return "", errors.Trace(err) - } - if resource.Err != nil { - return "", errors.Trace(resource.Err) - } - - for _, entry := range resource.Configs { - if entry.Key == configName { - log.Info("Kafka config item found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName), - zap.String("configValue", entry.MaybeValue())) - return entry.MaybeValue(), nil - } - } - - log.Warn("Kafka config item not found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName)) - return "", errors.ErrKafkaConfigNotFound.GenWithStack( - "cannot find the `%s` from the topic's configuration", configName) +func (a *franzAdminClientAdapter) GetTopicConfig(topicName string, configName string) (string, error) { + return a.inner.GetTopicConfig(topicName, configName) } -func (a *franzAdminClient) GetTopicsMeta( +func (a *franzAdminClientAdapter) GetTopicsMeta( topics []string, ignoreTopicError bool, ) (map[string]TopicDetail, error) { - if len(topics) == 0 { - return make(map[string]TopicDetail), nil - } - - ctx, cancel := a.newRequestContext() - defer cancel() - - meta, err := a.admin.Metadata(ctx, topics...) + meta, err := a.inner.GetTopicsMeta(topics, ignoreTopicError) if err != nil { - return nil, errors.Trace(err) + return nil, err } - result := make(map[string]TopicDetail, len(topics)) - for _, topic := range topics { - detail, ok := meta.Topics[topic] - if !ok { - continue - } - if detail.Err != nil { - if errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { - continue - } - if !ignoreTopicError { - return nil, errors.Trace(detail.Err) - } - log.Warn("fetch topic meta failed", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("topic", topic), - zap.Error(detail.Err)) - continue - } - + result := make(map[string]TopicDetail, len(meta)) + for topic, detail := range meta { result[topic] = TopicDetail{ - Name: topic, - NumPartitions: int32(len(detail.Partitions)), + Name: detail.Name, + NumPartitions: detail.NumPartitions, + ReplicationFactor: detail.ReplicationFactor, } } return result, nil } -func (a *franzAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - if len(topics) == 0 { - return make(map[string]int32), nil - } - - ctx, cancel := a.newRequestContext() - defer cancel() - - meta, err := a.admin.Metadata(ctx, topics...) - if err != nil { - return nil, errors.Trace(err) - } - - result := make(map[string]int32, len(topics)) - for _, topic := range topics { - detail, ok := meta.Topics[topic] - if !ok { - return nil, errors.Trace(kerr.UnknownTopicOrPartition) - } - if detail.Err != nil { - return nil, errors.Trace(detail.Err) - } - result[topic] = int32(len(detail.Partitions)) - } - return result, nil +func (a *franzAdminClientAdapter) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + return a.inner.GetTopicsPartitionsNum(topics) } -func (a *franzAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { - ctx, cancel := a.newRequestContext() - defer cancel() - - var ( - responses kadm.CreateTopicResponses - err error - ) - if validateOnly { - responses, err = a.admin.ValidateCreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) - } else { - responses, err = a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) - } - if err != nil { - return errors.Trace(err) - } - - resp, ok := responses[detail.Name] - if !ok { - return errors.ErrKafkaCreateTopic.GenWithStack("kafka topic create response is missing") +func (a *franzAdminClientAdapter) CreateTopic(detail *TopicDetail, validateOnly bool) error { + if detail == nil { + return a.inner.CreateTopic(nil, validateOnly) } - if resp.Err != nil { - if errors.Is(resp.Err, kerr.TopicAlreadyExists) { - return nil - } - return errors.Trace(resp.Err) + franzDetail := &kafkafranz.TopicDetail{ + Name: detail.Name, + NumPartitions: detail.NumPartitions, + ReplicationFactor: detail.ReplicationFactor, } - return nil + return a.inner.CreateTopic(franzDetail, validateOnly) } -func (a *franzAdminClient) Heartbeat() {} +func (a *franzAdminClientAdapter) Heartbeat() { + a.inner.Heartbeat() +} -func (a *franzAdminClient) Close() { - if a.admin != nil { - a.admin.Close() +func (a *franzAdminClientAdapter) Close() { + if a.inner != nil { + a.inner.Close() } } diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 33a85a4346..951a8942dd 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -15,67 +15,55 @@ package kafka import ( "context" - "crypto/tls" - "net/url" - "strings" - "time" - "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/security" - "github.com/twmb/franz-go/pkg/kgo" - "github.com/twmb/franz-go/pkg/kversion" - "github.com/twmb/franz-go/pkg/sasl" - "github.com/twmb/franz-go/pkg/sasl/oauth" - "github.com/twmb/franz-go/pkg/sasl/plain" - "github.com/twmb/franz-go/pkg/sasl/scram" - "go.uber.org/zap" - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" + kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" ) type franzFactory struct { changefeedID common.ChangeFeedID option *options - metricsHook *franzMetricsHook + metricsHook *kafkafranz.MetricsHook } // NewFranzFactory constructs a Factory with franz-go implementation. +// +// NOTE: The franz-go specific implementation details live in `pkg/sink/kafka/franz`. +// This function keeps the public API stable and adapts to the existing kafka package interfaces. func NewFranzFactory( ctx context.Context, o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - admin, err := newFranzAdminClient(ctx, changefeedID, o, nil) + adminInner, err := kafkafranz.NewAdminClient(ctx, changefeedID, newFranzOptions(o), nil) if err != nil { return nil, errors.Trace(err) } + admin := &franzAdminClientAdapter{inner: adminInner} defer admin.Close() - if err = adjustOptions(ctx, admin, o, o.Topic); err != nil { + if err := adjustOptions(ctx, admin, o, o.Topic); err != nil { return nil, errors.Trace(err) } - metricsHook := newFranzMetricsHook() - return &franzFactory{ changefeedID: changefeedID, option: o, - metricsHook: metricsHook, + metricsHook: kafkafranz.NewMetricsHook(), }, nil } func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - admin, err := newFranzAdminClient(ctx, f.changefeedID, f.option, f.metricsHook) + adminInner, err := kafkafranz.NewAdminClient(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } - return admin, nil + return &franzAdminClientAdapter{inner: adminInner}, nil } func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := newFranzSyncProducer(ctx, f.changefeedID, f.option, f.metricsHook) + producer, err := kafkafranz.NewSyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -83,7 +71,7 @@ func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { } func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := newFranzAsyncProducer(ctx, f.changefeedID, f.option, f.metricsHook) + producer, err := kafkafranz.NewAsyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -97,209 +85,28 @@ func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { } } -type noopMetricsCollector struct{} - -func (m *noopMetricsCollector) Run(_ context.Context) {} - -func buildFranzBaseOptions( - ctx context.Context, - o *options, - hook kgo.Hook, -) ([]kgo.Opt, error) { - timeoutOverhead := o.ReadTimeout - if o.WriteTimeout > timeoutOverhead { - timeoutOverhead = o.WriteTimeout - } - if timeoutOverhead <= 0 { - timeoutOverhead = 10 * time.Second - } - - opts := []kgo.Opt{ - kgo.WithContext(ctx), - kgo.SeedBrokers(o.BrokerEndpoints...), - kgo.ClientID(o.ClientID), - kgo.DialTimeout(o.DialTimeout), - kgo.RequestTimeoutOverhead(timeoutOverhead), - } - if hook != nil { - opts = append(opts, kgo.WithHooks(hook)) - } - - if o.IsAssignedVersion { - versions := kversion.FromString(o.Version) - if versions == nil { - return nil, errors.ErrKafkaInvalidVersion.GenWithStack("invalid kafka version %s", o.Version) - } - opts = append(opts, kgo.MaxVersions(versions)) - } - - if o.EnableTLS { - tlsConfig, err := buildFranzTLSConfig(o) - if err != nil { - return nil, errors.Trace(err) - } - opts = append(opts, kgo.DialTLSConfig(tlsConfig)) - } - - if o.SASL != nil && o.SASL.SASLMechanism != "" { - mechanism, err := buildFranzSaslMechanism(ctx, o) - if err != nil { - return nil, errors.Trace(err) - } - opts = append(opts, kgo.SASL(mechanism)) - } - - return opts, nil -} - -func buildFranzTLSConfig(o *options) (*tls.Config, error) { - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", "http/1.1"}, - } - - if o.Credential != nil && o.Credential.IsTLSEnabled() { - credentialTlsConfig, err := o.Credential.ToTLSConfig() - if err != nil { - return nil, errors.Trace(err) - } - tlsConfig = credentialTlsConfig - if tlsConfig.MinVersion == 0 { - tlsConfig.MinVersion = tls.VersionTLS12 - } - if len(tlsConfig.NextProtos) == 0 { - tlsConfig.NextProtos = []string{"h2", "http/1.1"} - } - } - - tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify - return tlsConfig, nil -} - -func buildFranzSaslMechanism(ctx context.Context, o *options) (sasl.Mechanism, error) { - if o.SASL == nil { - return nil, nil - } - - switch security.SASLMechanism(strings.ToUpper(string(o.SASL.SASLMechanism))) { - case security.PlainMechanism: - auth := plain.Auth{ - User: o.SASL.SASLUser, - Pass: o.SASL.SASLPassword, - } - return auth.AsMechanism(), nil - case security.SCRAM256Mechanism: - auth := scram.Auth{ - User: o.SASL.SASLUser, - Pass: o.SASL.SASLPassword, - } - return auth.AsSha256Mechanism(), nil - case security.SCRAM512Mechanism: - auth := scram.Auth{ - User: o.SASL.SASLUser, - Pass: o.SASL.SASLPassword, - } - return auth.AsSha512Mechanism(), nil - case security.OAuthMechanism: - tokenSource, err := buildFranzOauthTokenSource(ctx, o) - if err != nil { - return nil, errors.Trace(err) - } - return oauth.Oauth(func(context.Context) (oauth.Auth, error) { - token, err := tokenSource.Token() - if err != nil { - return oauth.Auth{}, errors.Trace(err) - } - return oauth.Auth{Token: token.AccessToken}, nil - }), nil - case security.GSSAPIMechanism: - return buildFranzGSSAPIMechanism(o.SASL.GSSAPI) - default: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) - } -} - -func buildFranzOauthTokenSource(ctx context.Context, o *options) (oauth2.TokenSource, error) { - endpointParams := url.Values{} - if o.SASL.OAuth2.GrantType != "" { - endpointParams.Set("grant_type", o.SASL.OAuth2.GrantType) - } - if o.SASL.OAuth2.Audience != "" { - endpointParams.Set("audience", o.SASL.OAuth2.Audience) - } - - tokenURL, err := url.Parse(o.SASL.OAuth2.TokenURL) - if err != nil { - return nil, errors.Trace(err) - } - - cfg := &clientcredentials.Config{ - ClientID: o.SASL.OAuth2.ClientID, - ClientSecret: o.SASL.OAuth2.ClientSecret, - TokenURL: tokenURL.String(), - EndpointParams: endpointParams, - Scopes: o.SASL.OAuth2.Scopes, - } - return cfg.TokenSource(ctx), nil -} - -func buildFranzProducerOptions( - o *options, - recordRetries int, -) ([]kgo.Opt, error) { - var acks kgo.Acks - switch o.RequiredAcks { - case WaitForAll: - acks = kgo.AllISRAcks() - case WaitForLocal: - acks = kgo.LeaderAck() - case NoResponse: - acks = kgo.NoAck() - default: - acks = kgo.AllISRAcks() - log.Warn("unknown required acks, use all isr acks", zap.Int16("requiredAcks", int16(o.RequiredAcks))) +func newFranzOptions(o *options) *kafkafranz.Options { + if o == nil { + return &kafkafranz.Options{} } + return &kafkafranz.Options{ + BrokerEndpoints: o.BrokerEndpoints, + ClientID: o.ClientID, - compressionOpt, err := buildFranzCompressionOption(o) - if err != nil { - return nil, errors.Trace(err) - } + Version: o.Version, + IsAssignedVersion: o.IsAssignedVersion, - produceTimeout := o.ReadTimeout - if produceTimeout < 100*time.Millisecond { - produceTimeout = 10 * time.Second - } + MaxMessageBytes: o.MaxMessageBytes, + Compression: o.Compression, + RequiredAcks: kafkafranz.RequiredAcks(o.RequiredAcks), - return []kgo.Opt{ - kgo.RecordPartitioner(kgo.ManualPartitioner()), - kgo.RequiredAcks(acks), - kgo.DisableIdempotentWrite(), - kgo.MaxProduceRequestsInflightPerBroker(1), - kgo.RecordRetries(recordRetries), - kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), - kgo.ProduceRequestTimeout(produceTimeout), - kgo.ProducerLinger(0), - compressionOpt, - }, nil -} + EnableTLS: o.EnableTLS, + Credential: o.Credential, + InsecureSkipVerify: o.InsecureSkipVerify, + SASL: o.SASL, -func buildFranzCompressionOption(o *options) (kgo.Opt, error) { - compression := strings.ToLower(strings.TrimSpace(o.Compression)) - switch compression { - case "none": - return kgo.ProducerBatchCompression(kgo.NoCompression()), nil - case "gzip": - return kgo.ProducerBatchCompression(kgo.GzipCompression()), nil - case "snappy": - return kgo.ProducerBatchCompression(kgo.SnappyCompression()), nil - case "lz4": - return kgo.ProducerBatchCompression(kgo.Lz4Compression()), nil - case "zstd": - return kgo.ProducerBatchCompression(kgo.ZstdCompression()), nil - case "": - return kgo.ProducerBatchCompression(kgo.NoCompression()), nil - default: - log.Warn("unsupported compression algorithm", zap.String("compression", o.Compression)) - return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + DialTimeout: o.DialTimeout, + WriteTimeout: o.WriteTimeout, + ReadTimeout: o.ReadTimeout, } } diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go index 83fdd5e397..be638a5b9f 100644 --- a/pkg/sink/kafka/franz_factory_test.go +++ b/pkg/sink/kafka/franz_factory_test.go @@ -14,73 +14,33 @@ package kafka import ( - "context" "testing" "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/security" + kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kgo" ) -func TestBuildFranzSaslMechanismGSSAPIUserAuth(t *testing.T) { +func TestFranzFactoryMetricsCollectorType(t *testing.T) { t.Parallel() - o := NewOptions() - o.SASL = &security.SASL{ - SASLMechanism: security.GSSAPIMechanism, - GSSAPI: security.GSSAPI{ - AuthType: security.UserAuth, - KerberosConfigPath: "/etc/krb5.conf", - ServiceName: "kafka", - Username: "alice", - Password: "pwd", - Realm: "EXAMPLE.COM", - }, - } - - mechanism, err := buildFranzSaslMechanism(context.Background(), o) - require.NoError(t, err) - require.Equal(t, "GSSAPI", mechanism.Name()) -} - -func TestBuildFranzSaslMechanismGSSAPIKeytabAuth(t *testing.T) { - t.Parallel() - - o := NewOptions() - o.SASL = &security.SASL{ - SASLMechanism: security.GSSAPIMechanism, - GSSAPI: security.GSSAPI{ - AuthType: security.KeyTabAuth, - KerberosConfigPath: "/etc/krb5.conf", - ServiceName: "kafka", - Username: "alice", - KeyTabPath: "/tmp/a.keytab", - Realm: "EXAMPLE.COM", - }, + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-metrics-type") + f := &franzFactory{ + changefeedID: changefeedID, + metricsHook: kafkafranz.NewMetricsHook(), } - - mechanism, err := buildFranzSaslMechanism(context.Background(), o) - require.NoError(t, err) - require.Equal(t, "GSSAPI", mechanism.Name()) -} - -func TestFranzFactoryMetricsCollectorIsNotNoop(t *testing.T) { - t.Parallel() - - f := &franzFactory{} collector := f.MetricsCollector(nil) - _, isNoop := collector.(*noopMetricsCollector) - require.False(t, isNoop) + _, ok := collector.(*franzMetricsCollector) + require.True(t, ok) } func TestFranzMetricsCollectorCollectMetrics(t *testing.T) { t.Parallel() - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceNamme, "franz-metrics") - hook := newFranzMetricsHook() + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-metrics") + hook := kafkafranz.NewMetricsHook() collector := &franzMetricsCollector{ changefeedID: changefeedID, hook: hook, @@ -89,13 +49,8 @@ func TestFranzMetricsCollectorCollectMetrics(t *testing.T) { collector.cleanupMetrics() }) - meta := kgo.BrokerMetadata{NodeID: 1} - hook.OnBrokerWrite(meta, 0, 128, 0, 0, nil) - hook.OnProduceBatchWritten(meta, "topic", 0, kgo.ProduceBatchMetrics{ - NumRecords: 8, - UncompressedBytes: 400, - CompressedBytes: 200, - }) + hook.RecordBrokerWrite(1, 128, nil) + hook.RecordProduceBatchWritten(8, 400, 200) collector.collectMetrics() diff --git a/pkg/sink/kafka/franz_metrics_collector.go b/pkg/sink/kafka/franz_metrics_collector.go index 3eaef5ba36..b4503d802f 100644 --- a/pkg/sink/kafka/franz_metrics_collector.go +++ b/pkg/sink/kafka/franz_metrics_collector.go @@ -16,146 +16,17 @@ package kafka import ( "context" "strconv" - "sync" - "sync/atomic" "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" - "github.com/rcrowley/go-metrics" - "github.com/twmb/franz-go/pkg/kgo" + kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" "go.uber.org/zap" ) -type franzBrokerMetrics struct { - outgoingByteRate metrics.Meter - requestRate metrics.Meter - requestLatency metrics.Histogram - responseRate metrics.Meter - inFlight int64 -} - -func newFranzBrokerMetrics() *franzBrokerMetrics { - return &franzBrokerMetrics{ - outgoingByteRate: metrics.NewMeter(), - requestRate: metrics.NewMeter(), - requestLatency: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), - responseRate: metrics.NewMeter(), - } -} - -type franzMetricsHook struct { - mu sync.RWMutex - brokers map[int32]*franzBrokerMetrics - - compressionRatio metrics.Histogram - recordsPerReq metrics.Histogram -} - -func newFranzMetricsHook() *franzMetricsHook { - return &franzMetricsHook{ - brokers: make(map[int32]*franzBrokerMetrics), - compressionRatio: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), - recordsPerReq: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), - } -} - -func (h *franzMetricsHook) getBroker(nodeID int32) *franzBrokerMetrics { - if nodeID < 0 { - return nil - } - - h.mu.RLock() - broker := h.brokers[nodeID] - h.mu.RUnlock() - if broker != nil { - return broker - } - - h.mu.Lock() - defer h.mu.Unlock() - broker = h.brokers[nodeID] - if broker != nil { - return broker - } - broker = newFranzBrokerMetrics() - h.brokers[nodeID] = broker - return broker -} - -func (h *franzMetricsHook) snapshotBrokers() map[int32]*franzBrokerMetrics { - h.mu.RLock() - defer h.mu.RUnlock() - result := make(map[int32]*franzBrokerMetrics, len(h.brokers)) - for id, broker := range h.brokers { - result[id] = broker - } - return result -} - -func (h *franzMetricsHook) OnBrokerWrite( - meta kgo.BrokerMetadata, - _ int16, - bytesWritten int, - _ time.Duration, - _ time.Duration, - err error, -) { - broker := h.getBroker(meta.NodeID) - if broker == nil { - return - } - - if bytesWritten > 0 { - broker.outgoingByteRate.Mark(int64(bytesWritten)) - } - broker.requestRate.Mark(1) - if err == nil { - atomic.AddInt64(&broker.inFlight, 1) - } -} - -func (h *franzMetricsHook) OnBrokerE2E( - meta kgo.BrokerMetadata, - _ int16, - e2e kgo.BrokerE2E, -) { - broker := h.getBroker(meta.NodeID) - if broker == nil { - return - } - - if e2e.WriteErr == nil { - if atomic.AddInt64(&broker.inFlight, -1) < 0 { - atomic.StoreInt64(&broker.inFlight, 0) - } - } - if e2e.BytesRead > 0 && e2e.ReadErr == nil { - broker.responseRate.Mark(1) - } - if e2e.Err() == nil { - broker.requestLatency.Update(e2e.DurationE2E().Microseconds()) - } -} - -func (h *franzMetricsHook) OnProduceBatchWritten( - _ kgo.BrokerMetadata, - _ string, - _ int32, - m kgo.ProduceBatchMetrics, -) { - if m.NumRecords > 0 { - h.recordsPerReq.Update(int64(m.NumRecords)) - } - if m.UncompressedBytes > 0 && m.CompressedBytes > 0 { - ratio := int64(float64(m.UncompressedBytes) / float64(m.CompressedBytes) * 100) - h.compressionRatio.Update(ratio) - } -} - type franzMetricsCollector struct { changefeedID common.ChangeFeedID - hook *franzMetricsHook + hook *kafkafranz.MetricsHook } func (m *franzMetricsCollector) Run(ctx context.Context) { @@ -182,34 +53,24 @@ func (m *franzMetricsCollector) collectMetrics() { keyspace := m.changefeedID.Keyspace() changefeedID := m.changefeedID.Name() - compressionSnapshot := m.hook.compressionRatio.Snapshot() - compressionRatioGauge.WithLabelValues(keyspace, changefeedID, avg).Set(compressionSnapshot.Mean()) - compressionRatioGauge.WithLabelValues(keyspace, changefeedID, p99).Set(compressionSnapshot.Percentile(0.99)) + snapshot := m.hook.Snapshot() + compressionRatioGauge.WithLabelValues(keyspace, changefeedID, avg).Set(snapshot.CompressionMean) + compressionRatioGauge.WithLabelValues(keyspace, changefeedID, p99).Set(snapshot.CompressionP99) + recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, avg).Set(snapshot.RecordsMean) + recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, p99).Set(snapshot.RecordsP99) - recordsSnapshot := m.hook.recordsPerReq.Snapshot() - recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, avg).Set(recordsSnapshot.Mean()) - recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, p99).Set(recordsSnapshot.Percentile(0.99)) - - for id, broker := range m.hook.snapshotBrokers() { + for id, broker := range snapshot.Brokers { brokerID := strconv.Itoa(int(id)) - OutgoingByteRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( - broker.outgoingByteRate.Snapshot().Rate1(), - ) - RequestRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( - broker.requestRate.Snapshot().Rate1(), - ) + OutgoingByteRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(broker.OutgoingByteRate) + RequestRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(broker.RequestRate) RequestLatencyGauge.WithLabelValues(keyspace, changefeedID, brokerID, avg).Set( - broker.requestLatency.Snapshot().Mean() / 1000, + broker.RequestLatencyMeanMic / 1000, ) RequestLatencyGauge.WithLabelValues(keyspace, changefeedID, brokerID, p99).Set( - broker.requestLatency.Snapshot().Percentile(0.99) / 1000, - ) - requestsInFlightGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( - float64(atomic.LoadInt64(&broker.inFlight)), - ) - responseRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set( - broker.responseRate.Snapshot().Rate1(), + broker.RequestLatencyP99Mic / 1000, ) + requestsInFlightGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(float64(broker.InFlight)) + responseRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(broker.ResponseRate) } } @@ -221,7 +82,8 @@ func (m *franzMetricsCollector) cleanupMetrics() { recordsPerRequestGauge.DeleteLabelValues(keyspace, changefeedID, avg) recordsPerRequestGauge.DeleteLabelValues(keyspace, changefeedID, p99) - for id := range m.hook.snapshotBrokers() { + snapshot := m.hook.Snapshot() + for id := range snapshot.Brokers { brokerID := strconv.Itoa(int(id)) OutgoingByteRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) RequestRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) diff --git a/pkg/sink/kafka/internal/logutil/logutil.go b/pkg/sink/kafka/internal/logutil/logutil.go new file mode 100644 index 0000000000..c38de351f6 --- /dev/null +++ b/pkg/sink/kafka/internal/logutil/logutil.go @@ -0,0 +1,107 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/pingcap/errors" + "github.com/pingcap/ticdc/pkg/sink/codec/common" +) + +// DetermineEventType infers the event type based on MessageLogInfo content. +func DetermineEventType(info *common.MessageLogInfo) string { + if info == nil { + return "unknown" + } + if info.DDL != nil { + return "ddl" + } + if info.Checkpoint != nil { + return "checkpoint" + } + if len(info.Rows) > 0 { + return "dml" + } + return "unknown" +} + +// BuildEventLogContext builds a textual representation of event info. +func BuildEventLogContext(keyspace, changefeed string, info *common.MessageLogInfo) string { + var sb strings.Builder + sb.WriteString("keyspace=") + sb.WriteString(keyspace) + sb.WriteString(", changefeed=") + sb.WriteString(changefeed) + sb.WriteString(", eventType=") + sb.WriteString(DetermineEventType(info)) + + if info == nil { + return sb.String() + } + + if len(info.Rows) > 0 { + if rowsStr := formatDMLInfo(info.Rows); rowsStr != "" { + sb.WriteString(", dmlInfo=") + sb.WriteString(rowsStr) + } + } + + if info.DDL != nil { + if info.DDL.Query != "" { + sb.WriteString(", ddlQuery=") + sb.WriteString(strconv.Quote(info.DDL.Query)) + } + if info.DDL.StartTs != 0 { + sb.WriteString(", ddlStartTs=") + sb.WriteString(strconv.FormatUint(info.DDL.StartTs, 10)) + } + if info.DDL.CommitTs != 0 { + sb.WriteString(", ddlCommitTs=") + sb.WriteString(strconv.FormatUint(info.DDL.CommitTs, 10)) + } + } + + if info.Checkpoint != nil && info.Checkpoint.CommitTs != 0 { + sb.WriteString(", checkpointTs=") + sb.WriteString(strconv.FormatUint(info.Checkpoint.CommitTs, 10)) + } + + return sb.String() +} + +// AnnotateEventError logs the event context and annotates the error with that context. +func AnnotateEventError( + keyspace, changefeed string, + info *common.MessageLogInfo, + err error, +) error { + if err == nil { + return nil + } + if contextStr := BuildEventLogContext(keyspace, changefeed, info); contextStr != "" { + return errors.Annotate(err, contextStr+"; ErrorInfo:"+err.Error()) + } + return err +} + +func formatDMLInfo(rows []common.RowLogInfo) string { + data, err := json.Marshal(rows) + if err != nil { + return "" + } + return string(data) +} diff --git a/pkg/sink/kafka/logutil.go b/pkg/sink/kafka/logutil.go index 8a90e7e3e9..3fb5dae4be 100644 --- a/pkg/sink/kafka/logutil.go +++ b/pkg/sink/kafka/logutil.go @@ -14,73 +14,18 @@ package kafka import ( - "encoding/json" - "strconv" - "strings" - - "github.com/pingcap/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/kafka/internal/logutil" ) // DetermineEventType infers the event type based on MessageLogInfo content. func DetermineEventType(info *common.MessageLogInfo) string { - if info == nil { - return "unknown" - } - if info.DDL != nil { - return "ddl" - } - if info.Checkpoint != nil { - return "checkpoint" - } - if len(info.Rows) > 0 { - return "dml" - } - return "unknown" + return logutil.DetermineEventType(info) } // BuildEventLogContext builds a textual representation of event info. func BuildEventLogContext(keyspace, changefeed string, info *common.MessageLogInfo) string { - var sb strings.Builder - sb.WriteString("keyspace=") - sb.WriteString(keyspace) - sb.WriteString(", changefeed=") - sb.WriteString(changefeed) - sb.WriteString(", eventType=") - sb.WriteString(DetermineEventType(info)) - - if info == nil { - return sb.String() - } - - if len(info.Rows) > 0 { - if rowsStr := formatDMLInfo(info.Rows); rowsStr != "" { - sb.WriteString(", dmlInfo=") - sb.WriteString(rowsStr) - } - } - - if info.DDL != nil { - if info.DDL.Query != "" { - sb.WriteString(", ddlQuery=") - sb.WriteString(strconv.Quote(info.DDL.Query)) - } - if info.DDL.StartTs != 0 { - sb.WriteString(", ddlStartTs=") - sb.WriteString(strconv.FormatUint(info.DDL.StartTs, 10)) - } - if info.DDL.CommitTs != 0 { - sb.WriteString(", ddlCommitTs=") - sb.WriteString(strconv.FormatUint(info.DDL.CommitTs, 10)) - } - } - - if info.Checkpoint != nil && info.Checkpoint.CommitTs != 0 { - sb.WriteString(", checkpointTs=") - sb.WriteString(strconv.FormatUint(info.Checkpoint.CommitTs, 10)) - } - - return sb.String() + return logutil.BuildEventLogContext(keyspace, changefeed, info) } // AnnotateEventError logs the event context and annotates the error with that context. @@ -89,19 +34,5 @@ func AnnotateEventError( info *common.MessageLogInfo, err error, ) error { - if err == nil { - return nil - } - if contextStr := BuildEventLogContext(keyspace, changefeed, info); contextStr != "" { - return errors.Annotate(err, contextStr+"; ErrorInfo:"+err.Error()) - } - return err -} - -func formatDMLInfo(rows []common.RowLogInfo) string { - data, err := json.Marshal(rows) - if err != nil { - return "" - } - return string(data) + return logutil.AnnotateEventError(keyspace, changefeed, info, err) } diff --git a/pkg/sink/kafka/logutil_test.go b/pkg/sink/kafka/logutil_test.go index eddaa41f32..982c3f4e1d 100644 --- a/pkg/sink/kafka/logutil_test.go +++ b/pkg/sink/kafka/logutil_test.go @@ -13,6 +13,7 @@ package kafka import ( + "encoding/json" "strings" "testing" @@ -48,7 +49,9 @@ func TestBuildEventLogContextRowsIncluded(t *testing.T) { } info := &common.MessageLogInfo{Rows: rows} ctx := BuildEventLogContext("ks", "cf", info) - expected := formatDMLInfo(rows) + data, err := json.Marshal(rows) + require.NoError(t, err) + expected := string(data) require.Contains(t, ctx, "dmlInfo="+expected) require.NotContains(t, ctx, "dmlInfoTruncated") require.NotContains(t, ctx, "truncatedRows") diff --git a/pkg/sink/kafka/main_test.go b/pkg/sink/kafka/main_test.go index 0e524e68ff..66978ada9e 100644 --- a/pkg/sink/kafka/main_test.go +++ b/pkg/sink/kafka/main_test.go @@ -17,8 +17,12 @@ import ( "testing" "github.com/pingcap/ticdc/pkg/leakutil" + "go.uber.org/goleak" ) func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m) + leakutil.SetUpLeakTest( + m, + goleak.IgnoreAnyFunction("github.com/godbus/dbus.(*Conn).inWorker"), + ) } From 58208ec0a46a6e2b31a2ca6d5bdfd69ebeab4c4c Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 9 Feb 2026 17:44:54 +0000 Subject: [PATCH 06/61] add more code --- pkg/sink/kafka/franz/admin_client.go | 64 ++++++++------------ pkg/sink/kafka/franz/async_producer.go | 43 +++++++------- pkg/sink/kafka/franz/factory.go | 71 +++++++---------------- pkg/sink/kafka/franz/factory_api_test.go | 30 ++++++++++ pkg/sink/kafka/franz/gssapi.go | 40 ++++++------- pkg/sink/kafka/franz/sync_producer.go | 18 ++---- pkg/sink/kafka/franz_admin_client.go | 6 +- pkg/sink/kafka/franz_admin_client_test.go | 30 ++++++++++ pkg/sink/kafka/franz_factory.go | 21 ++++--- 9 files changed, 166 insertions(+), 157 deletions(-) create mode 100644 pkg/sink/kafka/franz/factory_api_test.go create mode 100644 pkg/sink/kafka/franz_admin_client_test.go diff --git a/pkg/sink/kafka/franz/admin_client.go b/pkg/sink/kafka/franz/admin_client.go index 014417ec93..845c780748 100644 --- a/pkg/sink/kafka/franz/admin_client.go +++ b/pkg/sink/kafka/franz/admin_client.go @@ -34,11 +34,6 @@ type TopicDetail struct { ReplicationFactor int16 } -// Broker represents a Kafka broker. -type Broker struct { - ID int32 -} - type AdminClient struct { changefeed common.ChangeFeedID @@ -53,12 +48,12 @@ func NewAdminClient( o *Options, hook kgo.Hook, ) (*AdminClient, error) { - baseOpts, err := buildFranzBaseOptions(ctx, o, hook) + opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) } - client, err := kgo.NewClient(baseOpts...) + client, err := kgo.NewClient(opts...) if err != nil { return nil, errors.Trace(err) } @@ -83,7 +78,7 @@ func (a *AdminClient) newRequestContext() (context.Context, context.CancelFunc) return context.WithTimeout(a.client.Context(), a.timeout) } -func (a *AdminClient) GetAllBrokers() []Broker { +func (a *AdminClient) GetAllBrokers() []int32 { ctx, cancel := a.newRequestContext() defer cancel() @@ -95,15 +90,7 @@ func (a *AdminClient) GetAllBrokers() []Broker { zap.Error(err)) return nil } - - result := make([]Broker, 0, len(meta.Brokers)) - for _, broker := range meta.Brokers { - if broker.NodeID < 0 { - continue - } - result = append(result, Broker{ID: broker.NodeID}) - } - return result + return meta.Brokers.NodeIDs() } func (a *AdminClient) GetBrokerConfig(configName string) (string, error) { @@ -138,7 +125,7 @@ func (a *AdminClient) GetBrokerConfig(configName string) (string, error) { } } - log.Warn("Kafka config item not found", + log.Warn("Kafka broker config item not found", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.String("configName", configName)) @@ -165,7 +152,7 @@ func (a *AdminClient) GetTopicConfig(topicName string, configName string) (strin for _, entry := range resource.Configs { if entry.Key == configName { - log.Info("Kafka config item found", + log.Info("Kafka topic config item found", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.String("configName", configName), @@ -204,25 +191,22 @@ func (a *AdminClient) GetTopicsMeta( if !ok { continue } - if detail.Err != nil { - if errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { - continue + if detail.Err == nil { + result[topic] = TopicDetail{ + Name: topic, + NumPartitions: int32(len(detail.Partitions)), } - if !ignoreTopicError { - return nil, errors.Trace(detail.Err) - } - log.Warn("fetch topic meta failed", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("topic", topic), - zap.Error(detail.Err)) continue } - - result[topic] = TopicDetail{ - Name: topic, - NumPartitions: int32(len(detail.Partitions)), + if errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { + continue + } + if !ignoreTopicError { + return nil, errors.Trace(detail.Err) } + log.Warn("fetch topic meta failed", + zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), + zap.String("topic", topic), zap.Error(detail.Err)) } return result, nil } @@ -275,13 +259,13 @@ func (a *AdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error if !ok { return errors.ErrKafkaCreateTopic.GenWithStack("kafka topic create response is missing") } - if resp.Err != nil { - if errors.Is(resp.Err, kerr.TopicAlreadyExists) { - return nil - } - return errors.Trace(resp.Err) + if resp.Err == nil { + return nil + } + if errors.Is(resp.Err, kerr.TopicAlreadyExists) { + return nil } - return nil + return errors.Trace(resp.Err) } func (a *AdminClient) Heartbeat() {} diff --git a/pkg/sink/kafka/franz/async_producer.go b/pkg/sink/kafka/franz/async_producer.go index cca0185181..b8d5b0d9cb 100644 --- a/pkg/sink/kafka/franz/async_producer.go +++ b/pkg/sink/kafka/franz/async_producer.go @@ -28,8 +28,6 @@ import ( "go.uber.org/zap" ) -const franzAsyncRecordRetries = 3 - type AsyncProducer struct { client *kgo.Client changefeedID commonType.ChangeFeedID @@ -44,16 +42,12 @@ func NewAsyncProducer( o *Options, hook kgo.Hook, ) (*AsyncProducer, error) { - baseOpts, err := buildFranzBaseOptions(ctx, o, hook) - if err != nil { - return nil, errors.Trace(err) - } - producerOpts, err := buildFranzProducerOptions(o, franzAsyncRecordRetries) + opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) } - - client, err := kgo.NewClient(append(baseOpts, producerOpts...)...) + opts = append(opts, newProducerOptions(o)...) + client, err := kgo.NewClient(opts...) if err != nil { return nil, errors.Trace(err) } @@ -93,17 +87,21 @@ func (p *AsyncProducer) AsyncSend( select { case <-ctx.Done(): - return errors.Trace(ctx.Err()) + return context.Cause(ctx) default: } + var ( + keyspace = p.changefeedID.Keyspace() + changefeed = p.changefeedID.Name() + ) + failpoint.Inject("KafkaSinkAsyncSendError", func() { log.Info("KafkaSinkAsyncSendError error injected", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name())) + zap.String("keyspace", keyspace), zap.String("changefeed", changefeed)) errWithInfo := logutil.AnnotateEventError( - p.changefeedID.Keyspace(), - p.changefeedID.Name(), + keyspace, + changefeed, message.LogInfo, errors.New("kafka sink injected error"), ) @@ -114,9 +112,6 @@ func (p *AsyncProducer) AsyncSend( failpoint.Return(nil) }) - callback := message.Callback - logInfo := message.LogInfo - record := &kgo.Record{ Topic: topic, Partition: partition, @@ -124,16 +119,18 @@ func (p *AsyncProducer) AsyncSend( Value: message.Value, } - p.client.Produce(ctx, record, func(_ *kgo.Record, err error) { + callback := message.Callback + logInfo := message.LogInfo + promise := func(_ *kgo.Record, err error) { if err != nil { errWithInfo := logutil.AnnotateEventError( - p.changefeedID.Keyspace(), - p.changefeedID.Name(), + keyspace, changefeed, logInfo, err, ) select { case p.errCh <- errors.WrapError(errors.ErrKafkaAsyncSendMessage, errWithInfo): + // todo: remove this default after support dispatcher recover logic. default: } return @@ -141,8 +138,8 @@ func (p *AsyncProducer) AsyncSend( if callback != nil { callback() } - }) - + } + p.client.Produce(ctx, record, promise) return nil } @@ -156,7 +153,7 @@ func (p *AsyncProducer) AsyncRunCallback(ctx context.Context) error { log.Info("async producer exit since context is done", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name())) - return errors.Trace(ctx.Err()) + return context.Cause(ctx) case err := <-p.errCh: if err == nil { return nil diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/franz/factory.go index 603038a54d..5c331ca253 100644 --- a/pkg/sink/kafka/franz/factory.go +++ b/pkg/sink/kafka/franz/factory.go @@ -34,14 +34,6 @@ import ( "golang.org/x/oauth2/clientcredentials" ) -type RequiredAcks int16 - -const ( - NoResponse RequiredAcks = 0 - WaitForLocal RequiredAcks = 1 - WaitForAll RequiredAcks = -1 -) - type Options struct { BrokerEndpoints []string ClientID string @@ -51,7 +43,6 @@ type Options struct { MaxMessageBytes int Compression string - RequiredAcks RequiredAcks EnableTLS bool Credential *security.Credential @@ -63,7 +54,7 @@ type Options struct { ReadTimeout time.Duration } -func buildFranzBaseOptions( +func newOptions( ctx context.Context, o *Options, hook kgo.Hook, @@ -96,7 +87,7 @@ func buildFranzBaseOptions( } if o.EnableTLS { - tlsConfig, err := buildFranzTLSConfig(o) + tlsConfig, err := newTLSConfig(o) if err != nil { return nil, errors.Trace(err) } @@ -114,7 +105,7 @@ func buildFranzBaseOptions( return opts, nil } -func buildFranzTLSConfig(o *Options) (*tls.Config, error) { +func newTLSConfig(o *Options) (*tls.Config, error) { tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, NextProtos: []string{"h2", "http/1.1"}, @@ -163,7 +154,7 @@ func buildFranzSaslMechanism(ctx context.Context, o *Options) (sasl.Mechanism, e } return auth.AsSha512Mechanism(), nil case security.OAuthMechanism: - tokenSource, err := buildFranzOauthTokenSource(ctx, o) + tokenSource, err := newOauthTokenSource(ctx, o) if err != nil { return nil, errors.Trace(err) } @@ -175,13 +166,13 @@ func buildFranzSaslMechanism(ctx context.Context, o *Options) (sasl.Mechanism, e return oauth.Auth{Token: token.AccessToken}, nil }), nil case security.GSSAPIMechanism: - return buildFranzGSSAPIMechanism(o.SASL.GSSAPI) + return buildGSSAPIMechanism(o.SASL.GSSAPI) default: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) } + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) } -func buildFranzOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSource, error) { +func newOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSource, error) { endpointParams := url.Values{} if o.SASL.OAuth2.GrantType != "" { endpointParams.Set("grant_type", o.SASL.OAuth2.GrantType) @@ -205,28 +196,9 @@ func buildFranzOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSo return cfg.TokenSource(ctx), nil } -func buildFranzProducerOptions( +func newProducerOptions( o *Options, - recordRetries int, -) ([]kgo.Opt, error) { - var acks kgo.Acks - switch o.RequiredAcks { - case WaitForAll: - acks = kgo.AllISRAcks() - case WaitForLocal: - acks = kgo.LeaderAck() - case NoResponse: - acks = kgo.NoAck() - default: - acks = kgo.AllISRAcks() - log.Warn("unknown required acks, use all isr acks", zap.Int16("requiredAcks", int16(o.RequiredAcks))) - } - - compressionOpt, err := buildFranzCompressionOption(o) - if err != nil { - return nil, errors.Trace(err) - } - +) []kgo.Opt { produceTimeout := o.ReadTimeout if produceTimeout < 100*time.Millisecond { produceTimeout = 10 * time.Second @@ -234,34 +206,35 @@ func buildFranzProducerOptions( return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), - kgo.RequiredAcks(acks), + kgo.RequiredAcks(kgo.AllISRAcks()), kgo.DisableIdempotentWrite(), kgo.MaxProduceRequestsInflightPerBroker(1), - kgo.RecordRetries(recordRetries), + kgo.RecordRetries(5), kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), kgo.ProduceRequestTimeout(produceTimeout), kgo.ProducerLinger(0), - compressionOpt, - }, nil + newCompressionOption(o), + } } -func buildFranzCompressionOption(o *Options) (kgo.Opt, error) { +func newCompressionOption(o *Options) kgo.Opt { compression := strings.ToLower(strings.TrimSpace(o.Compression)) + var codec kgo.CompressionCodec switch compression { case "none": - return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + codec = kgo.NoCompression() case "gzip": - return kgo.ProducerBatchCompression(kgo.GzipCompression()), nil + codec = kgo.GzipCompression() case "snappy": - return kgo.ProducerBatchCompression(kgo.SnappyCompression()), nil + codec = kgo.SnappyCompression() case "lz4": - return kgo.ProducerBatchCompression(kgo.Lz4Compression()), nil + codec = kgo.Lz4Compression() case "zstd": - return kgo.ProducerBatchCompression(kgo.ZstdCompression()), nil + codec = kgo.ZstdCompression() case "": - return kgo.ProducerBatchCompression(kgo.NoCompression()), nil + codec = kgo.NoCompression() default: log.Warn("unsupported compression algorithm", zap.String("compression", o.Compression)) - return kgo.ProducerBatchCompression(kgo.NoCompression()), nil } + return kgo.ProducerBatchCompression(codec) } diff --git a/pkg/sink/kafka/franz/factory_api_test.go b/pkg/sink/kafka/franz/factory_api_test.go new file mode 100644 index 0000000000..fbefab76af --- /dev/null +++ b/pkg/sink/kafka/franz/factory_api_test.go @@ -0,0 +1,30 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import "testing" + +func TestBuildFranzCompressionOptionHasNoErrorReturn(t *testing.T) { + t.Parallel() + + o := &Options{} + _ = newCompressionOption(o) +} + +func TestBuildFranzProducerOptionsHasNoErrorReturn(t *testing.T) { + t.Parallel() + + o := &Options{} + _ = newProducerOptions(o, 3) +} diff --git a/pkg/sink/kafka/franz/gssapi.go b/pkg/sink/kafka/franz/gssapi.go index d1d1d2e4ad..72dcfffbd3 100644 --- a/pkg/sink/kafka/franz/gssapi.go +++ b/pkg/sink/kafka/franz/gssapi.go @@ -43,7 +43,7 @@ const ( gssAPIFinished = 3 ) -type franzKerberosClient interface { +type kerborosClient interface { Login() error GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) Domain() string @@ -51,19 +51,19 @@ type franzKerberosClient interface { Destroy() } -type franzGSSAPIMechanism struct { +type gssapiMechanism struct { config security.GSSAPI } -func (m *franzGSSAPIMechanism) Name() string { +func (m *gssapiMechanism) Name() string { return "GSSAPI" } -func (m *franzGSSAPIMechanism) Authenticate( +func (m *gssapiMechanism) Authenticate( _ context.Context, host string, ) (sasl.Session, []byte, error) { - client, err := newFranzKerberosClient(m.config) + client, err := newKerborosClient(m.config) if err != nil { return nil, nil, errors.Trace(err) } @@ -80,7 +80,7 @@ func (m *franzGSSAPIMechanism) Authenticate( return nil, nil, errors.Trace(err) } - session := &franzGSSAPISession{ + session := &gssapiSession{ client: client, ticket: ticket, encKey: encKey, @@ -94,8 +94,8 @@ func (m *franzGSSAPIMechanism) Authenticate( return session, firstMessage, nil } -type franzGSSAPISession struct { - client franzKerberosClient +type gssapiSession struct { + client kerborosClient ticket messages.Ticket encKey types.EncryptionKey step int @@ -103,7 +103,7 @@ type franzGSSAPISession struct { closeOnce sync.Once } -func (s *franzGSSAPISession) Challenge(challenge []byte) (bool, []byte, error) { +func (s *gssapiSession) Challenge(challenge []byte) (bool, []byte, error) { switch s.step { case gssAPIVerify: msg, err := s.nextMessage(challenge) @@ -124,7 +124,7 @@ func (s *franzGSSAPISession) Challenge(challenge []byte) (bool, []byte, error) { } } -func (s *franzGSSAPISession) close() { +func (s *gssapiSession) close() { s.closeOnce.Do(func() { if s.client != nil { s.client.Destroy() @@ -132,10 +132,10 @@ func (s *franzGSSAPISession) close() { }) } -func (s *franzGSSAPISession) nextMessage(challenge []byte) ([]byte, error) { +func (s *gssapiSession) nextMessage(challenge []byte) ([]byte, error) { switch s.step { case gssAPIInitial: - token, err := createKrb5Token(s.client.Domain(), s.client.CName(), s.ticket, s.encKey) + token, err := newKrb5Token(s.client.Domain(), s.client.CName(), s.ticket, s.encKey) if err != nil { return nil, errors.Trace(err) } @@ -165,7 +165,7 @@ func (s *franzGSSAPISession) nextMessage(challenge []byte) ([]byte, error) { } } -func buildFranzGSSAPIMechanism(g security.GSSAPI) (sasl.Mechanism, error) { +func buildGSSAPIMechanism(g security.GSSAPI) (sasl.Mechanism, error) { if g.ServiceName == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") @@ -199,22 +199,22 @@ func buildFranzGSSAPIMechanism(g security.GSSAPI) (sasl.Mechanism, error) { "unsupported sasl-gssapi-auth-type %d", g.AuthType) } - return &franzGSSAPIMechanism{config: g}, nil + return &gssapiMechanism{config: g}, nil } -type franzGoKrb5Client struct { +type krb5Client struct { krb5client.Client } -func (c *franzGoKrb5Client) Domain() string { +func (c *krb5Client) Domain() string { return c.Credentials.Domain() } -func (c *franzGoKrb5Client) CName() types.PrincipalName { +func (c *krb5Client) CName() types.PrincipalName { return c.Credentials.CName() } -func newFranzKerberosClient(g security.GSSAPI) (franzKerberosClient, error) { +func newKerborosClient(g security.GSSAPI) (kerborosClient, error) { cfg, err := krb5config.Load(g.KerberosConfigPath) if err != nil { return nil, errors.Trace(err) @@ -236,10 +236,10 @@ func newFranzKerberosClient(g security.GSSAPI) (franzKerberosClient, error) { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "unsupported sasl-gssapi-auth-type %d", g.AuthType) } - return &franzGoKrb5Client{*client}, nil + return &krb5Client{*client}, nil } -func createKrb5Token( +func newKrb5Token( domain string, cname types.PrincipalName, ticket messages.Ticket, diff --git a/pkg/sink/kafka/franz/sync_producer.go b/pkg/sink/kafka/franz/sync_producer.go index c234e4bf16..e7d6d87735 100644 --- a/pkg/sink/kafka/franz/sync_producer.go +++ b/pkg/sink/kafka/franz/sync_producer.go @@ -28,8 +28,6 @@ import ( "go.uber.org/zap" ) -const franzSyncRecordRetries = 5 - type SyncProducer struct { id commonType.ChangeFeedID @@ -44,25 +42,21 @@ func NewSyncProducer( o *Options, hook kgo.Hook, ) (*SyncProducer, error) { - baseOpts, err := buildFranzBaseOptions(ctx, o, hook) - if err != nil { - return nil, errors.Trace(err) - } - producerOpts, err := buildFranzProducerOptions(o, franzSyncRecordRetries) + opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) } + opts = append(opts, newProducerOptions(o)...) - client, err := kgo.NewClient(append(baseOpts, producerOpts...)...) + client, err := kgo.NewClient(opts...) if err != nil { return nil, errors.Trace(err) } - produceTimeout := o.ReadTimeout - if produceTimeout <= 0 { - produceTimeout = 10 * time.Second + timeout := o.ReadTimeout + if timeout <= 0 { + timeout = 10 * time.Second } - timeout := time.Duration(franzSyncRecordRetries+1) * produceTimeout return &SyncProducer{ id: changefeedID, diff --git a/pkg/sink/kafka/franz_admin_client.go b/pkg/sink/kafka/franz_admin_client.go index d1e8790063..ade2168b6c 100644 --- a/pkg/sink/kafka/franz_admin_client.go +++ b/pkg/sink/kafka/franz_admin_client.go @@ -14,7 +14,9 @@ package kafka import ( + "github.com/pingcap/ticdc/pkg/errors" kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" + "github.com/twmb/franz-go/pkg/kadm" ) // franzAdminClientAdapter adapts the franz-go admin client implementation to kafka.ClusterAdminClient. @@ -67,9 +69,9 @@ func (a *franzAdminClientAdapter) GetTopicsPartitionsNum(topics []string) (map[s func (a *franzAdminClientAdapter) CreateTopic(detail *TopicDetail, validateOnly bool) error { if detail == nil { - return a.inner.CreateTopic(nil, validateOnly) + return errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") } - franzDetail := &kafkafranz.TopicDetail{ + franzDetail := &kadm.TopicDetail{ Name: detail.Name, NumPartitions: detail.NumPartitions, ReplicationFactor: detail.ReplicationFactor, diff --git a/pkg/sink/kafka/franz_admin_client_test.go b/pkg/sink/kafka/franz_admin_client_test.go new file mode 100644 index 0000000000..34e47f402d --- /dev/null +++ b/pkg/sink/kafka/franz_admin_client_test.go @@ -0,0 +1,30 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFranzAdminClientAdapterCreateTopicNilDetailReturnsError(t *testing.T) { + t.Parallel() + + adapter := &franzAdminClientAdapter{inner: nil} + require.NotPanics(t, func() { + err := adapter.CreateTopic(nil, false) + require.Error(t, err) + }) +} diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 951a8942dd..cdcfbbccb6 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -18,13 +18,13 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" + "github.com/pingcap/ticdc/pkg/sink/kafka/franz" ) type franzFactory struct { changefeedID common.ChangeFeedID option *options - metricsHook *kafkafranz.MetricsHook + metricsHook *franz.MetricsHook } // NewFranzFactory constructs a Factory with franz-go implementation. @@ -36,7 +36,7 @@ func NewFranzFactory( o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - adminInner, err := kafkafranz.NewAdminClient(ctx, changefeedID, newFranzOptions(o), nil) + adminInner, err := franz.NewAdminClient(ctx, changefeedID, newFranzOptions(o), nil) if err != nil { return nil, errors.Trace(err) } @@ -50,12 +50,12 @@ func NewFranzFactory( return &franzFactory{ changefeedID: changefeedID, option: o, - metricsHook: kafkafranz.NewMetricsHook(), + metricsHook: franz.NewMetricsHook(), }, nil } func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - adminInner, err := kafkafranz.NewAdminClient(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) + adminInner, err := franz.NewAdminClient(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -63,7 +63,7 @@ func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, err } func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := kafkafranz.NewSyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) + producer, err := franz.NewSyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -71,7 +71,7 @@ func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { } func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := kafkafranz.NewAsyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) + producer, err := franz.NewAsyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -85,11 +85,11 @@ func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { } } -func newFranzOptions(o *options) *kafkafranz.Options { +func newFranzOptions(o *options) *franz.Options { if o == nil { - return &kafkafranz.Options{} + return &franz.Options{} } - return &kafkafranz.Options{ + return &franz.Options{ BrokerEndpoints: o.BrokerEndpoints, ClientID: o.ClientID, @@ -98,7 +98,6 @@ func newFranzOptions(o *options) *kafkafranz.Options { MaxMessageBytes: o.MaxMessageBytes, Compression: o.Compression, - RequiredAcks: kafkafranz.RequiredAcks(o.RequiredAcks), EnableTLS: o.EnableTLS, Credential: o.Credential, From 5eede91a02aa54f1a3d8664239ceee1d85598316 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 10 Feb 2026 07:48:23 +0000 Subject: [PATCH 07/61] adjust monitoring --- pkg/sink/kafka/franz/admin_client_test.go | 36 ++++ pkg/sink/kafka/franz/metrics_hook.go | 239 +++++++++++----------- pkg/sink/kafka/franz_admin_client.go | 5 +- pkg/sink/kafka/franz_factory.go | 23 ++- pkg/sink/kafka/franz_factory_test.go | 63 ++++-- pkg/sink/kafka/franz_metrics_collector.go | 95 --------- pkg/sink/kafka/metrics.go | 24 +++ 7 files changed, 248 insertions(+), 237 deletions(-) create mode 100644 pkg/sink/kafka/franz/admin_client_test.go delete mode 100644 pkg/sink/kafka/franz_metrics_collector.go diff --git a/pkg/sink/kafka/franz/admin_client_test.go b/pkg/sink/kafka/franz/admin_client_test.go new file mode 100644 index 0000000000..782e7ed83b --- /dev/null +++ b/pkg/sink/kafka/franz/admin_client_test.go @@ -0,0 +1,36 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetBrokerConfigControllerNotAvailableUsesDedicatedError(t *testing.T) { + t.Parallel() + + _, currentFile, _, ok := runtime.Caller(0) + require.True(t, ok) + dir := filepath.Dir(currentFile) + + source, err := os.ReadFile(filepath.Join(dir, "admin_client.go")) + require.NoError(t, err) + require.NotContains(t, string(source), `ErrKafkaInvalidConfig.GenWithStack("kafka controller is not available")`) + require.Contains(t, string(source), "ErrKafkaControllerNotAvailable") +} diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/franz/metrics_hook.go index 8e0bd415ea..e669c03505 100644 --- a/pkg/sink/kafka/franz/metrics_hook.go +++ b/pkg/sink/kafka/franz/metrics_hook.go @@ -14,142 +14,129 @@ package franz import ( + "context" + "strconv" "sync" - "sync/atomic" "time" - "github.com/rcrowley/go-metrics" + "github.com/prometheus/client_golang/prometheus" "github.com/twmb/franz-go/pkg/kgo" ) -type brokerMetrics struct { - outgoingByteRate metrics.Meter - requestRate metrics.Meter - requestLatency metrics.Histogram - responseRate metrics.Meter - inFlight int64 +type MetricsHook struct { + promMu sync.RWMutex + promBound bool + keyspace string + changefeed string + prom PrometheusMetrics } -func newBrokerMetrics() *brokerMetrics { - return &brokerMetrics{ - outgoingByteRate: metrics.NewMeter(), - requestRate: metrics.NewMeter(), - requestLatency: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), - responseRate: metrics.NewMeter(), - } +type PrometheusMetrics struct { + RequestsInFlight *prometheus.GaugeVec + OutgoingByteRate *prometheus.GaugeVec + RequestRate *prometheus.GaugeVec + RequestLatency *prometheus.HistogramVec + ResponseRate *prometheus.GaugeVec + CompressionRatio *prometheus.HistogramVec + RecordsPerRequest *prometheus.HistogramVec } -type MetricsHook struct { - mu sync.RWMutex - brokers map[int32]*brokerMetrics +func NewMetricsHook() *MetricsHook { + return &MetricsHook{} +} - compressionRatio metrics.Histogram - recordsPerReq metrics.Histogram +func (h *MetricsHook) BindPrometheusMetrics( + keyspace string, + changefeed string, + _ time.Duration, + metrics PrometheusMetrics, +) { + h.promMu.Lock() + defer h.promMu.Unlock() + + h.keyspace = keyspace + h.changefeed = changefeed + h.prom = metrics + h.promBound = true } -func NewMetricsHook() *MetricsHook { - return &MetricsHook{ - brokers: make(map[int32]*brokerMetrics), - compressionRatio: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), - recordsPerReq: metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)), +func (h *MetricsHook) loadPrometheusMetrics() (string, string, PrometheusMetrics, bool) { + h.promMu.RLock() + defer h.promMu.RUnlock() + + return h.keyspace, h.changefeed, h.prom, h.promBound +} + +func (h *MetricsHook) Run(ctx context.Context) { + _, _, _, bound := h.loadPrometheusMetrics() + + if !bound { + <-ctx.Done() + return } + + <-ctx.Done() + h.CleanupPrometheusMetrics() } -func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { - broker := h.getBroker(nodeID) - if broker == nil { +func (h *MetricsHook) FlushPrometheusMetrics(interval time.Duration) { + _ = interval +} + +func (h *MetricsHook) CleanupPrometheusMetrics() { + keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() + + if !bound { return } - if bytesWritten > 0 { - broker.outgoingByteRate.Mark(int64(bytesWritten)) + labels := prometheus.Labels{ + "namespace": keyspace, + "changefeed": changefeed, + } + if metrics.OutgoingByteRate != nil { + metrics.OutgoingByteRate.MetricVec.DeletePartialMatch(labels) + } + if metrics.RequestRate != nil { + metrics.RequestRate.MetricVec.DeletePartialMatch(labels) + } + if metrics.ResponseRate != nil { + metrics.ResponseRate.MetricVec.DeletePartialMatch(labels) } - broker.requestRate.Mark(1) - if err == nil { - atomic.AddInt64(&broker.inFlight, 1) + if metrics.RequestsInFlight != nil { + metrics.RequestsInFlight.MetricVec.DeletePartialMatch(labels) + } + if metrics.RequestLatency != nil { + metrics.RequestLatency.MetricVec.DeletePartialMatch(labels) + } + if metrics.CompressionRatio != nil { + metrics.CompressionRatio.MetricVec.DeletePartialMatch(labels) + } + if metrics.RecordsPerRequest != nil { + metrics.RecordsPerRequest.MetricVec.DeletePartialMatch(labels) } } -func (h *MetricsHook) getBroker(nodeID int32) *brokerMetrics { +func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { if nodeID < 0 { - return nil + return } - h.mu.RLock() - broker := h.brokers[nodeID] - h.mu.RUnlock() - if broker != nil { - return broker + keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() + if !bound { + return } + brokerID := strconv.Itoa(int(nodeID)) - h.mu.Lock() - defer h.mu.Unlock() - broker = h.brokers[nodeID] - if broker != nil { - return broker + if metrics.OutgoingByteRate != nil && bytesWritten > 0 { + metrics.OutgoingByteRate.WithLabelValues(keyspace, changefeed, brokerID).Add(float64(bytesWritten)) } - broker = newBrokerMetrics() - h.brokers[nodeID] = broker - return broker -} - -type BrokerMetricsSnapshot struct { - OutgoingByteRate float64 - RequestRate float64 - RequestLatencyMeanMic float64 - RequestLatencyP99Mic float64 - InFlight int64 - ResponseRate float64 -} - -type MetricsSnapshot struct { - CompressionMean float64 - CompressionP99 float64 - RecordsMean float64 - RecordsP99 float64 - Brokers map[int32]BrokerMetricsSnapshot -} - -func (h *MetricsHook) Snapshot() MetricsSnapshot { - h.mu.RLock() - brokers := make(map[int32]*brokerMetrics, len(h.brokers)) - for id, broker := range h.brokers { - brokers[id] = broker - } - compression := h.compressionRatio.Snapshot() - records := h.recordsPerReq.Snapshot() - h.mu.RUnlock() - - result := MetricsSnapshot{ - CompressionMean: compression.Mean(), - CompressionP99: compression.Percentile(0.99), - RecordsMean: records.Mean(), - RecordsP99: records.Percentile(0.99), - Brokers: make(map[int32]BrokerMetricsSnapshot, len(brokers)), - } - - for id, broker := range brokers { - latencySnapshot := broker.requestLatency.Snapshot() - result.Brokers[id] = BrokerMetricsSnapshot{ - OutgoingByteRate: broker.outgoingByteRate.Snapshot().Rate1(), - RequestRate: broker.requestRate.Snapshot().Rate1(), - RequestLatencyMeanMic: latencySnapshot.Mean(), - RequestLatencyP99Mic: latencySnapshot.Percentile(0.99), - InFlight: atomic.LoadInt64(&broker.inFlight), - ResponseRate: broker.responseRate.Snapshot().Rate1(), - } - } - return result -} - -func (h *MetricsHook) snapshotBrokers() map[int32]*brokerMetrics { - h.mu.RLock() - defer h.mu.RUnlock() - result := make(map[int32]*brokerMetrics, len(h.brokers)) - for id, broker := range h.brokers { - result[id] = broker + if metrics.RequestRate != nil { + metrics.RequestRate.WithLabelValues(keyspace, changefeed, brokerID).Add(1) + } + if err == nil && metrics.RequestsInFlight != nil { + metrics.RequestsInFlight.WithLabelValues(keyspace, changefeed, brokerID).Add(1) } - return result } func (h *MetricsHook) OnBrokerWrite( @@ -168,21 +155,25 @@ func (h *MetricsHook) OnBrokerE2E( _ int16, e2e kgo.BrokerE2E, ) { - broker := h.getBroker(meta.NodeID) - if broker == nil { + if meta.NodeID < 0 { + return + } + + keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() + if !bound { return } + brokerID := strconv.Itoa(int(meta.NodeID)) - if e2e.WriteErr == nil { - if atomic.AddInt64(&broker.inFlight, -1) < 0 { - atomic.StoreInt64(&broker.inFlight, 0) - } + if e2e.WriteErr == nil && metrics.RequestsInFlight != nil { + metrics.RequestsInFlight.WithLabelValues(keyspace, changefeed, brokerID).Add(-1) } - if e2e.BytesRead > 0 && e2e.ReadErr == nil { - broker.responseRate.Mark(1) + if e2e.BytesRead > 0 && e2e.ReadErr == nil && metrics.ResponseRate != nil { + metrics.ResponseRate.WithLabelValues(keyspace, changefeed, brokerID).Add(1) } - if e2e.Err() == nil { - broker.requestLatency.Update(e2e.DurationE2E().Microseconds()) + if e2e.Err() == nil && metrics.RequestLatency != nil { + latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 + metrics.RequestLatency.WithLabelValues(keyspace, changefeed, brokerID).Observe(latencyMs) } } @@ -196,11 +187,17 @@ func (h *MetricsHook) OnProduceBatchWritten( } func (h *MetricsHook) RecordProduceBatchWritten(numRecords int, uncompressedBytes int, compressedBytes int) { - if numRecords > 0 { - h.recordsPerReq.Update(int64(numRecords)) + keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() + if !bound { + return + } + + if metrics.RecordsPerRequest != nil && numRecords > 0 { + records := float64(numRecords) + metrics.RecordsPerRequest.WithLabelValues(keyspace, changefeed).Observe(records) } - if uncompressedBytes > 0 && compressedBytes > 0 { - ratio := int64(float64(uncompressedBytes) / float64(compressedBytes) * 100) - h.compressionRatio.Update(ratio) + if metrics.CompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { + ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 + metrics.CompressionRatio.WithLabelValues(keyspace, changefeed).Observe(ratio) } } diff --git a/pkg/sink/kafka/franz_admin_client.go b/pkg/sink/kafka/franz_admin_client.go index ade2168b6c..c11250aa03 100644 --- a/pkg/sink/kafka/franz_admin_client.go +++ b/pkg/sink/kafka/franz_admin_client.go @@ -16,7 +16,6 @@ package kafka import ( "github.com/pingcap/ticdc/pkg/errors" kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" - "github.com/twmb/franz-go/pkg/kadm" ) // franzAdminClientAdapter adapts the franz-go admin client implementation to kafka.ClusterAdminClient. @@ -30,7 +29,7 @@ func (a *franzAdminClientAdapter) GetAllBrokers() []Broker { brokers := a.inner.GetAllBrokers() result := make([]Broker, 0, len(brokers)) for _, b := range brokers { - result = append(result, Broker{ID: b.ID}) + result = append(result, Broker{ID: b}) } return result } @@ -71,7 +70,7 @@ func (a *franzAdminClientAdapter) CreateTopic(detail *TopicDetail, validateOnly if detail == nil { return errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") } - franzDetail := &kadm.TopicDetail{ + franzDetail := &kafkafranz.TopicDetail{ Name: detail.Name, NumPartitions: detail.NumPartitions, ReplicationFactor: detail.ReplicationFactor, diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index cdcfbbccb6..8aba9018ed 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -47,10 +47,26 @@ func NewFranzFactory( return nil, errors.Trace(err) } + metricsHook := franz.NewMetricsHook() + metricsHook.BindPrometheusMetrics( + changefeedID.Keyspace(), + changefeedID.Name(), + refreshMetricsInterval, + franz.PrometheusMetrics{ + RequestsInFlight: requestsInFlightGauge, + OutgoingByteRate: OutgoingByteRateGauge, + RequestRate: RequestRateGauge, + RequestLatency: franzRequestLatencyHistogram, + ResponseRate: responseRateGauge, + CompressionRatio: franzCompressionRatioHistogram, + RecordsPerRequest: franzRecordsPerRequestHistogram, + }, + ) + return &franzFactory{ changefeedID: changefeedID, option: o, - metricsHook: franz.NewMetricsHook(), + metricsHook: metricsHook, }, nil } @@ -79,10 +95,7 @@ func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) } func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { - return &franzMetricsCollector{ - changefeedID: f.changefeedID, - hook: f.metricsHook, - } + return f.metricsHook } func newFranzOptions(o *options) *franz.Options { diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go index be638a5b9f..0fe8b30fb6 100644 --- a/pkg/sink/kafka/franz_factory_test.go +++ b/pkg/sink/kafka/franz_factory_test.go @@ -15,11 +15,15 @@ package kafka import ( "testing" + "time" "github.com/pingcap/ticdc/pkg/common" kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" ) func TestFranzFactoryMetricsCollectorType(t *testing.T) { @@ -32,31 +36,64 @@ func TestFranzFactoryMetricsCollectorType(t *testing.T) { } collector := f.MetricsCollector(nil) - _, ok := collector.(*franzMetricsCollector) + _, ok := collector.(*kafkafranz.MetricsHook) require.True(t, ok) } -func TestFranzMetricsCollectorCollectMetrics(t *testing.T) { +func TestFranzMetricsHookWritePrometheusMetricsDirectly(t *testing.T) { t.Parallel() changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-metrics") hook := kafkafranz.NewMetricsHook() - collector := &franzMetricsCollector{ - changefeedID: changefeedID, - hook: hook, - } + hook.BindPrometheusMetrics( + changefeedID.Keyspace(), + changefeedID.Name(), + refreshMetricsInterval, + kafkafranz.PrometheusMetrics{ + RequestsInFlight: requestsInFlightGauge, + OutgoingByteRate: OutgoingByteRateGauge, + RequestRate: RequestRateGauge, + RequestLatency: franzRequestLatencyHistogram, + ResponseRate: responseRateGauge, + CompressionRatio: franzCompressionRatioHistogram, + RecordsPerRequest: franzRecordsPerRequestHistogram, + }, + ) t.Cleanup(func() { - collector.cleanupMetrics() + hook.CleanupPrometheusMetrics() }) hook.RecordBrokerWrite(1, 128, nil) - hook.RecordProduceBatchWritten(8, 400, 200) - - collector.collectMetrics() - keyspace := changefeedID.Keyspace() changefeed := changefeedID.Name() require.Equal(t, float64(1), testutil.ToFloat64(requestsInFlightGauge.WithLabelValues(keyspace, changefeed, "1"))) - require.Greater(t, testutil.ToFloat64(compressionRatioGauge.WithLabelValues(keyspace, changefeed, avg)), 0.0) - require.Greater(t, testutil.ToFloat64(recordsPerRequestGauge.WithLabelValues(keyspace, changefeed, avg)), 0.0) + require.Greater(t, testutil.ToFloat64(RequestRateGauge.WithLabelValues(keyspace, changefeed, "1")), 0.0) + require.Greater(t, testutil.ToFloat64(OutgoingByteRateGauge.WithLabelValues(keyspace, changefeed, "1")), 0.0) + + hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: 1}, 0, kgo.BrokerE2E{ + BytesRead: 128, + TimeToWrite: time.Millisecond, + ReadWait: time.Millisecond, + TimeToRead: time.Millisecond, + }) + require.Equal(t, float64(0), testutil.ToFloat64(requestsInFlightGauge.WithLabelValues(keyspace, changefeed, "1"))) + require.Greater(t, testutil.ToFloat64(responseRateGauge.WithLabelValues(keyspace, changefeed, "1")), 0.0) + require.Greater(t, histogramSampleCount(t, franzRequestLatencyHistogram.WithLabelValues(keyspace, changefeed, "1")), uint64(0)) + + hook.RecordProduceBatchWritten(8, 400, 200) + require.Greater(t, histogramSampleCount(t, franzCompressionRatioHistogram.WithLabelValues(keyspace, changefeed)), uint64(0)) + require.Greater(t, histogramSampleCount(t, franzRecordsPerRequestHistogram.WithLabelValues(keyspace, changefeed)), uint64(0)) +} + +func histogramSampleCount(t *testing.T, observer prometheus.Observer) uint64 { + t.Helper() + + histogram, ok := observer.(prometheus.Histogram) + require.True(t, ok) + + metric := &dto.Metric{} + require.NoError(t, histogram.Write(metric)) + require.NotNil(t, metric.Histogram) + + return metric.Histogram.GetSampleCount() } diff --git a/pkg/sink/kafka/franz_metrics_collector.go b/pkg/sink/kafka/franz_metrics_collector.go deleted file mode 100644 index b4503d802f..0000000000 --- a/pkg/sink/kafka/franz_metrics_collector.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "strconv" - "time" - - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" - kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" - "go.uber.org/zap" -) - -type franzMetricsCollector struct { - changefeedID common.ChangeFeedID - hook *kafkafranz.MetricsHook -} - -func (m *franzMetricsCollector) Run(ctx context.Context) { - ticker := time.NewTicker(refreshMetricsInterval) - defer func() { - ticker.Stop() - m.cleanupMetrics() - }() - - for { - select { - case <-ctx.Done(): - log.Info("franz kafka metrics collector stopped", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name())) - return - case <-ticker.C: - m.collectMetrics() - } - } -} - -func (m *franzMetricsCollector) collectMetrics() { - keyspace := m.changefeedID.Keyspace() - changefeedID := m.changefeedID.Name() - - snapshot := m.hook.Snapshot() - compressionRatioGauge.WithLabelValues(keyspace, changefeedID, avg).Set(snapshot.CompressionMean) - compressionRatioGauge.WithLabelValues(keyspace, changefeedID, p99).Set(snapshot.CompressionP99) - recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, avg).Set(snapshot.RecordsMean) - recordsPerRequestGauge.WithLabelValues(keyspace, changefeedID, p99).Set(snapshot.RecordsP99) - - for id, broker := range snapshot.Brokers { - brokerID := strconv.Itoa(int(id)) - OutgoingByteRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(broker.OutgoingByteRate) - RequestRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(broker.RequestRate) - RequestLatencyGauge.WithLabelValues(keyspace, changefeedID, brokerID, avg).Set( - broker.RequestLatencyMeanMic / 1000, - ) - RequestLatencyGauge.WithLabelValues(keyspace, changefeedID, brokerID, p99).Set( - broker.RequestLatencyP99Mic / 1000, - ) - requestsInFlightGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(float64(broker.InFlight)) - responseRateGauge.WithLabelValues(keyspace, changefeedID, brokerID).Set(broker.ResponseRate) - } -} - -func (m *franzMetricsCollector) cleanupMetrics() { - keyspace := m.changefeedID.Keyspace() - changefeedID := m.changefeedID.Name() - compressionRatioGauge.DeleteLabelValues(keyspace, changefeedID, avg) - compressionRatioGauge.DeleteLabelValues(keyspace, changefeedID, p99) - recordsPerRequestGauge.DeleteLabelValues(keyspace, changefeedID, avg) - recordsPerRequestGauge.DeleteLabelValues(keyspace, changefeedID, p99) - - snapshot := m.hook.Snapshot() - for id := range snapshot.Brokers { - brokerID := strconv.Itoa(int(id)) - OutgoingByteRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) - RequestRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) - RequestLatencyGauge.DeleteLabelValues(keyspace, changefeedID, brokerID, avg) - RequestLatencyGauge.DeleteLabelValues(keyspace, changefeedID, brokerID, p99) - requestsInFlightGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) - responseRateGauge.DeleteLabelValues(keyspace, changefeedID, brokerID) - } -} diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index d3f88055e2..21073235ec 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -53,6 +53,13 @@ var ( Name: "kafka_producer_request_latency", Help: "The request latency for all brokers.", }, []string{"namespace", "changefeed", "broker", "type"}) + franzRequestLatencyHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_producer_request_latency_histogram", + Help: "Request latency histogram for franz producer in milliseconds.", + }, []string{"namespace", "changefeed", "broker"}) // Histogram update by `compression-ratio`. compressionRatioGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -61,6 +68,13 @@ var ( Name: "kafka_producer_compression_ratio", Help: "The compression ratio times 100 of record batches for all topics.", }, []string{"namespace", "changefeed", "type"}) + franzCompressionRatioHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_producer_compression_ratio_histogram", + Help: "Compression ratio times 100 histogram for franz producer.", + }, []string{"namespace", "changefeed"}) // updated by `records-per-request`. recordsPerRequestGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -69,6 +83,13 @@ var ( Name: "kafka_producer_records_per_request", Help: "The number of records per request for all topics.", }, []string{"namespace", "changefeed", "type"}) + franzRecordsPerRequestHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_producer_records_per_request_histogram", + Help: "Records per request histogram for franz producer.", + }, []string{"namespace", "changefeed"}) // Meter mark by 1 once a response received. responseRateGauge = prometheus.NewGaugeVec( @@ -87,6 +108,9 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(OutgoingByteRateGauge) registry.MustRegister(RequestRateGauge) registry.MustRegister(RequestLatencyGauge) + registry.MustRegister(franzRequestLatencyHistogram) + registry.MustRegister(franzCompressionRatioHistogram) + registry.MustRegister(franzRecordsPerRequestHistogram) registry.MustRegister(requestsInFlightGauge) registry.MustRegister(responseRateGauge) From 91a9263f8f14cfe5552daeed62a18769d3882e1b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 10 Feb 2026 10:10:41 +0000 Subject: [PATCH 08/61] adjust code --- pkg/errors/error.go | 4 + pkg/errors/helper.go | 1 + pkg/sink/kafka/franz/admin_client.go | 14 ++- pkg/sink/kafka/franz/admin_client_test.go | 15 +++ pkg/sink/kafka/franz/factory.go | 33 +++++-- pkg/sink/kafka/franz/factory_api_test.go | 50 +++++++++- pkg/sink/kafka/franz/metrics_hook.go | 107 ++++++++++++---------- pkg/sink/kafka/franz/sync_producer.go | 67 +++++++------- pkg/sink/kafka/franz_factory.go | 1 - pkg/sink/kafka/franz_factory_test.go | 1 - 10 files changed, 190 insertions(+), 103 deletions(-) diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 3ad854eee1..e4565f202c 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -172,6 +172,10 @@ var ( "kafka config item not found", errors.RFCCodeText("CDC:ErrKafkaConfigNotFound"), ) + ErrKafkaControllerNotAvailable = errors.Normalize( + "kafka controller is not available", + errors.RFCCodeText("CDC:ErrKafkaControllerNotAvailable"), + ) ErrPulsarInvalidTopicExpression = errors.Normalize( "invalid topic expression", errors.RFCCodeText("CDC:ErrPulsarTopicExprInvalid"), diff --git a/pkg/errors/helper.go b/pkg/errors/helper.go index 3fd77ca3ba..671ba08479 100644 --- a/pkg/errors/helper.go +++ b/pkg/errors/helper.go @@ -115,6 +115,7 @@ var changefeedUnRetryableErrors = []*errors.Error{ ErrSinkURIInvalid, ErrKafkaInvalidConfig, + ErrKafkaControllerNotAvailable, ErrMySQLInvalidConfig, ErrStorageSinkInvalidConfig, diff --git a/pkg/sink/kafka/franz/admin_client.go b/pkg/sink/kafka/franz/admin_client.go index 845c780748..5a223c56da 100644 --- a/pkg/sink/kafka/franz/admin_client.go +++ b/pkg/sink/kafka/franz/admin_client.go @@ -48,6 +48,10 @@ func NewAdminClient( o *Options, hook kgo.Hook, ) (*AdminClient, error) { + if o == nil { + o = &Options{} + } + opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -58,13 +62,7 @@ func NewAdminClient( return nil, errors.Trace(err) } - timeout := o.ReadTimeout - if o.WriteTimeout > timeout { - timeout = o.WriteTimeout - } - if timeout <= 0 { - timeout = 10 * time.Second - } + timeout := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) return &AdminClient{ changefeed: changefeedID, @@ -102,7 +100,7 @@ func (a *AdminClient) GetBrokerConfig(configName string) (string, error) { return "", errors.Trace(err) } if meta.Controller < 0 { - return "", errors.ErrKafkaInvalidConfig.GenWithStack("kafka controller is not available") + return "", errors.ErrKafkaControllerNotAvailable.GenWithStackByArgs() } configs, err := a.admin.DescribeBrokerConfigs(ctx, meta.Controller) diff --git a/pkg/sink/kafka/franz/admin_client_test.go b/pkg/sink/kafka/franz/admin_client_test.go index 782e7ed83b..1e87eab9b2 100644 --- a/pkg/sink/kafka/franz/admin_client_test.go +++ b/pkg/sink/kafka/franz/admin_client_test.go @@ -14,11 +14,13 @@ package franz import ( + "context" "os" "path/filepath" "runtime" "testing" + "github.com/pingcap/ticdc/pkg/common" "github.com/stretchr/testify/require" ) @@ -34,3 +36,16 @@ func TestGetBrokerConfigControllerNotAvailableUsesDedicatedError(t *testing.T) { require.NotContains(t, string(source), `ErrKafkaInvalidConfig.GenWithStack("kafka controller is not available")`) require.Contains(t, string(source), "ErrKafkaControllerNotAvailable") } + +func TestNewAdminClientNilOptionsDoesNotPanic(t *testing.T) { + t.Parallel() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-admin-nil-options") + require.NotPanics(t, func() { + client, err := NewAdminClient(context.Background(), changefeedID, nil, nil) + if client != nil { + client.Close() + } + require.Error(t, err) + }) +} diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/franz/factory.go index 5c331ca253..5c379adf82 100644 --- a/pkg/sink/kafka/franz/factory.go +++ b/pkg/sink/kafka/franz/factory.go @@ -54,19 +54,30 @@ type Options struct { ReadTimeout time.Duration } +const defaultRequestTimeout = 10 * time.Second + +func maxTimeoutWithDefault(readTimeout, writeTimeout time.Duration) time.Duration { + timeout := readTimeout + if writeTimeout > timeout { + timeout = writeTimeout + } + if timeout <= 0 { + timeout = defaultRequestTimeout + } + return timeout +} + func newOptions( ctx context.Context, o *Options, hook kgo.Hook, ) ([]kgo.Opt, error) { - timeoutOverhead := o.ReadTimeout - if o.WriteTimeout > timeoutOverhead { - timeoutOverhead = o.WriteTimeout - } - if timeoutOverhead <= 0 { - timeoutOverhead = 10 * time.Second + if o == nil { + o = &Options{} } + timeoutOverhead := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) + opts := []kgo.Opt{ kgo.WithContext(ctx), kgo.SeedBrokers(o.BrokerEndpoints...), @@ -199,9 +210,13 @@ func newOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSource, e func newProducerOptions( o *Options, ) []kgo.Opt { + if o == nil { + o = &Options{} + } + produceTimeout := o.ReadTimeout if produceTimeout < 100*time.Millisecond { - produceTimeout = 10 * time.Second + produceTimeout = defaultRequestTimeout } return []kgo.Opt{ @@ -218,6 +233,10 @@ func newProducerOptions( } func newCompressionOption(o *Options) kgo.Opt { + if o == nil { + return kgo.ProducerBatchCompression(kgo.NoCompression()) + } + compression := strings.ToLower(strings.TrimSpace(o.Compression)) var codec kgo.CompressionCodec switch compression { diff --git a/pkg/sink/kafka/franz/factory_api_test.go b/pkg/sink/kafka/franz/factory_api_test.go index fbefab76af..533ff86fba 100644 --- a/pkg/sink/kafka/franz/factory_api_test.go +++ b/pkg/sink/kafka/franz/factory_api_test.go @@ -13,7 +13,13 @@ package franz -import "testing" +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/stretchr/testify/require" +) func TestBuildFranzCompressionOptionHasNoErrorReturn(t *testing.T) { t.Parallel() @@ -22,9 +28,49 @@ func TestBuildFranzCompressionOptionHasNoErrorReturn(t *testing.T) { _ = newCompressionOption(o) } +func TestBuildFranzCompressionOptionNilOption(t *testing.T) { + t.Parallel() + + require.NotPanics(t, func() { + _ = newCompressionOption(nil) + }) +} + func TestBuildFranzProducerOptionsHasNoErrorReturn(t *testing.T) { t.Parallel() o := &Options{} - _ = newProducerOptions(o, 3) + _ = newProducerOptions(o) +} + +func TestBuildFranzProducerOptionsNilOption(t *testing.T) { + t.Parallel() + + require.NotPanics(t, func() { + opts := newProducerOptions(nil) + require.NotEmpty(t, opts) + }) +} + +func TestBuildFranzClientOptionsNilOption(t *testing.T) { + t.Parallel() + + require.NotPanics(t, func() { + opts, err := newOptions(context.Background(), nil, nil) + require.NoError(t, err) + require.NotEmpty(t, opts) + }) +} + +func TestNewSyncProducerNilOptionsDoesNotPanic(t *testing.T) { + t.Parallel() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-sync-nil-options") + require.NotPanics(t, func() { + producer, err := NewSyncProducer(context.Background(), changefeedID, nil, nil) + if producer != nil { + producer.Close() + } + require.Error(t, err) + }) } diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/franz/metrics_hook.go index e669c03505..34629c5b88 100644 --- a/pkg/sink/kafka/franz/metrics_hook.go +++ b/pkg/sink/kafka/franz/metrics_hook.go @@ -48,7 +48,6 @@ func NewMetricsHook() *MetricsHook { func (h *MetricsHook) BindPrometheusMetrics( keyspace string, changefeed string, - _ time.Duration, metrics PrometheusMetrics, ) { h.promMu.Lock() @@ -79,10 +78,6 @@ func (h *MetricsHook) Run(ctx context.Context) { h.CleanupPrometheusMetrics() } -func (h *MetricsHook) FlushPrometheusMetrics(interval time.Duration) { - _ = interval -} - func (h *MetricsHook) CleanupPrometheusMetrics() { keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() @@ -94,27 +89,13 @@ func (h *MetricsHook) CleanupPrometheusMetrics() { "namespace": keyspace, "changefeed": changefeed, } - if metrics.OutgoingByteRate != nil { - metrics.OutgoingByteRate.MetricVec.DeletePartialMatch(labels) - } - if metrics.RequestRate != nil { - metrics.RequestRate.MetricVec.DeletePartialMatch(labels) - } - if metrics.ResponseRate != nil { - metrics.ResponseRate.MetricVec.DeletePartialMatch(labels) - } - if metrics.RequestsInFlight != nil { - metrics.RequestsInFlight.MetricVec.DeletePartialMatch(labels) - } - if metrics.RequestLatency != nil { - metrics.RequestLatency.MetricVec.DeletePartialMatch(labels) - } - if metrics.CompressionRatio != nil { - metrics.CompressionRatio.MetricVec.DeletePartialMatch(labels) - } - if metrics.RecordsPerRequest != nil { - metrics.RecordsPerRequest.MetricVec.DeletePartialMatch(labels) - } + deleteGaugeVecPartialMatch(metrics.OutgoingByteRate, labels) + deleteGaugeVecPartialMatch(metrics.RequestRate, labels) + deleteGaugeVecPartialMatch(metrics.ResponseRate, labels) + deleteGaugeVecPartialMatch(metrics.RequestsInFlight, labels) + deleteHistogramVecPartialMatch(metrics.RequestLatency, labels) + deleteHistogramVecPartialMatch(metrics.CompressionRatio, labels) + deleteHistogramVecPartialMatch(metrics.RecordsPerRequest, labels) } func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { @@ -122,20 +103,20 @@ func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err erro return } - keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() - if !bound { + ctx, ok := h.loadMetricsContext() + if !ok { return } brokerID := strconv.Itoa(int(nodeID)) - if metrics.OutgoingByteRate != nil && bytesWritten > 0 { - metrics.OutgoingByteRate.WithLabelValues(keyspace, changefeed, brokerID).Add(float64(bytesWritten)) + if ctx.metrics.OutgoingByteRate != nil && bytesWritten > 0 { + ctx.metrics.OutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(float64(bytesWritten)) } - if metrics.RequestRate != nil { - metrics.RequestRate.WithLabelValues(keyspace, changefeed, brokerID).Add(1) + if ctx.metrics.RequestRate != nil { + ctx.metrics.RequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) } - if err == nil && metrics.RequestsInFlight != nil { - metrics.RequestsInFlight.WithLabelValues(keyspace, changefeed, brokerID).Add(1) + if err == nil && ctx.metrics.RequestsInFlight != nil { + ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) } } @@ -159,21 +140,21 @@ func (h *MetricsHook) OnBrokerE2E( return } - keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() - if !bound { + ctx, ok := h.loadMetricsContext() + if !ok { return } brokerID := strconv.Itoa(int(meta.NodeID)) - if e2e.WriteErr == nil && metrics.RequestsInFlight != nil { - metrics.RequestsInFlight.WithLabelValues(keyspace, changefeed, brokerID).Add(-1) + if e2e.WriteErr == nil && ctx.metrics.RequestsInFlight != nil { + ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(-1) } - if e2e.BytesRead > 0 && e2e.ReadErr == nil && metrics.ResponseRate != nil { - metrics.ResponseRate.WithLabelValues(keyspace, changefeed, brokerID).Add(1) + if e2e.BytesRead > 0 && e2e.ReadErr == nil && ctx.metrics.ResponseRate != nil { + ctx.metrics.ResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) } - if e2e.Err() == nil && metrics.RequestLatency != nil { + if e2e.Err() == nil && ctx.metrics.RequestLatency != nil { latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 - metrics.RequestLatency.WithLabelValues(keyspace, changefeed, brokerID).Observe(latencyMs) + ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Observe(latencyMs) } } @@ -187,17 +168,47 @@ func (h *MetricsHook) OnProduceBatchWritten( } func (h *MetricsHook) RecordProduceBatchWritten(numRecords int, uncompressedBytes int, compressedBytes int) { - keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() - if !bound { + ctx, ok := h.loadMetricsContext() + if !ok { return } - if metrics.RecordsPerRequest != nil && numRecords > 0 { + if ctx.metrics.RecordsPerRequest != nil && numRecords > 0 { records := float64(numRecords) - metrics.RecordsPerRequest.WithLabelValues(keyspace, changefeed).Observe(records) + ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed).Observe(records) } - if metrics.CompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { + if ctx.metrics.CompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 - metrics.CompressionRatio.WithLabelValues(keyspace, changefeed).Observe(ratio) + ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed).Observe(ratio) + } +} + +type metricsContext struct { + keyspace string + changefeed string + metrics PrometheusMetrics +} + +func (h *MetricsHook) loadMetricsContext() (metricsContext, bool) { + keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() + if !bound { + return metricsContext{}, false + } + return metricsContext{ + keyspace: keyspace, + changefeed: changefeed, + metrics: metrics, + }, true +} + +func deleteGaugeVecPartialMatch(gaugeVec *prometheus.GaugeVec, labels prometheus.Labels) { + if gaugeVec != nil { + gaugeVec.MetricVec.DeletePartialMatch(labels) + } +} + +func deleteHistogramVecPartialMatch(histogramVec *prometheus.HistogramVec, labels prometheus.Labels) { + if histogramVec != nil { + histogramVec.MetricVec.DeletePartialMatch(labels) } } diff --git a/pkg/sink/kafka/franz/sync_producer.go b/pkg/sink/kafka/franz/sync_producer.go index e7d6d87735..aba5268d24 100644 --- a/pkg/sink/kafka/franz/sync_producer.go +++ b/pkg/sink/kafka/franz/sync_producer.go @@ -42,6 +42,10 @@ func NewSyncProducer( o *Options, hook kgo.Hook, ) (*SyncProducer, error) { + if o == nil { + o = &Options{} + } + opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -53,10 +57,7 @@ func NewSyncProducer( return nil, errors.Trace(err) } - timeout := o.ReadTimeout - if timeout <= 0 { - timeout = 10 * time.Second - } + timeout := maxTimeoutWithDefault(o.ReadTimeout, 0) return &SyncProducer{ id: changefeedID, @@ -78,27 +79,14 @@ func (p *SyncProducer) SendMessage(topic string, partitionNum int32, message *co ctx, cancel := p.newRequestContext() defer cancel() - record := &kgo.Record{ - Topic: topic, - Partition: partitionNum, - Key: message.Key, - Value: message.Value, - } + record := buildRecord(topic, partitionNum, message) err := p.client.ProduceSync(ctx, record).FirstErr() failpoint.Inject("KafkaSinkSyncSendMessageError", func() { err = errors.New("kafka sink sync send message injected error") }) - if err != nil { - err = logutil.AnnotateEventError( - p.id.Keyspace(), - p.id.Name(), - message.LogInfo, - err, - ) - } - return errors.WrapError(errors.ErrKafkaSendMessage, err) + return p.wrapSendError(message, err) } func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { @@ -108,12 +96,7 @@ func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *c records := make([]*kgo.Record, 0, partitionNum) for i := 0; i < int(partitionNum); i++ { - records = append(records, &kgo.Record{ - Topic: topic, - Partition: int32(i), - Key: message.Key, - Value: message.Value, - }) + records = append(records, buildRecord(topic, int32(i), message)) } ctx, cancel := p.newRequestContext() @@ -125,28 +108,19 @@ func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *c err = errors.New("kafka sink sync send messages injected error") }) - if err != nil { - err = logutil.AnnotateEventError( - p.id.Keyspace(), - p.id.Name(), - message.LogInfo, - err, - ) - } - return errors.WrapError(errors.ErrKafkaSendMessage, err) + return p.wrapSendError(message, err) } func (p *SyncProducer) Heartbeat() {} func (p *SyncProducer) Close() { - if p.closed.Load() { + if !p.closed.CompareAndSwap(false, true) { log.Warn("kafka DDL producer already closed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name())) return } - p.closed.Store(true) start := time.Now() p.client.Close() log.Info("Kafka DDL producer closed", @@ -154,3 +128,24 @@ func (p *SyncProducer) Close() { zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start))) } + +func buildRecord(topic string, partition int32, message *common.Message) *kgo.Record { + return &kgo.Record{ + Topic: topic, + Partition: partition, + Key: message.Key, + Value: message.Value, + } +} + +func (p *SyncProducer) wrapSendError(message *common.Message, err error) error { + if err != nil { + err = logutil.AnnotateEventError( + p.id.Keyspace(), + p.id.Name(), + message.LogInfo, + err, + ) + } + return errors.WrapError(errors.ErrKafkaSendMessage, err) +} diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 8aba9018ed..419ed7f283 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -51,7 +51,6 @@ func NewFranzFactory( metricsHook.BindPrometheusMetrics( changefeedID.Keyspace(), changefeedID.Name(), - refreshMetricsInterval, franz.PrometheusMetrics{ RequestsInFlight: requestsInFlightGauge, OutgoingByteRate: OutgoingByteRateGauge, diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go index 0fe8b30fb6..c4e94a713a 100644 --- a/pkg/sink/kafka/franz_factory_test.go +++ b/pkg/sink/kafka/franz_factory_test.go @@ -48,7 +48,6 @@ func TestFranzMetricsHookWritePrometheusMetricsDirectly(t *testing.T) { hook.BindPrometheusMetrics( changefeedID.Keyspace(), changefeedID.Name(), - refreshMetricsInterval, kafkafranz.PrometheusMetrics{ RequestsInFlight: requestsInFlightGauge, OutgoingByteRate: OutgoingByteRateGauge, From eb1ff686d96dd7772e00721fa3b9e47ae05fca3c Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 10 Feb 2026 11:25:14 +0000 Subject: [PATCH 09/61] fix require ack configuration --- pkg/sink/kafka/franz/factory.go | 21 ++++++++++++++++++- pkg/sink/kafka/franz/factory_api_test.go | 26 ++++++++++++++++++++++++ pkg/sink/kafka/franz_factory.go | 5 ++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/franz/factory.go index 5c379adf82..1a4b94ceed 100644 --- a/pkg/sink/kafka/franz/factory.go +++ b/pkg/sink/kafka/franz/factory.go @@ -43,6 +43,7 @@ type Options struct { MaxMessageBytes int Compression string + RequiredAcks int16 EnableTLS bool Credential *security.Credential @@ -221,7 +222,7 @@ func newProducerOptions( return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), - kgo.RequiredAcks(kgo.AllISRAcks()), + kgo.RequiredAcks(newRequiredAcks(o)), kgo.DisableIdempotentWrite(), kgo.MaxProduceRequestsInflightPerBroker(1), kgo.RecordRetries(5), @@ -232,6 +233,24 @@ func newProducerOptions( } } +func newRequiredAcks(o *Options) kgo.Acks { + if o == nil { + return kgo.AllISRAcks() + } + + switch o.RequiredAcks { + case -1: + return kgo.AllISRAcks() + case 1: + return kgo.LeaderAck() + case 0: + return kgo.NoAck() + default: + log.Warn("unsupported required acks", zap.Int16("requiredAcks", o.RequiredAcks)) + return kgo.AllISRAcks() + } +} + func newCompressionOption(o *Options) kgo.Opt { if o == nil { return kgo.ProducerBatchCompression(kgo.NoCompression()) diff --git a/pkg/sink/kafka/franz/factory_api_test.go b/pkg/sink/kafka/franz/factory_api_test.go index 533ff86fba..770d383643 100644 --- a/pkg/sink/kafka/franz/factory_api_test.go +++ b/pkg/sink/kafka/franz/factory_api_test.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" ) func TestBuildFranzCompressionOptionHasNoErrorReturn(t *testing.T) { @@ -74,3 +75,28 @@ func TestNewSyncProducerNilOptionsDoesNotPanic(t *testing.T) { require.Error(t, err) }) } + +func TestBuildFranzRequiredAcks(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + requiredAcks int16 + expected kgo.Acks + }{ + {name: "all", requiredAcks: -1, expected: kgo.AllISRAcks()}, + {name: "leader", requiredAcks: 1, expected: kgo.LeaderAck()}, + {name: "none", requiredAcks: 0, expected: kgo.NoAck()}, + {name: "invalid fallback all", requiredAcks: 2, expected: kgo.AllISRAcks()}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.expected, newRequiredAcks(&Options{RequiredAcks: tc.requiredAcks})) + }) + } + + require.Equal(t, kgo.AllISRAcks(), newRequiredAcks(nil)) +} diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 419ed7f283..3ec99ac9c5 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -99,7 +99,9 @@ func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { func newFranzOptions(o *options) *franz.Options { if o == nil { - return &franz.Options{} + return &franz.Options{ + RequiredAcks: int16(WaitForAll), + } } return &franz.Options{ BrokerEndpoints: o.BrokerEndpoints, @@ -110,6 +112,7 @@ func newFranzOptions(o *options) *franz.Options { MaxMessageBytes: o.MaxMessageBytes, Compression: o.Compression, + RequiredAcks: int16(o.RequiredAcks), EnableTLS: o.EnableTLS, Credential: o.Credential, From aa989430faf4e9284a62d162cd41ae2f79548062 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 10 Feb 2026 11:38:57 +0000 Subject: [PATCH 10/61] fix unit test --- pkg/sink/kafka/franz/admin_client_test.go | 30 ++------ pkg/sink/kafka/franz/factory_api_test.go | 83 +++++++-------------- pkg/sink/kafka/franz_admin_client_test.go | 6 +- pkg/sink/kafka/franz_factory_test.go | 89 ++++++----------------- 4 files changed, 56 insertions(+), 152 deletions(-) diff --git a/pkg/sink/kafka/franz/admin_client_test.go b/pkg/sink/kafka/franz/admin_client_test.go index 1e87eab9b2..c35c014f4a 100644 --- a/pkg/sink/kafka/franz/admin_client_test.go +++ b/pkg/sink/kafka/franz/admin_client_test.go @@ -15,37 +15,19 @@ package franz import ( "context" - "os" - "path/filepath" - "runtime" "testing" "github.com/pingcap/ticdc/pkg/common" "github.com/stretchr/testify/require" ) -func TestGetBrokerConfigControllerNotAvailableUsesDedicatedError(t *testing.T) { - t.Parallel() - - _, currentFile, _, ok := runtime.Caller(0) - require.True(t, ok) - dir := filepath.Dir(currentFile) - - source, err := os.ReadFile(filepath.Join(dir, "admin_client.go")) - require.NoError(t, err) - require.NotContains(t, string(source), `ErrKafkaInvalidConfig.GenWithStack("kafka controller is not available")`) - require.Contains(t, string(source), "ErrKafkaControllerNotAvailable") -} - -func TestNewAdminClientNilOptionsDoesNotPanic(t *testing.T) { +func TestNewAdminClientNilOptionsReturnsError(t *testing.T) { t.Parallel() changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-admin-nil-options") - require.NotPanics(t, func() { - client, err := NewAdminClient(context.Background(), changefeedID, nil, nil) - if client != nil { - client.Close() - } - require.Error(t, err) - }) + client, err := NewAdminClient(context.Background(), changefeedID, nil, nil) + if client != nil { + client.Close() + } + require.Error(t, err) } diff --git a/pkg/sink/kafka/franz/factory_api_test.go b/pkg/sink/kafka/franz/factory_api_test.go index 770d383643..02895ab3c1 100644 --- a/pkg/sink/kafka/franz/factory_api_test.go +++ b/pkg/sink/kafka/franz/factory_api_test.go @@ -14,69 +14,14 @@ package franz import ( - "context" "testing" + "time" - "github.com/pingcap/ticdc/pkg/common" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kgo" ) -func TestBuildFranzCompressionOptionHasNoErrorReturn(t *testing.T) { - t.Parallel() - - o := &Options{} - _ = newCompressionOption(o) -} - -func TestBuildFranzCompressionOptionNilOption(t *testing.T) { - t.Parallel() - - require.NotPanics(t, func() { - _ = newCompressionOption(nil) - }) -} - -func TestBuildFranzProducerOptionsHasNoErrorReturn(t *testing.T) { - t.Parallel() - - o := &Options{} - _ = newProducerOptions(o) -} - -func TestBuildFranzProducerOptionsNilOption(t *testing.T) { - t.Parallel() - - require.NotPanics(t, func() { - opts := newProducerOptions(nil) - require.NotEmpty(t, opts) - }) -} - -func TestBuildFranzClientOptionsNilOption(t *testing.T) { - t.Parallel() - - require.NotPanics(t, func() { - opts, err := newOptions(context.Background(), nil, nil) - require.NoError(t, err) - require.NotEmpty(t, opts) - }) -} - -func TestNewSyncProducerNilOptionsDoesNotPanic(t *testing.T) { - t.Parallel() - - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-sync-nil-options") - require.NotPanics(t, func() { - producer, err := NewSyncProducer(context.Background(), changefeedID, nil, nil) - if producer != nil { - producer.Close() - } - require.Error(t, err) - }) -} - -func TestBuildFranzRequiredAcks(t *testing.T) { +func TestNewRequiredAcks(t *testing.T) { t.Parallel() testCases := []struct { @@ -100,3 +45,27 @@ func TestBuildFranzRequiredAcks(t *testing.T) { require.Equal(t, kgo.AllISRAcks(), newRequiredAcks(nil)) } + +func TestMaxTimeoutWithDefault(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + readTimeout time.Duration + writeTimeout time.Duration + expected time.Duration + }{ + {name: "read timeout is max", readTimeout: 3 * time.Second, writeTimeout: 2 * time.Second, expected: 3 * time.Second}, + {name: "write timeout is max", readTimeout: 2 * time.Second, writeTimeout: 4 * time.Second, expected: 4 * time.Second}, + {name: "both zero use default", readTimeout: 0, writeTimeout: 0, expected: defaultRequestTimeout}, + {name: "both negative use default", readTimeout: -time.Second, writeTimeout: -2 * time.Second, expected: defaultRequestTimeout}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.expected, maxTimeoutWithDefault(tc.readTimeout, tc.writeTimeout)) + }) + } +} diff --git a/pkg/sink/kafka/franz_admin_client_test.go b/pkg/sink/kafka/franz_admin_client_test.go index 34e47f402d..cbf5865e18 100644 --- a/pkg/sink/kafka/franz_admin_client_test.go +++ b/pkg/sink/kafka/franz_admin_client_test.go @@ -23,8 +23,6 @@ func TestFranzAdminClientAdapterCreateTopicNilDetailReturnsError(t *testing.T) { t.Parallel() adapter := &franzAdminClientAdapter{inner: nil} - require.NotPanics(t, func() { - err := adapter.CreateTopic(nil, false) - require.Error(t, err) - }) + err := adapter.CreateTopic(nil, false) + require.Error(t, err) } diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go index c4e94a713a..9e2dd36bb4 100644 --- a/pkg/sink/kafka/franz_factory_test.go +++ b/pkg/sink/kafka/franz_factory_test.go @@ -15,84 +15,39 @@ package kafka import ( "testing" - "time" - "github.com/pingcap/ticdc/pkg/common" - kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kgo" ) -func TestFranzFactoryMetricsCollectorType(t *testing.T) { +func TestNewFranzOptionsNilUsesWaitForAll(t *testing.T) { t.Parallel() - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-metrics-type") - f := &franzFactory{ - changefeedID: changefeedID, - metricsHook: kafkafranz.NewMetricsHook(), - } - collector := f.MetricsCollector(nil) - - _, ok := collector.(*kafkafranz.MetricsHook) - require.True(t, ok) + options := newFranzOptions(nil) + require.Equal(t, int16(WaitForAll), options.RequiredAcks) } -func TestFranzMetricsHookWritePrometheusMetricsDirectly(t *testing.T) { +func TestNewFranzOptionsMapsRequiredAcks(t *testing.T) { t.Parallel() - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-metrics") - hook := kafkafranz.NewMetricsHook() - hook.BindPrometheusMetrics( - changefeedID.Keyspace(), - changefeedID.Name(), - kafkafranz.PrometheusMetrics{ - RequestsInFlight: requestsInFlightGauge, - OutgoingByteRate: OutgoingByteRateGauge, - RequestRate: RequestRateGauge, - RequestLatency: franzRequestLatencyHistogram, - ResponseRate: responseRateGauge, - CompressionRatio: franzCompressionRatioHistogram, - RecordsPerRequest: franzRecordsPerRequestHistogram, - }, - ) - t.Cleanup(func() { - hook.CleanupPrometheusMetrics() - }) - - hook.RecordBrokerWrite(1, 128, nil) - keyspace := changefeedID.Keyspace() - changefeed := changefeedID.Name() - require.Equal(t, float64(1), testutil.ToFloat64(requestsInFlightGauge.WithLabelValues(keyspace, changefeed, "1"))) - require.Greater(t, testutil.ToFloat64(RequestRateGauge.WithLabelValues(keyspace, changefeed, "1")), 0.0) - require.Greater(t, testutil.ToFloat64(OutgoingByteRateGauge.WithLabelValues(keyspace, changefeed, "1")), 0.0) - - hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: 1}, 0, kgo.BrokerE2E{ - BytesRead: 128, - TimeToWrite: time.Millisecond, - ReadWait: time.Millisecond, - TimeToRead: time.Millisecond, - }) - require.Equal(t, float64(0), testutil.ToFloat64(requestsInFlightGauge.WithLabelValues(keyspace, changefeed, "1"))) - require.Greater(t, testutil.ToFloat64(responseRateGauge.WithLabelValues(keyspace, changefeed, "1")), 0.0) - require.Greater(t, histogramSampleCount(t, franzRequestLatencyHistogram.WithLabelValues(keyspace, changefeed, "1")), uint64(0)) - - hook.RecordProduceBatchWritten(8, 400, 200) - require.Greater(t, histogramSampleCount(t, franzCompressionRatioHistogram.WithLabelValues(keyspace, changefeed)), uint64(0)) - require.Greater(t, histogramSampleCount(t, franzRecordsPerRequestHistogram.WithLabelValues(keyspace, changefeed)), uint64(0)) -} - -func histogramSampleCount(t *testing.T, observer prometheus.Observer) uint64 { - t.Helper() + testCases := []struct { + name string + requiredAcks RequiredAcks + }{ + {name: "wait for all", requiredAcks: WaitForAll}, + {name: "wait for local", requiredAcks: WaitForLocal}, + {name: "no response", requiredAcks: NoResponse}, + } - histogram, ok := observer.(prometheus.Histogram) - require.True(t, ok) + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - metric := &dto.Metric{} - require.NoError(t, histogram.Write(metric)) - require.NotNil(t, metric.Histogram) + options := NewOptions() + options.RequiredAcks = tc.requiredAcks - return metric.Histogram.GetSampleCount() + franzOptions := newFranzOptions(options) + require.Equal(t, int16(tc.requiredAcks), franzOptions.RequiredAcks) + }) + } } From 6e39cdd42bb255da439209e84a9bb739723b6710 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 11 Feb 2026 08:22:50 +0000 Subject: [PATCH 11/61] fix more code --- pkg/errors/error_test.go | 5 ++ pkg/errors/helper.go | 1 - pkg/sink/kafka/franz/admin_client.go | 48 ++++++++++++---- pkg/sink/kafka/franz/admin_metrics.go | 78 +++++++++++++++++++++++++ pkg/sink/kafka/franz/metrics_hook.go | 24 ++++---- pkg/sink/kafka/franz_factory.go | 79 +++++++++++++++++-------- pkg/sink/kafka/metrics.go | 83 +++++++++++++++++++-------- 7 files changed, 250 insertions(+), 68 deletions(-) create mode 100644 pkg/sink/kafka/franz/admin_metrics.go diff --git a/pkg/errors/error_test.go b/pkg/errors/error_test.go index 777570d547..fad42ab040 100644 --- a/pkg/errors/error_test.go +++ b/pkg/errors/error_test.go @@ -98,6 +98,11 @@ func TestShouldFailChangefeed(t *testing.T) { err: ErrKafkaInvalidConfig.GenWithStackByArgs("invalid config"), expected: true, }, + { + name: "ErrKafkaControllerNotAvailable should return false", + err: ErrKafkaControllerNotAvailable.GenWithStackByArgs(), + expected: false, + }, { name: "ErrMySQLInvalidConfig should return true", err: ErrMySQLInvalidConfig.GenWithStackByArgs("invalid config"), diff --git a/pkg/errors/helper.go b/pkg/errors/helper.go index 671ba08479..3fd77ca3ba 100644 --- a/pkg/errors/helper.go +++ b/pkg/errors/helper.go @@ -115,7 +115,6 @@ var changefeedUnRetryableErrors = []*errors.Error{ ErrSinkURIInvalid, ErrKafkaInvalidConfig, - ErrKafkaControllerNotAvailable, ErrMySQLInvalidConfig, ErrStorageSinkInvalidConfig, diff --git a/pkg/sink/kafka/franz/admin_client.go b/pkg/sink/kafka/franz/admin_client.go index 5a223c56da..06495d2fa4 100644 --- a/pkg/sink/kafka/franz/admin_client.go +++ b/pkg/sink/kafka/franz/admin_client.go @@ -77,21 +77,30 @@ func (a *AdminClient) newRequestContext() (context.Context, context.CancelFunc) } func (a *AdminClient) GetAllBrokers() []int32 { + startTime := time.Now() ctx, cancel := a.newRequestContext() defer cancel() meta, err := a.admin.BrokerMetadata(ctx) if err != nil { + observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetAllBrokers, err, time.Since(startTime)) log.Warn("Kafka admin client fetch broker metadata failed", zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), zap.Error(err)) return nil } + + observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetAllBrokers, nil, time.Since(startTime)) return meta.Brokers.NodeIDs() } -func (a *AdminClient) GetBrokerConfig(configName string) (string, error) { +func (a *AdminClient) GetBrokerConfig(configName string) (value string, err error) { + startTime := time.Now() + defer func() { + observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetBrokerConfig, err, time.Since(startTime)) + }() + ctx, cancel := a.newRequestContext() defer cancel() @@ -131,7 +140,12 @@ func (a *AdminClient) GetBrokerConfig(configName string) (string, error) { "cannot find the `%s` from the broker's configuration", configName) } -func (a *AdminClient) GetTopicConfig(topicName string, configName string) (string, error) { +func (a *AdminClient) GetTopicConfig(topicName string, configName string) (value string, err error) { + startTime := time.Now() + defer func() { + observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicConfig, err, time.Since(startTime)) + }() + ctx, cancel := a.newRequestContext() defer cancel() @@ -170,7 +184,12 @@ func (a *AdminClient) GetTopicConfig(topicName string, configName string) (strin func (a *AdminClient) GetTopicsMeta( topics []string, ignoreTopicError bool, -) (map[string]TopicDetail, error) { +) (result map[string]TopicDetail, err error) { + startTime := time.Now() + defer func() { + observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicsMeta, err, time.Since(startTime)) + }() + if len(topics) == 0 { return make(map[string]TopicDetail), nil } @@ -183,7 +202,7 @@ func (a *AdminClient) GetTopicsMeta( return nil, errors.Trace(err) } - result := make(map[string]TopicDetail, len(topics)) + result = make(map[string]TopicDetail, len(topics)) for _, topic := range topics { detail, ok := meta.Topics[topic] if !ok { @@ -209,7 +228,12 @@ func (a *AdminClient) GetTopicsMeta( return result, nil } -func (a *AdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { +func (a *AdminClient) GetTopicsPartitionsNum(topics []string) (result map[string]int32, err error) { + startTime := time.Now() + defer func() { + observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicsPartitions, err, time.Since(startTime)) + }() + if len(topics) == 0 { return make(map[string]int32), nil } @@ -222,7 +246,7 @@ func (a *AdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, return nil, errors.Trace(err) } - result := make(map[string]int32, len(topics)) + result = make(map[string]int32, len(topics)) for _, topic := range topics { detail, ok := meta.Topics[topic] if !ok { @@ -236,14 +260,16 @@ func (a *AdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, return result, nil } -func (a *AdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { +func (a *AdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) (err error) { + startTime := time.Now() + defer func() { + observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodCreateTopic, err, time.Since(startTime)) + }() + ctx, cancel := a.newRequestContext() defer cancel() - var ( - responses kadm.CreateTopicResponses - err error - ) + var responses kadm.CreateTopicResponses if validateOnly { responses, err = a.admin.ValidateCreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) } else { diff --git a/pkg/sink/kafka/franz/admin_metrics.go b/pkg/sink/kafka/franz/admin_metrics.go new file mode 100644 index 0000000000..baaee24729 --- /dev/null +++ b/pkg/sink/kafka/franz/admin_metrics.go @@ -0,0 +1,78 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + adminMethodGetAllBrokers = "get_all_brokers" + adminMethodGetBrokerConfig = "get_broker_config" + adminMethodGetTopicConfig = "get_topic_config" + adminMethodGetTopicsMeta = "get_topics_meta" + adminMethodGetTopicsPartitions = "get_topics_partitions_num" + adminMethodCreateTopic = "create_topic" + adminCallStatusOK = "ok" + adminCallStatusError = "error" +) + +var ( + adminCallCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_admin_call_total", + Help: "Total kafka admin calls by method and result.", + }, []string{"namespace", "changefeed", "method", "result"}) + adminCallLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_admin_call_duration_seconds", + Help: "Latency of kafka admin calls by method and result.", + Buckets: prometheus.DefBuckets, + }, []string{"namespace", "changefeed", "method", "result"}) +) + +func InitAdminMetrics(registry *prometheus.Registry) { + registry.MustRegister(adminCallCount) + registry.MustRegister(adminCallLatency) +} + +func CleanupAdminMetrics(keyspace string, changefeed string) { + labels := prometheus.Labels{ + "namespace": keyspace, + "changefeed": changefeed, + } + adminCallCount.MetricVec.DeletePartialMatch(labels) + adminCallLatency.MetricVec.DeletePartialMatch(labels) +} + +func observeAdminCall( + keyspace string, + changefeed string, + method string, + callErr error, + duration time.Duration, +) { + status := adminCallStatusOK + if callErr != nil { + status = adminCallStatusError + } + adminCallCount.WithLabelValues(keyspace, changefeed, method, status).Inc() + adminCallLatency.WithLabelValues(keyspace, changefeed, method, status).Observe(duration.Seconds()) +} diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/franz/metrics_hook.go index 34629c5b88..04e66441d8 100644 --- a/pkg/sink/kafka/franz/metrics_hook.go +++ b/pkg/sink/kafka/franz/metrics_hook.go @@ -28,6 +28,7 @@ type MetricsHook struct { promBound bool keyspace string changefeed string + clientType string prom PrometheusMetrics } @@ -41,8 +42,8 @@ type PrometheusMetrics struct { RecordsPerRequest *prometheus.HistogramVec } -func NewMetricsHook() *MetricsHook { - return &MetricsHook{} +func NewMetricsHook(clientType string) *MetricsHook { + return &MetricsHook{clientType: clientType} } func (h *MetricsHook) BindPrometheusMetrics( @@ -88,6 +89,7 @@ func (h *MetricsHook) CleanupPrometheusMetrics() { labels := prometheus.Labels{ "namespace": keyspace, "changefeed": changefeed, + "client": h.clientType, } deleteGaugeVecPartialMatch(metrics.OutgoingByteRate, labels) deleteGaugeVecPartialMatch(metrics.RequestRate, labels) @@ -110,13 +112,13 @@ func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err erro brokerID := strconv.Itoa(int(nodeID)) if ctx.metrics.OutgoingByteRate != nil && bytesWritten > 0 { - ctx.metrics.OutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(float64(bytesWritten)) + ctx.metrics.OutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(float64(bytesWritten)) } if ctx.metrics.RequestRate != nil { - ctx.metrics.RequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + ctx.metrics.RequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) } if err == nil && ctx.metrics.RequestsInFlight != nil { - ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) } } @@ -147,14 +149,14 @@ func (h *MetricsHook) OnBrokerE2E( brokerID := strconv.Itoa(int(meta.NodeID)) if e2e.WriteErr == nil && ctx.metrics.RequestsInFlight != nil { - ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(-1) + ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(-1) } if e2e.BytesRead > 0 && e2e.ReadErr == nil && ctx.metrics.ResponseRate != nil { - ctx.metrics.ResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + ctx.metrics.ResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) } if e2e.Err() == nil && ctx.metrics.RequestLatency != nil { latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 - ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Observe(latencyMs) + ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Observe(latencyMs) } } @@ -175,17 +177,18 @@ func (h *MetricsHook) RecordProduceBatchWritten(numRecords int, uncompressedByte if ctx.metrics.RecordsPerRequest != nil && numRecords > 0 { records := float64(numRecords) - ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed).Observe(records) + ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType).Observe(records) } if ctx.metrics.CompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 - ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed).Observe(ratio) + ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType).Observe(ratio) } } type metricsContext struct { keyspace string changefeed string + clientType string metrics PrometheusMetrics } @@ -197,6 +200,7 @@ func (h *MetricsHook) loadMetricsContext() (metricsContext, bool) { return metricsContext{ keyspace: keyspace, changefeed: changefeed, + clientType: h.clientType, metrics: metrics, }, true } diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 3ec99ac9c5..b800b3d081 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -21,10 +21,52 @@ import ( "github.com/pingcap/ticdc/pkg/sink/kafka/franz" ) +const ( + clientTypeAsyncProducer = "async_producer" + clientTypeSyncProducer = "sync_producer" + clientTypeAdminClient = "admin_client" +) + type franzFactory struct { changefeedID common.ChangeFeedID option *options - metricsHook *franz.MetricsHook + + asyncMetricsHook *franz.MetricsHook + syncMetricsHook *franz.MetricsHook + adminMetricsHook *franz.MetricsHook +} + +type franzMetricsCollector struct { + changefeedID common.ChangeFeedID + hooks []*franz.MetricsHook +} + +func (c *franzMetricsCollector) Run(ctx context.Context) { + <-ctx.Done() + for _, hook := range c.hooks { + if hook != nil { + hook.CleanupPrometheusMetrics() + } + } + franz.CleanupAdminMetrics(c.changefeedID.Keyspace(), c.changefeedID.Name()) +} + +func newFranzMetricsHook(changefeedID common.ChangeFeedID, clientType string) *franz.MetricsHook { + hook := franz.NewMetricsHook(clientType) + hook.BindPrometheusMetrics( + changefeedID.Keyspace(), + changefeedID.Name(), + franz.PrometheusMetrics{ + RequestsInFlight: franzRequestsInFlightByClientGauge, + OutgoingByteRate: franzOutgoingByteTotalByClientGauge, + RequestRate: franzRequestTotalByClientGauge, + RequestLatency: franzRequestLatencyHistogram, + ResponseRate: franzResponseTotalByClientGauge, + CompressionRatio: franzCompressionRatioHistogram, + RecordsPerRequest: franzRecordsPerRequestHistogram, + }, + ) + return hook } // NewFranzFactory constructs a Factory with franz-go implementation. @@ -47,30 +89,17 @@ func NewFranzFactory( return nil, errors.Trace(err) } - metricsHook := franz.NewMetricsHook() - metricsHook.BindPrometheusMetrics( - changefeedID.Keyspace(), - changefeedID.Name(), - franz.PrometheusMetrics{ - RequestsInFlight: requestsInFlightGauge, - OutgoingByteRate: OutgoingByteRateGauge, - RequestRate: RequestRateGauge, - RequestLatency: franzRequestLatencyHistogram, - ResponseRate: responseRateGauge, - CompressionRatio: franzCompressionRatioHistogram, - RecordsPerRequest: franzRecordsPerRequestHistogram, - }, - ) - return &franzFactory{ - changefeedID: changefeedID, - option: o, - metricsHook: metricsHook, + changefeedID: changefeedID, + option: o, + asyncMetricsHook: newFranzMetricsHook(changefeedID, clientTypeAsyncProducer), + syncMetricsHook: newFranzMetricsHook(changefeedID, clientTypeSyncProducer), + adminMetricsHook: newFranzMetricsHook(changefeedID, clientTypeAdminClient), }, nil } func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - adminInner, err := franz.NewAdminClient(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) + adminInner, err := franz.NewAdminClient(ctx, f.changefeedID, newFranzOptions(f.option), f.adminMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -78,7 +107,7 @@ func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, err } func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := franz.NewSyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) + producer, err := franz.NewSyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.syncMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -86,7 +115,7 @@ func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { } func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := franz.NewAsyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.metricsHook) + producer, err := franz.NewAsyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.asyncMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -94,7 +123,11 @@ func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) } func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { - return f.metricsHook + return &franzMetricsCollector{changefeedID: f.changefeedID, hooks: []*franz.MetricsHook{ + f.asyncMetricsHook, + f.syncMetricsHook, + f.adminMetricsHook, + }} } func newFranzOptions(o *options) *franz.Options { diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index 21073235ec..6e45419e90 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -16,6 +16,7 @@ package kafka import ( "github.com/pingcap/ticdc/pkg/sink/codec" "github.com/pingcap/ticdc/pkg/sink/kafka/claimcheck" + "github.com/pingcap/ticdc/pkg/sink/kafka/franz" "github.com/prometheus/client_golang/prometheus" ) @@ -53,13 +54,6 @@ var ( Name: "kafka_producer_request_latency", Help: "The request latency for all brokers.", }, []string{"namespace", "changefeed", "broker", "type"}) - franzRequestLatencyHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_producer_request_latency_histogram", - Help: "Request latency histogram for franz producer in milliseconds.", - }, []string{"namespace", "changefeed", "broker"}) // Histogram update by `compression-ratio`. compressionRatioGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -68,13 +62,6 @@ var ( Name: "kafka_producer_compression_ratio", Help: "The compression ratio times 100 of record batches for all topics.", }, []string{"namespace", "changefeed", "type"}) - franzCompressionRatioHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_producer_compression_ratio_histogram", - Help: "Compression ratio times 100 histogram for franz producer.", - }, []string{"namespace", "changefeed"}) // updated by `records-per-request`. recordsPerRequestGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -83,13 +70,6 @@ var ( Name: "kafka_producer_records_per_request", Help: "The number of records per request for all topics.", }, []string{"namespace", "changefeed", "type"}) - franzRecordsPerRequestHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_producer_records_per_request_histogram", - Help: "Records per request histogram for franz producer.", - }, []string{"namespace", "changefeed"}) // Meter mark by 1 once a response received. responseRateGauge = prometheus.NewGaugeVec( @@ -99,6 +79,57 @@ var ( Name: "kafka_producer_response_rate", Help: "Responses/second received from all brokers.", }, []string{"namespace", "changefeed", "broker"}) + + franzRequestsInFlightByClientGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_in_flight_requests", + Help: "Current number of in-flight requests by client type and broker.", + }, []string{"namespace", "changefeed", "client", "broker"}) + franzOutgoingByteTotalByClientGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_outgoing_byte_total", + Help: "Total bytes written by kafka sink clients.", + }, []string{"namespace", "changefeed", "client", "broker"}) + franzRequestTotalByClientGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_request_total", + Help: "Total requests sent by kafka sink clients.", + }, []string{"namespace", "changefeed", "client", "broker"}) + franzResponseTotalByClientGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_response_total", + Help: "Total responses received by kafka sink clients.", + }, []string{"namespace", "changefeed", "client", "broker"}) + + franzRequestLatencyHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_request_latency_histogram", + Help: "Request latency histogram for kafka producer in milliseconds.", + }, []string{"namespace", "changefeed", "client", "broker"}) + franzCompressionRatioHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_compression_ratio_histogram", + Help: "Compression ratio times 100 histogram for kafka producer.", + }, []string{"namespace", "changefeed", "client"}) + franzRecordsPerRequestHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_records_per_request_histogram", + Help: "Records per request histogram for kafka producer.", + }, []string{"namespace", "changefeed", "client"}) ) // InitMetrics registers all metrics in this file. @@ -108,12 +139,18 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(OutgoingByteRateGauge) registry.MustRegister(RequestRateGauge) registry.MustRegister(RequestLatencyGauge) + registry.MustRegister(requestsInFlightGauge) + registry.MustRegister(responseRateGauge) + + registry.MustRegister(franzRequestsInFlightByClientGauge) + registry.MustRegister(franzOutgoingByteTotalByClientGauge) + registry.MustRegister(franzRequestTotalByClientGauge) + registry.MustRegister(franzResponseTotalByClientGauge) registry.MustRegister(franzRequestLatencyHistogram) registry.MustRegister(franzCompressionRatioHistogram) registry.MustRegister(franzRecordsPerRequestHistogram) - registry.MustRegister(requestsInFlightGauge) - registry.MustRegister(responseRateGauge) + franz.InitAdminMetrics(registry) claimcheck.InitMetrics(registry) codec.InitMetrics(registry) } From 42a6a6ec0f25fa9367a9a54d39ce8066bcde73fc Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 26 Jun 2026 11:47:19 +0800 Subject: [PATCH 12/61] remove sarama, add franz-go --- .../topicmanager/kafka_topic_manager_test.go | 10 +- go.mod | 12 +- go.sum | 16 +- pkg/leakutil/leak_helper.go | 7 - pkg/logger/log.go | 40 +- pkg/security/sasl.go | 15 +- pkg/sink/codec/common/message.go | 8 +- pkg/sink/kafka/admin.go | 204 -------- pkg/sink/kafka/admin_mock.go | 175 ------- pkg/sink/kafka/admin_test.go | 64 --- pkg/sink/kafka/factory.go | 11 +- pkg/sink/kafka/factory_selector.go | 13 +- pkg/sink/kafka/franz/factory.go | 7 +- pkg/sink/kafka/franz/factory_api_test.go | 30 ++ pkg/sink/kafka/franz/metrics_hook.go | 55 +++ pkg/sink/kafka/franz_factory.go | 9 + pkg/sink/kafka/franz_factory_test.go | 10 + pkg/sink/kafka/metrics_collector.go | 195 +------- pkg/sink/kafka/oauth2_token_provider.go | 84 ---- pkg/sink/kafka/oauth2_token_provider_test.go | 74 --- pkg/sink/kafka/options.go | 15 +- pkg/sink/kafka/options_test.go | 32 +- pkg/sink/kafka/sarama_async_producer.go | 199 -------- pkg/sink/kafka/sarama_config.go | 274 ----------- pkg/sink/kafka/sarama_config_test.go | 448 ------------------ pkg/sink/kafka/sarama_factory.go | 174 ------- pkg/sink/kafka/sarama_sync_producer.go | 139 ------ pkg/sink/kafka/sarama_sync_producer_mock.go | 130 ----- pkg/sink/kafka/sarama_sync_producer_test.go | 59 --- scripts/generate-mock.sh | 2 - 30 files changed, 154 insertions(+), 2357 deletions(-) delete mode 100644 pkg/sink/kafka/admin.go delete mode 100644 pkg/sink/kafka/admin_mock.go delete mode 100644 pkg/sink/kafka/admin_test.go delete mode 100644 pkg/sink/kafka/oauth2_token_provider.go delete mode 100644 pkg/sink/kafka/oauth2_token_provider_test.go delete mode 100644 pkg/sink/kafka/sarama_async_producer.go delete mode 100644 pkg/sink/kafka/sarama_config.go delete mode 100644 pkg/sink/kafka/sarama_config_test.go delete mode 100644 pkg/sink/kafka/sarama_factory.go delete mode 100644 pkg/sink/kafka/sarama_sync_producer.go delete mode 100644 pkg/sink/kafka/sarama_sync_producer_mock.go delete mode 100644 pkg/sink/kafka/sarama_sync_producer_test.go diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index ae3ba10694..fb34b67bd1 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -15,9 +15,9 @@ package topicmanager import ( "context" + "errors" "testing" - "github.com/IBM/sarama" "github.com/golang/mock/gomock" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/sink/kafka" @@ -74,7 +74,7 @@ func TestCreateTopic(t *testing.T) { func(detail *kafka.TopicDetail, validateOnly bool) error { gotFailedTopicDetail = detail gotFailedTopicValidateOnly = validateOnly - return sarama.ErrInvalidReplicationFactor + return errors.New("invalid replication factor") }), ) @@ -121,7 +121,7 @@ func TestCreateTopic(t *testing.T) { _, err = manager.CreateTopicAndWaitUntilVisible(ctx, topic) require.Regexp( t, - "kafka create topic failed: kafka server: Replication-factor is invalid", + "kafka create topic failed: invalid replication factor", err, ) require.NotNil(t, gotFailedTopicDetail) @@ -155,9 +155,9 @@ func TestCreateTopicWaitsUntilVisible(t *testing.T) { return nil }), adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, sarama.ErrUnknownTopicOrPartition), + nil, errors.New("unknown topic or partition")), adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, sarama.ErrUnknownTopicOrPartition), + nil, errors.New("unknown topic or partition")), adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( map[string]kafka.TopicDetail{ topic: { diff --git a/go.mod b/go.mod index 256e4c8c29..23bef9899f 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 github.com/BurntSushi/toml v1.6.0 github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/IBM/sarama v1.41.2 github.com/KimMachineGun/automemlimit v0.2.4 github.com/agiledragon/gomonkey/v2 v2.11.0 github.com/apache/pulsar-client-go v0.13.0 @@ -47,7 +46,7 @@ require ( github.com/jcmturner/gofork v1.7.6 github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.18.5 + github.com/klauspost/compress v1.18.6 github.com/linkedin/goavro/v2 v2.14.0 github.com/mailru/easyjson v0.9.1 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 @@ -63,7 +62,6 @@ require ( github.com/pingcap/tiflow v0.0.0-20260610095716-97d622547231 github.com/prometheus/client_golang v1.23.0 github.com/r3labs/diff v1.1.0 - github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 github.com/robfig/cron v1.2.0 github.com/shirou/gopsutil/v3 v3.24.5 github.com/soheilhy/cmux v0.1.5 @@ -75,8 +73,8 @@ require ( github.com/tikv/pd v1.1.0-beta.0.20260604125942-9f1c47b1e851 github.com/tikv/pd/client v0.0.0-20260604125942-9f1c47b1e851 github.com/tinylib/msgp v1.5.0 - github.com/twmb/franz-go v1.20.6 - github.com/twmb/franz-go/pkg/kadm v1.17.1 + github.com/twmb/franz-go v1.21.4 + github.com/twmb/franz-go/pkg/kadm v1.18.0 github.com/uber-go/atomic v1.4.0 github.com/xdg/scram v1.0.5 github.com/zeebo/assert v1.3.0 @@ -126,6 +124,7 @@ require ( github.com/Azure/go-ntlmssp v0.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/DataDog/zstd v1.5.5 // indirect + github.com/IBM/sarama v1.41.2 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect @@ -307,6 +306,7 @@ require ( github.com/qri-io/jsonschema v0.2.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.55.0 // indirect + github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect @@ -340,7 +340,7 @@ require ( github.com/tklauser/numcpus v0.11.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/twmb/franz-go/pkg/kmsg v1.12.0 // indirect + github.com/twmb/franz-go/pkg/kmsg v1.13.1 // indirect github.com/twmb/murmur3 v1.1.6 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect diff --git a/go.sum b/go.sum index fef8340a56..68920db514 100644 --- a/go.sum +++ b/go.sum @@ -602,8 +602,8 @@ github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -965,12 +965,12 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/twmb/franz-go v1.20.6 h1:TpQTt4QcixJ1cHEmQGPOERvTzo99s8jAutmS7rbSD6w= -github.com/twmb/franz-go v1.20.6/go.mod h1:u+FzH2sInp7b9HNVv2cZN8AxdXy6y/AQ1Bkptu4c0FM= -github.com/twmb/franz-go/pkg/kadm v1.17.1 h1:Bt02Y/RLgnFO2NP2HVP1kd2TFtGRiJZx+fSArjZDtpw= -github.com/twmb/franz-go/pkg/kadm v1.17.1/go.mod h1:s4duQmrDbloVW9QTMXhs6mViTepze7JLG43xwPcAeTg= -github.com/twmb/franz-go/pkg/kmsg v1.12.0 h1:CbatD7ers1KzDNgJqPbKOq0Bz/WLBdsTH75wgzeVaPc= -github.com/twmb/franz-go/pkg/kmsg v1.12.0/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY= +github.com/twmb/franz-go v1.21.4 h1:skglTjGHOHHKxVdUG3A563gynBDhvSWFBBHXKOOMS8M= +github.com/twmb/franz-go v1.21.4/go.mod h1:rfoMTnVk7107fhTGxfEKIHP/e7tPe6oyij/ywzO0czk= +github.com/twmb/franz-go/pkg/kadm v1.18.0 h1:WRf/LZmDdcDXwX7WMbtDU++v+b3NzYh2bCGoPMmzirw= +github.com/twmb/franz-go/pkg/kadm v1.18.0/go.mod h1:XeLhGoLXLFzK8/ryv5FfpxPxGwj4oFEGpPJMB/x6KDE= +github.com/twmb/franz-go/pkg/kmsg v1.13.1 h1:fG5kItwysTk5UXqVwb64EpQEy3TydF3vYYK21nUQ+bI= +github.com/twmb/franz-go/pkg/kmsg v1.13.1/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= diff --git a/pkg/leakutil/leak_helper.go b/pkg/leakutil/leak_helper.go index 5e70e16786..a6e68c7291 100644 --- a/pkg/leakutil/leak_helper.go +++ b/pkg/leakutil/leak_helper.go @@ -28,13 +28,6 @@ var defaultOpts = []goleak.Option{ // The stack top is usually runtime_pollWait, so match by any-frame. goleak.IgnoreAnyFunction("github.com/godbus/dbus.(*Conn).inWorker"), goleak.IgnoreAnyFunction("github.com/godbus/dbus/v5.(*Conn).inWorker"), - // library used by sarama, ref: https://github.com/rcrowley/go-metrics/pull/266 - goleak.IgnoreTopFunction("github.com/rcrowley/go-metrics.(*meterArbiter).tick"), - // Because we close the sarama producer asynchronously, so we have to ignore these funcs. - goleak.IgnoreTopFunction("github.com/Shopify/sarama.(*client).backgroundMetadataUpdater"), - goleak.IgnoreTopFunction("github.com/Shopify/sarama.(*Broker).responseReceiver"), - goleak.IgnoreTopFunction("github.com/IBM/sarama.(*client).backgroundMetadataUpdater"), - goleak.IgnoreTopFunction("github.com/IBM/sarama.(*Broker).responseReceiver"), goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), } diff --git a/pkg/logger/log.go b/pkg/logger/log.go index 480e627c60..73ca4d129d 100644 --- a/pkg/logger/log.go +++ b/pkg/logger/log.go @@ -17,12 +17,10 @@ import ( "bytes" "context" "io" - stdlog "log" "os" "strconv" "strings" - "github.com/IBM/sarama" "github.com/gin-gonic/gin" "github.com/go-sql-driver/mysql" "github.com/pingcap/log" @@ -95,10 +93,9 @@ func IsDebugEnabled() bool { // loggerOp is the op for logger control type loggerOp struct { - isInitGRPCLogger bool - isInitSaramaLogger bool - isInitMySQLLogger bool - output zapcore.WriteSyncer + isInitGRPCLogger bool + isInitMySQLLogger bool + output zapcore.WriteSyncer } func (op *loggerOp) applyOpts(opts []LoggerOpt) { @@ -117,13 +114,6 @@ func WithInitGRPCLogger() LoggerOpt { } } -// WithInitSaramaLogger enables sarama logger initialization when initializes global logger -func WithInitSaramaLogger() LoggerOpt { - return func(op *loggerOp) { - op.isInitSaramaLogger = true - } -} - // WithInitMySQLLogger enables mysql logger initialization when initializes global logger func WithInitMySQLLogger() LoggerOpt { return func(op *loggerOp) { @@ -144,7 +134,6 @@ func InitLogger(cfg *Config, opts ...LoggerOpt) error { var op loggerOp opts = []LoggerOpt{ WithInitGRPCLogger(), - WithInitSaramaLogger(), WithInitMySQLLogger(), } op.applyOpts(opts) @@ -206,7 +195,7 @@ func InitLogger(cfg *Config, opts ...LoggerOpt) error { // initOptionalComponent initializes some optional components func initOptionalComponent(op *loggerOp, cfg *Config) error { var level zapcore.Level - if op.isInitGRPCLogger || op.isInitSaramaLogger { + if op.isInitGRPCLogger { err := level.UnmarshalText([]byte(cfg.Level)) if err != nil { return errors.Trace(err) @@ -219,12 +208,6 @@ func initOptionalComponent(op *loggerOp, cfg *Config) error { } } - if op.isInitSaramaLogger { - if err := initSaramaLogger(level); err != nil { - return err - } - } - if op.isInitMySQLLogger { if err := initMySQLLogger(); err != nil { return err @@ -256,21 +239,6 @@ func initMySQLLogger() error { return mysql.SetLogger(logger) } -// initSaramaLogger hacks logger used in sarama lib -func initSaramaLogger(level zapcore.Level) error { - if zapcore.InfoLevel.Enabled(level) { - sarama.Logger = stdlog.New(io.Discard, "[Sarama] ", stdlog.LstdFlags) - return nil - } - - logger, err := zap.NewStdLogAt(log.L().With(zap.String("component", "sarama")), level) - if err != nil { - return errors.Trace(err) - } - sarama.Logger = logger - return nil -} - type loggerWriter struct { logFunc func(msg string, fields ...zap.Field) } diff --git a/pkg/security/sasl.go b/pkg/security/sasl.go index 6b503b5bea..4f948f6ab9 100644 --- a/pkg/security/sasl.go +++ b/pkg/security/sasl.go @@ -16,7 +16,6 @@ package security import ( "strings" - "github.com/IBM/sarama" "github.com/pingcap/errors" ) @@ -28,15 +27,15 @@ const ( // UnknownMechanism means the SASL mechanism is unknown. UnknownMechanism SASLMechanism = "" // PlainMechanism means the SASL mechanism is plain. - PlainMechanism SASLMechanism = sarama.SASLTypePlaintext + PlainMechanism SASLMechanism = "PLAIN" // SCRAM256Mechanism means the SASL mechanism is SCRAM-SHA-256. - SCRAM256Mechanism SASLMechanism = sarama.SASLTypeSCRAMSHA256 + SCRAM256Mechanism SASLMechanism = "SCRAM-SHA-256" // SCRAM512Mechanism means the SASL mechanism is SCRAM-SHA-512. - SCRAM512Mechanism SASLMechanism = sarama.SASLTypeSCRAMSHA512 + SCRAM512Mechanism SASLMechanism = "SCRAM-SHA-512" // GSSAPIMechanism means the SASL mechanism is GSSAPI. - GSSAPIMechanism SASLMechanism = sarama.SASLTypeGSSAPI + GSSAPIMechanism SASLMechanism = "GSSAPI" // OAuthMechanism means the SASL mechanism is OAuth2. - OAuthMechanism SASLMechanism = sarama.SASLTypeOAuth + OAuthMechanism SASLMechanism = "OAUTHBEARER" ) // SASLMechanismFromString converts the string to SASL mechanism. @@ -109,9 +108,9 @@ const ( // UnknownAuth means the auth type is unknown. UnknownAuth GSSAPIAuthType = 0 // UserAuth means the auth type is user. - UserAuth GSSAPIAuthType = sarama.KRB5_USER_AUTH + UserAuth GSSAPIAuthType = 1 // KeyTabAuth means the auth type is keytab. - KeyTabAuth GSSAPIAuthType = sarama.KRB5_KEYTAB_AUTH + KeyTabAuth GSSAPIAuthType = 2 ) // AuthTypeFromString convent the string to GSSAPIAuthType. diff --git a/pkg/sink/codec/common/message.go b/pkg/sink/codec/common/message.go index 5cfd56b010..9c78be2fbc 100644 --- a/pkg/sink/codec/common/message.go +++ b/pkg/sink/codec/common/message.go @@ -18,11 +18,9 @@ import ( "encoding/json" ) -// MaxRecordOverhead is used to calculate message size by sarama kafka client. -// reference: https://github.com/IBM/sarama/blob/ -// 66521126c71c522c15a36663ae9cddc2b024c799/async_producer.go#L233 -// For TiCDC, minimum supported kafka version is `0.11.0.2`, -// which will be treated as `version = 2` by sarama producer. +// MaxRecordOverhead is used to calculate the expected Kafka record size. +// For TiCDC, minimum supported Kafka version is `0.11.0.2`, which uses record +// batch format v2 and varint encoded fields. const MaxRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 // MessageType is the type of message, which is used by MqSink and RedoLog. diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go deleted file mode 100644 index b8cfd1cfc3..0000000000 --- a/pkg/sink/kafka/admin.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "strconv" - "strings" - - "github.com/IBM/sarama" - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "go.uber.org/zap" -) - -type saramaAdminClient struct { - changefeed common.ChangeFeedID - - // client is the underlying sarama client created for this admin wrapper. - // It must be closed to stop background goroutines (e.g. metadata updater) and release memory. - client saramaClient - admin saramaClusterAdmin -} - -type saramaClient interface { - Brokers() []*sarama.Broker - Partitions(topic string) ([]int32, error) - Close() error -} - -type saramaClusterAdmin interface { - DescribeCluster() (brokers []*sarama.Broker, controllerID int32, err error) - DescribeConfig(resource sarama.ConfigResource) ([]sarama.ConfigEntry, error) - DescribeTopics(topics []string) (metadata []*sarama.TopicMetadata, err error) - CreateTopic(topic string, detail *sarama.TopicDetail, validateOnly bool) error - Close() error -} - -func (a *saramaAdminClient) GetAllBrokers() []Broker { - brokers := a.client.Brokers() - result := make([]Broker, 0, len(brokers)) - for _, broker := range brokers { - result = append(result, Broker{ - ID: broker.ID(), - }) - } - return result -} - -func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, error) { - _, controller, err := a.admin.DescribeCluster() - if err != nil { - return "", errors.Trace(err) - } - - configEntries, err := a.admin.DescribeConfig(sarama.ConfigResource{ - Type: sarama.BrokerResource, - Name: strconv.Itoa(int(controller)), - ConfigNames: []string{configName}, - }) - if err != nil { - return "", errors.Trace(err) - } - - // For compatibility with KOP, we checked all return values. - // 1. Kafka only returns requested configs. - // 2. Kop returns all configs. - for _, entry := range configEntries { - if entry.Name == configName { - return entry.Value, nil - } - } - - log.Warn("Kafka config item not found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName)) - return "", errors.ErrKafkaConfigNotFound.GenWithStack( - "cannot find the `%s` from the broker's configuration", configName) -} - -func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) (string, error) { - configEntries, err := a.admin.DescribeConfig(sarama.ConfigResource{ - Type: sarama.TopicResource, - Name: topicName, - ConfigNames: []string{configName}, - }) - if err != nil { - return "", errors.Trace(err) - } - - // For compatibility with KOP, we checked all return values. - // 1. Kafka only returns requested configs. - // 2. Kop returns all configs. - for _, entry := range configEntries { - if entry.Name == configName { - log.Info("Kafka config item found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName), - zap.String("configValue", entry.Value)) - return entry.Value, nil - } - } - - log.Warn("Kafka config item not found", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("configName", configName)) - return "", errors.ErrKafkaConfigNotFound.GenWithStack( - "cannot find the `%s` from the topic's configuration", configName) -} - -func (a *saramaAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { - result := make(map[string]TopicDetail, len(topics)) - - metaList, err := a.admin.DescribeTopics(topics) - if err != nil { - return nil, errors.Trace(err) - } - - for _, meta := range metaList { - if meta.Err != sarama.ErrNoError { - if meta.Err == sarama.ErrUnknownTopicOrPartition { - continue - } - if !ignoreTopicError { - return nil, meta.Err - } - log.Warn("fetch topic meta failed", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.String("topic", meta.Name), - zap.Error(meta.Err)) - continue - } - result[meta.Name] = TopicDetail{ - Name: meta.Name, - NumPartitions: int32(len(meta.Partitions)), - } - } - return result, nil -} - -func (a *saramaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - result := make(map[string]int32, len(topics)) - for _, topic := range topics { - partition, err := a.client.Partitions(topic) - if err != nil { - return nil, errors.Trace(err) - } - result[topic] = int32(len(partition)) - } - - return result, nil -} - -func (a *saramaAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { - request := &sarama.TopicDetail{ - NumPartitions: detail.NumPartitions, - ReplicationFactor: detail.ReplicationFactor, - } - - err := a.admin.CreateTopic(detail.Name, request, validateOnly) - // Ignore the already exists error because it's not harmful. - if err != nil && !strings.Contains(err.Error(), sarama.ErrTopicAlreadyExists.Error()) { - return err - } - return nil -} - -func (a *saramaAdminClient) Close() { - // For admins created via sarama.NewClusterAdminFromClient, admin.Close() takes care - // of closing the underlying client as well. Fall back to closing the client directly - // only when admin is unexpectedly nil. - if a.admin != nil { - if err := a.admin.Close(); err != nil { - log.Warn("close admin client meet error", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.Error(err)) - } - return - } - if a.client != nil { - if err := a.client.Close(); err != nil { - log.Warn("close kafka client meet error", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.Error(err)) - } - } -} diff --git a/pkg/sink/kafka/admin_mock.go b/pkg/sink/kafka/admin_mock.go deleted file mode 100644 index 588e08986b..0000000000 --- a/pkg/sink/kafka/admin_mock.go +++ /dev/null @@ -1,175 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/admin.go - -// Package kafka is a generated GoMock package. -package kafka - -import ( - reflect "reflect" - - sarama "github.com/IBM/sarama" - gomock "github.com/golang/mock/gomock" -) - -// MocksaramaClient is a mock of saramaClient interface. -type MocksaramaClient struct { - ctrl *gomock.Controller - recorder *MocksaramaClientMockRecorder -} - -// MocksaramaClientMockRecorder is the mock recorder for MocksaramaClient. -type MocksaramaClientMockRecorder struct { - mock *MocksaramaClient -} - -// NewMocksaramaClient creates a new mock instance. -func NewMocksaramaClient(ctrl *gomock.Controller) *MocksaramaClient { - mock := &MocksaramaClient{ctrl: ctrl} - mock.recorder = &MocksaramaClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MocksaramaClient) EXPECT() *MocksaramaClientMockRecorder { - return m.recorder -} - -// Brokers mocks base method. -func (m *MocksaramaClient) Brokers() []*sarama.Broker { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Brokers") - ret0, _ := ret[0].([]*sarama.Broker) - return ret0 -} - -// Brokers indicates an expected call of Brokers. -func (mr *MocksaramaClientMockRecorder) Brokers() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Brokers", reflect.TypeOf((*MocksaramaClient)(nil).Brokers)) -} - -// Close mocks base method. -func (m *MocksaramaClient) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MocksaramaClientMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaClient)(nil).Close)) -} - -// Partitions mocks base method. -func (m *MocksaramaClient) Partitions(topic string) ([]int32, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Partitions", topic) - ret0, _ := ret[0].([]int32) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Partitions indicates an expected call of Partitions. -func (mr *MocksaramaClientMockRecorder) Partitions(topic interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Partitions", reflect.TypeOf((*MocksaramaClient)(nil).Partitions), topic) -} - -// MocksaramaClusterAdmin is a mock of saramaClusterAdmin interface. -type MocksaramaClusterAdmin struct { - ctrl *gomock.Controller - recorder *MocksaramaClusterAdminMockRecorder -} - -// MocksaramaClusterAdminMockRecorder is the mock recorder for MocksaramaClusterAdmin. -type MocksaramaClusterAdminMockRecorder struct { - mock *MocksaramaClusterAdmin -} - -// NewMocksaramaClusterAdmin creates a new mock instance. -func NewMocksaramaClusterAdmin(ctrl *gomock.Controller) *MocksaramaClusterAdmin { - mock := &MocksaramaClusterAdmin{ctrl: ctrl} - mock.recorder = &MocksaramaClusterAdminMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MocksaramaClusterAdmin) EXPECT() *MocksaramaClusterAdminMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MocksaramaClusterAdmin) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MocksaramaClusterAdminMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).Close)) -} - -// CreateTopic mocks base method. -func (m *MocksaramaClusterAdmin) CreateTopic(topic string, detail *sarama.TopicDetail, validateOnly bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTopic", topic, detail, validateOnly) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateTopic indicates an expected call of CreateTopic. -func (mr *MocksaramaClusterAdminMockRecorder) CreateTopic(topic, detail, validateOnly interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).CreateTopic), topic, detail, validateOnly) -} - -// DescribeCluster mocks base method. -func (m *MocksaramaClusterAdmin) DescribeCluster() ([]*sarama.Broker, int32, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeCluster") - ret0, _ := ret[0].([]*sarama.Broker) - ret1, _ := ret[1].(int32) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// DescribeCluster indicates an expected call of DescribeCluster. -func (mr *MocksaramaClusterAdminMockRecorder) DescribeCluster() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeCluster", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).DescribeCluster)) -} - -// DescribeConfig mocks base method. -func (m *MocksaramaClusterAdmin) DescribeConfig(resource sarama.ConfigResource) ([]sarama.ConfigEntry, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeConfig", resource) - ret0, _ := ret[0].([]sarama.ConfigEntry) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DescribeConfig indicates an expected call of DescribeConfig. -func (mr *MocksaramaClusterAdminMockRecorder) DescribeConfig(resource interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeConfig", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).DescribeConfig), resource) -} - -// DescribeTopics mocks base method. -func (m *MocksaramaClusterAdmin) DescribeTopics(topics []string) ([]*sarama.TopicMetadata, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeTopics", topics) - ret0, _ := ret[0].([]*sarama.TopicMetadata) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DescribeTopics indicates an expected call of DescribeTopics. -func (mr *MocksaramaClusterAdminMockRecorder) DescribeTopics(topics interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTopics", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).DescribeTopics), topics) -} diff --git a/pkg/sink/kafka/admin_test.go b/pkg/sink/kafka/admin_test.go deleted file mode 100644 index c2e3f90e37..0000000000 --- a/pkg/sink/kafka/admin_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "testing" - - "github.com/golang/mock/gomock" - "github.com/pingcap/ticdc/pkg/common" - "github.com/stretchr/testify/require" -) - -func TestAdminClientClose(t *testing.T) { - tests := []struct { - name string - setup func(*gomock.Controller) *saramaAdminClient - }{ - { - name: "uses admin close", - setup: func(ctrl *gomock.Controller) *saramaAdminClient { - client := NewMocksaramaClient(ctrl) - admin := NewMocksaramaClusterAdmin(ctrl) - admin.EXPECT().Close().Return(nil) - client.EXPECT().Close().Times(0) - return &saramaAdminClient{ - changefeed: common.NewChangeFeedIDWithName("test", "default"), - client: client, - admin: admin, - } - }, - }, - { - name: "falls back to client when admin is nil", - setup: func(ctrl *gomock.Controller) *saramaAdminClient { - client := NewMocksaramaClient(ctrl) - client.EXPECT().Close().Return(nil) - return &saramaAdminClient{ - changefeed: common.NewChangeFeedIDWithName("test", "default"), - client: client, - } - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - adminClient := test.setup(ctrl) - - require.NotPanics(t, func() { adminClient.Close() }) - }) - } -} diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 72a458a508..5c827932d1 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -48,19 +48,14 @@ type SyncProducer interface { // SendMessages will return an error. SendMessages(topic string, partitionNum int32, message *common.Message) error - // Close shuts down the producer; you must call this function before a producer - // object passes out of scope, as it may otherwise leak memory. - // You must call this before calling Close on the underlying client. + // Close shuts down the producer and releases its Kafka client resources. Close() } // AsyncProducer is the kafka async producer type AsyncProducer interface { - // Close shuts down the producer and waits for any buffered messages to be - // flushed. You must call this function before a producer object passes out of - // scope, as it may otherwise leak memory. You must call this before process - // shutting down, or you may lose messages. You must call this before calling - // Close on the underlying client. + // Close shuts down the producer asynchronously and releases its Kafka client + // resources. It does not wait for buffered messages to be flushed. Close() // AsyncSend is the input channel for the user to write messages to that they diff --git a/pkg/sink/kafka/factory_selector.go b/pkg/sink/kafka/factory_selector.go index 0aa47abe25..be599a29cf 100644 --- a/pkg/sink/kafka/factory_selector.go +++ b/pkg/sink/kafka/factory_selector.go @@ -15,24 +15,15 @@ package kafka import ( "context" - "strings" "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" ) -// NewFactory selects a Kafka client implementation based on options. +// NewFactory creates the Kafka client factory. func NewFactory( ctx context.Context, o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - switch strings.ToLower(strings.TrimSpace(o.KafkaClient)) { - case "", "franz": - return NewFranzFactory(ctx, o, changefeedID) - case "sarama": - return NewSaramaFactory(ctx, o, changefeedID) - default: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported kafka client %s", o.KafkaClient) - } + return NewFranzFactory(ctx, o, changefeedID) } diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/franz/factory.go index 1a4b94ceed..48c8a6bfd2 100644 --- a/pkg/sink/kafka/franz/factory.go +++ b/pkg/sink/kafka/franz/factory.go @@ -42,6 +42,7 @@ type Options struct { IsAssignedVersion bool MaxMessageBytes int + MaxRetry int Compression string RequiredAcks int16 @@ -56,6 +57,7 @@ type Options struct { } const defaultRequestTimeout = 10 * time.Second +const defaultRecordRetries = 5 func maxTimeoutWithDefault(readTimeout, writeTimeout time.Duration) time.Duration { timeout := readTimeout @@ -211,8 +213,11 @@ func newOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSource, e func newProducerOptions( o *Options, ) []kgo.Opt { + recordRetries := defaultRecordRetries if o == nil { o = &Options{} + } else { + recordRetries = o.MaxRetry } produceTimeout := o.ReadTimeout @@ -225,7 +230,7 @@ func newProducerOptions( kgo.RequiredAcks(newRequiredAcks(o)), kgo.DisableIdempotentWrite(), kgo.MaxProduceRequestsInflightPerBroker(1), - kgo.RecordRetries(5), + kgo.RecordRetries(recordRetries), kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), kgo.ProduceRequestTimeout(produceTimeout), kgo.ProducerLinger(0), diff --git a/pkg/sink/kafka/franz/factory_api_test.go b/pkg/sink/kafka/franz/factory_api_test.go index 02895ab3c1..be359d6fe5 100644 --- a/pkg/sink/kafka/franz/factory_api_test.go +++ b/pkg/sink/kafka/franz/factory_api_test.go @@ -14,9 +14,11 @@ package franz import ( + "context" "testing" "time" + "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kgo" ) @@ -69,3 +71,31 @@ func TestMaxTimeoutWithDefault(t *testing.T) { }) } } + +func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { + t.Parallel() + + opts, err := newOptions(context.Background(), &Options{ + Version: "invalid", + IsAssignedVersion: true, + }, nil) + require.Nil(t, opts) + require.ErrorContains(t, err, "invalid kafka version invalid") +} + +func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { + t.Parallel() + + _, err := newOauthTokenSource(context.Background(), &Options{ + SASL: &security.SASL{ + OAuth2: security.OAuth2{ + ClientID: "client-id", + ClientSecret: "client-secret", + TokenURL: "http://test.com/Segment%%2815197306101420000%29", + Scopes: []string{"scope1", "scope2"}, + GrantType: "client_credentials", + }, + }, + }) + require.ErrorContains(t, err, "invalid URL escape") +} diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/franz/metrics_hook.go index 04e66441d8..e57f270d04 100644 --- a/pkg/sink/kafka/franz/metrics_hook.go +++ b/pkg/sink/kafka/franz/metrics_hook.go @@ -40,8 +40,21 @@ type PrometheusMetrics struct { ResponseRate *prometheus.GaugeVec CompressionRatio *prometheus.HistogramVec RecordsPerRequest *prometheus.HistogramVec + + LegacyRequestsInFlight *prometheus.GaugeVec + LegacyOutgoingByteRate *prometheus.GaugeVec + LegacyRequestRate *prometheus.GaugeVec + LegacyRequestLatency *prometheus.GaugeVec + LegacyResponseRate *prometheus.GaugeVec + LegacyCompressionRatio *prometheus.GaugeVec + LegacyRecordsPerRequest *prometheus.GaugeVec } +const ( + legacyMetricAvg = "avg" + legacyMetricP99 = "p99" +) + func NewMetricsHook(clientType string) *MetricsHook { return &MetricsHook{clientType: clientType} } @@ -98,6 +111,18 @@ func (h *MetricsHook) CleanupPrometheusMetrics() { deleteHistogramVecPartialMatch(metrics.RequestLatency, labels) deleteHistogramVecPartialMatch(metrics.CompressionRatio, labels) deleteHistogramVecPartialMatch(metrics.RecordsPerRequest, labels) + + legacyLabels := prometheus.Labels{ + "namespace": keyspace, + "changefeed": changefeed, + } + deleteGaugeVecPartialMatch(metrics.LegacyOutgoingByteRate, legacyLabels) + deleteGaugeVecPartialMatch(metrics.LegacyRequestRate, legacyLabels) + deleteGaugeVecPartialMatch(metrics.LegacyResponseRate, legacyLabels) + deleteGaugeVecPartialMatch(metrics.LegacyRequestsInFlight, legacyLabels) + deleteGaugeVecPartialMatch(metrics.LegacyRequestLatency, legacyLabels) + deleteGaugeVecPartialMatch(metrics.LegacyCompressionRatio, legacyLabels) + deleteGaugeVecPartialMatch(metrics.LegacyRecordsPerRequest, legacyLabels) } func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { @@ -114,12 +139,21 @@ func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err erro if ctx.metrics.OutgoingByteRate != nil && bytesWritten > 0 { ctx.metrics.OutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(float64(bytesWritten)) } + if ctx.metrics.LegacyOutgoingByteRate != nil && bytesWritten > 0 { + ctx.metrics.LegacyOutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(float64(bytesWritten)) + } if ctx.metrics.RequestRate != nil { ctx.metrics.RequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) } + if ctx.metrics.LegacyRequestRate != nil { + ctx.metrics.LegacyRequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + } if err == nil && ctx.metrics.RequestsInFlight != nil { ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) } + if err == nil && ctx.metrics.LegacyRequestsInFlight != nil { + ctx.metrics.LegacyRequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + } } func (h *MetricsHook) OnBrokerWrite( @@ -151,13 +185,24 @@ func (h *MetricsHook) OnBrokerE2E( if e2e.WriteErr == nil && ctx.metrics.RequestsInFlight != nil { ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(-1) } + if e2e.WriteErr == nil && ctx.metrics.LegacyRequestsInFlight != nil { + ctx.metrics.LegacyRequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(-1) + } if e2e.BytesRead > 0 && e2e.ReadErr == nil && ctx.metrics.ResponseRate != nil { ctx.metrics.ResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) } + if e2e.BytesRead > 0 && e2e.ReadErr == nil && ctx.metrics.LegacyResponseRate != nil { + ctx.metrics.LegacyResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + } if e2e.Err() == nil && ctx.metrics.RequestLatency != nil { latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Observe(latencyMs) } + if e2e.Err() == nil && ctx.metrics.LegacyRequestLatency != nil { + latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 + ctx.metrics.LegacyRequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricAvg).Set(latencyMs) + ctx.metrics.LegacyRequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricP99).Set(latencyMs) + } } func (h *MetricsHook) OnProduceBatchWritten( @@ -179,10 +224,20 @@ func (h *MetricsHook) RecordProduceBatchWritten(numRecords int, uncompressedByte records := float64(numRecords) ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType).Observe(records) } + if ctx.metrics.LegacyRecordsPerRequest != nil && numRecords > 0 { + records := float64(numRecords) + ctx.metrics.LegacyRecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(records) + ctx.metrics.LegacyRecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(records) + } if ctx.metrics.CompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType).Observe(ratio) } + if ctx.metrics.LegacyCompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { + ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 + ctx.metrics.LegacyCompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(ratio) + ctx.metrics.LegacyCompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(ratio) + } } type metricsContext struct { diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index b800b3d081..f4d164130e 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -64,6 +64,14 @@ func newFranzMetricsHook(changefeedID common.ChangeFeedID, clientType string) *f ResponseRate: franzResponseTotalByClientGauge, CompressionRatio: franzCompressionRatioHistogram, RecordsPerRequest: franzRecordsPerRequestHistogram, + + LegacyRequestsInFlight: requestsInFlightGauge, + LegacyOutgoingByteRate: OutgoingByteRateGauge, + LegacyRequestRate: RequestRateGauge, + LegacyRequestLatency: RequestLatencyGauge, + LegacyResponseRate: responseRateGauge, + LegacyCompressionRatio: compressionRatioGauge, + LegacyRecordsPerRequest: recordsPerRequestGauge, }, ) return hook @@ -144,6 +152,7 @@ func newFranzOptions(o *options) *franz.Options { IsAssignedVersion: o.IsAssignedVersion, MaxMessageBytes: o.MaxMessageBytes, + MaxRetry: o.MaxRetry, Compression: o.Compression, RequiredAcks: int16(o.RequiredAcks), diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go index 9e2dd36bb4..b7844229d5 100644 --- a/pkg/sink/kafka/franz_factory_test.go +++ b/pkg/sink/kafka/franz_factory_test.go @@ -51,3 +51,13 @@ func TestNewFranzOptionsMapsRequiredAcks(t *testing.T) { }) } } + +func TestNewFranzOptionsMapsMaxRetry(t *testing.T) { + t.Parallel() + + options := NewOptions() + options.MaxRetry = 7 + + franzOptions := newFranzOptions(options) + require.Equal(t, 7, franzOptions.MaxRetry) +} diff --git a/pkg/sink/kafka/metrics_collector.go b/pkg/sink/kafka/metrics_collector.go index 96f969dd39..0146d744c3 100644 --- a/pkg/sink/kafka/metrics_collector.go +++ b/pkg/sink/kafka/metrics_collector.go @@ -13,202 +13,9 @@ package kafka -import ( - "context" - "strconv" - "time" - - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" - "github.com/rcrowley/go-metrics" - "go.uber.org/zap" -) +import "context" // MetricsCollector is the interface for kafka metrics collector. type MetricsCollector interface { Run(ctx context.Context) } - -const ( - // refreshMetricsInterval specifies the interval of refresh kafka client metrics. - refreshMetricsInterval = 5 * time.Second - // refreshClusterMetaInterval specifies the interval of refresh kafka cluster meta. - // Do not set it too small, because it will cause too many requests to kafka cluster. - // Every request will get all topics and all brokers information. - refreshClusterMetaInterval = 30 * time.Minute -) - -// Sarama metrics names, see https://pkg.go.dev/github.com/IBM/sarama#pkg-overview. -const ( - // Producer level. - compressionRatioMetricName = "compression-ratio" - recordsPerRequestMetricName = "records-per-request" - - // Broker level. - outgoingByteRateMetricNamePrefix = "outgoing-byte-rate-for-broker-" - requestRateMetricNamePrefix = "request-rate-for-broker-" - requestLatencyInMsMetricNamePrefix = "request-latency-in-ms-for-broker-" - requestsInFlightMetricNamePrefix = "requests-in-flight-for-broker-" - responseRateMetricNamePrefix = "response-rate-for-broker-" - - p99 = "p99" - avg = "avg" -) - -type saramaMetricsCollector struct { - changefeedID common.ChangeFeedID - // adminClient is used to get broker infos from broker. - adminClient ClusterAdminClient - brokers map[int32]struct{} - registry metrics.Registry -} - -func (m *saramaMetricsCollector) Run(ctx context.Context) { - // Initialize brokers. - m.updateBrokers(ctx) - - refreshMetricsTicker := time.NewTicker(refreshMetricsInterval) - refreshClusterMetaTicker := time.NewTicker(refreshClusterMetaInterval) - defer func() { - refreshMetricsTicker.Stop() - refreshClusterMetaTicker.Stop() - m.cleanupMetrics() - }() - - for { - select { - case <-ctx.Done(): - log.Info("kafka metrics collector stopped", - zap.String("keyspace", m.changefeedID.Keyspace()), - zap.String("changefeed", m.changefeedID.Name())) - return - case <-refreshMetricsTicker.C: - m.collectBrokerMetrics() - m.collectProducerMetrics() - case <-refreshClusterMetaTicker.C: - m.updateBrokers(ctx) - } - } -} - -func (m *saramaMetricsCollector) updateBrokers(ctx context.Context) { - brokers := m.adminClient.GetAllBrokers() - for _, b := range brokers { - m.brokers[b.ID] = struct{}{} - } -} - -func (m *saramaMetricsCollector) collectProducerMetrics() { - keyspace := m.changefeedID.Keyspace() - changefeedID := m.changefeedID.Name() - compressionRatioMetric := m.registry.Get(compressionRatioMetricName) - if histogram, ok := compressionRatioMetric.(metrics.Histogram); ok { - compressionRatioGauge. - WithLabelValues(keyspace, changefeedID, avg). - Set(histogram.Snapshot().Mean()) - compressionRatioGauge.WithLabelValues(keyspace, changefeedID, p99). - Set(histogram.Snapshot().Percentile(0.99)) - } - - recordsPerRequestMetric := m.registry.Get(recordsPerRequestMetricName) - if histogram, ok := recordsPerRequestMetric.(metrics.Histogram); ok { - recordsPerRequestGauge. - WithLabelValues(keyspace, changefeedID, avg). - Set(histogram.Snapshot().Mean()) - recordsPerRequestGauge. - WithLabelValues(keyspace, changefeedID, p99). - Set(histogram.Snapshot().Percentile(0.99)) - } -} - -func (m *saramaMetricsCollector) collectBrokerMetrics() { - keyspace := m.changefeedID.Keyspace() - changefeedID := m.changefeedID.Name() - for id := range m.brokers { - brokerID := strconv.Itoa(int(id)) - outgoingByteRateMetric := m.registry.Get( - getBrokerMetricName(outgoingByteRateMetricNamePrefix, brokerID)) - if meter, ok := outgoingByteRateMetric.(metrics.Meter); ok { - OutgoingByteRateGauge. - WithLabelValues(keyspace, changefeedID, brokerID). - Set(meter.Snapshot().Rate1()) - } - - requestRateMetric := m.registry.Get( - getBrokerMetricName(requestRateMetricNamePrefix, brokerID)) - if meter, ok := requestRateMetric.(metrics.Meter); ok { - RequestRateGauge. - WithLabelValues(keyspace, changefeedID, brokerID). - Set(meter.Snapshot().Rate1()) - } - - requestLatencyMetric := m.registry.Get( - getBrokerMetricName(requestLatencyInMsMetricNamePrefix, brokerID)) - if histogram, ok := requestLatencyMetric.(metrics.Histogram); ok { - RequestLatencyGauge. - WithLabelValues(keyspace, changefeedID, brokerID, avg). - Set(histogram.Snapshot().Mean() / 1000) - RequestLatencyGauge. - WithLabelValues(keyspace, changefeedID, brokerID, p99). - Set(histogram.Snapshot().Percentile(0.99) / 1000) - } - - requestsInFlightMetric := m.registry.Get(getBrokerMetricName( - requestsInFlightMetricNamePrefix, brokerID)) - if counter, ok := requestsInFlightMetric.(metrics.Counter); ok { - requestsInFlightGauge. - WithLabelValues(keyspace, changefeedID, brokerID). - Set(float64(counter.Snapshot().Count())) - } - - responseRateMetric := m.registry.Get(getBrokerMetricName( - responseRateMetricNamePrefix, brokerID)) - if meter, ok := responseRateMetric.(metrics.Meter); ok { - responseRateGauge. - WithLabelValues(keyspace, changefeedID, brokerID). - Set(meter.Snapshot().Rate1()) - } - } -} - -func getBrokerMetricName(prefix, brokerID string) string { - return prefix + brokerID -} - -func (m *saramaMetricsCollector) cleanupProducerMetrics() { - compressionRatioGauge. - DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), avg) - compressionRatioGauge. - DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), p99) - - recordsPerRequestGauge. - DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), avg) - recordsPerRequestGauge. - DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), p99) -} - -func (m *saramaMetricsCollector) cleanupBrokerMetrics() { - keyspace := m.changefeedID.Keyspace() - changefeedID := m.changefeedID.Name() - for id := range m.brokers { - brokerID := strconv.Itoa(int(id)) - OutgoingByteRateGauge. - DeleteLabelValues(keyspace, changefeedID, brokerID) - RequestRateGauge. - DeleteLabelValues(keyspace, changefeedID, brokerID) - RequestLatencyGauge. - DeleteLabelValues(keyspace, changefeedID, brokerID, avg) - RequestLatencyGauge. - DeleteLabelValues(keyspace, changefeedID, brokerID, p99) - requestsInFlightGauge. - DeleteLabelValues(keyspace, changefeedID, brokerID) - responseRateGauge. - DeleteLabelValues(keyspace, changefeedID, brokerID) - - } -} - -func (m *saramaMetricsCollector) cleanupMetrics() { - m.cleanupProducerMetrics() - m.cleanupBrokerMetrics() -} diff --git a/pkg/sink/kafka/oauth2_token_provider.go b/pkg/sink/kafka/oauth2_token_provider.go deleted file mode 100644 index dd25b3ff31..0000000000 --- a/pkg/sink/kafka/oauth2_token_provider.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "net/url" - - "github.com/IBM/sarama" - "github.com/pingcap/errors" - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" -) - -// tokenProvider is a user-defined callback for generating -// access tokens for SASL/OAUTHBEARER auth. -type tokenProvider struct { - tokenSource oauth2.TokenSource -} - -var _ sarama.AccessTokenProvider = (*tokenProvider)(nil) - -// Token implements the sarama.AccessTokenProvider interface. -// Token returns an access token. The implementation should ensure token -// reuse so that multiple calls at connect time do not create multiple -// tokens. The implementation should also periodically refresh the token in -// order to guarantee that each call returns an unexpired token. This -// method should not block indefinitely--a timeout error should be returned -// after a short period of inactivity so that the broker connection logic -// can log debugging information and retry. -func (t *tokenProvider) Token() (*sarama.AccessToken, error) { - token, err := t.tokenSource.Token() - if err != nil { - // Errors will result in Sarama retrying the broker connection and logging - // the transient error, with a Broker connection error surfacing after retry - // attempts have been exhausted. - return nil, err - } - - return &sarama.AccessToken{Token: token.AccessToken}, nil -} - -func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvider, error) { - // grant_type is by default going to be set to 'client_credentials' by the - // client credentials library as defined by the spec, however non-compliant - // auth server implementations may want a custom type - endpointParams := url.Values{} - if o.SASL.OAuth2.GrantType != "" { - endpointParams.Set("grant_type", o.SASL.OAuth2.GrantType) - } - - // audience is an optional parameter that can be used to specify the - // intended audience of the token. - if o.SASL.OAuth2.Audience != "" { - endpointParams.Set("audience", o.SASL.OAuth2.Audience) - } - - tokenURL, err := url.Parse(o.SASL.OAuth2.TokenURL) - if err != nil { - return nil, errors.Trace(err) - } - - cfg := clientcredentials.Config{ - ClientID: o.SASL.OAuth2.ClientID, - ClientSecret: o.SASL.OAuth2.ClientSecret, - TokenURL: tokenURL.String(), - EndpointParams: endpointParams, - Scopes: o.SASL.OAuth2.Scopes, - } - return &tokenProvider{ - tokenSource: cfg.TokenSource(ctx), - }, nil -} diff --git a/pkg/sink/kafka/oauth2_token_provider_test.go b/pkg/sink/kafka/oauth2_token_provider_test.go deleted file mode 100644 index 4438377824..0000000000 --- a/pkg/sink/kafka/oauth2_token_provider_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "testing" - - "github.com/pingcap/ticdc/pkg/security" - "github.com/stretchr/testify/require" -) - -func TestNewTokenProvider(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - options *options - expectedErr string - }{ - { - name: "valid", - options: &options{ - SASL: &security.SASL{ - OAuth2: security.OAuth2{ - ClientID: "client-id", - ClientSecret: "client-secret", - TokenURL: "http://localhost:8080/oauth2/token", - Scopes: []string{"scope1", "scope2"}, - GrantType: "client_credentials", - }, - }, - }, - }, - { - name: "invalid token URL", - options: &options{ - SASL: &security.SASL{ - OAuth2: security.OAuth2{ - ClientID: "client-id", - ClientSecret: "client-secret", - TokenURL: "http://test.com/Segment%%2815197306101420000%29", - Scopes: []string{"scope1", "scope2"}, - GrantType: "client_credentials", - }, - }, - }, - expectedErr: "invalid URL escape", - }, - } { - ts := test - t.Run(ts.name, func(t *testing.T) { - t.Parallel() - _, err := newTokenProvider(context.TODO(), ts.options) - if ts.expectedErr == "" { - require.NoError(t, err) - } else { - require.Error(t, err) - require.Contains(t, err.Error(), ts.expectedErr) - } - }) - } -} diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index dd43c0305f..d7bd0bd643 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -119,7 +119,6 @@ type urlConfig struct { MaxMessageBytes *int `form:"max-message-bytes"` MaxRetry *int `form:"max-retry"` Compression *string `form:"compression"` - KafkaClient *string `form:"kafka-client"` KafkaClientID *string `form:"kafka-client-id"` AutoCreateTopic *bool `form:"auto-create-topic"` DialTimeout *string `form:"dial-timeout"` @@ -148,7 +147,6 @@ type urlConfig struct { type options struct { Topic string BrokerEndpoints []string - KafkaClient string // control whether to create topic AutoCreate bool @@ -185,7 +183,6 @@ func NewOptions() *options { Version: "2.4.0", // MaxMessageBytes will be used to initialize producer MaxMessageBytes: config.DefaultMaxMessageBytes, - KafkaClient: "franz", MaxRetry: defaultMaxRetry, ReplicationFactor: 1, Compression: "none", @@ -272,16 +269,6 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, o.Compression = *urlParameter.Compression } - if urlParameter.KafkaClient != nil && *urlParameter.KafkaClient != "" { - kafkaClient := strings.ToLower(strings.TrimSpace(*urlParameter.KafkaClient)) - switch kafkaClient { - case "sarama", "franz": - o.KafkaClient = kafkaClient - default: - return cerror.ErrKafkaInvalidConfig.GenWithStack("unsupported kafka client %s", kafkaClient) - } - } - var kafkaClientID string if urlParameter.KafkaClientID != nil { kafkaClientID = *urlParameter.KafkaClientID @@ -588,7 +575,7 @@ func NewKafkaClientID(captureAddr string, return } -// adjustOptions adjust the `options` and `sarama.Config` by condition. +// adjustOptions adjusts Kafka sink options by broker and topic metadata. func adjustOptions( ctx context.Context, admin ClusterAdminClient, diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 79729df7f1..0ffbcf6b86 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -15,6 +15,7 @@ package kafka import ( "context" + stdErrors "errors" "fmt" "net/url" "strconv" @@ -22,7 +23,6 @@ import ( "testing" "time" - "github.com/IBM/sarama" "github.com/aws/aws-sdk-go-v2/aws" "github.com/golang/mock/gomock" commonType "github.com/pingcap/ticdc/pkg/common" @@ -133,11 +133,11 @@ func (f *kafkaAdminFixture) getTopicConfig(topicName string, configName string) func (f *kafkaAdminFixture) createTopic(detail *TopicDetail, _ bool) error { if detail.ReplicationFactor > mockClusterReplicationFactor { - return sarama.ErrInvalidReplicationFactor + return stdErrors.New("invalid replication factor") } if _, ok := f.brokerConfig[MinInsyncReplicasConfigName]; !ok && detail.ReplicationFactor != mockClusterReplicationFactor { - return sarama.ErrPolicyViolation + return stdErrors.New("policy violation") } f.topics[detail.Name] = *detail return nil @@ -289,13 +289,6 @@ func TestCompleteOptions(t *testing.T) { require.Equal(t, defaultMaxRetry, options.MaxRetry) } -func TestNewOptionsDefaultKafkaClient(t *testing.T) { - t.Parallel() - - options := NewOptions() - require.Equal(t, "franz", options.KafkaClient) -} - func TestSetPartitionNum(t *testing.T) { options := NewOptions() err := options.setPartitionNum(2) @@ -427,11 +420,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t err = adjustOptions(ctx, adminClient, options, topicName) require.NoError(t, err) - saramaConfig, err := newSaramaConfig(ctx, options) - require.NoError(t, err) - require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, expectedMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) }) } } @@ -469,7 +458,7 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { Name: topicName, ReplicationFactor: 1, }, false) - require.ErrorIs(t, err, sarama.ErrPolicyViolation) + require.ErrorContains(t, err, "policy violation") // Report an error if the replication-factor is less than min.insync.replicas // when the topic does exist. @@ -513,15 +502,6 @@ func TestSkipAdjustConfigMinInsyncReplicasWhenRequiredAcksIsNotWailAll(t *testin require.Nil(t, err, "Should not report an error when `required-acks` is not `all`") } -func TestCreateProducerFailed(t *testing.T) { - options := NewOptions() - options.Version = "invalid" - options.IsAssignedVersion = true - saramaConfig, err := newSaramaConfig(context.Background(), options) - require.Regexp(t, "invalid version.*", errors.Cause(err)) - require.Nil(t, saramaConfig) -} - func TestConfigurationCombinations(t *testing.T) { combinations := []struct { name string @@ -699,10 +679,6 @@ func TestConfigurationCombinations(t *testing.T) { require.Nil(t, err) require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) - saramaConfig, err := newSaramaConfig(ctx, options) - require.Nil(t, err) - require.Equal(t, expectedMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) - encoderConfig := common.NewConfig(config.ProtocolOpen) err = encoderConfig.Apply(sinkURI, &config.SinkConfig{ KafkaConfig: &config.KafkaConfig{ diff --git a/pkg/sink/kafka/sarama_async_producer.go b/pkg/sink/kafka/sarama_async_producer.go deleted file mode 100644 index 9c0bd82104..0000000000 --- a/pkg/sink/kafka/sarama_async_producer.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "time" - - "github.com/IBM/sarama" - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - commonType "github.com/pingcap/ticdc/pkg/common" - cerror "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/codec/common" - "go.uber.org/atomic" - "go.uber.org/zap" -) - -type saramaAsyncProducer struct { - client sarama.Client - producer sarama.AsyncProducer - changefeedID commonType.ChangeFeedID - - closed *atomic.Bool - failpointCh chan *sarama.ProducerError -} - -type messageMetadata struct { - callback func() - logInfo *common.MessageLogInfo -} - -func (p *saramaAsyncProducer) Close() { - p.closed.Store(true) - go func() { - // We need to close it asynchronously. Otherwise, we might get stuck - // with an unhealthy(i.e. Network jitter, isolation) state of Kafka. - // Safety: - // * If the kafka cluster is running well, it will be closed as soon as possible. - // Also, we cancel all table pipelines before closed, so it's safe. - // * If there is a problem with the kafka cluster, it will shut down the client first, - // which means no more data will be sent because the connection to the broker is dropped. - // Also, we cancel all table pipelines before closed, so it's safe. - // * For Kafka Sink, duplicate data is acceptable. - // * There is a risk of goroutine leakage, but it is acceptable and our main - // goal is not to get stuck with the processor tick. - - // `client` is mainly used by `asyncProducer` to fetch metadata and perform other related - // operations. When we close the `kafkaSaramaProducer`, - // there is no need for TiCDC to make sure that all buffered messages are flushed. - // Consider the situation where the broker is irresponsive. If the client were not - // closed, `asyncProducer.Close()` would waste a mount of time to try flush all messages. - // To prevent the scenario mentioned above, close the client first. - start := time.Now() - if err := p.client.Close(); err != nil { - log.Warn("Close kafka async producer client error", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.Duration("duration", time.Since(start)), - zap.Error(err)) - } else { - log.Info("Close kafka async producer client success", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.Duration("duration", time.Since(start))) - } - - start = time.Now() - if err := p.producer.Close(); err != nil { - log.Warn("Close kafka async producer error", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.Duration("duration", time.Since(start)), - zap.Error(err)) - } else { - log.Info("Close kafka async producer success", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.Duration("duration", time.Since(start))) - } - }() -} - -func (p *saramaAsyncProducer) AsyncRunCallback( - ctx context.Context, -) error { - defer p.closed.Store(true) - for { - select { - case <-ctx.Done(): - log.Info("async producer exit since context is done", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name())) - return errors.Trace(ctx.Err()) - case err := <-p.failpointCh: - log.Warn("Receive from failpoint chan in kafka DML producer", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.Error(err)) - return p.handleProducerError(err) - case ack := <-p.producer.Successes(): - if ack != nil { - switch meta := ack.Metadata.(type) { - case *messageMetadata: - if meta != nil && meta.callback != nil { - meta.callback() - } - default: - log.Error("unknown message metadata type in async producer", - zap.Any("metadata", ack.Metadata)) - } - } - case err := <-p.producer.Errors(): - // We should not wrap a nil pointer if the pointer - // is of a subtype of `error` because Go would store the type info - // and the resulted `error` variable would not be nil, - // which will cause the pkg/error library to malfunction. - // See: https://go.dev/doc/faq#nil_error - if err == nil { - return nil - } - return p.handleProducerError(err) - } - } -} - -func (p *saramaAsyncProducer) handleProducerError(err *sarama.ProducerError) error { - errWithInfo := AnnotateEventError( - p.changefeedID.Keyspace(), - p.changefeedID.Name(), - extractLogInfo(err.Msg), - err.Err, - ) - return cerror.WrapError(cerror.ErrKafkaAsyncSendMessage, errWithInfo) -} - -// AsyncSend is the input channel for the user to write messages to that they -// wish to send. -func (p *saramaAsyncProducer) AsyncSend( - ctx context.Context, topic string, partition int32, message *common.Message, -) error { - if p.closed.Load() { - return cerror.ErrKafkaProducerClosed.GenWithStackByArgs() - } - failpoint.Inject("KafkaSinkAsyncSendError", func() { - // simulate sending message to input channel successfully but flushing - // message to Kafka meets error - log.Info("KafkaSinkAsyncSendError error injected", zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name())) - p.failpointCh <- &sarama.ProducerError{ - Err: errors.New("kafka sink injected error"), - Msg: &sarama.ProducerMessage{Metadata: &messageMetadata{ - callback: message.Callback, - logInfo: message.LogInfo, - }}, - } - failpoint.Return(nil) - }) - meta := &messageMetadata{ - callback: message.Callback, - logInfo: message.LogInfo, - } - msg := &sarama.ProducerMessage{ - Topic: topic, - Partition: partition, - Key: sarama.StringEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), - Metadata: meta, - } - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - case p.producer.Input() <- msg: - } - return nil -} - -func extractLogInfo(msg *sarama.ProducerMessage) *common.MessageLogInfo { - if msg == nil { - return nil - } - meta, ok := msg.Metadata.(*messageMetadata) - if !ok || meta == nil { - return nil - } - return meta.logInfo -} diff --git a/pkg/sink/kafka/sarama_config.go b/pkg/sink/kafka/sarama_config.go deleted file mode 100644 index b53dc47b37..0000000000 --- a/pkg/sink/kafka/sarama_config.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "crypto/tls" - "math/rand" - "strings" - "time" - - "github.com/IBM/sarama" - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/security" - "go.uber.org/zap" -) - -var ( - defaultKafkaVersion = sarama.V2_0_0_0 - maxKafkaVersion = sarama.V2_8_0_0 -) - -// newSaramaConfig return the default config and set the according version and metrics -func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { - config := sarama.NewConfig() - config.ClientID = o.ClientID - var err error - // Admin client would refresh metadata periodically, - // if metadata cannot be refreshed easily, this would indicate the network condition between the - // capture server and kafka broker is not good. - // Set the timeout to 2 minutes to ensure that the underlying client does not retry for too long. - // If retrying to obtain the metadata fails, simply return the error and let sinkManager rebuild the sink. - config.Metadata.Retry.Max = 10 - config.Metadata.Retry.Backoff = 200 * time.Millisecond - config.Metadata.Timeout = 2 * time.Minute - config.Admin.Retry.Max = 10 - config.Admin.Retry.Backoff = 200 * time.Millisecond - // This timeout control the request timeout for each admin request. - // set it as the read timeout. - config.Admin.Timeout = 10 * time.Second - - // Keep a bounded producer retry budget to tolerate transient broker-side - // connection failures such as stale connections or broken pipe errors. - // The PingCAP Sarama fork includes the partition-muting ordering fix, while - // Net.MaxOpenRequests=1 below remains an extra ordering guard. - config.Producer.Retry.Max = o.MaxRetry - config.Producer.Retry.Backoff = 100 * time.Millisecond - - // make sure sarama producer flush messages as soon as possible. - config.Producer.Flush.Bytes = 0 - config.Producer.Flush.Messages = 0 - config.Producer.Flush.Frequency = time.Duration(0) - config.Producer.Flush.MaxMessages = o.MaxMessages - - config.Net.MaxOpenRequests = 1 - config.Net.DialTimeout = o.DialTimeout - config.Net.WriteTimeout = o.WriteTimeout - config.Net.ReadTimeout = o.ReadTimeout - - config.Producer.Partitioner = sarama.NewManualPartitioner - config.Producer.MaxMessageBytes = o.MaxMessageBytes - config.Producer.Return.Successes = true - config.Producer.Return.Errors = true - config.Producer.RequiredAcks = sarama.RequiredAcks(o.RequiredAcks) - compression := strings.ToLower(strings.TrimSpace(o.Compression)) - switch compression { - case "none": - config.Producer.Compression = sarama.CompressionNone - case "gzip": - config.Producer.Compression = sarama.CompressionGZIP - case "snappy": - config.Producer.Compression = sarama.CompressionSnappy - case "lz4": - config.Producer.Compression = sarama.CompressionLZ4 - case "zstd": - config.Producer.Compression = sarama.CompressionZSTD - default: - log.Warn("Unsupported compression algorithm", zap.String("compression", o.Compression)) - config.Producer.Compression = sarama.CompressionNone - } - if config.Producer.Compression != sarama.CompressionNone { - log.Info("Kafka producer uses " + compression + " compression algorithm") - } - - if o.EnableTLS { - // for SSL encryption with a trust CA certificate, we must populate the - // following two params of config.Net.TLS - config.Net.TLS.Enable = true - config.Net.TLS.Config = &tls.Config{ - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", "http/1.1"}, - } - - // for SSL encryption with self-signed CA certificate, we reassign the - // config.Net.TLS.Config using the relevant credential files. - if o.Credential != nil && o.Credential.IsTLSEnabled() { - config.Net.TLS.Config, err = o.Credential.ToTLSConfig() - if err != nil { - return nil, errors.Trace(err) - } - } - - config.Net.TLS.Config.InsecureSkipVerify = o.InsecureSkipVerify - } - - err = completeSaramaSASLConfig(ctx, config, o) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - - kafkaVersion, err := getKafkaVersion(config, o) - if err != nil { - log.Warn("Can't get Kafka version by broker. ticdc will use default version", - zap.String("defaultVersion", kafkaVersion.String())) - } - config.Version = kafkaVersion - - if o.IsAssignedVersion { - version, err := sarama.ParseKafkaVersion(o.Version) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidVersion, err) - } - config.Version = version - if !version.IsAtLeast(maxKafkaVersion) && version.String() != kafkaVersion.String() { - log.Warn("The Kafka version you assigned may not be correct. "+ - "Please assign a version equal to or less than the specified version", - zap.String("assignedVersion", version.String()), - zap.String("desiredVersion", kafkaVersion.String())) - } - } - return config, nil -} - -func completeSaramaSASLConfig(ctx context.Context, config *sarama.Config, o *options) error { - if o.SASL != nil && o.SASL.SASLMechanism != "" { - config.Net.SASL.Enable = true - config.Net.SASL.Mechanism = sarama.SASLMechanism(o.SASL.SASLMechanism) - switch o.SASL.SASLMechanism { - case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512, SASLTypePlaintext: - config.Net.SASL.User = o.SASL.SASLUser - config.Net.SASL.Password = o.SASL.SASLPassword - if strings.EqualFold(string(o.SASL.SASLMechanism), SASLTypeSCRAMSHA256) { - config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { - return &security.XDGSCRAMClient{HashGeneratorFcn: security.SHA256} - } - } else if strings.EqualFold(string(o.SASL.SASLMechanism), SASLTypeSCRAMSHA512) { - config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { - return &security.XDGSCRAMClient{HashGeneratorFcn: security.SHA512} - } - } - case SASLTypeGSSAPI: - config.Net.SASL.GSSAPI.AuthType = int(o.SASL.GSSAPI.AuthType) - config.Net.SASL.GSSAPI.Username = o.SASL.GSSAPI.Username - config.Net.SASL.GSSAPI.ServiceName = o.SASL.GSSAPI.ServiceName - config.Net.SASL.GSSAPI.KerberosConfigPath = o.SASL.GSSAPI.KerberosConfigPath - config.Net.SASL.GSSAPI.Realm = o.SASL.GSSAPI.Realm - config.Net.SASL.GSSAPI.DisablePAFXFAST = o.SASL.GSSAPI.DisablePAFXFAST - switch o.SASL.GSSAPI.AuthType { - case security.UserAuth: - config.Net.SASL.GSSAPI.Password = o.SASL.GSSAPI.Password - case security.KeyTabAuth: - config.Net.SASL.GSSAPI.KeyTabPath = o.SASL.GSSAPI.KeyTabPath - } - - case SASLTypeOAuth: - p, err := newTokenProvider(ctx, o) - if err != nil { - return errors.Trace(err) - } - config.Net.SASL.TokenProvider = p - } - } - - return nil -} - -func getKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, error) { - addrs := o.BrokerEndpoints - if len(addrs) > 1 { - // Shuffle the list of addresses to randomize the order in which - // connections are attempted. This prevents routing all connections - // to the first broker (which will usually succeed). - rand.Shuffle(len(addrs), func(i, j int) { - addrs[i], addrs[j] = addrs[j], addrs[i] - }) - } - - var ( - err error - targetVersion sarama.KafkaVersion - ) - for i := range addrs { - targetVersion, err = getKafkaVersionFromBroker(config, o.RequestVersion, addrs[i]) - if err == nil { - break - } - } - if err != nil { - log.Warn("kafka sink use the default kafka version since cannot find it from the brokers", - zap.String("defaultVersion", defaultKafkaVersion.String())) - targetVersion = defaultKafkaVersion - } - - if o.IsAssignedVersion { - assignedVersion, err := sarama.ParseKafkaVersion(o.Version) - if err != nil { - return assignedVersion, errors.WrapError(errors.ErrKafkaInvalidVersion, err) - } - if !assignedVersion.IsAtLeast(maxKafkaVersion) && assignedVersion.String() != targetVersion.String() { - log.Warn("The Kafka version you assigned may not be correct. "+ - "Please assign a version equal to or less than the specified version", - zap.String("assignedVersion", assignedVersion.String()), - zap.String("desiredVersion", targetVersion.String())) - } - targetVersion = assignedVersion - } - return targetVersion, nil -} - -func getKafkaVersionFromBroker(config *sarama.Config, requestVersion int16, addr string) (sarama.KafkaVersion, error) { - KafkaVersion := defaultKafkaVersion - broker := sarama.NewBroker(addr) - err := broker.Open(config) - defer func() { - _ = broker.Close() - }() - if err != nil { - log.Warn("Kafka fail to open broker", zap.String("addr", addr), zap.Error(err)) - return KafkaVersion, err - } - apiResponse, err := broker.ApiVersions(&sarama.ApiVersionsRequest{Version: requestVersion}) - if err != nil { - log.Warn("Kafka fail to get ApiVersions", zap.String("addr", addr), zap.Error(err)) - return KafkaVersion, err - } - // ApiKey method - // 0 Produce - // 3 Metadata (default) - version := apiResponse.ApiKeys[3].MaxVersion - if version >= 10 { - KafkaVersion = sarama.V2_8_0_0 - } else if version >= 9 { - KafkaVersion = sarama.V2_4_0_0 - } else if version >= 8 { - KafkaVersion = sarama.V2_3_0_0 - } else if version >= 7 { - KafkaVersion = sarama.V2_1_0_0 - } else if version >= 6 { - KafkaVersion = sarama.V2_0_0_0 - } else if version >= 5 { - KafkaVersion = sarama.V1_0_0_0 - } else if version >= 3 { - KafkaVersion = sarama.V0_11_0_0 - } else if version >= 2 { - KafkaVersion = sarama.V0_10_1_0 - } else if version >= 1 { - KafkaVersion = sarama.V0_10_0_0 - } else if version >= 0 { - KafkaVersion = sarama.V0_8_2_0 - } - return KafkaVersion, nil -} diff --git a/pkg/sink/kafka/sarama_config_test.go b/pkg/sink/kafka/sarama_config_test.go deleted file mode 100644 index d12595bfe9..0000000000 --- a/pkg/sink/kafka/sarama_config_test.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "net/http" - "net/url" - "testing" - - "github.com/IBM/sarama" - "github.com/gin-gonic/gin/binding" - "github.com/pingcap/errors" - commonType "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/security" - "github.com/stretchr/testify/require" -) - -func TestNewSaramaConfig(t *testing.T) { - options := NewOptions() - options.Version = "invalid" - options.IsAssignedVersion = true - ctx := context.Background() - _, err := newSaramaConfig(ctx, options) - require.Regexp(t, "invalid version.*", errors.Cause(err)) - options.Version = "2.6.0" - - options.ClientID = "test-kafka-client" - compressionCases := []struct { - algorithm string - expected sarama.CompressionCodec - }{ - {"none", sarama.CompressionNone}, - {"gzip", sarama.CompressionGZIP}, - {"snappy", sarama.CompressionSnappy}, - {"lz4", sarama.CompressionLZ4}, - {"zstd", sarama.CompressionZSTD}, - {"others", sarama.CompressionNone}, - } - for _, cc := range compressionCases { - options.Compression = cc.algorithm - cfg, err := newSaramaConfig(ctx, options) - require.NoError(t, err) - require.Equal(t, cc.expected, cfg.Producer.Compression) - } - cfg, err := newSaramaConfig(ctx, options) - require.NoError(t, err) - require.Equal(t, defaultMaxRetry, cfg.Producer.Retry.Max) - - options.EnableTLS = true - options.Credential = &security.Credential{ - CAPath: "/invalid/ca/path", - CertPath: "/invalid/cert/path", - KeyPath: "/invalid/key/path", - } - _, err = newSaramaConfig(ctx, options) - require.Regexp(t, ".*no such file or directory", errors.Cause(err)) - - saslOptions := NewOptions() - saslOptions.Version = "2.6.0" - saslOptions.ClientID = "test-sasl-scram" - saslOptions.SASL = &security.SASL{ - SASLUser: "user", - SASLPassword: "password", - SASLMechanism: sarama.SASLTypeSCRAMSHA256, - } - - cfg, err = newSaramaConfig(ctx, saslOptions) - require.NoError(t, err) - require.NotNil(t, cfg) - require.Equal(t, "user", cfg.Net.SASL.User) - require.Equal(t, "password", cfg.Net.SASL.Password) - require.Equal(t, sarama.SASLMechanism("SCRAM-SHA-256"), cfg.Net.SASL.Mechanism) -} - -func TestNewSaramaConfigMaxRetryFromSinkURI(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - sinkURI string - expected int - }{ - { - name: "default max retry", - sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&kafka-client-id=unit-test", - expected: defaultMaxRetry, - }, - { - name: "set max retry", - sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0" + - "&kafka-client-id=unit-test&max-retry=7", - expected: 7, - }, - { - name: "zero max retry", - sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0" + - "&kafka-client-id=unit-test&max-retry=0", - expected: 0, - }, - { - name: "negative max retry", - sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0" + - "&kafka-client-id=unit-test&max-retry=-1", - expected: defaultMaxRetry, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - options := NewOptions() - sinkURI, err := url.Parse(test.sinkURI) - require.NoError(t, err) - err = options.Apply( - commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), - sinkURI, - config.GetDefaultReplicaConfig().Sink, - ) - require.NoError(t, err) - - cfg, err := newSaramaConfig(context.Background(), options) - require.NoError(t, err) - require.Equal(t, test.expected, cfg.Producer.Retry.Max) - }) - } -} - -func TestApplySASL(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - URI string - replicaConfig func() *config.ReplicaConfig - exceptErr string - }{ - { - name: "no params", - URI: "kafka://127.0.0.1:9092/abc", - replicaConfig: config.GetDefaultReplicaConfig, - exceptErr: "", - }, - { - name: "valid PLAIN SASL", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-user=user&sasl-password=password&sasl-mechanism=plain", - replicaConfig: config.GetDefaultReplicaConfig, - exceptErr: "", - }, - { - name: "valid SCRAM SASL", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-user=user&sasl-password=password&sasl-mechanism=SCRAM-SHA-512", - replicaConfig: config.GetDefaultReplicaConfig, - exceptErr: "", - }, - { - name: "valid GSSAPI user auth SASL", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-mechanism=GSSAPI&sasl-gssapi-auth-type=USER" + - "&sasl-gssapi-kerberos-config-path=/root/config" + - "&sasl-gssapi-service-name=a&sasl-gssapi-user=user" + - "&sasl-gssapi-password=pwd" + - "&sasl-gssapi-realm=realm&sasl-gssapi-disable-pafxfast=false", - replicaConfig: config.GetDefaultReplicaConfig, - exceptErr: "", - }, - { - name: "valid GSSAPI keytab auth SASL", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-mechanism=GSSAPI&sasl-gssapi-auth-type=keytab" + - "&sasl-gssapi-kerberos-config-path=/root/config" + - "&sasl-gssapi-service-name=a&sasl-gssapi-user=user" + - "&sasl-gssapi-keytab-path=/root/keytab" + - "&sasl-gssapi-realm=realm&sasl-gssapi-disable-pafxfast=false", - replicaConfig: config.GetDefaultReplicaConfig, - exceptErr: "", - }, - { - name: "invalid mechanism", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-mechanism=a", - replicaConfig: config.GetDefaultReplicaConfig, - exceptErr: "unknown a SASL mechanism", - }, - { - name: "invalid GSSAPI auth type", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-mechanism=gssapi&sasl-gssapi-auth-type=keyta1b", - replicaConfig: config.GetDefaultReplicaConfig, - exceptErr: "unknown keyta1b auth type", - }, - { - name: "valid OAUTHBEARER SASL", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0&sasl-mechanism=OAUTHBEARER", - replicaConfig: func() *config.ReplicaConfig { - cfg := config.GetDefaultReplicaConfig() - oauthMechanism := string(security.OAuthMechanism) - clientID := "client_id" - clientSecret := "Y2xpZW50X3NlY3JldA==" // base64(client_secret) - tokenURL := "127.0.0.1:9093/token" - cfg.Sink.KafkaConfig = &config.KafkaConfig{ - SASLMechanism: &oauthMechanism, - SASLOAuthClientID: &clientID, - SASLOAuthClientSecret: &clientSecret, - SASLOAuthTokenURL: &tokenURL, - } - return cfg - }, - exceptErr: "", - }, - { - name: "invalid OAUTHBEARER SASL: missing client id", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0&sasl-mechanism=OAUTHBEARER", - replicaConfig: func() *config.ReplicaConfig { - cfg := config.GetDefaultReplicaConfig() - oauthMechanism := string(security.OAuthMechanism) - clientSecret := "Y2xpZW50X3NlY3JldA==" // base64(client_secret) - tokenURL := "127.0.0.1:9093/token" - cfg.Sink.KafkaConfig = &config.KafkaConfig{ - SASLMechanism: &oauthMechanism, - SASLOAuthClientSecret: &clientSecret, - SASLOAuthTokenURL: &tokenURL, - } - return cfg - }, - exceptErr: "OAuth2 client id is empty", - }, - { - name: "invalid OAUTHBEARER SASL: missing client secret", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0&sasl-mechanism=OAUTHBEARER", - replicaConfig: func() *config.ReplicaConfig { - cfg := config.GetDefaultReplicaConfig() - oauthMechanism := string(security.OAuthMechanism) - clientID := "client_id" - tokenURL := "127.0.0.1:9093/token" - cfg.Sink.KafkaConfig = &config.KafkaConfig{ - SASLMechanism: &oauthMechanism, - SASLOAuthClientID: &clientID, - SASLOAuthTokenURL: &tokenURL, - } - return cfg - }, - exceptErr: "OAuth2 client secret is empty", - }, - { - name: "invalid OAUTHBEARER SASL: missing token url", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0&sasl-mechanism=OAUTHBEARER", - replicaConfig: func() *config.ReplicaConfig { - cfg := config.GetDefaultReplicaConfig() - oauthMechanism := string(security.OAuthMechanism) - clientID := "client_id" - clientSecret := "Y2xpZW50X3NlY3JldA==" // base64(client_secret) - cfg.Sink.KafkaConfig = &config.KafkaConfig{ - SASLMechanism: &oauthMechanism, - SASLOAuthClientID: &clientID, - SASLOAuthClientSecret: &clientSecret, - } - return cfg - }, - exceptErr: "OAuth2 token url is empty", - }, - { - name: "invalid OAUTHBEARER SASL: non base64 client secret", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0&sasl-mechanism=OAUTHBEARER", - replicaConfig: func() *config.ReplicaConfig { - cfg := config.GetDefaultReplicaConfig() - oauthMechanism := string(security.OAuthMechanism) - clientID := "client_id" - clientSecret := "client_secret" - tokenURL := "127.0.0.1:9093/token" - cfg.Sink.KafkaConfig = &config.KafkaConfig{ - SASLMechanism: &oauthMechanism, - SASLOAuthClientID: &clientID, - SASLOAuthClientSecret: &clientSecret, - SASLOAuthTokenURL: &tokenURL, - } - return cfg - }, - exceptErr: "OAuth2 client secret is not base64 encoded", - }, - { - name: "invalid OAUTHBEARER SASL: wrong mechanism", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0&sasl-mechanism=GSSAPI", - replicaConfig: func() *config.ReplicaConfig { - cfg := config.GetDefaultReplicaConfig() - oauthMechanism := string(security.OAuthMechanism) - clientID := "client_id" - clientSecret := "Y2xpZW50X3NlY3JldA==" // base64(client_secret) - tokenURL := "127.0.0.1:9093/token" - cfg.Sink.KafkaConfig = &config.KafkaConfig{ - SASLMechanism: &oauthMechanism, - SASLOAuthClientID: &clientID, - SASLOAuthClientSecret: &clientSecret, - SASLOAuthTokenURL: &tokenURL, - } - return cfg - }, - exceptErr: "OAuth2 is only supported with SASL mechanism type OAUTHBEARER", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - options := NewOptions() - sinkURI, err := url.Parse(test.URI) - require.NoError(t, err) - req := &http.Request{URL: sinkURI} - urlParameter := &urlConfig{} - err = binding.Query.Bind(req, urlParameter) - require.NoError(t, err) - if test.exceptErr == "" { - require.Nil(t, options.applySASL(urlParameter, test.replicaConfig().Sink)) - } else { - require.Regexp(t, test.exceptErr, - options.applySASL(urlParameter, test.replicaConfig().Sink).Error()) - } - }) - } -} - -func TestApplyTLS(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - URI string - tlsEnabled bool - exceptErr string - }{ - { - name: "tls config with 'enable-tls' set to true", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-user=user&sasl-password=password&sasl-mechanism=plain&enable-tls=true", - tlsEnabled: true, - exceptErr: "", - }, - { - name: "tls config with no 'enable-tls', and credential files are supplied", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-user=user&sasl-password=password&sasl-mechanism=plain" + - "&ca=/root/ca.file&cert=/root/cert.file&key=/root/key.file", - tlsEnabled: true, - exceptErr: "", - }, - { - name: "tls config with no 'enable-tls', and credential files are not supplied", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-user=user&sasl-password=password&sasl-mechanism=plain", - tlsEnabled: false, - exceptErr: "", - }, - { - name: "tls config with 'enable-tls' set to false, and credential files are supplied", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-user=user&sasl-password=password&sasl-mechanism=plain&enable-tls=false" + - "&ca=/root/ca&cert=/root/cert&key=/root/key", - tlsEnabled: false, - exceptErr: "credential files are supplied, but 'enable-tls' is set to false", - }, - { - name: "tls config with 'enable-tls' set to true, and some of " + - "the credential files are not supplied ", - URI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" + - "&sasl-user=user&sasl-password=password&sasl-mechanism=plain&enable-tls=true" + - "&ca=/root/ca&cert=/root/cert&", - tlsEnabled: false, - exceptErr: "ca, cert and key files should all be supplied", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - options := NewOptions() - sinkURI, err := url.Parse(test.URI) - require.NoError(t, err) - req := &http.Request{URL: sinkURI} - urlParameter := &urlConfig{} - err = binding.Query.Bind(req, urlParameter) - require.NoError(t, err) - if test.exceptErr == "" { - require.Nil(t, options.applyTLS(urlParameter)) - } else { - require.Regexp(t, test.exceptErr, options.applyTLS(urlParameter).Error()) - } - require.Equal(t, test.tlsEnabled, options.EnableTLS) - }) - } -} - -func TestCompleteSaramaSASLConfig(t *testing.T) { - t.Parallel() - - // Test that SASL is turned on correctly. - options := NewOptions() - options.SASL = &security.SASL{ - SASLUser: "user", - SASLPassword: "password", - SASLMechanism: "", - GSSAPI: security.GSSAPI{}, - } - ctx := context.Background() - saramaConfig := sarama.NewConfig() - completeSaramaSASLConfig(ctx, saramaConfig, options) - require.False(t, saramaConfig.Net.SASL.Enable) - options.SASL.SASLMechanism = "plain" - completeSaramaSASLConfig(ctx, saramaConfig, options) - require.True(t, saramaConfig.Net.SASL.Enable) - // Test that the SCRAMClientGeneratorFunc is set up correctly. - options = NewOptions() - options.SASL = &security.SASL{ - SASLUser: "user", - SASLPassword: "password", - SASLMechanism: "plain", - GSSAPI: security.GSSAPI{}, - } - saramaConfig = sarama.NewConfig() - completeSaramaSASLConfig(ctx, saramaConfig, options) - require.Nil(t, saramaConfig.Net.SASL.SCRAMClientGeneratorFunc) - options.SASL.SASLMechanism = "SCRAM-SHA-512" - completeSaramaSASLConfig(ctx, saramaConfig, options) - require.NotNil(t, saramaConfig.Net.SASL.SCRAMClientGeneratorFunc) -} - -func TestSaramaTimeout(t *testing.T) { - options := NewOptions() - saramaConfig, err := newSaramaConfig(context.Background(), options) - require.NoError(t, err) - require.Equal(t, options.DialTimeout, saramaConfig.Net.DialTimeout) - require.Equal(t, options.WriteTimeout, saramaConfig.Net.WriteTimeout) - require.Equal(t, options.ReadTimeout, saramaConfig.Net.ReadTimeout) -} diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go deleted file mode 100644 index 650f346b4c..0000000000 --- a/pkg/sink/kafka/sarama_factory.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "time" - - "github.com/IBM/sarama" - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/rcrowley/go-metrics" - "go.uber.org/atomic" - "go.uber.org/zap" -) - -type saramaFactory struct { - changefeedID common.ChangeFeedID - option *options - metricRegistry metrics.Registry -} - -// NewSaramaFactory constructs a Factory with sarama implementation. -func NewSaramaFactory( - ctx context.Context, - o *options, - changefeedID common.ChangeFeedID, -) (Factory, error) { - start := time.Now() - config, err := newSaramaConfig(ctx, o) - duration := time.Since(start).Seconds() - if duration > 2 { - log.Warn("new sarama config cost too much time", - zap.Stringer("changefeedID", changefeedID), zap.Any("duration", duration)) - } - if err != nil { - return nil, errors.Trace(err) - } - - admin, err := newAdminClient(changefeedID, o.BrokerEndpoints, config) - if err != nil { - return nil, errors.Trace(err) - } - defer func() { - admin.Close() - }() - - if err = adjustOptions(ctx, admin, o, o.Topic); err != nil { - return nil, errors.Trace(err) - } - - return &saramaFactory{ - changefeedID: changefeedID, - option: o, - metricRegistry: metrics.NewRegistry(), - }, nil -} - -func newAdminClient(changefeedID common.ChangeFeedID, endpoints []string, config *sarama.Config) (ClusterAdminClient, error) { - start := time.Now() - client, err := sarama.NewClient(endpoints, config) - duration := time.Since(start).Seconds() - if duration > 2 { - log.Warn("new sarama client cost too much time", - zap.Any("duration", duration), zap.Stringer("changefeedID", changefeedID)) - } - if err != nil { - return nil, errors.Trace(err) - } - - start = time.Now() - admin, err := sarama.NewClusterAdminFromClient(client) - duration = time.Since(start).Seconds() - if duration > 2 { - log.Warn("new sarama cluster admin cost too much time", - zap.Any("duration", duration), zap.Stringer("changefeedID", changefeedID)) - } - if err != nil { - // `sarama.NewClusterAdminFromClient` does not take ownership of the client, - // so we need to close it on failures to avoid leaking background goroutines. - _ = client.Close() - return nil, errors.Trace(err) - } - return &saramaAdminClient{ - client: client, - admin: admin, - changefeed: changefeedID, - }, nil -} - -func (f *saramaFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - config, err := newSaramaConfig(ctx, f.option) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - return newAdminClient(f.changefeedID, f.option.BrokerEndpoints, config) -} - -// SyncProducer returns a Sync SyncProducer, -// it should be the caller's responsibility to close the producer -func (f *saramaFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - config, err := newSaramaConfig(ctx, f.option) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - config.MetricRegistry = f.metricRegistry - - client, err := sarama.NewClient(f.option.BrokerEndpoints, config) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - - p, err := sarama.NewSyncProducerFromClient(client) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - - return &saramaSyncProducer{ - id: f.changefeedID, - client: client, - producer: p, - closed: atomic.NewBool(false), - }, nil -} - -// AsyncProducer return an Async SyncProducer, -// it should be the caller's responsibility to close the producer -func (f *saramaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - config, err := newSaramaConfig(ctx, f.option) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - config.MetricRegistry = f.metricRegistry - - client, err := sarama.NewClient(f.option.BrokerEndpoints, config) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - - p, err := sarama.NewAsyncProducerFromClient(client) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - return &saramaAsyncProducer{ - client: client, - producer: p, - changefeedID: f.changefeedID, - closed: atomic.NewBool(false), - failpointCh: make(chan *sarama.ProducerError, 1), - }, nil -} - -func (f *saramaFactory) MetricsCollector( - adminClient ClusterAdminClient, -) MetricsCollector { - return &saramaMetricsCollector{ - changefeedID: f.changefeedID, - adminClient: adminClient, - brokers: make(map[int32]struct{}), - registry: f.metricRegistry, - } -} diff --git a/pkg/sink/kafka/sarama_sync_producer.go b/pkg/sink/kafka/sarama_sync_producer.go deleted file mode 100644 index 9d5efdfb0b..0000000000 --- a/pkg/sink/kafka/sarama_sync_producer.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "time" - - "github.com/IBM/sarama" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - commonType "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/codec/common" - "go.uber.org/atomic" - "go.uber.org/zap" -) - -type saramaSyncClient interface { - Brokers() []*sarama.Broker - Close() error -} - -type saramaSyncProducerClient interface { - SendMessage(msg *sarama.ProducerMessage) (partition int32, offset int64, err error) - SendMessages(msgs []*sarama.ProducerMessage) error - Close() error -} - -type saramaSyncProducer struct { - id commonType.ChangeFeedID - client saramaSyncClient - producer saramaSyncProducerClient - closed *atomic.Bool -} - -func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { - if p.closed.Load() { - return errors.ErrKafkaProducerClosed.GenWithStackByArgs() - } - - msg := &sarama.ProducerMessage{ - Topic: topic, - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), - Partition: partitionNum, - } - _, _, err := p.producer.SendMessage(msg) - - failpoint.Inject("KafkaSinkSyncSendMessageError", func() { - err = errors.WrapError(errors.ErrKafkaSendMessage, errors.New("kafka sink sync send message injected error")) - }) - if err != nil { - err = AnnotateEventError( - p.id.Keyspace(), - p.id.Name(), - message.LogInfo, - err, - ) - } - return errors.WrapError(errors.ErrKafkaSendMessage, err) -} - -func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { - if p.closed.Load() { - return errors.ErrKafkaProducerClosed.GenWithStackByArgs() - } - - msgs := make([]*sarama.ProducerMessage, partitionNum) - for i := 0; i < int(partitionNum); i++ { - msgs[i] = &sarama.ProducerMessage{ - Topic: topic, - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), - Partition: int32(i), - } - } - err := p.producer.SendMessages(msgs) - - failpoint.Inject("KafkaSinkSyncSendMessagesError", func() { - err = errors.WrapError(errors.ErrKafkaSendMessage, errors.New("kafka sink sync send messages injected error")) - }) - if err != nil { - err = AnnotateEventError( - p.id.Keyspace(), - p.id.Name(), - message.LogInfo, - err, - ) - } - return errors.WrapError(errors.ErrKafkaSendMessage, err) -} - -func (p *saramaSyncProducer) Close() { - if p.closed.Load() { - log.Warn("kafka DDL producer already closed", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name())) - return - } - - p.closed.Store(true) - start := time.Now() - // sarama.NewSyncProducerFromClient wraps the provided client with a nopCloserClient, - // so producer.Close() alone won't release the underlying client resources. - if p.client != nil { - if err := p.client.Close(); err != nil { - log.Warn("Close Kafka DDL producer client with error", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name()), - zap.Duration("duration", time.Since(start)), - zap.Error(err)) - } - } - if p.producer != nil { - if err := p.producer.Close(); err != nil { - log.Error("Close Kafka DDL producer with error", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name()), - zap.Duration("duration", time.Since(start)), - zap.Error(err)) - return - } - } - log.Info("Kafka DDL producer closed", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name()), - zap.Duration("duration", time.Since(start))) -} diff --git a/pkg/sink/kafka/sarama_sync_producer_mock.go b/pkg/sink/kafka/sarama_sync_producer_mock.go deleted file mode 100644 index 78671e02f2..0000000000 --- a/pkg/sink/kafka/sarama_sync_producer_mock.go +++ /dev/null @@ -1,130 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/sarama_sync_producer.go - -// Package kafka is a generated GoMock package. -package kafka - -import ( - reflect "reflect" - - sarama "github.com/IBM/sarama" - gomock "github.com/golang/mock/gomock" -) - -// MocksaramaSyncClient is a mock of saramaSyncClient interface. -type MocksaramaSyncClient struct { - ctrl *gomock.Controller - recorder *MocksaramaSyncClientMockRecorder -} - -// MocksaramaSyncClientMockRecorder is the mock recorder for MocksaramaSyncClient. -type MocksaramaSyncClientMockRecorder struct { - mock *MocksaramaSyncClient -} - -// NewMocksaramaSyncClient creates a new mock instance. -func NewMocksaramaSyncClient(ctrl *gomock.Controller) *MocksaramaSyncClient { - mock := &MocksaramaSyncClient{ctrl: ctrl} - mock.recorder = &MocksaramaSyncClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MocksaramaSyncClient) EXPECT() *MocksaramaSyncClientMockRecorder { - return m.recorder -} - -// Brokers mocks base method. -func (m *MocksaramaSyncClient) Brokers() []*sarama.Broker { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Brokers") - ret0, _ := ret[0].([]*sarama.Broker) - return ret0 -} - -// Brokers indicates an expected call of Brokers. -func (mr *MocksaramaSyncClientMockRecorder) Brokers() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Brokers", reflect.TypeOf((*MocksaramaSyncClient)(nil).Brokers)) -} - -// Close mocks base method. -func (m *MocksaramaSyncClient) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MocksaramaSyncClientMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaSyncClient)(nil).Close)) -} - -// MocksaramaSyncProducerClient is a mock of saramaSyncProducerClient interface. -type MocksaramaSyncProducerClient struct { - ctrl *gomock.Controller - recorder *MocksaramaSyncProducerClientMockRecorder -} - -// MocksaramaSyncProducerClientMockRecorder is the mock recorder for MocksaramaSyncProducerClient. -type MocksaramaSyncProducerClientMockRecorder struct { - mock *MocksaramaSyncProducerClient -} - -// NewMocksaramaSyncProducerClient creates a new mock instance. -func NewMocksaramaSyncProducerClient(ctrl *gomock.Controller) *MocksaramaSyncProducerClient { - mock := &MocksaramaSyncProducerClient{ctrl: ctrl} - mock.recorder = &MocksaramaSyncProducerClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MocksaramaSyncProducerClient) EXPECT() *MocksaramaSyncProducerClientMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MocksaramaSyncProducerClient) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MocksaramaSyncProducerClientMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaSyncProducerClient)(nil).Close)) -} - -// SendMessage mocks base method. -func (m *MocksaramaSyncProducerClient) SendMessage(msg *sarama.ProducerMessage) (int32, int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessage", msg) - ret0, _ := ret[0].(int32) - ret1, _ := ret[1].(int64) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// SendMessage indicates an expected call of SendMessage. -func (mr *MocksaramaSyncProducerClientMockRecorder) SendMessage(msg interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MocksaramaSyncProducerClient)(nil).SendMessage), msg) -} - -// SendMessages mocks base method. -func (m *MocksaramaSyncProducerClient) SendMessages(msgs []*sarama.ProducerMessage) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessages", msgs) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMessages indicates an expected call of SendMessages. -func (mr *MocksaramaSyncProducerClientMockRecorder) SendMessages(msgs interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MocksaramaSyncProducerClient)(nil).SendMessages), msgs) -} diff --git a/pkg/sink/kafka/sarama_sync_producer_test.go b/pkg/sink/kafka/sarama_sync_producer_test.go deleted file mode 100644 index 37285419c6..0000000000 --- a/pkg/sink/kafka/sarama_sync_producer_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "errors" - "testing" - - "github.com/golang/mock/gomock" - "github.com/pingcap/ticdc/pkg/common" - "go.uber.org/atomic" -) - -func TestSyncProducerClose(t *testing.T) { - tests := []struct { - name string - clientCloseErr error - }{ - { - name: "closes client and producer", - }, - { - name: "still closes producer when client close fails", - clientCloseErr: errors.New("boom"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - client := NewMocksaramaSyncClient(ctrl) - producer := NewMocksaramaSyncProducerClient(ctrl) - gomock.InOrder( - client.EXPECT().Close().Return(test.clientCloseErr), - producer.EXPECT().Close().Return(nil), - ) - - p := &saramaSyncProducer{ - id: common.NewChangeFeedIDWithName("test", "default"), - client: client, - producer: producer, - closed: atomic.NewBool(false), - } - - p.Close() - }) - } -} diff --git a/scripts/generate-mock.sh b/scripts/generate-mock.sh index c825e85946..bf10fdeaa0 100755 --- a/scripts/generate-mock.sh +++ b/scripts/generate-mock.sh @@ -37,8 +37,6 @@ fi "$MOCKGEN" -source pkg/sink/kafka/cluster_admin_client.go -destination pkg/sink/kafka/cluster_admin_client_mock.go -package kafka "$MOCKGEN" -source pkg/sink/kafka/factory.go -destination pkg/sink/kafka/factory_mock.go -package kafka "$MOCKGEN" -source pkg/sink/kafka/metrics_collector.go -destination pkg/sink/kafka/metrics_collector_mock.go -package kafka -"$MOCKGEN" -source pkg/sink/kafka/admin.go -destination pkg/sink/kafka/admin_mock.go -package kafka -"$MOCKGEN" -source pkg/sink/kafka/sarama_sync_producer.go -destination pkg/sink/kafka/sarama_sync_producer_mock.go -package kafka "$MOCKGEN" -source pkg/keyspace/keyspace_manager.go -destination pkg/keyspace/keyspace_manager_mock.go -package keyspace "$MOCKGEN" -source pkg/txnutil/gc/gc_manager.go -destination pkg/txnutil/gc/gc_manager_mock.go -package gc "$MOCKGEN" -source pkg/txnutil/gc/gc_client.go -destination pkg/txnutil/gc/gc_client_mock.go -package gc From d238110a5c640da9aacff1d87a7d513c59f9377a Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 26 Jun 2026 13:23:45 +0800 Subject: [PATCH 13/61] remove keyworkd franz and prometheus --- pkg/sink/kafka/{franz => }/admin_client.go | 51 +++++---- .../kafka/{franz => }/admin_client_test.go | 17 ++- pkg/sink/kafka/{franz => }/admin_metrics.go | 10 +- pkg/sink/kafka/{franz => }/async_producer.go | 64 +++++++---- pkg/sink/kafka/async_producer_test.go | 88 +++++++++++++++ .../{franz/factory.go => client_options.go} | 24 ++-- ...ory_api_test.go => client_options_test.go} | 8 +- pkg/sink/kafka/factory_selector.go | 2 +- pkg/sink/kafka/failpoint.go | 45 ++++++++ pkg/sink/kafka/failpoint_test.go | 41 +++++++ pkg/sink/kafka/franz_admin_client.go | 89 --------------- pkg/sink/kafka/franz_admin_client_test.go | 28 ----- pkg/sink/kafka/{franz => }/gssapi.go | 4 +- .../{franz_factory.go => kafka_factory.go} | 83 +++++++------- ..._factory_test.go => kafka_factory_test.go} | 16 +-- pkg/sink/kafka/metrics.go | 45 ++++---- pkg/sink/kafka/{franz => }/metrics_hook.go | 68 ++++++------ pkg/sink/kafka/metrics_hook_test.go | 73 ++++++++++++ pkg/sink/kafka/{franz => }/sasl_test.go | 14 +-- pkg/sink/kafka/{franz => }/sync_producer.go | 34 +++--- pkg/sink/kafka/sync_producer_test.go | 104 ++++++++++++++++++ 21 files changed, 590 insertions(+), 318 deletions(-) rename pkg/sink/kafka/{franz => }/admin_client.go (86%) rename pkg/sink/kafka/{franz => }/admin_client_test.go (70%) rename pkg/sink/kafka/{franz => }/admin_metrics.go (90%) rename pkg/sink/kafka/{franz => }/async_producer.go (73%) create mode 100644 pkg/sink/kafka/async_producer_test.go rename pkg/sink/kafka/{franz/factory.go => client_options.go} (92%) rename pkg/sink/kafka/{franz/factory_api_test.go => client_options_test.go} (92%) create mode 100644 pkg/sink/kafka/failpoint.go create mode 100644 pkg/sink/kafka/failpoint_test.go delete mode 100644 pkg/sink/kafka/franz_admin_client.go delete mode 100644 pkg/sink/kafka/franz_admin_client_test.go rename pkg/sink/kafka/{franz => }/gssapi.go (98%) rename pkg/sink/kafka/{franz_factory.go => kafka_factory.go} (53%) rename pkg/sink/kafka/{franz_factory_test.go => kafka_factory_test.go} (74%) rename pkg/sink/kafka/{franz => }/metrics_hook.go (85%) create mode 100644 pkg/sink/kafka/metrics_hook_test.go rename pkg/sink/kafka/{franz => }/sasl_test.go (82%) rename pkg/sink/kafka/{franz => }/sync_producer.go (77%) create mode 100644 pkg/sink/kafka/sync_producer_test.go diff --git a/pkg/sink/kafka/franz/admin_client.go b/pkg/sink/kafka/admin_client.go similarity index 86% rename from pkg/sink/kafka/franz/admin_client.go rename to pkg/sink/kafka/admin_client.go index 06495d2fa4..51bedf262d 100644 --- a/pkg/sink/kafka/franz/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -27,14 +27,7 @@ import ( "go.uber.org/zap" ) -// TopicDetail represent a topic's detail information. -type TopicDetail struct { - Name string - NumPartitions int32 - ReplicationFactor int16 -} - -type AdminClient struct { +type kafkaAdminClient struct { changefeed common.ChangeFeedID client *kgo.Client @@ -42,14 +35,14 @@ type AdminClient struct { timeout time.Duration } -func NewAdminClient( +func newAdminClient( ctx context.Context, changefeedID common.ChangeFeedID, - o *Options, + o *clientOptions, hook kgo.Hook, -) (*AdminClient, error) { +) (*kafkaAdminClient, error) { if o == nil { - o = &Options{} + o = &clientOptions{} } opts, err := newOptions(ctx, o, hook) @@ -64,7 +57,7 @@ func NewAdminClient( timeout := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) - return &AdminClient{ + return &kafkaAdminClient{ changefeed: changefeedID, client: client, admin: kadm.NewClient(client), @@ -72,11 +65,11 @@ func NewAdminClient( }, nil } -func (a *AdminClient) newRequestContext() (context.Context, context.CancelFunc) { +func (a *kafkaAdminClient) newRequestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(a.client.Context(), a.timeout) } -func (a *AdminClient) GetAllBrokers() []int32 { +func (a *kafkaAdminClient) GetAllBrokers() []Broker { startTime := time.Now() ctx, cancel := a.newRequestContext() defer cancel() @@ -92,10 +85,15 @@ func (a *AdminClient) GetAllBrokers() []int32 { } observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetAllBrokers, nil, time.Since(startTime)) - return meta.Brokers.NodeIDs() + brokerIDs := meta.Brokers.NodeIDs() + brokers := make([]Broker, 0, len(brokerIDs)) + for _, brokerID := range brokerIDs { + brokers = append(brokers, Broker{ID: brokerID}) + } + return brokers } -func (a *AdminClient) GetBrokerConfig(configName string) (value string, err error) { +func (a *kafkaAdminClient) GetBrokerConfig(configName string) (value string, err error) { startTime := time.Now() defer func() { observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetBrokerConfig, err, time.Since(startTime)) @@ -140,7 +138,7 @@ func (a *AdminClient) GetBrokerConfig(configName string) (value string, err erro "cannot find the `%s` from the broker's configuration", configName) } -func (a *AdminClient) GetTopicConfig(topicName string, configName string) (value string, err error) { +func (a *kafkaAdminClient) GetTopicConfig(topicName string, configName string) (value string, err error) { startTime := time.Now() defer func() { observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicConfig, err, time.Since(startTime)) @@ -181,7 +179,7 @@ func (a *AdminClient) GetTopicConfig(topicName string, configName string) (value "cannot find the `%s` from the topic's configuration", configName) } -func (a *AdminClient) GetTopicsMeta( +func (a *kafkaAdminClient) GetTopicsMeta( topics []string, ignoreTopicError bool, ) (result map[string]TopicDetail, err error) { @@ -228,7 +226,7 @@ func (a *AdminClient) GetTopicsMeta( return result, nil } -func (a *AdminClient) GetTopicsPartitionsNum(topics []string) (result map[string]int32, err error) { +func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (result map[string]int32, err error) { startTime := time.Now() defer func() { observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicsPartitions, err, time.Since(startTime)) @@ -260,12 +258,17 @@ func (a *AdminClient) GetTopicsPartitionsNum(topics []string) (result map[string return result, nil } -func (a *AdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) (err error) { +func (a *kafkaAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) (err error) { startTime := time.Now() defer func() { observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodCreateTopic, err, time.Since(startTime)) }() + if detail == nil { + err = errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") + return err + } + ctx, cancel := a.newRequestContext() defer cancel() @@ -292,9 +295,9 @@ func (a *AdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) (err e return errors.Trace(resp.Err) } -func (a *AdminClient) Heartbeat() {} +func (a *kafkaAdminClient) Heartbeat() {} -func (a *AdminClient) Close() { +func (a *kafkaAdminClient) Close() { if a.admin != nil { a.admin.Close() } diff --git a/pkg/sink/kafka/franz/admin_client_test.go b/pkg/sink/kafka/admin_client_test.go similarity index 70% rename from pkg/sink/kafka/franz/admin_client_test.go rename to pkg/sink/kafka/admin_client_test.go index c35c014f4a..75fa99d8fd 100644 --- a/pkg/sink/kafka/franz/admin_client_test.go +++ b/pkg/sink/kafka/admin_client_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -24,10 +24,21 @@ import ( func TestNewAdminClientNilOptionsReturnsError(t *testing.T) { t.Parallel() - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz-admin-nil-options") - client, err := NewAdminClient(context.Background(), changefeedID, nil, nil) + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "kafka-admin-nil-options") + client, err := newAdminClient(context.Background(), changefeedID, nil, nil) if client != nil { client.Close() } require.Error(t, err) } + +func TestAdminClientCreateTopicNilDetailReturnsError(t *testing.T) { + t.Parallel() + + client := &kafkaAdminClient{} + + err := client.CreateTopic(nil, false) + + require.Error(t, err) + require.Contains(t, err.Error(), "topic detail must not be nil") +} diff --git a/pkg/sink/kafka/franz/admin_metrics.go b/pkg/sink/kafka/admin_metrics.go similarity index 90% rename from pkg/sink/kafka/franz/admin_metrics.go rename to pkg/sink/kafka/admin_metrics.go index baaee24729..5c869600c0 100644 --- a/pkg/sink/kafka/franz/admin_metrics.go +++ b/pkg/sink/kafka/admin_metrics.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "time" @@ -35,25 +35,25 @@ var ( prometheus.CounterOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_admin_call_total", + Name: "kafka_client_admin_call_total", Help: "Total kafka admin calls by method and result.", }, []string{"namespace", "changefeed", "method", "result"}) adminCallLatency = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_admin_call_duration_seconds", + Name: "kafka_client_admin_call_duration_seconds", Help: "Latency of kafka admin calls by method and result.", Buckets: prometheus.DefBuckets, }, []string{"namespace", "changefeed", "method", "result"}) ) -func InitAdminMetrics(registry *prometheus.Registry) { +func initAdminMetrics(registry *prometheus.Registry) { registry.MustRegister(adminCallCount) registry.MustRegister(adminCallLatency) } -func CleanupAdminMetrics(keyspace string, changefeed string) { +func cleanupAdminMetrics(keyspace string, changefeed string) { labels := prometheus.Labels{ "namespace": keyspace, "changefeed": changefeed, diff --git a/pkg/sink/kafka/franz/async_producer.go b/pkg/sink/kafka/async_producer.go similarity index 73% rename from pkg/sink/kafka/franz/async_producer.go rename to pkg/sink/kafka/async_producer.go index b8d5b0d9cb..69b148bf95 100644 --- a/pkg/sink/kafka/franz/async_producer.go +++ b/pkg/sink/kafka/async_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -28,7 +28,7 @@ import ( "go.uber.org/zap" ) -type AsyncProducer struct { +type kafkaAsyncProducer struct { client *kgo.Client changefeedID commonType.ChangeFeedID @@ -36,12 +36,12 @@ type AsyncProducer struct { errCh chan error } -func NewAsyncProducer( +func newAsyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, - o *Options, + o *clientOptions, hook kgo.Hook, -) (*AsyncProducer, error) { +) (*kafkaAsyncProducer, error) { opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -52,7 +52,7 @@ func NewAsyncProducer( return nil, errors.Trace(err) } - return &AsyncProducer{ + return &kafkaAsyncProducer{ client: client, changefeedID: changefeedID, closed: atomic.NewBool(false), @@ -60,7 +60,7 @@ func NewAsyncProducer( }, nil } -func (p *AsyncProducer) Close() { +func (p *kafkaAsyncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { return } @@ -75,7 +75,7 @@ func (p *AsyncProducer) Close() { }() } -func (p *AsyncProducer) AsyncSend( +func (p *kafkaAsyncProducer) AsyncSend( ctx context.Context, topic string, partition int32, @@ -96,19 +96,27 @@ func (p *AsyncProducer) AsyncSend( changefeed = p.changefeedID.Name() ) + if legacyKafkaSinkFailpointEnabled(kafkaSinkAsyncSendErrorFailpoint) { + log.Info("KafkaSinkAsyncSendError error injected", + zap.String("keyspace", keyspace), zap.String("changefeed", changefeed)) + p.enqueueAsyncSendError( + keyspace, + changefeed, + message.LogInfo, + errors.New("kafka sink injected error"), + ) + return nil + } + failpoint.Inject("KafkaSinkAsyncSendError", func() { log.Info("KafkaSinkAsyncSendError error injected", zap.String("keyspace", keyspace), zap.String("changefeed", changefeed)) - errWithInfo := logutil.AnnotateEventError( + p.enqueueAsyncSendError( keyspace, changefeed, message.LogInfo, errors.New("kafka sink injected error"), ) - select { - case p.errCh <- errors.WrapError(errors.ErrKafkaAsyncSendMessage, errWithInfo): - default: - } failpoint.Return(nil) }) @@ -123,16 +131,11 @@ func (p *AsyncProducer) AsyncSend( logInfo := message.LogInfo promise := func(_ *kgo.Record, err error) { if err != nil { - errWithInfo := logutil.AnnotateEventError( + p.enqueueAsyncSendError( keyspace, changefeed, logInfo, err, ) - select { - case p.errCh <- errors.WrapError(errors.ErrKafkaAsyncSendMessage, errWithInfo): - // todo: remove this default after support dispatcher recover logic. - default: - } return } if callback != nil { @@ -143,9 +146,28 @@ func (p *AsyncProducer) AsyncSend( return nil } -func (p *AsyncProducer) Heartbeat() {} +func (p *kafkaAsyncProducer) enqueueAsyncSendError( + keyspace string, + changefeed string, + logInfo *common.MessageLogInfo, + err error, +) { + errWithInfo := logutil.AnnotateEventError( + keyspace, + changefeed, + logInfo, + err, + ) + select { + case p.errCh <- errors.WrapError(errors.ErrKafkaAsyncSendMessage, errWithInfo): + // todo: remove this default after support dispatcher recover logic. + default: + } +} + +func (p *kafkaAsyncProducer) Heartbeat() {} -func (p *AsyncProducer) AsyncRunCallback(ctx context.Context) error { +func (p *kafkaAsyncProducer) AsyncRunCallback(ctx context.Context) error { defer p.closed.Store(true) for { select { diff --git a/pkg/sink/kafka/async_producer_test.go b/pkg/sink/kafka/async_producer_test.go new file mode 100644 index 0000000000..b58ec544c2 --- /dev/null +++ b/pkg/sink/kafka/async_producer_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + stderrors "errors" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + cerror "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" +) + +func TestAsyncSendClosedProducer(t *testing.T) { + producer := &kafkaAsyncProducer{closed: atomic.NewBool(true)} + + err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) + + require.ErrorIs(t, err, cerror.ErrKafkaProducerClosed) +} + +func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { + producer := &kafkaAsyncProducer{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), + closed: atomic.NewBool(false), + errCh: make(chan error, 1), + } + producer.errCh <- stderrors.New("queued async error") + + err := producer.AsyncRunCallback(context.Background()) + + require.ErrorContains(t, err, "queued async error") + require.True(t, producer.closed.Load()) +} + +func TestAsyncSendLegacyFailpointAnnotatesDMLContext(t *testing.T) { + enableLegacyKafkaSinkFailpointForTest(t, kafkaSinkAsyncSendErrorFailpoint) + + producer := &kafkaAsyncProducer{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-legacy-failpoint"), + closed: atomic.NewBool(false), + errCh: make(chan error, 1), + } + message := &codeccommon.Message{ + Key: []byte("key"), + Value: []byte("value"), + LogInfo: &codeccommon.MessageLogInfo{Rows: []codeccommon.RowLogInfo{ + { + Type: "insert", + Database: "db", + Table: "t", + StartTs: 1, + CommitTs: 2, + PrimaryKeys: []codeccommon.ColumnLogInfo{ + {Name: "id", Value: 1}, + }, + }, + }}, + } + + err := producer.AsyncSend(context.Background(), "topic", 0, message) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err = producer.AsyncRunCallback(ctx) + + require.ErrorContains(t, err, "kafka sink injected error") + require.ErrorContains(t, err, "keyspace=default") + require.ErrorContains(t, err, "changefeed=async-legacy-failpoint") + require.ErrorContains(t, err, "eventType=dml") + require.ErrorContains(t, err, `"Table":"t"`) +} diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/client_options.go similarity index 92% rename from pkg/sink/kafka/franz/factory.go rename to pkg/sink/kafka/client_options.go index 48c8a6bfd2..ecf4f263a1 100644 --- a/pkg/sink/kafka/franz/factory.go +++ b/pkg/sink/kafka/client_options.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -34,7 +34,7 @@ import ( "golang.org/x/oauth2/clientcredentials" ) -type Options struct { +type clientOptions struct { BrokerEndpoints []string ClientID string @@ -72,11 +72,11 @@ func maxTimeoutWithDefault(readTimeout, writeTimeout time.Duration) time.Duratio func newOptions( ctx context.Context, - o *Options, + o *clientOptions, hook kgo.Hook, ) ([]kgo.Opt, error) { if o == nil { - o = &Options{} + o = &clientOptions{} } timeoutOverhead := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) @@ -109,7 +109,7 @@ func newOptions( } if o.SASL != nil && o.SASL.SASLMechanism != "" { - mechanism, err := buildFranzSaslMechanism(ctx, o) + mechanism, err := buildSaslMechanism(ctx, o) if err != nil { return nil, errors.Trace(err) } @@ -119,7 +119,7 @@ func newOptions( return opts, nil } -func newTLSConfig(o *Options) (*tls.Config, error) { +func newTLSConfig(o *clientOptions) (*tls.Config, error) { tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, NextProtos: []string{"h2", "http/1.1"}, @@ -143,7 +143,7 @@ func newTLSConfig(o *Options) (*tls.Config, error) { return tlsConfig, nil } -func buildFranzSaslMechanism(ctx context.Context, o *Options) (sasl.Mechanism, error) { +func buildSaslMechanism(ctx context.Context, o *clientOptions) (sasl.Mechanism, error) { if o.SASL == nil { return nil, nil } @@ -186,7 +186,7 @@ func buildFranzSaslMechanism(ctx context.Context, o *Options) (sasl.Mechanism, e return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) } -func newOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSource, error) { +func newOauthTokenSource(ctx context.Context, o *clientOptions) (oauth2.TokenSource, error) { endpointParams := url.Values{} if o.SASL.OAuth2.GrantType != "" { endpointParams.Set("grant_type", o.SASL.OAuth2.GrantType) @@ -211,11 +211,11 @@ func newOauthTokenSource(ctx context.Context, o *Options) (oauth2.TokenSource, e } func newProducerOptions( - o *Options, + o *clientOptions, ) []kgo.Opt { recordRetries := defaultRecordRetries if o == nil { - o = &Options{} + o = &clientOptions{} } else { recordRetries = o.MaxRetry } @@ -238,7 +238,7 @@ func newProducerOptions( } } -func newRequiredAcks(o *Options) kgo.Acks { +func newRequiredAcks(o *clientOptions) kgo.Acks { if o == nil { return kgo.AllISRAcks() } @@ -256,7 +256,7 @@ func newRequiredAcks(o *Options) kgo.Acks { } } -func newCompressionOption(o *Options) kgo.Opt { +func newCompressionOption(o *clientOptions) kgo.Opt { if o == nil { return kgo.ProducerBatchCompression(kgo.NoCompression()) } diff --git a/pkg/sink/kafka/franz/factory_api_test.go b/pkg/sink/kafka/client_options_test.go similarity index 92% rename from pkg/sink/kafka/franz/factory_api_test.go rename to pkg/sink/kafka/client_options_test.go index be359d6fe5..28e8e58d5b 100644 --- a/pkg/sink/kafka/franz/factory_api_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -41,7 +41,7 @@ func TestNewRequiredAcks(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - require.Equal(t, tc.expected, newRequiredAcks(&Options{RequiredAcks: tc.requiredAcks})) + require.Equal(t, tc.expected, newRequiredAcks(&clientOptions{RequiredAcks: tc.requiredAcks})) }) } @@ -75,7 +75,7 @@ func TestMaxTimeoutWithDefault(t *testing.T) { func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { t.Parallel() - opts, err := newOptions(context.Background(), &Options{ + opts, err := newOptions(context.Background(), &clientOptions{ Version: "invalid", IsAssignedVersion: true, }, nil) @@ -86,7 +86,7 @@ func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { t.Parallel() - _, err := newOauthTokenSource(context.Background(), &Options{ + _, err := newOauthTokenSource(context.Background(), &clientOptions{ SASL: &security.SASL{ OAuth2: security.OAuth2{ ClientID: "client-id", diff --git a/pkg/sink/kafka/factory_selector.go b/pkg/sink/kafka/factory_selector.go index be599a29cf..d8f12688d4 100644 --- a/pkg/sink/kafka/factory_selector.go +++ b/pkg/sink/kafka/factory_selector.go @@ -25,5 +25,5 @@ func NewFactory( o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - return NewFranzFactory(ctx, o, changefeedID) + return NewKafkaFactory(ctx, o, changefeedID) } diff --git a/pkg/sink/kafka/failpoint.go b/pkg/sink/kafka/failpoint.go new file mode 100644 index 0000000000..faca9a64b3 --- /dev/null +++ b/pkg/sink/kafka/failpoint.go @@ -0,0 +1,45 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "os" + "sync/atomic" + + "github.com/pingcap/failpoint" +) + +const ( + legacyKafkaSinkFailpointPrefix = "github.com/pingcap/ticdc/pkg/sink/kafka/" + + kafkaSinkAsyncSendErrorFailpoint = "KafkaSinkAsyncSendError" + kafkaSinkSyncSendMessageErrorFailpoint = "KafkaSinkSyncSendMessageError" + kafkaSinkSyncSendMessagesErrorFailpoint = "KafkaSinkSyncSendMessagesError" +) + +var legacyKafkaSinkFailpointsRuntimeEnabled atomic.Bool + +func init() { + legacyKafkaSinkFailpointsRuntimeEnabled.Store( + os.Getenv("GO_FAILPOINTS") != "" || os.Getenv("GO_FAILPOINTS_HTTP") != "", + ) +} + +func legacyKafkaSinkFailpointEnabled(name string) bool { + if !legacyKafkaSinkFailpointsRuntimeEnabled.Load() { + return false + } + _, err := failpoint.Eval(legacyKafkaSinkFailpointPrefix + name) + return err == nil +} diff --git a/pkg/sink/kafka/failpoint_test.go b/pkg/sink/kafka/failpoint_test.go new file mode 100644 index 0000000000..7de5934351 --- /dev/null +++ b/pkg/sink/kafka/failpoint_test.go @@ -0,0 +1,41 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "testing" + + "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" +) + +func enableLegacyKafkaSinkFailpointForTest(t *testing.T, name string) { + t.Helper() + + previous := legacyKafkaSinkFailpointsRuntimeEnabled.Load() + legacyKafkaSinkFailpointsRuntimeEnabled.Store(true) + + failpointPath := legacyKafkaSinkFailpointPrefix + name + require.NoError(t, failpoint.Enable(failpointPath, "return(true)")) + t.Cleanup(func() { + _ = failpoint.Disable(failpointPath) + legacyKafkaSinkFailpointsRuntimeEnabled.Store(previous) + }) +} + +func TestLegacyKafkaSinkFailpointEnabled(t *testing.T) { + enableLegacyKafkaSinkFailpointForTest(t, kafkaSinkSyncSendMessageErrorFailpoint) + + require.True(t, legacyKafkaSinkFailpointEnabled(kafkaSinkSyncSendMessageErrorFailpoint)) +} diff --git a/pkg/sink/kafka/franz_admin_client.go b/pkg/sink/kafka/franz_admin_client.go deleted file mode 100644 index c11250aa03..0000000000 --- a/pkg/sink/kafka/franz_admin_client.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "github.com/pingcap/ticdc/pkg/errors" - kafkafranz "github.com/pingcap/ticdc/pkg/sink/kafka/franz" -) - -// franzAdminClientAdapter adapts the franz-go admin client implementation to kafka.ClusterAdminClient. -// It intentionally lives in the kafka package to reuse existing option adjustment logic without -// introducing an import cycle (kafka -> franz -> kafka). -type franzAdminClientAdapter struct { - inner *kafkafranz.AdminClient -} - -func (a *franzAdminClientAdapter) GetAllBrokers() []Broker { - brokers := a.inner.GetAllBrokers() - result := make([]Broker, 0, len(brokers)) - for _, b := range brokers { - result = append(result, Broker{ID: b}) - } - return result -} - -func (a *franzAdminClientAdapter) GetBrokerConfig(configName string) (string, error) { - return a.inner.GetBrokerConfig(configName) -} - -func (a *franzAdminClientAdapter) GetTopicConfig(topicName string, configName string) (string, error) { - return a.inner.GetTopicConfig(topicName, configName) -} - -func (a *franzAdminClientAdapter) GetTopicsMeta( - topics []string, - ignoreTopicError bool, -) (map[string]TopicDetail, error) { - meta, err := a.inner.GetTopicsMeta(topics, ignoreTopicError) - if err != nil { - return nil, err - } - - result := make(map[string]TopicDetail, len(meta)) - for topic, detail := range meta { - result[topic] = TopicDetail{ - Name: detail.Name, - NumPartitions: detail.NumPartitions, - ReplicationFactor: detail.ReplicationFactor, - } - } - return result, nil -} - -func (a *franzAdminClientAdapter) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - return a.inner.GetTopicsPartitionsNum(topics) -} - -func (a *franzAdminClientAdapter) CreateTopic(detail *TopicDetail, validateOnly bool) error { - if detail == nil { - return errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") - } - franzDetail := &kafkafranz.TopicDetail{ - Name: detail.Name, - NumPartitions: detail.NumPartitions, - ReplicationFactor: detail.ReplicationFactor, - } - return a.inner.CreateTopic(franzDetail, validateOnly) -} - -func (a *franzAdminClientAdapter) Heartbeat() { - a.inner.Heartbeat() -} - -func (a *franzAdminClientAdapter) Close() { - if a.inner != nil { - a.inner.Close() - } -} diff --git a/pkg/sink/kafka/franz_admin_client_test.go b/pkg/sink/kafka/franz_admin_client_test.go deleted file mode 100644 index cbf5865e18..0000000000 --- a/pkg/sink/kafka/franz_admin_client_test.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestFranzAdminClientAdapterCreateTopicNilDetailReturnsError(t *testing.T) { - t.Parallel() - - adapter := &franzAdminClientAdapter{inner: nil} - err := adapter.CreateTopic(nil, false) - require.Error(t, err) -} diff --git a/pkg/sink/kafka/franz/gssapi.go b/pkg/sink/kafka/gssapi.go similarity index 98% rename from pkg/sink/kafka/franz/gssapi.go rename to pkg/sink/kafka/gssapi.go index 72dcfffbd3..00838b9302 100644 --- a/pkg/sink/kafka/franz/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -112,7 +112,7 @@ func (s *gssapiSession) Challenge(challenge []byte) (bool, []byte, error) { return false, nil, errors.Trace(err) } // Return a final payload while marking done=true. - // franz-go will write this message and finish the auth flow. + // The Kafka client writes this message and finishes the auth flow. s.close() return true, msg, nil case gssAPIFinished: diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/kafka_factory.go similarity index 53% rename from pkg/sink/kafka/franz_factory.go rename to pkg/sink/kafka/kafka_factory.go index f4d164130e..236b716cf5 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -18,7 +18,6 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/kafka/franz" ) const ( @@ -27,43 +26,43 @@ const ( clientTypeAdminClient = "admin_client" ) -type franzFactory struct { +type kafkaFactory struct { changefeedID common.ChangeFeedID option *options - asyncMetricsHook *franz.MetricsHook - syncMetricsHook *franz.MetricsHook - adminMetricsHook *franz.MetricsHook + asyncMetricsHook *metricsHook + syncMetricsHook *metricsHook + adminMetricsHook *metricsHook } -type franzMetricsCollector struct { +type kafkaMetricsCollector struct { changefeedID common.ChangeFeedID - hooks []*franz.MetricsHook + hooks []*metricsHook } -func (c *franzMetricsCollector) Run(ctx context.Context) { +func (c *kafkaMetricsCollector) Run(ctx context.Context) { <-ctx.Done() for _, hook := range c.hooks { if hook != nil { - hook.CleanupPrometheusMetrics() + hook.cleanupMetrics() } } - franz.CleanupAdminMetrics(c.changefeedID.Keyspace(), c.changefeedID.Name()) + cleanupAdminMetrics(c.changefeedID.Keyspace(), c.changefeedID.Name()) } -func newFranzMetricsHook(changefeedID common.ChangeFeedID, clientType string) *franz.MetricsHook { - hook := franz.NewMetricsHook(clientType) - hook.BindPrometheusMetrics( +func newKafkaMetricsHook(changefeedID common.ChangeFeedID, clientType string) *metricsHook { + hook := newMetricsHook(clientType) + hook.bindMetrics( changefeedID.Keyspace(), changefeedID.Name(), - franz.PrometheusMetrics{ - RequestsInFlight: franzRequestsInFlightByClientGauge, - OutgoingByteRate: franzOutgoingByteTotalByClientGauge, - RequestRate: franzRequestTotalByClientGauge, - RequestLatency: franzRequestLatencyHistogram, - ResponseRate: franzResponseTotalByClientGauge, - CompressionRatio: franzCompressionRatioHistogram, - RecordsPerRequest: franzRecordsPerRequestHistogram, + metricVectors{ + RequestsInFlight: kafkaClientRequestsInFlightGauge, + OutgoingByteRate: kafkaClientOutgoingByteTotalGauge, + RequestRate: kafkaClientRequestTotalGauge, + RequestLatency: kafkaClientRequestLatencyHistogram, + ResponseRate: kafkaClientResponseTotalGauge, + CompressionRatio: kafkaClientCompressionRatioHistogram, + RecordsPerRequest: kafkaClientRecordsPerRequestHistogram, LegacyRequestsInFlight: requestsInFlightGauge, LegacyOutgoingByteRate: OutgoingByteRateGauge, @@ -77,74 +76,70 @@ func newFranzMetricsHook(changefeedID common.ChangeFeedID, clientType string) *f return hook } -// NewFranzFactory constructs a Factory with franz-go implementation. -// -// NOTE: The franz-go specific implementation details live in `pkg/sink/kafka/franz`. -// This function keeps the public API stable and adapts to the existing kafka package interfaces. -func NewFranzFactory( +// NewKafkaFactory constructs a Factory. +func NewKafkaFactory( ctx context.Context, o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - adminInner, err := franz.NewAdminClient(ctx, changefeedID, newFranzOptions(o), nil) + admin, err := newAdminClient(ctx, changefeedID, newKafkaOptions(o), nil) if err != nil { return nil, errors.Trace(err) } - admin := &franzAdminClientAdapter{inner: adminInner} defer admin.Close() if err := adjustOptions(ctx, admin, o, o.Topic); err != nil { return nil, errors.Trace(err) } - return &franzFactory{ + return &kafkaFactory{ changefeedID: changefeedID, option: o, - asyncMetricsHook: newFranzMetricsHook(changefeedID, clientTypeAsyncProducer), - syncMetricsHook: newFranzMetricsHook(changefeedID, clientTypeSyncProducer), - adminMetricsHook: newFranzMetricsHook(changefeedID, clientTypeAdminClient), + asyncMetricsHook: newKafkaMetricsHook(changefeedID, clientTypeAsyncProducer), + syncMetricsHook: newKafkaMetricsHook(changefeedID, clientTypeSyncProducer), + adminMetricsHook: newKafkaMetricsHook(changefeedID, clientTypeAdminClient), }, nil } -func (f *franzFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - adminInner, err := franz.NewAdminClient(ctx, f.changefeedID, newFranzOptions(f.option), f.adminMetricsHook) +func (f *kafkaFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { + admin, err := newAdminClient(ctx, f.changefeedID, newKafkaOptions(f.option), f.adminMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } - return &franzAdminClientAdapter{inner: adminInner}, nil + return admin, nil } -func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := franz.NewSyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.syncMetricsHook) +func (f *kafkaFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { + producer, err := newSyncProducer(ctx, f.changefeedID, newKafkaOptions(f.option), f.syncMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } return producer, nil } -func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := franz.NewAsyncProducer(ctx, f.changefeedID, newFranzOptions(f.option), f.asyncMetricsHook) +func (f *kafkaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { + producer, err := newAsyncProducer(ctx, f.changefeedID, newKafkaOptions(f.option), f.asyncMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } return producer, nil } -func (f *franzFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { - return &franzMetricsCollector{changefeedID: f.changefeedID, hooks: []*franz.MetricsHook{ +func (f *kafkaFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { + return &kafkaMetricsCollector{changefeedID: f.changefeedID, hooks: []*metricsHook{ f.asyncMetricsHook, f.syncMetricsHook, f.adminMetricsHook, }} } -func newFranzOptions(o *options) *franz.Options { +func newKafkaOptions(o *options) *clientOptions { if o == nil { - return &franz.Options{ + return &clientOptions{ RequiredAcks: int16(WaitForAll), } } - return &franz.Options{ + return &clientOptions{ BrokerEndpoints: o.BrokerEndpoints, ClientID: o.ClientID, diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/kafka_factory_test.go similarity index 74% rename from pkg/sink/kafka/franz_factory_test.go rename to pkg/sink/kafka/kafka_factory_test.go index b7844229d5..75c0924040 100644 --- a/pkg/sink/kafka/franz_factory_test.go +++ b/pkg/sink/kafka/kafka_factory_test.go @@ -19,14 +19,14 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewFranzOptionsNilUsesWaitForAll(t *testing.T) { +func TestNewKafkaOptionsNilUsesWaitForAll(t *testing.T) { t.Parallel() - options := newFranzOptions(nil) + options := newKafkaOptions(nil) require.Equal(t, int16(WaitForAll), options.RequiredAcks) } -func TestNewFranzOptionsMapsRequiredAcks(t *testing.T) { +func TestNewKafkaOptionsMapsRequiredAcks(t *testing.T) { t.Parallel() testCases := []struct { @@ -46,18 +46,18 @@ func TestNewFranzOptionsMapsRequiredAcks(t *testing.T) { options := NewOptions() options.RequiredAcks = tc.requiredAcks - franzOptions := newFranzOptions(options) - require.Equal(t, int16(tc.requiredAcks), franzOptions.RequiredAcks) + kafkaOptions := newKafkaOptions(options) + require.Equal(t, int16(tc.requiredAcks), kafkaOptions.RequiredAcks) }) } } -func TestNewFranzOptionsMapsMaxRetry(t *testing.T) { +func TestNewKafkaOptionsMapsMaxRetry(t *testing.T) { t.Parallel() options := NewOptions() options.MaxRetry = 7 - franzOptions := newFranzOptions(options) - require.Equal(t, 7, franzOptions.MaxRetry) + kafkaOptions := newKafkaOptions(options) + require.Equal(t, 7, kafkaOptions.MaxRetry) } diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index 6e45419e90..545a3bb798 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -16,7 +16,6 @@ package kafka import ( "github.com/pingcap/ticdc/pkg/sink/codec" "github.com/pingcap/ticdc/pkg/sink/kafka/claimcheck" - "github.com/pingcap/ticdc/pkg/sink/kafka/franz" "github.com/prometheus/client_golang/prometheus" ) @@ -80,54 +79,54 @@ var ( Help: "Responses/second received from all brokers.", }, []string{"namespace", "changefeed", "broker"}) - franzRequestsInFlightByClientGauge = prometheus.NewGaugeVec( + kafkaClientRequestsInFlightGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_producer_in_flight_requests", + Name: "kafka_client_producer_in_flight_requests", Help: "Current number of in-flight requests by client type and broker.", }, []string{"namespace", "changefeed", "client", "broker"}) - franzOutgoingByteTotalByClientGauge = prometheus.NewGaugeVec( + kafkaClientOutgoingByteTotalGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_producer_outgoing_byte_total", + Name: "kafka_client_producer_outgoing_byte_total", Help: "Total bytes written by kafka sink clients.", }, []string{"namespace", "changefeed", "client", "broker"}) - franzRequestTotalByClientGauge = prometheus.NewGaugeVec( + kafkaClientRequestTotalGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_producer_request_total", + Name: "kafka_client_producer_request_total", Help: "Total requests sent by kafka sink clients.", }, []string{"namespace", "changefeed", "client", "broker"}) - franzResponseTotalByClientGauge = prometheus.NewGaugeVec( + kafkaClientResponseTotalGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_producer_response_total", + Name: "kafka_client_producer_response_total", Help: "Total responses received by kafka sink clients.", }, []string{"namespace", "changefeed", "client", "broker"}) - franzRequestLatencyHistogram = prometheus.NewHistogramVec( + kafkaClientRequestLatencyHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_producer_request_latency_histogram", + Name: "kafka_client_producer_request_latency_histogram", Help: "Request latency histogram for kafka producer in milliseconds.", }, []string{"namespace", "changefeed", "client", "broker"}) - franzCompressionRatioHistogram = prometheus.NewHistogramVec( + kafkaClientCompressionRatioHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_producer_compression_ratio_histogram", + Name: "kafka_client_producer_compression_ratio_histogram", Help: "Compression ratio times 100 histogram for kafka producer.", }, []string{"namespace", "changefeed", "client"}) - franzRecordsPerRequestHistogram = prometheus.NewHistogramVec( + kafkaClientRecordsPerRequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_franz_producer_records_per_request_histogram", + Name: "kafka_client_producer_records_per_request_histogram", Help: "Records per request histogram for kafka producer.", }, []string{"namespace", "changefeed", "client"}) ) @@ -142,15 +141,15 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(requestsInFlightGauge) registry.MustRegister(responseRateGauge) - registry.MustRegister(franzRequestsInFlightByClientGauge) - registry.MustRegister(franzOutgoingByteTotalByClientGauge) - registry.MustRegister(franzRequestTotalByClientGauge) - registry.MustRegister(franzResponseTotalByClientGauge) - registry.MustRegister(franzRequestLatencyHistogram) - registry.MustRegister(franzCompressionRatioHistogram) - registry.MustRegister(franzRecordsPerRequestHistogram) + registry.MustRegister(kafkaClientRequestsInFlightGauge) + registry.MustRegister(kafkaClientOutgoingByteTotalGauge) + registry.MustRegister(kafkaClientRequestTotalGauge) + registry.MustRegister(kafkaClientResponseTotalGauge) + registry.MustRegister(kafkaClientRequestLatencyHistogram) + registry.MustRegister(kafkaClientCompressionRatioHistogram) + registry.MustRegister(kafkaClientRecordsPerRequestHistogram) - franz.InitAdminMetrics(registry) + initAdminMetrics(registry) claimcheck.InitMetrics(registry) codec.InitMetrics(registry) } diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go similarity index 85% rename from pkg/sink/kafka/franz/metrics_hook.go rename to pkg/sink/kafka/metrics_hook.go index e57f270d04..35f74e75ad 100644 --- a/pkg/sink/kafka/franz/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -23,16 +23,16 @@ import ( "github.com/twmb/franz-go/pkg/kgo" ) -type MetricsHook struct { - promMu sync.RWMutex - promBound bool - keyspace string - changefeed string - clientType string - prom PrometheusMetrics +type metricsHook struct { + metricsMu sync.RWMutex + metricsBound bool + keyspace string + changefeed string + clientType string + metrics metricVectors } -type PrometheusMetrics struct { +type metricVectors struct { RequestsInFlight *prometheus.GaugeVec OutgoingByteRate *prometheus.GaugeVec RequestRate *prometheus.GaugeVec @@ -55,33 +55,33 @@ const ( legacyMetricP99 = "p99" ) -func NewMetricsHook(clientType string) *MetricsHook { - return &MetricsHook{clientType: clientType} +func newMetricsHook(clientType string) *metricsHook { + return &metricsHook{clientType: clientType} } -func (h *MetricsHook) BindPrometheusMetrics( +func (h *metricsHook) bindMetrics( keyspace string, changefeed string, - metrics PrometheusMetrics, + metrics metricVectors, ) { - h.promMu.Lock() - defer h.promMu.Unlock() + h.metricsMu.Lock() + defer h.metricsMu.Unlock() h.keyspace = keyspace h.changefeed = changefeed - h.prom = metrics - h.promBound = true + h.metrics = metrics + h.metricsBound = true } -func (h *MetricsHook) loadPrometheusMetrics() (string, string, PrometheusMetrics, bool) { - h.promMu.RLock() - defer h.promMu.RUnlock() +func (h *metricsHook) loadMetrics() (string, string, metricVectors, bool) { + h.metricsMu.RLock() + defer h.metricsMu.RUnlock() - return h.keyspace, h.changefeed, h.prom, h.promBound + return h.keyspace, h.changefeed, h.metrics, h.metricsBound } -func (h *MetricsHook) Run(ctx context.Context) { - _, _, _, bound := h.loadPrometheusMetrics() +func (h *metricsHook) Run(ctx context.Context) { + _, _, _, bound := h.loadMetrics() if !bound { <-ctx.Done() @@ -89,11 +89,11 @@ func (h *MetricsHook) Run(ctx context.Context) { } <-ctx.Done() - h.CleanupPrometheusMetrics() + h.cleanupMetrics() } -func (h *MetricsHook) CleanupPrometheusMetrics() { - keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() +func (h *metricsHook) cleanupMetrics() { + keyspace, changefeed, metrics, bound := h.loadMetrics() if !bound { return @@ -125,7 +125,7 @@ func (h *MetricsHook) CleanupPrometheusMetrics() { deleteGaugeVecPartialMatch(metrics.LegacyRecordsPerRequest, legacyLabels) } -func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { +func (h *metricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { if nodeID < 0 { return } @@ -156,7 +156,7 @@ func (h *MetricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err erro } } -func (h *MetricsHook) OnBrokerWrite( +func (h *metricsHook) OnBrokerWrite( meta kgo.BrokerMetadata, _ int16, bytesWritten int, @@ -167,7 +167,7 @@ func (h *MetricsHook) OnBrokerWrite( h.RecordBrokerWrite(meta.NodeID, bytesWritten, err) } -func (h *MetricsHook) OnBrokerE2E( +func (h *metricsHook) OnBrokerE2E( meta kgo.BrokerMetadata, _ int16, e2e kgo.BrokerE2E, @@ -205,7 +205,7 @@ func (h *MetricsHook) OnBrokerE2E( } } -func (h *MetricsHook) OnProduceBatchWritten( +func (h *metricsHook) OnProduceBatchWritten( _ kgo.BrokerMetadata, _ string, _ int32, @@ -214,7 +214,7 @@ func (h *MetricsHook) OnProduceBatchWritten( h.RecordProduceBatchWritten(m.NumRecords, m.UncompressedBytes, m.CompressedBytes) } -func (h *MetricsHook) RecordProduceBatchWritten(numRecords int, uncompressedBytes int, compressedBytes int) { +func (h *metricsHook) RecordProduceBatchWritten(numRecords int, uncompressedBytes int, compressedBytes int) { ctx, ok := h.loadMetricsContext() if !ok { return @@ -244,11 +244,11 @@ type metricsContext struct { keyspace string changefeed string clientType string - metrics PrometheusMetrics + metrics metricVectors } -func (h *MetricsHook) loadMetricsContext() (metricsContext, bool) { - keyspace, changefeed, metrics, bound := h.loadPrometheusMetrics() +func (h *metricsHook) loadMetricsContext() (metricsContext, bool) { + keyspace, changefeed, metrics, bound := h.loadMetrics() if !bound { return metricsContext{}, false } diff --git a/pkg/sink/kafka/metrics_hook_test.go b/pkg/sink/kafka/metrics_hook_test.go new file mode 100644 index 0000000000..1bc498c1bd --- /dev/null +++ b/pkg/sink/kafka/metrics_hook_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestMetricsHookRecordsLegacyMetricsAndCleanup(t *testing.T) { + outgoingByteRate := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "kafka_producer_outgoing_byte_rate"}, + []string{"namespace", "changefeed", "broker"}, + ) + requestRate := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "kafka_producer_request_rate"}, + []string{"namespace", "changefeed", "broker"}, + ) + requestsInFlight := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "kafka_producer_request_in_flight"}, + []string{"namespace", "changefeed", "broker"}, + ) + recordsPerRequest := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "kafka_producer_records_per_request"}, + []string{"namespace", "changefeed", "type"}, + ) + compressionRatio := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "kafka_producer_compression_ratio"}, + []string{"namespace", "changefeed", "type"}, + ) + + hook := newMetricsHook("async_producer") + hook.bindMetrics("default", "cf", metricVectors{ + LegacyOutgoingByteRate: outgoingByteRate, + LegacyRequestRate: requestRate, + LegacyRequestsInFlight: requestsInFlight, + LegacyRecordsPerRequest: recordsPerRequest, + LegacyCompressionRatio: compressionRatio, + }) + + hook.RecordBrokerWrite(1, 42, nil) + hook.RecordProduceBatchWritten(3, 100, 50) + + require.Equal(t, float64(42), testutil.ToFloat64(outgoingByteRate.WithLabelValues("default", "cf", "1"))) + require.Equal(t, float64(1), testutil.ToFloat64(requestRate.WithLabelValues("default", "cf", "1"))) + require.Equal(t, float64(1), testutil.ToFloat64(requestsInFlight.WithLabelValues("default", "cf", "1"))) + require.Equal(t, float64(3), testutil.ToFloat64(recordsPerRequest.WithLabelValues("default", "cf", legacyMetricAvg))) + require.Equal(t, float64(3), testutil.ToFloat64(recordsPerRequest.WithLabelValues("default", "cf", legacyMetricP99))) + require.Equal(t, float64(200), testutil.ToFloat64(compressionRatio.WithLabelValues("default", "cf", legacyMetricAvg))) + require.Equal(t, float64(200), testutil.ToFloat64(compressionRatio.WithLabelValues("default", "cf", legacyMetricP99))) + + hook.cleanupMetrics() + + require.Equal(t, 0, testutil.CollectAndCount(outgoingByteRate)) + require.Equal(t, 0, testutil.CollectAndCount(requestRate)) + require.Equal(t, 0, testutil.CollectAndCount(requestsInFlight)) + require.Equal(t, 0, testutil.CollectAndCount(recordsPerRequest)) + require.Equal(t, 0, testutil.CollectAndCount(compressionRatio)) +} diff --git a/pkg/sink/kafka/franz/sasl_test.go b/pkg/sink/kafka/sasl_test.go similarity index 82% rename from pkg/sink/kafka/franz/sasl_test.go rename to pkg/sink/kafka/sasl_test.go index 7961125c88..183d3bbead 100644 --- a/pkg/sink/kafka/franz/sasl_test.go +++ b/pkg/sink/kafka/sasl_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -21,10 +21,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestBuildFranzSaslMechanismGSSAPIUserAuth(t *testing.T) { +func TestBuildSaslMechanismGSSAPIUserAuth(t *testing.T) { t.Parallel() - o := &Options{ + o := &clientOptions{ SASL: &security.SASL{ SASLMechanism: security.GSSAPIMechanism, GSSAPI: security.GSSAPI{ @@ -38,15 +38,15 @@ func TestBuildFranzSaslMechanismGSSAPIUserAuth(t *testing.T) { }, } - mechanism, err := buildFranzSaslMechanism(context.Background(), o) + mechanism, err := buildSaslMechanism(context.Background(), o) require.NoError(t, err) require.Equal(t, "GSSAPI", mechanism.Name()) } -func TestBuildFranzSaslMechanismGSSAPIKeytabAuth(t *testing.T) { +func TestBuildSaslMechanismGSSAPIKeytabAuth(t *testing.T) { t.Parallel() - o := &Options{ + o := &clientOptions{ SASL: &security.SASL{ SASLMechanism: security.GSSAPIMechanism, GSSAPI: security.GSSAPI{ @@ -60,7 +60,7 @@ func TestBuildFranzSaslMechanismGSSAPIKeytabAuth(t *testing.T) { }, } - mechanism, err := buildFranzSaslMechanism(context.Background(), o) + mechanism, err := buildSaslMechanism(context.Background(), o) require.NoError(t, err) require.Equal(t, "GSSAPI", mechanism.Name()) } diff --git a/pkg/sink/kafka/franz/sync_producer.go b/pkg/sink/kafka/sync_producer.go similarity index 77% rename from pkg/sink/kafka/franz/sync_producer.go rename to pkg/sink/kafka/sync_producer.go index aba5268d24..7425dc9530 100644 --- a/pkg/sink/kafka/franz/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -28,7 +28,7 @@ import ( "go.uber.org/zap" ) -type SyncProducer struct { +type kafkaSyncProducer struct { id commonType.ChangeFeedID client *kgo.Client @@ -36,14 +36,14 @@ type SyncProducer struct { timeout time.Duration } -func NewSyncProducer( +func newSyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, - o *Options, + o *clientOptions, hook kgo.Hook, -) (*SyncProducer, error) { +) (*kafkaSyncProducer, error) { if o == nil { - o = &Options{} + o = &clientOptions{} } opts, err := newOptions(ctx, o, hook) @@ -59,7 +59,7 @@ func NewSyncProducer( timeout := maxTimeoutWithDefault(o.ReadTimeout, 0) - return &SyncProducer{ + return &kafkaSyncProducer{ id: changefeedID, client: client, closed: atomic.NewBool(false), @@ -67,11 +67,11 @@ func NewSyncProducer( }, nil } -func (p *SyncProducer) newRequestContext() (context.Context, context.CancelFunc) { +func (p *kafkaSyncProducer) newRequestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(p.client.Context(), p.timeout) } -func (p *SyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { +func (p *kafkaSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { if p.closed.Load() { return errors.ErrKafkaProducerClosed.GenWithStackByArgs() } @@ -82,6 +82,10 @@ func (p *SyncProducer) SendMessage(topic string, partitionNum int32, message *co record := buildRecord(topic, partitionNum, message) err := p.client.ProduceSync(ctx, record).FirstErr() + if legacyKafkaSinkFailpointEnabled(kafkaSinkSyncSendMessageErrorFailpoint) { + err = errors.New("kafka sink sync send message injected error") + } + failpoint.Inject("KafkaSinkSyncSendMessageError", func() { err = errors.New("kafka sink sync send message injected error") }) @@ -89,7 +93,7 @@ func (p *SyncProducer) SendMessage(topic string, partitionNum int32, message *co return p.wrapSendError(message, err) } -func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { +func (p *kafkaSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { if p.closed.Load() { return errors.ErrKafkaProducerClosed.GenWithStackByArgs() } @@ -104,6 +108,10 @@ func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *c err := p.client.ProduceSync(ctx, records...).FirstErr() + if legacyKafkaSinkFailpointEnabled(kafkaSinkSyncSendMessagesErrorFailpoint) { + err = errors.New("kafka sink sync send messages injected error") + } + failpoint.Inject("KafkaSinkSyncSendMessagesError", func() { err = errors.New("kafka sink sync send messages injected error") }) @@ -111,9 +119,9 @@ func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *c return p.wrapSendError(message, err) } -func (p *SyncProducer) Heartbeat() {} +func (p *kafkaSyncProducer) Heartbeat() {} -func (p *SyncProducer) Close() { +func (p *kafkaSyncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { log.Warn("kafka DDL producer already closed", zap.String("keyspace", p.id.Keyspace()), @@ -138,7 +146,7 @@ func buildRecord(topic string, partition int32, message *common.Message) *kgo.Re } } -func (p *SyncProducer) wrapSendError(message *common.Message, err error) error { +func (p *kafkaSyncProducer) wrapSendError(message *common.Message, err error) error { if err != nil { err = logutil.AnnotateEventError( p.id.Keyspace(), diff --git a/pkg/sink/kafka/sync_producer_test.go b/pkg/sink/kafka/sync_producer_test.go new file mode 100644 index 0000000000..0c5d05d99f --- /dev/null +++ b/pkg/sink/kafka/sync_producer_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + stderrors "errors" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + cerror "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" +) + +func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { + producer := &kafkaSyncProducer{closed: atomic.NewBool(true)} + + err := producer.SendMessage("topic", 1, &codeccommon.Message{}) + require.ErrorIs(t, err, cerror.ErrKafkaProducerClosed) + + err = producer.SendMessages("topic", 1, &codeccommon.Message{}) + require.ErrorIs(t, err, cerror.ErrKafkaProducerClosed) +} + +func TestBuildRecord(t *testing.T) { + message := &codeccommon.Message{ + Key: []byte("key"), + Value: []byte("value"), + } + + record := buildRecord("topic", 3, message) + + require.Equal(t, "topic", record.Topic) + require.Equal(t, int32(3), record.Partition) + require.Equal(t, []byte("key"), record.Key) + require.Equal(t, []byte("value"), record.Value) +} + +func TestSyncProducerWrapSendErrorAnnotatesEventContext(t *testing.T) { + producer := &kafkaSyncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync-error"), + } + + testCases := []struct { + name string + message *codeccommon.Message + contains []string + }{ + { + name: "ddl", + message: &codeccommon.Message{LogInfo: &codeccommon.MessageLogInfo{ + DDL: &codeccommon.DDLLogInfo{ + Query: "create table t(id int primary key)", + StartTs: 10, + CommitTs: 20, + }, + }}, + contains: []string{ + "keyspace=default", + "changefeed=sync-error", + "eventType=ddl", + `ddlQuery="create table t(id int primary key)"`, + "ddlStartTs=10", + "ddlCommitTs=20", + }, + }, + { + name: "checkpoint", + message: &codeccommon.Message{LogInfo: &codeccommon.MessageLogInfo{ + Checkpoint: &codeccommon.CheckpointLogInfo{CommitTs: 30}, + }}, + contains: []string{ + "keyspace=default", + "changefeed=sync-error", + "eventType=checkpoint", + "checkpointTs=30", + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + err := producer.wrapSendError(tc.message, stderrors.New("send failed")) + require.ErrorIs(t, err, cerror.ErrKafkaSendMessage) + require.ErrorContains(t, err, "send failed") + for _, expected := range tc.contains { + require.ErrorContains(t, err, expected) + } + }) + } +} From 0dd9a73c36c86518a75e84f80a98aeb670b96387 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 26 Jun 2026 13:42:10 +0800 Subject: [PATCH 14/61] make fmt --- pkg/sink/kafka/client_options.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index ecf4f263a1..f573b91c72 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -56,8 +56,10 @@ type clientOptions struct { ReadTimeout time.Duration } -const defaultRequestTimeout = 10 * time.Second -const defaultRecordRetries = 5 +const ( + defaultRequestTimeout = 10 * time.Second + defaultRecordRetries = 5 +) func maxTimeoutWithDefault(readTimeout, writeTimeout time.Duration) time.Duration { timeout := readTimeout From da5dad674380688b17616753d5e8a35674fe3899 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 30 Jun 2026 22:48:57 +0800 Subject: [PATCH 15/61] fix the code --- pkg/sink/kafka/client_options.go | 4 ++++ tests/integration_tests/kafka_compression/run.sh | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index f573b91c72..d841df9043 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -280,6 +280,10 @@ func newCompressionOption(o *clientOptions) kgo.Opt { codec = kgo.NoCompression() default: log.Warn("unsupported compression algorithm", zap.String("compression", o.Compression)) + codec = kgo.NoCompression() + } + if codec != kgo.NoCompression() { + log.Info("Kafka producer uses " + compression + " compression algorithm") } return kgo.ProducerBatchCompression(codec) } diff --git a/tests/integration_tests/kafka_compression/run.sh b/tests/integration_tests/kafka_compression/run.sh index e0f85df648..4f241dc6dd 100755 --- a/tests/integration_tests/kafka_compression/run.sh +++ b/tests/integration_tests/kafka_compression/run.sh @@ -18,9 +18,8 @@ function test_compression() { run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&version=${KAFKA_VERSION}&enable-tidb-extension=true" run_sql_file $CUR/data/$1_data.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} - compression_algorithm=$(grep "Kafka producer uses $1 compression algorithm" "$WORK_DIR/cdc.log") - if [[ "$compression_algorithm" -ne 1 ]]; then - echo "can't found producer compression algorithm" + if ! grep -q "Kafka producer uses $1 compression algorithm" "$WORK_DIR/cdc.log"; then + echo "can't find producer compression algorithm" exit 1 fi check_table_exists test.$1_finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 From da278d8bc8dae321b03c0e0ec0478df693e42a57 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 2 Jul 2026 23:19:18 +0800 Subject: [PATCH 16/61] remove the overhead --- pkg/sink/kafka/options.go | 35 ++++++++-------------------------- pkg/sink/kafka/options_test.go | 7 +++---- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index d7bd0bd643..ecff7a9642 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -40,13 +40,6 @@ const ( defaultPartitionNum = 3 // defaultMaxRetry is the default retry budget for Kafka producers. defaultMaxRetry = 5 - - // the `max-message-bytes` is set equal to topic's `max.message.bytes`, and is used to check - // whether the message is larger than the max size limit. It's found some message pass the message - // size limit check at the client side and failed at the broker side since message enlarged during - // the network transmission. so we set the `max-message-bytes` to a smaller value to avoid this problem. - // maxMessageBytesOverhead is used to reduce the `max-message-bytes`. - maxMessageBytesOverhead = 128 ) const ( @@ -601,7 +594,7 @@ func adjustOptions( // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. if exists { - // make sure that producer's `MaxMessageBytes` smaller than topic's `max.message.bytes` + // make sure that producer's `MaxMessageBytes` is not larger than topic's `max.message.bytes`. topicMaxMessageBytesStr, err := getTopicConfig( ctx, admin, info.Name, TopicMaxMessageBytesConfigName, @@ -615,18 +608,12 @@ func adjustOptions( return errors.Trace(err) } - maxMessageBytes := topicMaxMessageBytes - maxMessageBytesOverhead - if topicMaxMessageBytes <= options.MaxMessageBytes { + if topicMaxMessageBytes < options.MaxMessageBytes { log.Warn("topic's `max.message.bytes` less than the `max-message-bytes`,"+ "use topic's `max.message.bytes` to initialize the Kafka producer", zap.Int("max.message.bytes", topicMaxMessageBytes), - zap.Int("max-message-bytes", options.MaxMessageBytes), - zap.Int("real-max-message-bytes", maxMessageBytes)) - options.MaxMessageBytes = maxMessageBytes - } else { - if maxMessageBytes < options.MaxMessageBytes { - options.MaxMessageBytes = maxMessageBytes - } + zap.Int("max-message-bytes", options.MaxMessageBytes)) + options.MaxMessageBytes = topicMaxMessageBytes } // no need to create the topic, @@ -655,20 +642,14 @@ func adjustOptions( // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - // TiCDC need to make sure that the producer's `MaxMessageBytes` won't larger than + // TiCDC need to make sure that the producer's `MaxMessageBytes` won't be larger than // broker's `message.max.bytes`. - maxMessageBytes := brokerMessageMaxBytes - maxMessageBytesOverhead - if brokerMessageMaxBytes <= options.MaxMessageBytes { + if brokerMessageMaxBytes < options.MaxMessageBytes { log.Warn("broker's `message.max.bytes` less than the `max-message-bytes`,"+ "use broker's `message.max.bytes` to initialize the Kafka producer", zap.Int("message.max.bytes", brokerMessageMaxBytes), - zap.Int("max-message-bytes", options.MaxMessageBytes), - zap.Int("real-max-message-bytes", maxMessageBytes)) - options.MaxMessageBytes = maxMessageBytes - } else { - if maxMessageBytes < options.MaxMessageBytes { - options.MaxMessageBytes = maxMessageBytes - } + zap.Int("max-message-bytes", options.MaxMessageBytes)) + options.MaxMessageBytes = brokerMessageMaxBytes } // topic not exists yet, and user does not specify the `partition-num` in the sink uri. diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 0ffbcf6b86..d96cee8a1e 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -159,7 +159,6 @@ func (f *kafkaAdminFixture) setMessageMaxBytes(brokerValue, topicValue string) { } func expectedAdjustedMaxMessageBytes(configuredMaxMessageBytes, sourceMaxMessageBytes int) int { - sourceMaxMessageBytes -= maxMessageBytesOverhead if configuredMaxMessageBytes < sourceMaxMessageBytes { return configuredMaxMessageBytes } @@ -381,7 +380,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t }, }, { - name: "uses broker limit when configured value is within overhead", + name: "keeps configured value below broker limit by one byte", configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { return f.brokerMessageMaxBytes() - 1 }, @@ -546,7 +545,7 @@ func TestConfigurationCombinations(t *testing.T) { mockTopicMessageMaxBytes, }, { - "new topic broker overhead below user", + "new topic broker below user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", []any{"not-created-topic", strconv.Itoa(1024*1024 + 1)}, mockBrokerMessageMaxBytes, @@ -612,7 +611,7 @@ func TestConfigurationCombinations(t *testing.T) { strconv.Itoa(config.DefaultMaxMessageBytes + 1), }, { - "existing topic topic overhead below user", + "existing topic topic below user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", []any{defaultMockTopicName, strconv.Itoa(1024*1024 + 1)}, mockBrokerMessageMaxBytes, From 3e04dac249aff9a020b27bb5edcd38a35ddf143f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 3 Jul 2026 10:13:26 +0800 Subject: [PATCH 17/61] try to fix code --- .../codec/canal/canal_json_txn_encoder.go | 2 +- pkg/sink/codec/common/message.go | 16 ++---- pkg/sink/codec/common/message_test.go | 29 +++++++++++ pkg/sink/codec/open/codec.go | 2 +- pkg/sink/codec/open/encoder_test.go | 4 +- pkg/sink/kafka/client_options.go | 15 ++++-- pkg/sink/kafka/client_options_test.go | 25 +++++++++ pkg/sink/kafka/kafka_factory.go | 9 ++-- pkg/sink/kafka/options.go | 52 +++++++++++-------- pkg/sink/kafka/options_test.go | 18 ++++++- 10 files changed, 122 insertions(+), 50 deletions(-) create mode 100644 pkg/sink/codec/common/message_test.go diff --git a/pkg/sink/codec/canal/canal_json_txn_encoder.go b/pkg/sink/codec/canal/canal_json_txn_encoder.go index 0af6f4f3f2..4d0e7300ae 100644 --- a/pkg/sink/codec/canal/canal_json_txn_encoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_encoder.go @@ -64,7 +64,7 @@ func (j *JSONTxnEventEncoder) AppendTxnEvent(event *commonEvent.DMLEvent) error if err != nil { return err } - length := len(value) + common.MaxRecordOverhead + length := len(value) // For single message that is longer than max-message-bytes, do not send it. if length > j.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json", diff --git a/pkg/sink/codec/common/message.go b/pkg/sink/codec/common/message.go index 9c78be2fbc..773c65ed1e 100644 --- a/pkg/sink/codec/common/message.go +++ b/pkg/sink/codec/common/message.go @@ -13,15 +13,7 @@ package common -import ( - "encoding/binary" - "encoding/json" -) - -// MaxRecordOverhead is used to calculate the expected Kafka record size. -// For TiCDC, minimum supported Kafka version is `0.11.0.2`, which uses record -// batch format v2 and varint encoded fields. -const MaxRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 +import "encoding/json" // MessageType is the type of message, which is used by MqSink and RedoLog. type MessageType int @@ -85,11 +77,9 @@ type CheckpointLogInfo struct { CommitTs uint64 } -// Length returns the expected size of the Kafka message -// We didn't append any `Headers` when send the message, so ignore the calculations related to it. -// If `ProducerMessage` Headers fields used, this method should also adjust. +// Length returns the encoded key/value payload size of the sink message. func (m *Message) Length() int { - return len(m.Key) + len(m.Value) + MaxRecordOverhead + return len(m.Key) + len(m.Value) } // GetRowsCount returns the number of rows batched in one Message diff --git a/pkg/sink/codec/common/message_test.go b/pkg/sink/codec/common/message_test.go new file mode 100644 index 0000000000..d0492bda25 --- /dev/null +++ b/pkg/sink/codec/common/message_test.go @@ -0,0 +1,29 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMessageLengthIsPayloadSize(t *testing.T) { + message := &Message{ + Key: []byte("key"), + Value: []byte("value"), + } + + require.Equal(t, len(message.Key)+len(message.Value), message.Length()) +} diff --git a/pkg/sink/codec/open/codec.go b/pkg/sink/codec/open/codec.go index 454cb0d372..9a7a7b633f 100644 --- a/pkg/sink/codec/open/codec.go +++ b/pkg/sink/codec/open/codec.go @@ -114,7 +114,7 @@ func encodeRowChangedEvent( // for single message that is longer than max-message-bytes // 16 is the length of `keyLenByte` and `valueLenByte`, 8 is the length of `versionHead` - length := len(key) + len(valueCompressed) + common.MaxRecordOverhead + 16 + 8 + length := len(key) + len(valueCompressed) + 16 + 8 return key, valueCompressed, length, nil } diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index 9b02366709..18f148f866 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -907,7 +907,7 @@ func TestLargeMessageWithHandleEnableHandleKeyOnly(t *testing.T) { } ctx := context.Background() - codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(168) + codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(130) codecConfig.LargeMessageHandle.LargeMessageHandleOption = config.LargeMessageHandleOptionHandleKeyOnly encoder, err := NewBatchEncoder(ctx, codecConfig) require.NoError(t, err) @@ -947,7 +947,7 @@ func TestLargeMessageWithHandleEnableHandleKeyOnly(t *testing.T) { func TestLargeMessageWithoutHandle(t *testing.T) { ctx := context.Background() - codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(150) + codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(100) codecConfig.LargeMessageHandle.LargeMessageHandleOption = config.LargeMessageHandleOptionHandleKeyOnly encoder, err := NewBatchEncoder(ctx, codecConfig) require.NoError(t, err) diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index d841df9043..d86ef14ee6 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -41,10 +41,11 @@ type clientOptions struct { Version string IsAssignedVersion bool - MaxMessageBytes int - MaxRetry int - Compression string - RequiredAcks int16 + MaxMessageBytes int + ProducerBatchMaxBytes int + MaxRetry int + Compression string + RequiredAcks int16 EnableTLS bool Credential *security.Credential @@ -226,6 +227,10 @@ func newProducerOptions( if produceTimeout < 100*time.Millisecond { produceTimeout = defaultRequestTimeout } + producerBatchMaxBytes := o.ProducerBatchMaxBytes + if producerBatchMaxBytes <= 0 { + producerBatchMaxBytes = o.MaxMessageBytes + } return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), @@ -233,7 +238,7 @@ func newProducerOptions( kgo.DisableIdempotentWrite(), kgo.MaxProduceRequestsInflightPerBroker(1), kgo.RecordRetries(recordRetries), - kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), + kgo.ProducerBatchMaxBytes(int32(producerBatchMaxBytes)), kgo.ProduceRequestTimeout(produceTimeout), kgo.ProducerLinger(0), newCompressionOption(o), diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index 28e8e58d5b..8f328b0dc1 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -83,6 +83,31 @@ func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { require.ErrorContains(t, err, "invalid kafka version invalid") } +func TestNewProducerOptionsUsesProducerBatchMaxBytes(t *testing.T) { + t.Parallel() + + const ( + encoderMaxMessageBytes = 800 + producerBatchMaxBytes = 1048588 + ) + o := &clientOptions{ + BrokerEndpoints: []string{"127.0.0.1:9092"}, + MaxMessageBytes: encoderMaxMessageBytes, + ProducerBatchMaxBytes: producerBatchMaxBytes, + MaxRetry: defaultRecordRetries, + RequiredAcks: int16(WaitForAll), + } + + opts, err := newOptions(context.Background(), o, nil) + require.NoError(t, err) + opts = append(opts, newProducerOptions(o)...) + client, err := kgo.NewClient(opts...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, int32(producerBatchMaxBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) +} + func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { t.Parallel() diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go index 236b716cf5..c90273f897 100644 --- a/pkg/sink/kafka/kafka_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -146,10 +146,11 @@ func newKafkaOptions(o *options) *clientOptions { Version: o.Version, IsAssignedVersion: o.IsAssignedVersion, - MaxMessageBytes: o.MaxMessageBytes, - MaxRetry: o.MaxRetry, - Compression: o.Compression, - RequiredAcks: int16(o.RequiredAcks), + MaxMessageBytes: o.MaxMessageBytes, + ProducerBatchMaxBytes: o.ProducerBatchMaxBytes, + MaxRetry: o.MaxRetry, + Compression: o.Compression, + RequiredAcks: int16(o.RequiredAcks), EnableTLS: o.EnableTLS, Credential: o.Credential, diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index ecff7a9642..f3cab22c27 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -149,11 +149,15 @@ type options struct { Version string IsAssignedVersion bool RequestVersion int16 - MaxMessageBytes int - MaxRetry int - Compression string - ClientID string - RequiredAcks RequiredAcks + // MaxMessageBytes is the TiCDC encoder payload limit used by batching and + // large-message handling. + MaxMessageBytes int + // ProducerBatchMaxBytes is the Kafka record batch limit used by franz-go. + ProducerBatchMaxBytes int + MaxRetry int + Compression string + ClientID string + RequiredAcks RequiredAcks // Only for test. User can not set this value. // The current prod default value is 0. MaxMessages int @@ -173,20 +177,20 @@ type options struct { // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ - Version: "2.4.0", - // MaxMessageBytes will be used to initialize producer - MaxMessageBytes: config.DefaultMaxMessageBytes, - MaxRetry: defaultMaxRetry, - ReplicationFactor: 1, - Compression: "none", - RequiredAcks: WaitForAll, - Credential: &security.Credential{}, - InsecureSkipVerify: false, - SASL: &security.SASL{}, - AutoCreate: true, - DialTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - ReadTimeout: 10 * time.Second, + Version: "2.4.0", + MaxMessageBytes: config.DefaultMaxMessageBytes, + ProducerBatchMaxBytes: config.DefaultMaxMessageBytes, + MaxRetry: defaultMaxRetry, + ReplicationFactor: 1, + Compression: "none", + RequiredAcks: WaitForAll, + Credential: &security.Credential{}, + InsecureSkipVerify: false, + SASL: &security.SASL{}, + AutoCreate: true, + DialTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, } } @@ -594,7 +598,7 @@ func adjustOptions( // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. if exists { - // make sure that producer's `MaxMessageBytes` is not larger than topic's `max.message.bytes`. + // Make sure the encoder does not generate messages larger than topic's `max.message.bytes`. topicMaxMessageBytesStr, err := getTopicConfig( ctx, admin, info.Name, TopicMaxMessageBytesConfigName, @@ -607,10 +611,11 @@ func adjustOptions( if err != nil { return errors.Trace(err) } + options.ProducerBatchMaxBytes = topicMaxMessageBytes if topicMaxMessageBytes < options.MaxMessageBytes { log.Warn("topic's `max.message.bytes` less than the `max-message-bytes`,"+ - "use topic's `max.message.bytes` to initialize the Kafka producer", + "use topic's `max.message.bytes` as max-message-bytes", zap.Int("max.message.bytes", topicMaxMessageBytes), zap.Int("max-message-bytes", options.MaxMessageBytes)) options.MaxMessageBytes = topicMaxMessageBytes @@ -642,11 +647,12 @@ func adjustOptions( // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - // TiCDC need to make sure that the producer's `MaxMessageBytes` won't be larger than + options.ProducerBatchMaxBytes = brokerMessageMaxBytes + // TiCDC needs to make sure the encoder does not generate messages larger than // broker's `message.max.bytes`. if brokerMessageMaxBytes < options.MaxMessageBytes { log.Warn("broker's `message.max.bytes` less than the `max-message-bytes`,"+ - "use broker's `message.max.bytes` to initialize the Kafka producer", + "use broker's `message.max.bytes` as max-message-bytes", zap.Int("message.max.bytes", brokerMessageMaxBytes), zap.Int("max-message-bytes", options.MaxMessageBytes)) options.MaxMessageBytes = brokerMessageMaxBytes diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index d96cee8a1e..4eba75bcb9 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -414,12 +414,14 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t options.MaxMessageBytes, adminFixture.brokerMessageMaxBytes(), ) + expectedProducerBatchMaxBytes := adminFixture.brokerMessageMaxBytes() ctx := context.Background() err = adjustOptions(ctx, adminClient, options, topicName) require.NoError(t, err) require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, expectedProducerBatchMaxBytes, options.ProducerBatchMaxBytes) }) } } @@ -537,6 +539,13 @@ func TestConfigurationCombinations(t *testing.T) { mockBrokerMessageMaxBytes, mockTopicMessageMaxBytes, }, + { + "new topic claim check threshold below broker", + "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", + []any{"not-created-topic", "800"}, + mockBrokerMessageMaxBytes, + mockTopicMessageMaxBytes, + }, { "new topic user below default below broker", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", @@ -600,6 +609,13 @@ func TestConfigurationCombinations(t *testing.T) { mockBrokerMessageMaxBytes, mockTopicMessageMaxBytes, }, + { + "existing topic claim check threshold below topic", + "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", + []any{defaultMockTopicName, "800"}, + mockBrokerMessageMaxBytes, + mockTopicMessageMaxBytes, + }, { "existing topic user below default below topic", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", @@ -677,6 +693,7 @@ func TestConfigurationCombinations(t *testing.T) { err = adjustOptions(ctx, adminClient, options, topic) require.Nil(t, err) require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, sourceMaxMessageBytes, options.ProducerBatchMaxBytes) encoderConfig := common.NewConfig(config.ProtocolOpen) err = encoderConfig.Apply(sinkURI, &config.SinkConfig{ @@ -690,7 +707,6 @@ func TestConfigurationCombinations(t *testing.T) { err = encoderConfig.Validate() require.Nil(t, err) - // producer's `MaxMessageBytes` = encoder's `MaxMessageBytes`. require.Equal(t, expectedMaxMessageBytes, encoderConfig.MaxMessageBytes) adminClient.Close() From 840b21dc9caf3237ecbb7193ed5699d84af921ce Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 3 Jul 2026 10:21:15 +0800 Subject: [PATCH 18/61] yagni the code --- cmd/kafka-consumer/option.go | 8 +- downstreamadapter/sink/kafka/sink.go | 2 +- downstreamadapter/sink/kafka/sink_test.go | 2 +- pkg/sink/kafka/admin_client.go | 24 ----- pkg/sink/kafka/admin_metrics.go | 1 - pkg/sink/kafka/async_producer.go | 3 +- pkg/sink/kafka/cluster_admin_client.go | 8 -- pkg/sink/kafka/cluster_admin_client_mock.go | 14 --- pkg/sink/kafka/factory.go | 2 +- pkg/sink/kafka/factory_mock.go | 8 +- pkg/sink/kafka/internal/logutil/logutil.go | 107 -------------------- pkg/sink/kafka/kafka_factory.go | 7 +- pkg/sink/kafka/logutil.go | 77 +++++++++++++- pkg/sink/kafka/metrics_hook.go | 103 ++++++------------- pkg/sink/kafka/metrics_hook_test.go | 3 +- pkg/sink/kafka/sync_producer.go | 3 +- 16 files changed, 123 insertions(+), 249 deletions(-) delete mode 100644 pkg/sink/kafka/internal/logutil/logutil.go diff --git a/cmd/kafka-consumer/option.go b/cmd/kafka-consumer/option.go index ae7ba1f198..0113d0d3f3 100644 --- a/cmd/kafka-consumer/option.go +++ b/cmd/kafka-consumer/option.go @@ -122,11 +122,11 @@ func (o *option) Adjust(upstreamURIStr string, configFile string) { } o.partitionNum = int32(c) } - partitionNum, err := getPartitionNum(o) - if err != nil { - log.Panic("cannot get the partition number", zap.String("topic", o.topic), zap.Error(err)) - } if o.partitionNum == 0 { + partitionNum, err := getPartitionNum(o) + if err != nil { + log.Panic("cannot get the partition number", zap.String("topic", o.topic), zap.Error(err)) + } o.partitionNum = partitionNum } diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 2f5f7dd005..21fcd2d41b 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -120,7 +120,7 @@ func newWithComponents( changefeedID: changefeedID, dmlProducer: asyncProducer, ddlProducer: syncProducer, - metricsCollector: comp.factory.MetricsCollector(comp.adminClient), + metricsCollector: comp.factory.MetricsCollector(), partitionRule: helper.GetDDLDispatchRule(protocol), protocol: protocol, diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 7d46286e27..9557de7a53 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -87,7 +87,7 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, factory := kafka.NewMockFactory(ctrl) factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(syncProducer, nil) - factory.EXPECT().MetricsCollector(adminClient).Return(metricsCollector) + factory.EXPECT().MetricsCollector().Return(metricsCollector) eventRouter, err := eventrouter.NewEventRouter(sinkConfig, topic, false, false) if err != nil { diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index 51bedf262d..3024f06984 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -69,30 +69,6 @@ func (a *kafkaAdminClient) newRequestContext() (context.Context, context.CancelF return context.WithTimeout(a.client.Context(), a.timeout) } -func (a *kafkaAdminClient) GetAllBrokers() []Broker { - startTime := time.Now() - ctx, cancel := a.newRequestContext() - defer cancel() - - meta, err := a.admin.BrokerMetadata(ctx) - if err != nil { - observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetAllBrokers, err, time.Since(startTime)) - log.Warn("Kafka admin client fetch broker metadata failed", - zap.String("keyspace", a.changefeed.Keyspace()), - zap.String("changefeed", a.changefeed.Name()), - zap.Error(err)) - return nil - } - - observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetAllBrokers, nil, time.Since(startTime)) - brokerIDs := meta.Brokers.NodeIDs() - brokers := make([]Broker, 0, len(brokerIDs)) - for _, brokerID := range brokerIDs { - brokers = append(brokers, Broker{ID: brokerID}) - } - return brokers -} - func (a *kafkaAdminClient) GetBrokerConfig(configName string) (value string, err error) { startTime := time.Now() defer func() { diff --git a/pkg/sink/kafka/admin_metrics.go b/pkg/sink/kafka/admin_metrics.go index 5c869600c0..8e44bb9302 100644 --- a/pkg/sink/kafka/admin_metrics.go +++ b/pkg/sink/kafka/admin_metrics.go @@ -20,7 +20,6 @@ import ( ) const ( - adminMethodGetAllBrokers = "get_all_brokers" adminMethodGetBrokerConfig = "get_broker_config" adminMethodGetTopicConfig = "get_topic_config" adminMethodGetTopicsMeta = "get_topics_meta" diff --git a/pkg/sink/kafka/async_producer.go b/pkg/sink/kafka/async_producer.go index 69b148bf95..869b92f5f1 100644 --- a/pkg/sink/kafka/async_producer.go +++ b/pkg/sink/kafka/async_producer.go @@ -22,7 +22,6 @@ import ( commonType "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" - "github.com/pingcap/ticdc/pkg/sink/kafka/internal/logutil" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/atomic" "go.uber.org/zap" @@ -152,7 +151,7 @@ func (p *kafkaAsyncProducer) enqueueAsyncSendError( logInfo *common.MessageLogInfo, err error, ) { - errWithInfo := logutil.AnnotateEventError( + errWithInfo := AnnotateEventError( keyspace, changefeed, logInfo, diff --git a/pkg/sink/kafka/cluster_admin_client.go b/pkg/sink/kafka/cluster_admin_client.go index 3c6c331c88..d559b53931 100644 --- a/pkg/sink/kafka/cluster_admin_client.go +++ b/pkg/sink/kafka/cluster_admin_client.go @@ -20,17 +20,9 @@ type TopicDetail struct { ReplicationFactor int16 } -// Broker represents a Kafka broker. -type Broker struct { - ID int32 -} - // ClusterAdminClient is the administrative client for Kafka, // which supports managing and inspecting topics, brokers, configurations and ACLs. type ClusterAdminClient interface { - // GetAllBrokers return all brokers among the cluster - GetAllBrokers() []Broker - // GetBrokerConfig return the broker level configuration with the `configName` GetBrokerConfig(configName string) (string, error) diff --git a/pkg/sink/kafka/cluster_admin_client_mock.go b/pkg/sink/kafka/cluster_admin_client_mock.go index 9a67b63520..72536d5084 100644 --- a/pkg/sink/kafka/cluster_admin_client_mock.go +++ b/pkg/sink/kafka/cluster_admin_client_mock.go @@ -59,20 +59,6 @@ func (mr *MockClusterAdminClientMockRecorder) CreateTopic(detail, validateOnly i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockClusterAdminClient)(nil).CreateTopic), detail, validateOnly) } -// GetAllBrokers mocks base method. -func (m *MockClusterAdminClient) GetAllBrokers() []Broker { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllBrokers") - ret0, _ := ret[0].([]Broker) - return ret0 -} - -// GetAllBrokers indicates an expected call of GetAllBrokers. -func (mr *MockClusterAdminClientMockRecorder) GetAllBrokers() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrokers", reflect.TypeOf((*MockClusterAdminClient)(nil).GetAllBrokers)) -} - // GetBrokerConfig mocks base method. func (m *MockClusterAdminClient) GetBrokerConfig(configName string) (string, error) { m.ctrl.T.Helper() diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 5c827932d1..d80e6b0cce 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -29,7 +29,7 @@ type Factory interface { // AsyncProducer creates an async producer to writer message to kafka AsyncProducer(ctx context.Context) (AsyncProducer, error) // MetricsCollector returns the kafka metrics collector - MetricsCollector(adminClient ClusterAdminClient) MetricsCollector + MetricsCollector() MetricsCollector } // FactoryCreator defines the type of factory creator. diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index d2e8358943..866a85113f 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -66,17 +66,17 @@ func (mr *MockFactoryMockRecorder) AsyncProducer(ctx interface{}) *gomock.Call { } // MetricsCollector mocks base method. -func (m *MockFactory) MetricsCollector(adminClient ClusterAdminClient) MetricsCollector { +func (m *MockFactory) MetricsCollector() MetricsCollector { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MetricsCollector", adminClient) + ret := m.ctrl.Call(m, "MetricsCollector") ret0, _ := ret[0].(MetricsCollector) return ret0 } // MetricsCollector indicates an expected call of MetricsCollector. -func (mr *MockFactoryMockRecorder) MetricsCollector(adminClient interface{}) *gomock.Call { +func (mr *MockFactoryMockRecorder) MetricsCollector() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MetricsCollector", reflect.TypeOf((*MockFactory)(nil).MetricsCollector), adminClient) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MetricsCollector", reflect.TypeOf((*MockFactory)(nil).MetricsCollector)) } // SyncProducer mocks base method. diff --git a/pkg/sink/kafka/internal/logutil/logutil.go b/pkg/sink/kafka/internal/logutil/logutil.go deleted file mode 100644 index c38de351f6..0000000000 --- a/pkg/sink/kafka/internal/logutil/logutil.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package logutil - -import ( - "encoding/json" - "strconv" - "strings" - - "github.com/pingcap/errors" - "github.com/pingcap/ticdc/pkg/sink/codec/common" -) - -// DetermineEventType infers the event type based on MessageLogInfo content. -func DetermineEventType(info *common.MessageLogInfo) string { - if info == nil { - return "unknown" - } - if info.DDL != nil { - return "ddl" - } - if info.Checkpoint != nil { - return "checkpoint" - } - if len(info.Rows) > 0 { - return "dml" - } - return "unknown" -} - -// BuildEventLogContext builds a textual representation of event info. -func BuildEventLogContext(keyspace, changefeed string, info *common.MessageLogInfo) string { - var sb strings.Builder - sb.WriteString("keyspace=") - sb.WriteString(keyspace) - sb.WriteString(", changefeed=") - sb.WriteString(changefeed) - sb.WriteString(", eventType=") - sb.WriteString(DetermineEventType(info)) - - if info == nil { - return sb.String() - } - - if len(info.Rows) > 0 { - if rowsStr := formatDMLInfo(info.Rows); rowsStr != "" { - sb.WriteString(", dmlInfo=") - sb.WriteString(rowsStr) - } - } - - if info.DDL != nil { - if info.DDL.Query != "" { - sb.WriteString(", ddlQuery=") - sb.WriteString(strconv.Quote(info.DDL.Query)) - } - if info.DDL.StartTs != 0 { - sb.WriteString(", ddlStartTs=") - sb.WriteString(strconv.FormatUint(info.DDL.StartTs, 10)) - } - if info.DDL.CommitTs != 0 { - sb.WriteString(", ddlCommitTs=") - sb.WriteString(strconv.FormatUint(info.DDL.CommitTs, 10)) - } - } - - if info.Checkpoint != nil && info.Checkpoint.CommitTs != 0 { - sb.WriteString(", checkpointTs=") - sb.WriteString(strconv.FormatUint(info.Checkpoint.CommitTs, 10)) - } - - return sb.String() -} - -// AnnotateEventError logs the event context and annotates the error with that context. -func AnnotateEventError( - keyspace, changefeed string, - info *common.MessageLogInfo, - err error, -) error { - if err == nil { - return nil - } - if contextStr := BuildEventLogContext(keyspace, changefeed, info); contextStr != "" { - return errors.Annotate(err, contextStr+"; ErrorInfo:"+err.Error()) - } - return err -} - -func formatDMLInfo(rows []common.RowLogInfo) string { - data, err := json.Marshal(rows) - if err != nil { - return "" - } - return string(data) -} diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go index c90273f897..5e5a48e52c 100644 --- a/pkg/sink/kafka/kafka_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -51,10 +51,10 @@ func (c *kafkaMetricsCollector) Run(ctx context.Context) { } func newKafkaMetricsHook(changefeedID common.ChangeFeedID, clientType string) *metricsHook { - hook := newMetricsHook(clientType) - hook.bindMetrics( + return newMetricsHook( changefeedID.Keyspace(), changefeedID.Name(), + clientType, metricVectors{ RequestsInFlight: kafkaClientRequestsInFlightGauge, OutgoingByteRate: kafkaClientOutgoingByteTotalGauge, @@ -73,7 +73,6 @@ func newKafkaMetricsHook(changefeedID common.ChangeFeedID, clientType string) *m LegacyRecordsPerRequest: recordsPerRequestGauge, }, ) - return hook } // NewKafkaFactory constructs a Factory. @@ -125,7 +124,7 @@ func (f *kafkaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) return producer, nil } -func (f *kafkaFactory) MetricsCollector(_ ClusterAdminClient) MetricsCollector { +func (f *kafkaFactory) MetricsCollector() MetricsCollector { return &kafkaMetricsCollector{changefeedID: f.changefeedID, hooks: []*metricsHook{ f.asyncMetricsHook, f.syncMetricsHook, diff --git a/pkg/sink/kafka/logutil.go b/pkg/sink/kafka/logutil.go index 3fb5dae4be..8a90e7e3e9 100644 --- a/pkg/sink/kafka/logutil.go +++ b/pkg/sink/kafka/logutil.go @@ -14,18 +14,73 @@ package kafka import ( + "encoding/json" + "strconv" + "strings" + + "github.com/pingcap/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" - "github.com/pingcap/ticdc/pkg/sink/kafka/internal/logutil" ) // DetermineEventType infers the event type based on MessageLogInfo content. func DetermineEventType(info *common.MessageLogInfo) string { - return logutil.DetermineEventType(info) + if info == nil { + return "unknown" + } + if info.DDL != nil { + return "ddl" + } + if info.Checkpoint != nil { + return "checkpoint" + } + if len(info.Rows) > 0 { + return "dml" + } + return "unknown" } // BuildEventLogContext builds a textual representation of event info. func BuildEventLogContext(keyspace, changefeed string, info *common.MessageLogInfo) string { - return logutil.BuildEventLogContext(keyspace, changefeed, info) + var sb strings.Builder + sb.WriteString("keyspace=") + sb.WriteString(keyspace) + sb.WriteString(", changefeed=") + sb.WriteString(changefeed) + sb.WriteString(", eventType=") + sb.WriteString(DetermineEventType(info)) + + if info == nil { + return sb.String() + } + + if len(info.Rows) > 0 { + if rowsStr := formatDMLInfo(info.Rows); rowsStr != "" { + sb.WriteString(", dmlInfo=") + sb.WriteString(rowsStr) + } + } + + if info.DDL != nil { + if info.DDL.Query != "" { + sb.WriteString(", ddlQuery=") + sb.WriteString(strconv.Quote(info.DDL.Query)) + } + if info.DDL.StartTs != 0 { + sb.WriteString(", ddlStartTs=") + sb.WriteString(strconv.FormatUint(info.DDL.StartTs, 10)) + } + if info.DDL.CommitTs != 0 { + sb.WriteString(", ddlCommitTs=") + sb.WriteString(strconv.FormatUint(info.DDL.CommitTs, 10)) + } + } + + if info.Checkpoint != nil && info.Checkpoint.CommitTs != 0 { + sb.WriteString(", checkpointTs=") + sb.WriteString(strconv.FormatUint(info.Checkpoint.CommitTs, 10)) + } + + return sb.String() } // AnnotateEventError logs the event context and annotates the error with that context. @@ -34,5 +89,19 @@ func AnnotateEventError( info *common.MessageLogInfo, err error, ) error { - return logutil.AnnotateEventError(keyspace, changefeed, info, err) + if err == nil { + return nil + } + if contextStr := BuildEventLogContext(keyspace, changefeed, info); contextStr != "" { + return errors.Annotate(err, contextStr+"; ErrorInfo:"+err.Error()) + } + return err +} + +func formatDMLInfo(rows []common.RowLogInfo) string { + data, err := json.Marshal(rows) + if err != nil { + return "" + } + return string(data) } diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go index 35f74e75ad..93c04e2520 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -14,9 +14,7 @@ package kafka import ( - "context" "strconv" - "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -24,12 +22,10 @@ import ( ) type metricsHook struct { - metricsMu sync.RWMutex - metricsBound bool - keyspace string - changefeed string - clientType string - metrics metricVectors + keyspace string + changefeed string + clientType string + metrics metricVectors } type metricVectors struct { @@ -55,74 +51,45 @@ const ( legacyMetricP99 = "p99" ) -func newMetricsHook(clientType string) *metricsHook { - return &metricsHook{clientType: clientType} -} - -func (h *metricsHook) bindMetrics( +func newMetricsHook( keyspace string, changefeed string, + clientType string, metrics metricVectors, -) { - h.metricsMu.Lock() - defer h.metricsMu.Unlock() - - h.keyspace = keyspace - h.changefeed = changefeed - h.metrics = metrics - h.metricsBound = true -} - -func (h *metricsHook) loadMetrics() (string, string, metricVectors, bool) { - h.metricsMu.RLock() - defer h.metricsMu.RUnlock() - - return h.keyspace, h.changefeed, h.metrics, h.metricsBound -} - -func (h *metricsHook) Run(ctx context.Context) { - _, _, _, bound := h.loadMetrics() - - if !bound { - <-ctx.Done() - return +) *metricsHook { + return &metricsHook{ + keyspace: keyspace, + changefeed: changefeed, + clientType: clientType, + metrics: metrics, } - - <-ctx.Done() - h.cleanupMetrics() } func (h *metricsHook) cleanupMetrics() { - keyspace, changefeed, metrics, bound := h.loadMetrics() - - if !bound { - return - } - labels := prometheus.Labels{ - "namespace": keyspace, - "changefeed": changefeed, + "namespace": h.keyspace, + "changefeed": h.changefeed, "client": h.clientType, } - deleteGaugeVecPartialMatch(metrics.OutgoingByteRate, labels) - deleteGaugeVecPartialMatch(metrics.RequestRate, labels) - deleteGaugeVecPartialMatch(metrics.ResponseRate, labels) - deleteGaugeVecPartialMatch(metrics.RequestsInFlight, labels) - deleteHistogramVecPartialMatch(metrics.RequestLatency, labels) - deleteHistogramVecPartialMatch(metrics.CompressionRatio, labels) - deleteHistogramVecPartialMatch(metrics.RecordsPerRequest, labels) + deleteGaugeVecPartialMatch(h.metrics.OutgoingByteRate, labels) + deleteGaugeVecPartialMatch(h.metrics.RequestRate, labels) + deleteGaugeVecPartialMatch(h.metrics.ResponseRate, labels) + deleteGaugeVecPartialMatch(h.metrics.RequestsInFlight, labels) + deleteHistogramVecPartialMatch(h.metrics.RequestLatency, labels) + deleteHistogramVecPartialMatch(h.metrics.CompressionRatio, labels) + deleteHistogramVecPartialMatch(h.metrics.RecordsPerRequest, labels) legacyLabels := prometheus.Labels{ - "namespace": keyspace, - "changefeed": changefeed, + "namespace": h.keyspace, + "changefeed": h.changefeed, } - deleteGaugeVecPartialMatch(metrics.LegacyOutgoingByteRate, legacyLabels) - deleteGaugeVecPartialMatch(metrics.LegacyRequestRate, legacyLabels) - deleteGaugeVecPartialMatch(metrics.LegacyResponseRate, legacyLabels) - deleteGaugeVecPartialMatch(metrics.LegacyRequestsInFlight, legacyLabels) - deleteGaugeVecPartialMatch(metrics.LegacyRequestLatency, legacyLabels) - deleteGaugeVecPartialMatch(metrics.LegacyCompressionRatio, legacyLabels) - deleteGaugeVecPartialMatch(metrics.LegacyRecordsPerRequest, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.LegacyOutgoingByteRate, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.LegacyRequestRate, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.LegacyResponseRate, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.LegacyRequestsInFlight, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.LegacyRequestLatency, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.LegacyCompressionRatio, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.LegacyRecordsPerRequest, legacyLabels) } func (h *metricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { @@ -248,15 +215,11 @@ type metricsContext struct { } func (h *metricsHook) loadMetricsContext() (metricsContext, bool) { - keyspace, changefeed, metrics, bound := h.loadMetrics() - if !bound { - return metricsContext{}, false - } return metricsContext{ - keyspace: keyspace, - changefeed: changefeed, + keyspace: h.keyspace, + changefeed: h.changefeed, clientType: h.clientType, - metrics: metrics, + metrics: h.metrics, }, true } diff --git a/pkg/sink/kafka/metrics_hook_test.go b/pkg/sink/kafka/metrics_hook_test.go index 1bc498c1bd..addb5b2831 100644 --- a/pkg/sink/kafka/metrics_hook_test.go +++ b/pkg/sink/kafka/metrics_hook_test.go @@ -43,8 +43,7 @@ func TestMetricsHookRecordsLegacyMetricsAndCleanup(t *testing.T) { []string{"namespace", "changefeed", "type"}, ) - hook := newMetricsHook("async_producer") - hook.bindMetrics("default", "cf", metricVectors{ + hook := newMetricsHook("default", "cf", "async_producer", metricVectors{ LegacyOutgoingByteRate: outgoingByteRate, LegacyRequestRate: requestRate, LegacyRequestsInFlight: requestsInFlight, diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index 7425dc9530..9cb60cbcc0 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -22,7 +22,6 @@ import ( commonType "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" - "github.com/pingcap/ticdc/pkg/sink/kafka/internal/logutil" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/atomic" "go.uber.org/zap" @@ -148,7 +147,7 @@ func buildRecord(topic string, partition int32, message *common.Message) *kgo.Re func (p *kafkaSyncProducer) wrapSendError(message *common.Message, err error) error { if err != nil { - err = logutil.AnnotateEventError( + err = AnnotateEventError( p.id.Keyspace(), p.id.Name(), message.LogInfo, From b22e5727fe6237b56e3b11b68c088f1641831a52 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 3 Jul 2026 11:56:21 +0800 Subject: [PATCH 19/61] adjust the kafka compression case --- pkg/sink/kafka/client_options_test.go | 30 +++++++++++++++++ .../kafka_compression/data/gzip_data.sql | 21 ------------ .../kafka_compression/data/lz4_data.sql | 21 ------------ .../kafka_compression/data/snappy_data.sql | 21 ------------ .../kafka_compression/data/zstd_data.sql | 21 ------------ .../kafka_compression/run.sh | 32 ++++++++++++------- 6 files changed, 51 insertions(+), 95 deletions(-) delete mode 100644 tests/integration_tests/kafka_compression/data/gzip_data.sql delete mode 100644 tests/integration_tests/kafka_compression/data/lz4_data.sql delete mode 100644 tests/integration_tests/kafka_compression/data/snappy_data.sql delete mode 100644 tests/integration_tests/kafka_compression/data/zstd_data.sql diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index 8f328b0dc1..5b7a449a41 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -108,6 +108,36 @@ func TestNewProducerOptionsUsesProducerBatchMaxBytes(t *testing.T) { require.Equal(t, int32(producerBatchMaxBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) } +func TestNewCompressionOptionMapsToProducerBatchCompression(t *testing.T) { + t.Parallel() + + testCases := []struct { + compression string + expected kgo.CompressionCodec + }{ + {compression: "gzip", expected: kgo.GzipCompression()}, + {compression: "snappy", expected: kgo.SnappyCompression()}, + {compression: "lz4", expected: kgo.Lz4Compression()}, + {compression: "zstd", expected: kgo.ZstdCompression()}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.compression, func(t *testing.T) { + t.Parallel() + + client, err := kgo.NewClient( + kgo.SeedBrokers("127.0.0.1:9092"), + newCompressionOption(&clientOptions{Compression: tc.compression}), + ) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, []kgo.CompressionCodec{tc.expected}, client.OptValue(kgo.ProducerBatchCompression)) + }) + } +} + func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { t.Parallel() diff --git a/tests/integration_tests/kafka_compression/data/gzip_data.sql b/tests/integration_tests/kafka_compression/data/gzip_data.sql deleted file mode 100644 index f1c7671010..0000000000 --- a/tests/integration_tests/kafka_compression/data/gzip_data.sql +++ /dev/null @@ -1,21 +0,0 @@ -use test; - -create table tp_int_gzip -( - id int auto_increment, - c_tinyint tinyint null, - c_smallint smallint null, - c_mediumint mediumint null, - c_int int null, - c_bigint bigint null, - constraint pk - primary key (id) -); - -insert into tp_int_gzip(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) -values (1, 2, 3, 4, 5); - -create table gzip_finish_mark -( - id int PRIMARY KEY -); diff --git a/tests/integration_tests/kafka_compression/data/lz4_data.sql b/tests/integration_tests/kafka_compression/data/lz4_data.sql deleted file mode 100644 index 6f00b24faa..0000000000 --- a/tests/integration_tests/kafka_compression/data/lz4_data.sql +++ /dev/null @@ -1,21 +0,0 @@ -use test; - -create table tp_int_lz4 -( - id int auto_increment, - c_tinyint tinyint null, - c_smallint smallint null, - c_mediumint mediumint null, - c_int int null, - c_bigint bigint null, - constraint pk - primary key (id) -); - -insert into tp_int_lz4(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) -values (1, 2, 3, 4, 5); - -create table lz4_finish_mark -( - id int PRIMARY KEY -); diff --git a/tests/integration_tests/kafka_compression/data/snappy_data.sql b/tests/integration_tests/kafka_compression/data/snappy_data.sql deleted file mode 100644 index 435e9f6f7f..0000000000 --- a/tests/integration_tests/kafka_compression/data/snappy_data.sql +++ /dev/null @@ -1,21 +0,0 @@ -use test; - -create table tp_int_snappy -( - id int auto_increment, - c_tinyint tinyint null, - c_smallint smallint null, - c_mediumint mediumint null, - c_int int null, - c_bigint bigint null, - constraint pk - primary key (id) -); - -insert into tp_int_snappy(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) -values (1, 2, 3, 4, 5); - -create table snappy_finish_mark -( - id int PRIMARY KEY -); diff --git a/tests/integration_tests/kafka_compression/data/zstd_data.sql b/tests/integration_tests/kafka_compression/data/zstd_data.sql deleted file mode 100644 index 82b78dc6e4..0000000000 --- a/tests/integration_tests/kafka_compression/data/zstd_data.sql +++ /dev/null @@ -1,21 +0,0 @@ -use test; - -create table tp_int_zstd -( - id int auto_increment, - c_tinyint tinyint null, - c_smallint smallint null, - c_mediumint mediumint null, - c_int int null, - c_bigint bigint null, - constraint pk - primary key (id) -); - -insert into tp_int_zstd(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) -values (1, 2, 3, 4, 5); - -create table zstd_finish_mark -( - id int PRIMARY KEY -); diff --git a/tests/integration_tests/kafka_compression/run.sh b/tests/integration_tests/kafka_compression/run.sh index 4f241dc6dd..de9f6cd7e0 100755 --- a/tests/integration_tests/kafka_compression/run.sh +++ b/tests/integration_tests/kafka_compression/run.sh @@ -9,23 +9,33 @@ CDC_BINARY=cdc.test SINK_TYPE=$1 function test_compression() { + local compression=$1 + # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) - TOPIC_NAME="ticdc-kafka-compression-$1-test-$RANDOM" - SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true&kafka-version=${KAFKA_VERSION}&compression=$1" - cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" -c $1 + TOPIC_NAME="ticdc-kafka-compression-$compression-test-$RANDOM" + SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true&kafka-version=${KAFKA_VERSION}&compression=$compression" + cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" -c $compression run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&version=${KAFKA_VERSION}&enable-tidb-extension=true" - run_sql_file $CUR/data/$1_data.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} - if ! grep -q "Kafka producer uses $1 compression algorithm" "$WORK_DIR/cdc.log"; then - echo "can't find producer compression algorithm" - exit 1 - fi - check_table_exists test.$1_finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 + run_sql "CREATE TABLE test.tp_int_$compression ( + id INT AUTO_INCREMENT, + c_tinyint TINYINT NULL, + c_smallint SMALLINT NULL, + c_mediumint MEDIUMINT NULL, + c_int INT NULL, + c_bigint BIGINT NULL, + PRIMARY KEY (id) + );" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "INSERT INTO test.tp_int_$compression(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) + VALUES (1, 2, 3, 4, 5);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "CREATE TABLE test.${compression}_finish_mark (id INT PRIMARY KEY);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + + check_table_exists test.${compression}_finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - cdc_cli_changefeed pause -c $1 - cdc_cli_changefeed remove -c $1 + cdc_cli_changefeed pause -c $compression + cdc_cli_changefeed remove -c $compression } function run() { From d0e6cbcbe60009b6c35662d294e825adabe9ddc7 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 3 Jul 2026 15:29:05 +0800 Subject: [PATCH 20/61] fix a lot of code --- cmd/kafka-consumer/consumer.go | 3 +- .../dispatcher_orchestrator.go | 2 +- .../topicmanager/kafka_topic_manager_test.go | 2 +- maintainer/operator/operator_controller.go | 2 -- pkg/sink/kafka/admin_metrics.go | 4 +-- pkg/sink/kafka/async_producer_test.go | 7 ++--- pkg/sink/kafka/client_options.go | 5 +--- pkg/sink/kafka/client_options_test.go | 3 -- pkg/sink/kafka/gssapi.go | 4 +-- pkg/sink/kafka/kafka_factory_test.go | 1 - pkg/sink/kafka/metrics_hook.go | 4 +-- pkg/sink/kafka/options_test.go | 5 ++-- pkg/sink/kafka/sync_producer_test.go | 30 +++++++++---------- 13 files changed, 30 insertions(+), 42 deletions(-) diff --git a/cmd/kafka-consumer/consumer.go b/cmd/kafka-consumer/consumer.go index 4e78582f9e..8616296803 100644 --- a/cmd/kafka-consumer/consumer.go +++ b/cmd/kafka-consumer/consumer.go @@ -46,7 +46,8 @@ func getPartitionNum(o *option) (int32, error) { for i := 0; i <= 30; i++ { resp, err := admin.GetMetadata(&o.topic, false, timeout) if err != nil { - if err.(kafka.Error).Code() == kafka.ErrTransport { + var kafkaErr kafka.Error + if errors.As(err, &kafkaErr) && kafkaErr.Code() == kafka.ErrTransport { log.Info("retry get partition number", zap.Int("retryTime", i), zap.Int("timeout", timeout)) timeout += 100 continue diff --git a/downstreamadapter/dispatcherorchestrator/dispatcher_orchestrator.go b/downstreamadapter/dispatcherorchestrator/dispatcher_orchestrator.go index 04c41524ad..54b9df3b80 100644 --- a/downstreamadapter/dispatcherorchestrator/dispatcher_orchestrator.go +++ b/downstreamadapter/dispatcherorchestrator/dispatcher_orchestrator.go @@ -638,7 +638,7 @@ func retrieveOperatorsForBootstrapResponse( manager *dispatchermanager.DispatcherManager, response *heartbeatpb.MaintainerBootstrapResponse, ) { - manager.GetCurrentOperatorMap().Range(func(key, value any) bool { + manager.GetCurrentOperatorMap().Range(func(_, value any) bool { req := value.(dispatchermanager.SchedulerDispatcherRequest) dispatcherID := common.NewDispatcherIDFromPB(req.Config.DispatcherID) if common.IsRedoMode(req.Config.Mode) { diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index fb34b67bd1..db4143239e 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -15,11 +15,11 @@ package topicmanager import ( "context" - "errors" "testing" "github.com/golang/mock/gomock" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/stretchr/testify/require" ) diff --git a/maintainer/operator/operator_controller.go b/maintainer/operator/operator_controller.go index f876b41dd6..4691ff6c01 100644 --- a/maintainer/operator/operator_controller.go +++ b/maintainer/operator/operator_controller.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/maintainer/replica" "github.com/pingcap/ticdc/maintainer/span" - "github.com/pingcap/ticdc/maintainer/split" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/messaging" @@ -51,7 +50,6 @@ type Controller struct { messageCenter messaging.MessageCenter spanController *span.Controller nodeManager *watcher.NodeManager - splitter *split.Splitter // admissionMu serializes removing-mode quiesce with normal operator side effects. // A normal operator must hold the read side from its final allow check through diff --git a/pkg/sink/kafka/admin_metrics.go b/pkg/sink/kafka/admin_metrics.go index 8e44bb9302..1b4f763b66 100644 --- a/pkg/sink/kafka/admin_metrics.go +++ b/pkg/sink/kafka/admin_metrics.go @@ -57,8 +57,8 @@ func cleanupAdminMetrics(keyspace string, changefeed string) { "namespace": keyspace, "changefeed": changefeed, } - adminCallCount.MetricVec.DeletePartialMatch(labels) - adminCallLatency.MetricVec.DeletePartialMatch(labels) + adminCallCount.DeletePartialMatch(labels) + adminCallLatency.DeletePartialMatch(labels) } func observeAdminCall( diff --git a/pkg/sink/kafka/async_producer_test.go b/pkg/sink/kafka/async_producer_test.go index b58ec544c2..4f4d89aa0e 100644 --- a/pkg/sink/kafka/async_producer_test.go +++ b/pkg/sink/kafka/async_producer_test.go @@ -15,12 +15,11 @@ package kafka import ( "context" - stderrors "errors" "testing" "time" "github.com/pingcap/ticdc/pkg/common" - cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" "go.uber.org/atomic" @@ -31,7 +30,7 @@ func TestAsyncSendClosedProducer(t *testing.T) { err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) - require.ErrorIs(t, err, cerror.ErrKafkaProducerClosed) + require.ErrorIs(t, err, errors.ErrKafkaProducerClosed) } func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { @@ -40,7 +39,7 @@ func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { closed: atomic.NewBool(false), errCh: make(chan error, 1), } - producer.errCh <- stderrors.New("queued async error") + producer.errCh <- errors.New("queued async error") err := producer.AsyncRunCallback(context.Background()) diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index d86ef14ee6..faa1a14f32 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -63,10 +63,7 @@ const ( ) func maxTimeoutWithDefault(readTimeout, writeTimeout time.Duration) time.Duration { - timeout := readTimeout - if writeTimeout > timeout { - timeout = writeTimeout - } + timeout := max(readTimeout, writeTimeout) if timeout <= 0 { timeout = defaultRequestTimeout } diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index 5b7a449a41..f91678361c 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -38,7 +38,6 @@ func TestNewRequiredAcks(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() require.Equal(t, tc.expected, newRequiredAcks(&clientOptions{RequiredAcks: tc.requiredAcks})) @@ -64,7 +63,6 @@ func TestMaxTimeoutWithDefault(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() require.Equal(t, tc.expected, maxTimeoutWithDefault(tc.readTimeout, tc.writeTimeout)) @@ -122,7 +120,6 @@ func TestNewCompressionOptionMapsToProducerBatchCompression(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.compression, func(t *testing.T) { t.Parallel() diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/gssapi.go index 00838b9302..29b47d4c70 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -259,12 +259,12 @@ func newKrb5Token( return nil, errors.Trace(err) } - prefix := make([]byte, 2) - binary.BigEndian.PutUint16(prefix, tokIDKrbAPReq) body, err := apReq.Marshal() if err != nil { return nil, errors.Trace(err) } + prefix := make([]byte, 2, 2+len(body)) + binary.BigEndian.PutUint16(prefix, tokIDKrbAPReq) return append(prefix, body...), nil } diff --git a/pkg/sink/kafka/kafka_factory_test.go b/pkg/sink/kafka/kafka_factory_test.go index 75c0924040..a5a23924dc 100644 --- a/pkg/sink/kafka/kafka_factory_test.go +++ b/pkg/sink/kafka/kafka_factory_test.go @@ -39,7 +39,6 @@ func TestNewKafkaOptionsMapsRequiredAcks(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go index 93c04e2520..fbed2a8d8f 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -225,12 +225,12 @@ func (h *metricsHook) loadMetricsContext() (metricsContext, bool) { func deleteGaugeVecPartialMatch(gaugeVec *prometheus.GaugeVec, labels prometheus.Labels) { if gaugeVec != nil { - gaugeVec.MetricVec.DeletePartialMatch(labels) + gaugeVec.DeletePartialMatch(labels) } } func deleteHistogramVecPartialMatch(histogramVec *prometheus.HistogramVec, labels prometheus.Labels) { if histogramVec != nil { - histogramVec.MetricVec.DeletePartialMatch(labels) + histogramVec.DeletePartialMatch(labels) } } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 4eba75bcb9..c3cba1edfc 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -15,7 +15,6 @@ package kafka import ( "context" - stdErrors "errors" "fmt" "net/url" "strconv" @@ -133,11 +132,11 @@ func (f *kafkaAdminFixture) getTopicConfig(topicName string, configName string) func (f *kafkaAdminFixture) createTopic(detail *TopicDetail, _ bool) error { if detail.ReplicationFactor > mockClusterReplicationFactor { - return stdErrors.New("invalid replication factor") + return errors.New("invalid replication factor") } if _, ok := f.brokerConfig[MinInsyncReplicasConfigName]; !ok && detail.ReplicationFactor != mockClusterReplicationFactor { - return stdErrors.New("policy violation") + return errors.New("policy violation") } f.topics[detail.Name] = *detail return nil diff --git a/pkg/sink/kafka/sync_producer_test.go b/pkg/sink/kafka/sync_producer_test.go index 0c5d05d99f..47dbdd79e9 100644 --- a/pkg/sink/kafka/sync_producer_test.go +++ b/pkg/sink/kafka/sync_producer_test.go @@ -14,12 +14,11 @@ package kafka import ( - stderrors "errors" "testing" "github.com/pingcap/ticdc/pkg/common" - cerror "github.com/pingcap/ticdc/pkg/errors" - codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/errors" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" "go.uber.org/atomic" ) @@ -27,15 +26,15 @@ import ( func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { producer := &kafkaSyncProducer{closed: atomic.NewBool(true)} - err := producer.SendMessage("topic", 1, &codeccommon.Message{}) - require.ErrorIs(t, err, cerror.ErrKafkaProducerClosed) + err := producer.SendMessage("topic", 1, &codecCommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaProducerClosed) - err = producer.SendMessages("topic", 1, &codeccommon.Message{}) - require.ErrorIs(t, err, cerror.ErrKafkaProducerClosed) + err = producer.SendMessages("topic", 1, &codecCommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaProducerClosed) } func TestBuildRecord(t *testing.T) { - message := &codeccommon.Message{ + message := &codecCommon.Message{ Key: []byte("key"), Value: []byte("value"), } @@ -55,13 +54,13 @@ func TestSyncProducerWrapSendErrorAnnotatesEventContext(t *testing.T) { testCases := []struct { name string - message *codeccommon.Message + message *codecCommon.Message contains []string }{ { name: "ddl", - message: &codeccommon.Message{LogInfo: &codeccommon.MessageLogInfo{ - DDL: &codeccommon.DDLLogInfo{ + message: &codecCommon.Message{LogInfo: &codecCommon.MessageLogInfo{ + DDL: &codecCommon.DDLLogInfo{ Query: "create table t(id int primary key)", StartTs: 10, CommitTs: 20, @@ -78,8 +77,8 @@ func TestSyncProducerWrapSendErrorAnnotatesEventContext(t *testing.T) { }, { name: "checkpoint", - message: &codeccommon.Message{LogInfo: &codeccommon.MessageLogInfo{ - Checkpoint: &codeccommon.CheckpointLogInfo{CommitTs: 30}, + message: &codecCommon.Message{LogInfo: &codecCommon.MessageLogInfo{ + Checkpoint: &codecCommon.CheckpointLogInfo{CommitTs: 30}, }}, contains: []string{ "keyspace=default", @@ -91,10 +90,9 @@ func TestSyncProducerWrapSendErrorAnnotatesEventContext(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { - err := producer.wrapSendError(tc.message, stderrors.New("send failed")) - require.ErrorIs(t, err, cerror.ErrKafkaSendMessage) + err := producer.wrapSendError(tc.message, errors.New("send failed")) + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) require.ErrorContains(t, err, "send failed") for _, expected := range tc.contains { require.ErrorContains(t, err, expected) From 2d559bc467ed3222cc30fac1b791cd9d32aeef03 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 3 Jul 2026 15:33:34 +0800 Subject: [PATCH 21/61] add golangci --- .golangci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 4b8ae72311..3f2b324652 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -81,12 +81,13 @@ linters: - name: unreachable-code - name: unused-parameter - name: var-declaration - - name: var-naming # G104: Audit errors not checked (duplicates errcheck). + # G115: Integer conversion overflow warnings are too noisy for bounded protocol values. gosec: excludes: - G104 + - G115 # ST1000: don't require package comments. # ST1003: don't enforce naming conventions on legacy identifiers. From 32190aa95af25fb3dd04cfa035455d9a2b889c7e Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 18:39:32 +0800 Subject: [PATCH 22/61] remove design doc --- docs/design/2026-01-09-franz-go-kafka-sink.md | 279 ------------------ 1 file changed, 279 deletions(-) delete mode 100644 docs/design/2026-01-09-franz-go-kafka-sink.md diff --git a/docs/design/2026-01-09-franz-go-kafka-sink.md b/docs/design/2026-01-09-franz-go-kafka-sink.md deleted file mode 100644 index 6fc01aa0bf..0000000000 --- a/docs/design/2026-01-09-franz-go-kafka-sink.md +++ /dev/null @@ -1,279 +0,0 @@ -# Kafka Sink 基于 franz-go 的实现设计与实施计划 - -## Status - -- Status: Proposed -- Date: 2026-01-09 -- Owner: TiCDC Team - -## Background / Context - -TiCDC 的 Kafka sink 负责把上游产生的 DML/DDL/checkpoint 事件编码并写入 Kafka，写入成功后通过 callback 将“已落盘”信息回传给上游推进进度。当前实现分为两层： - -- 使用层：`downstreamadapter/sink/kafka`（事件路由、编码流水线、发送调度） -- 客户端抽象层：`pkg/sink/kafka`（Factory、AdminClient、AsyncProducer、SyncProducer 以及 metrics 采集） - -`pkg/sink/kafka` 当前以 Sarama 为主要实现（`sarama_factory.go` / `sarama_*_producer.go` / `admin.go` / `sarama_config.go`）。本设计的目标是引入并基于 `github.com/twmb/franz-go`（`kgo` + `kadm`）实现等价的 Kafka 客户端层，使 `downstreamadapter/sink/kafka` 的使用方式尽可能不变，并支持渐进式切换与回滚。 - -## Problem Statement - -在不破坏既有 Kafka sink 行为与配置的前提下，引入 franz-go 作为 Kafka client 实现，满足： - -- 兼容现有 sink-uri 参数（topic、partition-num、required-acks、compression、TLS、SASL 等） -- 兼容现有模块边界与接口：`pkg/sink/kafka/factory.go`、`pkg/sink/kafka/cluster_admin_client.go` -- 可灰度、可回滚（支持 Sarama 与 franz-go 并存，按配置选择） -- 性能与稳定性不劣于现有实现，并为后续优化留出空间 - -## Goals / Non-Goals - -### Goals - -- 在 `pkg/sink/kafka` 内新增 franz-go 实现：Factory、AdminClient、AsyncProducer、SyncProducer。 -- `downstreamadapter/sink/kafka` 仅做最小化改动（最好只改 factory 选择逻辑）。 -- 覆盖安全连接能力：TLS、SASL PLAIN / SCRAM / OAuth（与现有选项对齐）。 -- 错误语义与诊断信息对齐：保留 `pkg/sink/kafka/logutil.go: AnnotateEventError(...)` 的日志上下文能力。 -- 支持渐进式验证：单元测试 + 复用现有集成测试（通过切换参数跑两套）。 - -### Non-Goals - -- 不修改事件编码协议与路由语义（`downstreamadapter/sink/eventrouter`、`pkg/sink/codec` 不在本设计范围）。 -- 不实现 Kafka consumer 能力（仅生产端与 admin 能力）。 -- 不在第一阶段追求 metrics 完全等价（可先保证功能正确，再补齐指标采集）。 - -## Current State (as-is) - -### 关键接口与调用路径 - -- `pkg/sink/kafka/factory.go: type Factory`：为上层提供 - - `AdminClient(ctx)` - - `AsyncProducer(ctx)`（DML） - - `SyncProducer(ctx)`（DDL/checkpoint） - - `MetricsCollector(adminClient)` -- `downstreamadapter/sink/kafka/helper.go: newKafkaSinkComponent(...)`：默认使用 `kafka.NewSaramaFactory` -- `downstreamadapter/sink/kafka/sink.go`： - - DML：编码后调用 `AsyncProducer.AsyncSend(ctx, topic, partition, message)`，并在 `AsyncRunCallback` 中消费 ack/error - - DDL/checkpoint：调用 `SyncProducer.SendMessage/SendMessages` - - 心跳：每 5s 调用一次 `Producer.Heartbeat()`（DML 与 DDL 分别一个 ticker） -- Topic 管理依赖 admin：`downstreamadapter/sink/topicmanager/kafka_topic_manager.go` -- 配置自适应：`pkg/sink/kafka/options.go: adjustOptions(...)` 通过 `ClusterAdminClient` 读取 topic/broker 配置并调整 `MaxMessageBytes`、`PartitionNum`、`KeepConnAliveInterval` 等 - -## Proposed Design (to-be) - -### 总体架构 - -保持 `downstreamadapter/sink/kafka` 逻辑基本不变，仅将 `pkg/sink/kafka` 的 Sarama 实现扩展为“多实现可选”： - -``` -downstreamadapter/sink/kafka - └─ uses pkg/sink/kafka.Factory - ├─ Sarama (existing): saramaFactory / saramaAdminClient / sarama{Async,Sync}Producer - └─ Franz (new): franzFactory / franzAdminClient / franz{Async,Sync}Producer -``` - -### 组件与职责 - -#### 1) `franzFactory`（新增） - -- 文件建议：`pkg/sink/kafka/franz_factory.go` -- 责任： - - 从 `options` 构造 `kgo.Opt` 集合（seed brokers、TLS、SASL、超时、ack、压缩、producer 行为等） - - 复用现有自适应逻辑：创建临时 admin client → 调用 `pkg/sink/kafka/options.go: adjustOptions(...)` → 关闭临时 admin → 保存调整后的 `options` - - 提供 `AdminClient/AsyncProducer/SyncProducer/MetricsCollector` 的 franz-go 实现 - -#### 2) `franzAdminClient`（新增） - -- 文件建议：`pkg/sink/kafka/franz_admin_client.go` -- 内部使用： - - `kgo.Client`（底层连接与请求） - - `kadm.Client`（admin API 封装） -- 需要实现 `pkg/sink/kafka/cluster_admin_client.go: ClusterAdminClient`： - - `GetAllBrokers()`：`kadm.Client.ListBrokers(ctx)` 或 `BrokerMetadata(ctx)` 解析 broker id - - `GetTopicsMeta(...)` / `GetTopicsPartitionsNum(...)`：`kadm.Client.Metadata(ctx, topics...)` - - `CreateTopic(...)`：`kadm.Client.CreateTopics(ctx, partitions, rf, configs, topic)`；对 “topic already exists” 做兼容性忽略 - - `GetBrokerConfig(...)`：`kadm.Client.BrokerMetadata(ctx)` 获取 controller id，再 `DescribeBrokerConfigs(ctx, controllerID)` - - `GetTopicConfig(...)`：`kadm.Client.DescribeTopicConfigs(ctx, topic)` - - `Heartbeat()`：可实现为 no-op，依赖 `kgo` 的自动重连与 producer 的重试能力；必要时再引入 `Ping`（短超时）的实现以辅助排障 - -#### 3) `franzAsyncProducer`（新增，DML） - -- 文件建议：`pkg/sink/kafka/franz_async_producer.go` -- 对齐上层语义： - - `AsyncSend(ctx, topic, partition, message)`：调用 `kgo.Client.Produce`，record 的 `Topic/Partition/Key/Value` 来自现有路由与编码结果 - - `AsyncRunCallback(ctx)`：阻塞等待第一条 produce error 或 ctx.Done；对齐 Sarama 行为（发生错误导致 sink 退出重建） - - `message.Callback`：在 produce 回调成功时执行（与 Sarama 成功通道消费一致） - - 错误：立刻在边界处包装为带 stack 的错误，并通过 `AnnotateEventError(...)` 附带 message 的 `LogInfo` - - `Heartbeat()`：可实现为 no-op；通过 `kgo.RecordRetries` 在网络抖动、连接被 broker 关闭等场景下提升鲁棒性 - -#### 4) `franzSyncProducer`（新增，DDL/checkpoint） - -- 文件建议：`pkg/sink/kafka/franz_sync_producer.go` -- 对齐上层语义： - - `SendMessage`：构造 1 条 record，`ProduceSync` 并返回错误 - - `SendMessages`：按 partitionNum 构造 N 条 record（与当前逻辑一致），`ProduceSync` 等待全部返回，聚合错误 - - `Heartbeat()`：可实现为 no-op - -#### 5) 选择机制（灰度） - -建议增加一个可选 sink-uri 参数来选择 Kafka client 实现，默认保持 Sarama： - -- 新增参数：`kafka-client=sarama|franz`（默认 sarama） -- 影响范围： - - `pkg/sink/kafka/options.go: urlConfig` 增加字段 - - `downstreamadapter/sink/kafka/helper.go: newKafkaSinkComponent(...)` 根据 options 选择 `kafka.NewSaramaFactory` 或 `kafka.NewFranzFactory` - -该机制允许： - -- CI/集成测试中在不改代码的情况下切换实现 -- 线上灰度（按 changefeed 配置逐个切换） -- 快速回滚（改回 sarama） - -## Detailed Design - -### 配置映射（options → franz-go） - -建议以“对齐现有行为”为优先原则，主要映射如下（示例为概念性描述，具体以实现为准）： - -- Brokers：`options.BrokerEndpoints` → `kgo.SeedBrokers(...)` -- ClientID：`options.ClientID` → `kgo.ClientID(...)` -- Dial timeout：`options.DialTimeout` → `kgo.DialTimeout(...)` -- TLS： - - `options.EnableTLS` / `options.Credential` / `options.InsecureSkipVerify` - - → 构造 `tls.Config` 后 `kgo.DialTLSConfig(tlsConf)` -- SASL（注意能力差异）： - - PLAIN / SCRAM：`kgo.SASL(...)`（基于 `github.com/twmb/franz-go/pkg/sasl/plain`、`.../scram`） - - OAuth：基于 `github.com/twmb/franz-go/pkg/sasl/oauth`，把现有 token provider 适配为 franz-go 的 oauth provider - - GSSAPI：franz-go 默认包不提供现成实现（当前 `pkg/sink/kafka/sarama_config.go: completeSaramaSASLConfig(...)` 支持）。第一阶段建议：若检测到 `sasl-mechanism=GSSAPI` 则强制走 Sarama，或直接返回“暂不支持”的显式错误。 -- RequiredAcks：`options.RequiredAcks` → - - `WaitForAll` → `kgo.RequiredAcks(kgo.AllISRAcks())` - - `WaitForLocal` → `kgo.RequiredAcks(kgo.LeaderAck())` - - `NoResponse` → `kgo.RequiredAcks(kgo.NoAck())` -- Compression：`options.Compression` → `kgo.ProducerBatchCompression(...)`（`SnappyCompression/GzipCompression/Lz4Compression/ZstdCompression/NoCompression`） -- MaxMessageBytes：`options.MaxMessageBytes` → `kgo.ProducerBatchMaxBytes(int32(...))` - -### Producer 行为对齐（重试、顺序、幂等） - -Sarama 现状（见 `pkg/sink/kafka/sarama_config.go`）： - -- DML async：`Producer.Retry.Max = 0`，`Net.MaxOpenRequests = 1`（偏向“顺序安全 + fail fast”） -- DDL/checkpoint sync：`Producer.Retry.Max = 3`（偏向“关键控制面更稳健”） - -franz-go 默认行为差异较大（默认 recordRetries 近似无限、默认开启幂等写），因此需要显式对齐： - -- DML async（建议第一阶段）： - - `kgo.DisableIdempotentWrite()`：避免引入 `IDEMPOTENT_WRITE` ACL 依赖，保持与 Sarama 默认一致 - - `kgo.MaxProduceRequestsInflightPerBroker(1)`：对齐顺序与可预期性 - - `kgo.RecordRetries(N)`：设置一个合理重试次数以提升鲁棒性（例如 N=3 或 5），并依赖 franz-go 的“gapless ordering”语义避免在单分区内越过失败记录继续成功写入 - - `kgo.ProduceRequestTimeout(...)`：与现有 `options.WriteTimeout/ReadTimeout` 对齐，避免重试导致长时间阻塞 - - `kgo.ProducerLinger(0)`：对齐“尽快 flush” -- DDL/checkpoint sync： - - 可采用更保守的重试策略（例如 `RecordRetries(5)`），以提升控制面事件（DDL/checkpoint）的成功率 - - 需要用内部超时兜底，避免在 `SyncProducer` 接口缺少 ctx 的情况下无限阻塞 - -后续如需提升吞吐，可在不影响语义的前提下评估： - -- 允许更大的 in-flight（可能导致乱序） -- 打开幂等写（需评估权限、配额与 broker 版本） -- 适度增加 linger（吞吐上升，延迟增加） - -### 错误处理与诊断信息 - -边界层（franz-go → TiCDC）要做到： - -- franz-go / kadm 返回的错误属于第三方错误：在最接近发生点立即用 TiCDC 的 errors 包装以获得 stack trace -- 附带事件上下文：使用 `pkg/sink/kafka/logutil.go: AnnotateEventError(...)` 把 `MessageLogInfo` 拼入错误，便于定位是哪类事件（dml/ddl/checkpoint）以及表信息、ts 等 -- 上层 caller 对已包装错误不再重复 wrap（减少噪音与重复堆栈） - -### Close 语义与资源管理 - -Sarama 版本中每个 producer/admin 都持有独立 client；close 顺序也写入了注释（先关 client 再关 producer，避免阻塞 flush）。franz-go 可以选择两种实现方式： - -1) **与现状一致：每个组件一个 kgo.Client**（实现简单、行为可控，代价是连接数略多） -2) **同一个 factory 共享一个 kgo.Client**（连接更少、资源更省，但需要引用计数与更严格的 close 协议） - -第一阶段建议采用方案 (1)，降低引入风险；后续可在确认稳定后再做共享优化。 - -## Performance Considerations - -franz-go 的优势通常来自： - -- 更紧凑的编码与更少的反射/分配 -- 统一 client 能力（produce/admin/consume 一套基础设施） -- 可通过 hooks/telemetry 获取更丰富的请求级信息 - -但在 TiCDC Kafka sink 场景，真正的性能瓶颈往往在“上层编码与调度”，并非单纯 client 库。引入 franz-go 后仍需重点关注： - -- `downstreamadapter/sink/kafka` 的无限队列与 per-row 分配（不在本设计范围，但可在后续优化） -- Producer 参数对吞吐/延迟/乱序的权衡（linger、batch、in-flight、retries） -- 若 franz 实现的 `Heartbeat()` 为 no-op，可考虑后续把上层 5s ticker 变为按需或配置化，减少无效调用 - -## Testing Strategy - -### Unit Tests - -- 配置映射测试：给定 `options`，断言构造出的 franz-go 配置与预期一致（acks/compression/TLS/SASL 等）。 -- admin wrapper 行为测试：对 `GetTopicsMeta/GetTopicConfig/GetBrokerConfig/CreateTopic` 的错误处理、已存在 topic 的兼容性处理等。 - -### Integration / E2E - -复用现有 Kafka 集成测试，通过 sink-uri 参数切换实现： - -- 现有测试用例（示例）： - - `tests/integration_tests/kafka_log_info/run.sh`（依赖 failpoint 注入错误与日志上下文） - - `tests/integration_tests/mq_sink_error_resume/run.sh`（错误恢复） -- 新增运行方式： - - 在 sink-uri 增加 `kafka-client=franz`，并确保 failpoint 名称在 franz 实现中兼容（或新增等价 failpoint） - -### 性能回归 - -- A/B 对比：同一 workload 下对比 Sarama 与 franz-go 的吞吐、端到端延迟、CPU、内存、Kafka 请求数量。 -- 关注场景：高并发 DML、批量 DDL、checkpoint 广播、topic 自动创建/metadata 刷新。 - -## Observability / Operations - -- 日志：错误日志必须包含 changefeed 维度（keyspace/changefeed）和事件上下文（eventType/table/ts），但避免在日志文本中拼接函数名与多余格式噪音。 -- Metrics（阶段性计划）： - - 第一阶段：可先保持 `MetricsCollector` 为 no-op（功能优先） - - 第二阶段：基于 franz-go hooks 或 client telemetry 把关键指标接入现有 Prometheus 指标体系（例如 request latency、in-flight、吞吐等） - -## Rollout Plan - -1) **实现与编译通过** - - 新增 `kafka-client=franz` 选项，默认仍为 sarama - - 引入 `NewFranzFactory` 与相关实现文件 -2) **功能验证** - - 单元测试覆盖关键映射与错误处理 - - 本地/CI 跑现有 Kafka 集成测试，分别用 sarama 与 franz-go 跑一遍 -3) **灰度** - - 选取少量 changefeed 开启 franz-go - - 对比关键指标与故障率 -4) **扩大与默认切换** - - 确认稳定后逐步扩大覆盖面 - - 视情况将默认实现切换为 franz-go，并保留 sarama 回滚窗口 - -## Alternatives Considered - -- 继续使用 Sarama：稳定但维护与性能空间受限，且部分行为（如 metadata/连接管理）需要更多定制补丁。 -- 其他 Go Kafka client（如 kafka-go）：API/语义与现有实现差异较大，迁移成本与回归风险更高。 - -## Open Questions / Future Work - -- SASL GSSAPI（Kerberos）在 franz-go 体系下的实现方案（自定义 sasl.Mechanism vs 继续走 Sarama）。 -- franz-go metrics / hooks 与现有 `pkg/sink/kafka/metrics_collector.go` 指标体系的对齐方案与成本评估。 -- 是否要在 factory 内共享 `kgo.Client`（资源更省）以及如何保证 close 语义与并发安全。 - -## References - -- franz-go：`github.com/twmb/franz-go`（核心 `pkg/kgo`） -- kadm：admin 封装 `github.com/twmb/franz-go/pkg/kadm` -- 现有 TiCDC Kafka sink： - - `downstreamadapter/sink/kafka/helper.go` - - `downstreamadapter/sink/kafka/sink.go` - - `pkg/sink/kafka/factory.go` - - `pkg/sink/kafka/cluster_admin_client.go` - - `pkg/sink/kafka/options.go` - - `pkg/sink/kafka/sarama_factory.go` - - `pkg/sink/kafka/sarama_config.go` - - `pkg/sink/kafka/admin.go` - - `pkg/sink/kafka/sarama_async_producer.go` - - `pkg/sink/kafka/sarama_sync_producer.go` - - `pkg/sink/kafka/logutil.go` From 28de04939bd10a866435968195d64dc928eeebe3 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 18:46:19 +0800 Subject: [PATCH 23/61] adjust the code --- pkg/sink/kafka/factory_selector.go | 29 ----------------------------- pkg/sink/kafka/kafka_factory.go | 16 ++++++++-------- 2 files changed, 8 insertions(+), 37 deletions(-) delete mode 100644 pkg/sink/kafka/factory_selector.go diff --git a/pkg/sink/kafka/factory_selector.go b/pkg/sink/kafka/factory_selector.go deleted file mode 100644 index d8f12688d4..0000000000 --- a/pkg/sink/kafka/factory_selector.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - - "github.com/pingcap/ticdc/pkg/common" -) - -// NewFactory creates the Kafka client factory. -func NewFactory( - ctx context.Context, - o *options, - changefeedID common.ChangeFeedID, -) (Factory, error) { - return NewKafkaFactory(ctx, o, changefeedID) -} diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go index 5e5a48e52c..889d252e45 100644 --- a/pkg/sink/kafka/kafka_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -26,7 +26,7 @@ const ( clientTypeAdminClient = "admin_client" ) -type kafkaFactory struct { +type factory struct { changefeedID common.ChangeFeedID option *options @@ -75,8 +75,8 @@ func newKafkaMetricsHook(changefeedID common.ChangeFeedID, clientType string) *m ) } -// NewKafkaFactory constructs a Factory. -func NewKafkaFactory( +// NewFactory constructs a Factory. +func NewFactory( ctx context.Context, o *options, changefeedID common.ChangeFeedID, @@ -91,7 +91,7 @@ func NewKafkaFactory( return nil, errors.Trace(err) } - return &kafkaFactory{ + return &factory{ changefeedID: changefeedID, option: o, asyncMetricsHook: newKafkaMetricsHook(changefeedID, clientTypeAsyncProducer), @@ -100,7 +100,7 @@ func NewKafkaFactory( }, nil } -func (f *kafkaFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { +func (f *factory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { admin, err := newAdminClient(ctx, f.changefeedID, newKafkaOptions(f.option), f.adminMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) @@ -108,7 +108,7 @@ func (f *kafkaFactory) AdminClient(ctx context.Context) (ClusterAdminClient, err return admin, nil } -func (f *kafkaFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { +func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { producer, err := newSyncProducer(ctx, f.changefeedID, newKafkaOptions(f.option), f.syncMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) @@ -116,7 +116,7 @@ func (f *kafkaFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { return producer, nil } -func (f *kafkaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { +func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { producer, err := newAsyncProducer(ctx, f.changefeedID, newKafkaOptions(f.option), f.asyncMetricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) @@ -124,7 +124,7 @@ func (f *kafkaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) return producer, nil } -func (f *kafkaFactory) MetricsCollector() MetricsCollector { +func (f *factory) MetricsCollector() MetricsCollector { return &kafkaMetricsCollector{changefeedID: f.changefeedID, hooks: []*metricsHook{ f.asyncMetricsHook, f.syncMetricsHook, From adc37423a5c088c18e9901b78d50d8c2ad34b72b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 18:54:09 +0800 Subject: [PATCH 24/61] simplify the metrics hook --- pkg/sink/kafka/kafka_factory.go | 37 ++++-------- pkg/sink/kafka/metrics.go | 59 ------------------ pkg/sink/kafka/metrics_hook.go | 93 ++++++----------------------- pkg/sink/kafka/metrics_hook_test.go | 41 ++++++++++--- 4 files changed, 60 insertions(+), 170 deletions(-) diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go index 889d252e45..c00d2e191d 100644 --- a/pkg/sink/kafka/kafka_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -20,12 +20,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" ) -const ( - clientTypeAsyncProducer = "async_producer" - clientTypeSyncProducer = "sync_producer" - clientTypeAdminClient = "admin_client" -) - type factory struct { changefeedID common.ChangeFeedID option *options @@ -50,27 +44,18 @@ func (c *kafkaMetricsCollector) Run(ctx context.Context) { cleanupAdminMetrics(c.changefeedID.Keyspace(), c.changefeedID.Name()) } -func newKafkaMetricsHook(changefeedID common.ChangeFeedID, clientType string) *metricsHook { +func newKafkaMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { return newMetricsHook( changefeedID.Keyspace(), changefeedID.Name(), - clientType, metricVectors{ - RequestsInFlight: kafkaClientRequestsInFlightGauge, - OutgoingByteRate: kafkaClientOutgoingByteTotalGauge, - RequestRate: kafkaClientRequestTotalGauge, - RequestLatency: kafkaClientRequestLatencyHistogram, - ResponseRate: kafkaClientResponseTotalGauge, - CompressionRatio: kafkaClientCompressionRatioHistogram, - RecordsPerRequest: kafkaClientRecordsPerRequestHistogram, - - LegacyRequestsInFlight: requestsInFlightGauge, - LegacyOutgoingByteRate: OutgoingByteRateGauge, - LegacyRequestRate: RequestRateGauge, - LegacyRequestLatency: RequestLatencyGauge, - LegacyResponseRate: responseRateGauge, - LegacyCompressionRatio: compressionRatioGauge, - LegacyRecordsPerRequest: recordsPerRequestGauge, + RequestsInFlight: requestsInFlightGauge, + OutgoingByteRate: OutgoingByteRateGauge, + RequestRate: RequestRateGauge, + RequestLatency: RequestLatencyGauge, + ResponseRate: responseRateGauge, + CompressionRatio: compressionRatioGauge, + RecordsPerRequest: recordsPerRequestGauge, }, ) } @@ -94,9 +79,9 @@ func NewFactory( return &factory{ changefeedID: changefeedID, option: o, - asyncMetricsHook: newKafkaMetricsHook(changefeedID, clientTypeAsyncProducer), - syncMetricsHook: newKafkaMetricsHook(changefeedID, clientTypeSyncProducer), - adminMetricsHook: newKafkaMetricsHook(changefeedID, clientTypeAdminClient), + asyncMetricsHook: newKafkaMetricsHook(changefeedID), + syncMetricsHook: newKafkaMetricsHook(changefeedID), + adminMetricsHook: newKafkaMetricsHook(changefeedID), }, nil } diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index 545a3bb798..6d33c678c5 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -78,57 +78,6 @@ var ( Name: "kafka_producer_response_rate", Help: "Responses/second received from all brokers.", }, []string{"namespace", "changefeed", "broker"}) - - kafkaClientRequestsInFlightGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_producer_in_flight_requests", - Help: "Current number of in-flight requests by client type and broker.", - }, []string{"namespace", "changefeed", "client", "broker"}) - kafkaClientOutgoingByteTotalGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_producer_outgoing_byte_total", - Help: "Total bytes written by kafka sink clients.", - }, []string{"namespace", "changefeed", "client", "broker"}) - kafkaClientRequestTotalGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_producer_request_total", - Help: "Total requests sent by kafka sink clients.", - }, []string{"namespace", "changefeed", "client", "broker"}) - kafkaClientResponseTotalGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_producer_response_total", - Help: "Total responses received by kafka sink clients.", - }, []string{"namespace", "changefeed", "client", "broker"}) - - kafkaClientRequestLatencyHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_producer_request_latency_histogram", - Help: "Request latency histogram for kafka producer in milliseconds.", - }, []string{"namespace", "changefeed", "client", "broker"}) - kafkaClientCompressionRatioHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_producer_compression_ratio_histogram", - Help: "Compression ratio times 100 histogram for kafka producer.", - }, []string{"namespace", "changefeed", "client"}) - kafkaClientRecordsPerRequestHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_producer_records_per_request_histogram", - Help: "Records per request histogram for kafka producer.", - }, []string{"namespace", "changefeed", "client"}) ) // InitMetrics registers all metrics in this file. @@ -141,14 +90,6 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(requestsInFlightGauge) registry.MustRegister(responseRateGauge) - registry.MustRegister(kafkaClientRequestsInFlightGauge) - registry.MustRegister(kafkaClientOutgoingByteTotalGauge) - registry.MustRegister(kafkaClientRequestTotalGauge) - registry.MustRegister(kafkaClientResponseTotalGauge) - registry.MustRegister(kafkaClientRequestLatencyHistogram) - registry.MustRegister(kafkaClientCompressionRatioHistogram) - registry.MustRegister(kafkaClientRecordsPerRequestHistogram) - initAdminMetrics(registry) claimcheck.InitMetrics(registry) codec.InitMetrics(registry) diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go index fbed2a8d8f..9653bef9ae 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -24,7 +24,6 @@ import ( type metricsHook struct { keyspace string changefeed string - clientType string metrics metricVectors } @@ -32,18 +31,10 @@ type metricVectors struct { RequestsInFlight *prometheus.GaugeVec OutgoingByteRate *prometheus.GaugeVec RequestRate *prometheus.GaugeVec - RequestLatency *prometheus.HistogramVec + RequestLatency *prometheus.GaugeVec ResponseRate *prometheus.GaugeVec - CompressionRatio *prometheus.HistogramVec - RecordsPerRequest *prometheus.HistogramVec - - LegacyRequestsInFlight *prometheus.GaugeVec - LegacyOutgoingByteRate *prometheus.GaugeVec - LegacyRequestRate *prometheus.GaugeVec - LegacyRequestLatency *prometheus.GaugeVec - LegacyResponseRate *prometheus.GaugeVec - LegacyCompressionRatio *prometheus.GaugeVec - LegacyRecordsPerRequest *prometheus.GaugeVec + CompressionRatio *prometheus.GaugeVec + RecordsPerRequest *prometheus.GaugeVec } const ( @@ -54,13 +45,11 @@ const ( func newMetricsHook( keyspace string, changefeed string, - clientType string, metrics metricVectors, ) *metricsHook { return &metricsHook{ keyspace: keyspace, changefeed: changefeed, - clientType: clientType, metrics: metrics, } } @@ -69,27 +58,14 @@ func (h *metricsHook) cleanupMetrics() { labels := prometheus.Labels{ "namespace": h.keyspace, "changefeed": h.changefeed, - "client": h.clientType, } deleteGaugeVecPartialMatch(h.metrics.OutgoingByteRate, labels) deleteGaugeVecPartialMatch(h.metrics.RequestRate, labels) deleteGaugeVecPartialMatch(h.metrics.ResponseRate, labels) deleteGaugeVecPartialMatch(h.metrics.RequestsInFlight, labels) - deleteHistogramVecPartialMatch(h.metrics.RequestLatency, labels) - deleteHistogramVecPartialMatch(h.metrics.CompressionRatio, labels) - deleteHistogramVecPartialMatch(h.metrics.RecordsPerRequest, labels) - - legacyLabels := prometheus.Labels{ - "namespace": h.keyspace, - "changefeed": h.changefeed, - } - deleteGaugeVecPartialMatch(h.metrics.LegacyOutgoingByteRate, legacyLabels) - deleteGaugeVecPartialMatch(h.metrics.LegacyRequestRate, legacyLabels) - deleteGaugeVecPartialMatch(h.metrics.LegacyResponseRate, legacyLabels) - deleteGaugeVecPartialMatch(h.metrics.LegacyRequestsInFlight, legacyLabels) - deleteGaugeVecPartialMatch(h.metrics.LegacyRequestLatency, legacyLabels) - deleteGaugeVecPartialMatch(h.metrics.LegacyCompressionRatio, legacyLabels) - deleteGaugeVecPartialMatch(h.metrics.LegacyRecordsPerRequest, legacyLabels) + deleteGaugeVecPartialMatch(h.metrics.RequestLatency, labels) + deleteGaugeVecPartialMatch(h.metrics.CompressionRatio, labels) + deleteGaugeVecPartialMatch(h.metrics.RecordsPerRequest, labels) } func (h *metricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { @@ -104,22 +80,13 @@ func (h *metricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err erro brokerID := strconv.Itoa(int(nodeID)) if ctx.metrics.OutgoingByteRate != nil && bytesWritten > 0 { - ctx.metrics.OutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(float64(bytesWritten)) - } - if ctx.metrics.LegacyOutgoingByteRate != nil && bytesWritten > 0 { - ctx.metrics.LegacyOutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(float64(bytesWritten)) + ctx.metrics.OutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(float64(bytesWritten)) } if ctx.metrics.RequestRate != nil { - ctx.metrics.RequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) - } - if ctx.metrics.LegacyRequestRate != nil { - ctx.metrics.LegacyRequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + ctx.metrics.RequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) } if err == nil && ctx.metrics.RequestsInFlight != nil { - ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) - } - if err == nil && ctx.metrics.LegacyRequestsInFlight != nil { - ctx.metrics.LegacyRequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) } } @@ -150,25 +117,15 @@ func (h *metricsHook) OnBrokerE2E( brokerID := strconv.Itoa(int(meta.NodeID)) if e2e.WriteErr == nil && ctx.metrics.RequestsInFlight != nil { - ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(-1) - } - if e2e.WriteErr == nil && ctx.metrics.LegacyRequestsInFlight != nil { - ctx.metrics.LegacyRequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(-1) + ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(-1) } if e2e.BytesRead > 0 && e2e.ReadErr == nil && ctx.metrics.ResponseRate != nil { - ctx.metrics.ResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Add(1) - } - if e2e.BytesRead > 0 && e2e.ReadErr == nil && ctx.metrics.LegacyResponseRate != nil { - ctx.metrics.LegacyResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + ctx.metrics.ResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) } if e2e.Err() == nil && ctx.metrics.RequestLatency != nil { latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 - ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType, brokerID).Observe(latencyMs) - } - if e2e.Err() == nil && ctx.metrics.LegacyRequestLatency != nil { - latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 - ctx.metrics.LegacyRequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricAvg).Set(latencyMs) - ctx.metrics.LegacyRequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricP99).Set(latencyMs) + ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricAvg).Set(latencyMs) + ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricP99).Set(latencyMs) } } @@ -189,28 +146,19 @@ func (h *metricsHook) RecordProduceBatchWritten(numRecords int, uncompressedByte if ctx.metrics.RecordsPerRequest != nil && numRecords > 0 { records := float64(numRecords) - ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType).Observe(records) - } - if ctx.metrics.LegacyRecordsPerRequest != nil && numRecords > 0 { - records := float64(numRecords) - ctx.metrics.LegacyRecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(records) - ctx.metrics.LegacyRecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(records) + ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(records) + ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(records) } if ctx.metrics.CompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 - ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, ctx.clientType).Observe(ratio) - } - if ctx.metrics.LegacyCompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { - ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 - ctx.metrics.LegacyCompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(ratio) - ctx.metrics.LegacyCompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(ratio) + ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(ratio) + ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(ratio) } } type metricsContext struct { keyspace string changefeed string - clientType string metrics metricVectors } @@ -218,7 +166,6 @@ func (h *metricsHook) loadMetricsContext() (metricsContext, bool) { return metricsContext{ keyspace: h.keyspace, changefeed: h.changefeed, - clientType: h.clientType, metrics: h.metrics, }, true } @@ -228,9 +175,3 @@ func deleteGaugeVecPartialMatch(gaugeVec *prometheus.GaugeVec, labels prometheus gaugeVec.DeletePartialMatch(labels) } } - -func deleteHistogramVecPartialMatch(histogramVec *prometheus.HistogramVec, labels prometheus.Labels) { - if histogramVec != nil { - histogramVec.DeletePartialMatch(labels) - } -} diff --git a/pkg/sink/kafka/metrics_hook_test.go b/pkg/sink/kafka/metrics_hook_test.go index addb5b2831..726eb80518 100644 --- a/pkg/sink/kafka/metrics_hook_test.go +++ b/pkg/sink/kafka/metrics_hook_test.go @@ -15,13 +15,15 @@ package kafka import ( "testing" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" ) -func TestMetricsHookRecordsLegacyMetricsAndCleanup(t *testing.T) { +func TestMetricsHookRecordsMetricsAndCleanup(t *testing.T) { outgoingByteRate := prometheus.NewGaugeVec( prometheus.GaugeOpts{Name: "kafka_producer_outgoing_byte_rate"}, []string{"namespace", "changefeed", "broker"}, @@ -31,9 +33,17 @@ func TestMetricsHookRecordsLegacyMetricsAndCleanup(t *testing.T) { []string{"namespace", "changefeed", "broker"}, ) requestsInFlight := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_request_in_flight"}, + prometheus.GaugeOpts{Name: "kafka_producer_in_flight_requests"}, []string{"namespace", "changefeed", "broker"}, ) + responseRate := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "kafka_producer_response_rate"}, + []string{"namespace", "changefeed", "broker"}, + ) + requestLatency := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "kafka_producer_request_latency"}, + []string{"namespace", "changefeed", "broker", "type"}, + ) recordsPerRequest := prometheus.NewGaugeVec( prometheus.GaugeOpts{Name: "kafka_producer_records_per_request"}, []string{"namespace", "changefeed", "type"}, @@ -43,20 +53,31 @@ func TestMetricsHookRecordsLegacyMetricsAndCleanup(t *testing.T) { []string{"namespace", "changefeed", "type"}, ) - hook := newMetricsHook("default", "cf", "async_producer", metricVectors{ - LegacyOutgoingByteRate: outgoingByteRate, - LegacyRequestRate: requestRate, - LegacyRequestsInFlight: requestsInFlight, - LegacyRecordsPerRequest: recordsPerRequest, - LegacyCompressionRatio: compressionRatio, + hook := newMetricsHook("default", "cf", metricVectors{ + OutgoingByteRate: outgoingByteRate, + RequestRate: requestRate, + RequestsInFlight: requestsInFlight, + ResponseRate: responseRate, + RequestLatency: requestLatency, + RecordsPerRequest: recordsPerRequest, + CompressionRatio: compressionRatio, }) hook.RecordBrokerWrite(1, 42, nil) + hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: 1}, 0, kgo.BrokerE2E{ + BytesRead: 42, + TimeToWrite: 10 * time.Millisecond, + ReadWait: 20 * time.Millisecond, + TimeToRead: 30 * time.Millisecond, + }) hook.RecordProduceBatchWritten(3, 100, 50) require.Equal(t, float64(42), testutil.ToFloat64(outgoingByteRate.WithLabelValues("default", "cf", "1"))) require.Equal(t, float64(1), testutil.ToFloat64(requestRate.WithLabelValues("default", "cf", "1"))) - require.Equal(t, float64(1), testutil.ToFloat64(requestsInFlight.WithLabelValues("default", "cf", "1"))) + require.Equal(t, float64(0), testutil.ToFloat64(requestsInFlight.WithLabelValues("default", "cf", "1"))) + require.Equal(t, float64(1), testutil.ToFloat64(responseRate.WithLabelValues("default", "cf", "1"))) + require.Equal(t, float64(60), testutil.ToFloat64(requestLatency.WithLabelValues("default", "cf", "1", legacyMetricAvg))) + require.Equal(t, float64(60), testutil.ToFloat64(requestLatency.WithLabelValues("default", "cf", "1", legacyMetricP99))) require.Equal(t, float64(3), testutil.ToFloat64(recordsPerRequest.WithLabelValues("default", "cf", legacyMetricAvg))) require.Equal(t, float64(3), testutil.ToFloat64(recordsPerRequest.WithLabelValues("default", "cf", legacyMetricP99))) require.Equal(t, float64(200), testutil.ToFloat64(compressionRatio.WithLabelValues("default", "cf", legacyMetricAvg))) @@ -67,6 +88,8 @@ func TestMetricsHookRecordsLegacyMetricsAndCleanup(t *testing.T) { require.Equal(t, 0, testutil.CollectAndCount(outgoingByteRate)) require.Equal(t, 0, testutil.CollectAndCount(requestRate)) require.Equal(t, 0, testutil.CollectAndCount(requestsInFlight)) + require.Equal(t, 0, testutil.CollectAndCount(responseRate)) + require.Equal(t, 0, testutil.CollectAndCount(requestLatency)) require.Equal(t, 0, testutil.CollectAndCount(recordsPerRequest)) require.Equal(t, 0, testutil.CollectAndCount(compressionRatio)) } From f7a2d2216eb6209c17dfb77a6155070757ae2b01 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 18:58:47 +0800 Subject: [PATCH 25/61] move sasl to the kafka package --- pkg/sink/kafka/client_options.go | 14 +++++++------- pkg/sink/kafka/client_options_test.go | 5 ++--- pkg/sink/kafka/gssapi.go | 15 +++++++-------- pkg/sink/kafka/options.go | 10 +++++----- .../sasl.go => sink/kafka/sasl_config.go} | 2 +- .../kafka/sasl_config_test.go} | 2 +- pkg/sink/kafka/sasl_test.go | 17 ++++++++--------- 7 files changed, 31 insertions(+), 34 deletions(-) rename pkg/{security/sasl.go => sink/kafka/sasl_config.go} (99%) rename pkg/{security/sasl_test.go => sink/kafka/sasl_config_test.go} (99%) diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index faa1a14f32..621ad0ea13 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -50,7 +50,7 @@ type clientOptions struct { EnableTLS bool Credential *security.Credential InsecureSkipVerify bool - SASL *security.SASL + SASL *SASL DialTimeout time.Duration WriteTimeout time.Duration @@ -148,26 +148,26 @@ func buildSaslMechanism(ctx context.Context, o *clientOptions) (sasl.Mechanism, return nil, nil } - switch security.SASLMechanism(strings.ToUpper(string(o.SASL.SASLMechanism))) { - case security.PlainMechanism: + switch SASLMechanism(strings.ToUpper(string(o.SASL.SASLMechanism))) { + case PlainMechanism: auth := plain.Auth{ User: o.SASL.SASLUser, Pass: o.SASL.SASLPassword, } return auth.AsMechanism(), nil - case security.SCRAM256Mechanism: + case SCRAM256Mechanism: auth := scram.Auth{ User: o.SASL.SASLUser, Pass: o.SASL.SASLPassword, } return auth.AsSha256Mechanism(), nil - case security.SCRAM512Mechanism: + case SCRAM512Mechanism: auth := scram.Auth{ User: o.SASL.SASLUser, Pass: o.SASL.SASLPassword, } return auth.AsSha512Mechanism(), nil - case security.OAuthMechanism: + case OAuthMechanism: tokenSource, err := newOauthTokenSource(ctx, o) if err != nil { return nil, errors.Trace(err) @@ -179,7 +179,7 @@ func buildSaslMechanism(ctx context.Context, o *clientOptions) (sasl.Mechanism, } return oauth.Auth{Token: token.AccessToken}, nil }), nil - case security.GSSAPIMechanism: + case GSSAPIMechanism: return buildGSSAPIMechanism(o.SASL.GSSAPI) default: } diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index f91678361c..19a88cbfe1 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -18,7 +18,6 @@ import ( "testing" "time" - "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kgo" ) @@ -139,8 +138,8 @@ func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { t.Parallel() _, err := newOauthTokenSource(context.Background(), &clientOptions{ - SASL: &security.SASL{ - OAuth2: security.OAuth2{ + SASL: &SASL{ + OAuth2: OAuth2{ ClientID: "client-id", ClientSecret: "client-secret", TokenURL: "http://test.com/Segment%%2815197306101420000%29", diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/gssapi.go index 29b47d4c70..1c3c6b7c41 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -31,7 +31,6 @@ import ( "github.com/jcmturner/gokrb5/v8/messages" "github.com/jcmturner/gokrb5/v8/types" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/security" "github.com/twmb/franz-go/pkg/sasl" ) @@ -52,7 +51,7 @@ type kerborosClient interface { } type gssapiMechanism struct { - config security.GSSAPI + config GSSAPI } func (m *gssapiMechanism) Name() string { @@ -165,7 +164,7 @@ func (s *gssapiSession) nextMessage(challenge []byte) ([]byte, error) { } } -func buildGSSAPIMechanism(g security.GSSAPI) (sasl.Mechanism, error) { +func buildGSSAPIMechanism(g GSSAPI) (sasl.Mechanism, error) { if g.ServiceName == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") @@ -184,12 +183,12 @@ func buildGSSAPIMechanism(g security.GSSAPI) (sasl.Mechanism, error) { } switch g.AuthType { - case security.UserAuth: + case UserAuth: if g.Password == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") } - case security.KeyTabAuth: + case KeyTabAuth: if g.KeyTabPath == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") @@ -214,7 +213,7 @@ func (c *krb5Client) CName() types.PrincipalName { return c.Credentials.CName() } -func newKerborosClient(g security.GSSAPI) (kerborosClient, error) { +func newKerborosClient(g GSSAPI) (kerborosClient, error) { cfg, err := krb5config.Load(g.KerberosConfigPath) if err != nil { return nil, errors.Trace(err) @@ -222,14 +221,14 @@ func newKerborosClient(g security.GSSAPI) (kerborosClient, error) { var client *krb5client.Client switch g.AuthType { - case security.KeyTabAuth: + case KeyTabAuth: kt, err := keytab.Load(g.KeyTabPath) if err != nil { return nil, errors.Trace(err) } client = krb5client.NewWithKeytab( g.Username, g.Realm, kt, cfg, krb5client.DisablePAFXFAST(g.DisablePAFXFAST)) - case security.UserAuth: + case UserAuth: client = krb5client.NewWithPassword( g.Username, g.Realm, g.Password, cfg, krb5client.DisablePAFXFAST(g.DisablePAFXFAST)) default: diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index f3cab22c27..d932edd425 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -166,7 +166,7 @@ type options struct { EnableTLS bool Credential *security.Credential InsecureSkipVerify bool - SASL *security.SASL + SASL *SASL // Timeout for network configurations, default to `10s` DialTimeout time.Duration @@ -186,7 +186,7 @@ func NewOptions() *options { RequiredAcks: WaitForAll, Credential: &security.Credential{}, InsecureSkipVerify: false, - SASL: &security.SASL{}, + SASL: &SASL{}, AutoCreate: true, DialTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, @@ -426,7 +426,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf } if urlParameter.SASLMechanism != nil && *urlParameter.SASLMechanism != "" { - mechanism, err := security.SASLMechanismFromString(*urlParameter.SASLMechanism) + mechanism, err := SASLMechanismFromString(*urlParameter.SASLMechanism) if err != nil { return cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) } @@ -434,7 +434,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf } if urlParameter.SASLGssAPIAuthType != nil && *urlParameter.SASLGssAPIAuthType != "" { - authType, err := security.AuthTypeFromString(*urlParameter.SASLGssAPIAuthType) + authType, err := AuthTypeFromString(*urlParameter.SASLGssAPIAuthType) if err != nil { return cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) } @@ -506,7 +506,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf } if o.SASL.OAuth2.IsEnable() { - if o.SASL.SASLMechanism != security.OAuthMechanism { + if o.SASL.SASLMechanism != OAuthMechanism { return cerror.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 is only supported with SASL mechanism type OAUTHBEARER, but got %s", o.SASL.SASLMechanism) diff --git a/pkg/security/sasl.go b/pkg/sink/kafka/sasl_config.go similarity index 99% rename from pkg/security/sasl.go rename to pkg/sink/kafka/sasl_config.go index 4f948f6ab9..6236432ffb 100644 --- a/pkg/security/sasl.go +++ b/pkg/sink/kafka/sasl_config.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package security +package kafka import ( "strings" diff --git a/pkg/security/sasl_test.go b/pkg/sink/kafka/sasl_config_test.go similarity index 99% rename from pkg/security/sasl_test.go rename to pkg/sink/kafka/sasl_config_test.go index 71f3c90754..73ff7b61ae 100644 --- a/pkg/security/sasl_test.go +++ b/pkg/sink/kafka/sasl_config_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package security +package kafka import ( "testing" diff --git a/pkg/sink/kafka/sasl_test.go b/pkg/sink/kafka/sasl_test.go index 183d3bbead..a3f71ab1a9 100644 --- a/pkg/sink/kafka/sasl_test.go +++ b/pkg/sink/kafka/sasl_test.go @@ -17,7 +17,6 @@ import ( "context" "testing" - "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" ) @@ -25,10 +24,10 @@ func TestBuildSaslMechanismGSSAPIUserAuth(t *testing.T) { t.Parallel() o := &clientOptions{ - SASL: &security.SASL{ - SASLMechanism: security.GSSAPIMechanism, - GSSAPI: security.GSSAPI{ - AuthType: security.UserAuth, + SASL: &SASL{ + SASLMechanism: GSSAPIMechanism, + GSSAPI: GSSAPI{ + AuthType: UserAuth, KerberosConfigPath: "/etc/krb5.conf", ServiceName: "kafka", Username: "alice", @@ -47,10 +46,10 @@ func TestBuildSaslMechanismGSSAPIKeytabAuth(t *testing.T) { t.Parallel() o := &clientOptions{ - SASL: &security.SASL{ - SASLMechanism: security.GSSAPIMechanism, - GSSAPI: security.GSSAPI{ - AuthType: security.KeyTabAuth, + SASL: &SASL{ + SASLMechanism: GSSAPIMechanism, + GSSAPI: GSSAPI{ + AuthType: KeyTabAuth, KerberosConfigPath: "/etc/krb5.conf", ServiceName: "kafka", Username: "alice", From 92480ab030ad5b3931f325275044f3ab10921fba Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 19:18:39 +0800 Subject: [PATCH 26/61] simplify more code --- pkg/sink/kafka/admin_client.go | 4 --- pkg/sink/kafka/admin_client_test.go | 6 ++-- pkg/sink/kafka/client_options.go | 22 +------------- pkg/sink/kafka/client_options_test.go | 4 +-- pkg/sink/kafka/kafka_factory.go | 43 +++++++++------------------ pkg/sink/kafka/kafka_factory_test.go | 15 +++------- pkg/sink/kafka/sync_producer.go | 4 --- 7 files changed, 23 insertions(+), 75 deletions(-) diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index 3024f06984..b795c36372 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -41,10 +41,6 @@ func newAdminClient( o *clientOptions, hook kgo.Hook, ) (*kafkaAdminClient, error) { - if o == nil { - o = &clientOptions{} - } - opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) diff --git a/pkg/sink/kafka/admin_client_test.go b/pkg/sink/kafka/admin_client_test.go index 75fa99d8fd..9154f34e0f 100644 --- a/pkg/sink/kafka/admin_client_test.go +++ b/pkg/sink/kafka/admin_client_test.go @@ -21,11 +21,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewAdminClientNilOptionsReturnsError(t *testing.T) { +func TestNewAdminClientEmptyOptionsReturnsError(t *testing.T) { t.Parallel() - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "kafka-admin-nil-options") - client, err := newAdminClient(context.Background(), changefeedID, nil, nil) + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "kafka-admin-empty-options") + client, err := newAdminClient(context.Background(), changefeedID, &clientOptions{}, nil) if client != nil { client.Close() } diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index 621ad0ea13..aab0e5d3e3 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -59,7 +59,6 @@ type clientOptions struct { const ( defaultRequestTimeout = 10 * time.Second - defaultRecordRetries = 5 ) func maxTimeoutWithDefault(readTimeout, writeTimeout time.Duration) time.Duration { @@ -75,10 +74,6 @@ func newOptions( o *clientOptions, hook kgo.Hook, ) ([]kgo.Opt, error) { - if o == nil { - o = &clientOptions{} - } - timeoutOverhead := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) opts := []kgo.Opt{ @@ -213,13 +208,6 @@ func newOauthTokenSource(ctx context.Context, o *clientOptions) (oauth2.TokenSou func newProducerOptions( o *clientOptions, ) []kgo.Opt { - recordRetries := defaultRecordRetries - if o == nil { - o = &clientOptions{} - } else { - recordRetries = o.MaxRetry - } - produceTimeout := o.ReadTimeout if produceTimeout < 100*time.Millisecond { produceTimeout = defaultRequestTimeout @@ -234,7 +222,7 @@ func newProducerOptions( kgo.RequiredAcks(newRequiredAcks(o)), kgo.DisableIdempotentWrite(), kgo.MaxProduceRequestsInflightPerBroker(1), - kgo.RecordRetries(recordRetries), + kgo.RecordRetries(o.MaxRetry), kgo.ProducerBatchMaxBytes(int32(producerBatchMaxBytes)), kgo.ProduceRequestTimeout(produceTimeout), kgo.ProducerLinger(0), @@ -243,10 +231,6 @@ func newProducerOptions( } func newRequiredAcks(o *clientOptions) kgo.Acks { - if o == nil { - return kgo.AllISRAcks() - } - switch o.RequiredAcks { case -1: return kgo.AllISRAcks() @@ -261,10 +245,6 @@ func newRequiredAcks(o *clientOptions) kgo.Acks { } func newCompressionOption(o *clientOptions) kgo.Opt { - if o == nil { - return kgo.ProducerBatchCompression(kgo.NoCompression()) - } - compression := strings.ToLower(strings.TrimSpace(o.Compression)) var codec kgo.CompressionCodec switch compression { diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index 19a88cbfe1..e0d4e6f96f 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -42,8 +42,6 @@ func TestNewRequiredAcks(t *testing.T) { require.Equal(t, tc.expected, newRequiredAcks(&clientOptions{RequiredAcks: tc.requiredAcks})) }) } - - require.Equal(t, kgo.AllISRAcks(), newRequiredAcks(nil)) } func TestMaxTimeoutWithDefault(t *testing.T) { @@ -91,7 +89,7 @@ func TestNewProducerOptionsUsesProducerBatchMaxBytes(t *testing.T) { BrokerEndpoints: []string{"127.0.0.1:9092"}, MaxMessageBytes: encoderMaxMessageBytes, ProducerBatchMaxBytes: producerBatchMaxBytes, - MaxRetry: defaultRecordRetries, + MaxRetry: defaultMaxRetry, RequiredAcks: int16(WaitForAll), } diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go index c00d2e191d..733c99f1ca 100644 --- a/pkg/sink/kafka/kafka_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -22,24 +22,20 @@ import ( type factory struct { changefeedID common.ChangeFeedID - option *options + clientOption *clientOptions - asyncMetricsHook *metricsHook - syncMetricsHook *metricsHook - adminMetricsHook *metricsHook + metricsHook *metricsHook } type kafkaMetricsCollector struct { changefeedID common.ChangeFeedID - hooks []*metricsHook + hook *metricsHook } func (c *kafkaMetricsCollector) Run(ctx context.Context) { <-ctx.Done() - for _, hook := range c.hooks { - if hook != nil { - hook.cleanupMetrics() - } + if c.hook != nil { + c.hook.cleanupMetrics() } cleanupAdminMetrics(c.changefeedID.Keyspace(), c.changefeedID.Name()) } @@ -66,7 +62,7 @@ func NewFactory( o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - admin, err := newAdminClient(ctx, changefeedID, newKafkaOptions(o), nil) + admin, err := newAdminClient(ctx, changefeedID, newClientOption(o), nil) if err != nil { return nil, errors.Trace(err) } @@ -77,16 +73,14 @@ func NewFactory( } return &factory{ - changefeedID: changefeedID, - option: o, - asyncMetricsHook: newKafkaMetricsHook(changefeedID), - syncMetricsHook: newKafkaMetricsHook(changefeedID), - adminMetricsHook: newKafkaMetricsHook(changefeedID), + changefeedID: changefeedID, + clientOption: newClientOption(o), + metricsHook: newKafkaMetricsHook(changefeedID), }, nil } func (f *factory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - admin, err := newAdminClient(ctx, f.changefeedID, newKafkaOptions(f.option), f.adminMetricsHook) + admin, err := newAdminClient(ctx, f.changefeedID, f.clientOption, f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -94,7 +88,7 @@ func (f *factory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { } func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := newSyncProducer(ctx, f.changefeedID, newKafkaOptions(f.option), f.syncMetricsHook) + producer, err := newSyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -102,7 +96,7 @@ func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { } func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := newAsyncProducer(ctx, f.changefeedID, newKafkaOptions(f.option), f.asyncMetricsHook) + producer, err := newAsyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -110,19 +104,10 @@ func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { } func (f *factory) MetricsCollector() MetricsCollector { - return &kafkaMetricsCollector{changefeedID: f.changefeedID, hooks: []*metricsHook{ - f.asyncMetricsHook, - f.syncMetricsHook, - f.adminMetricsHook, - }} + return &kafkaMetricsCollector{changefeedID: f.changefeedID, hook: f.metricsHook} } -func newKafkaOptions(o *options) *clientOptions { - if o == nil { - return &clientOptions{ - RequiredAcks: int16(WaitForAll), - } - } +func newClientOption(o *options) *clientOptions { return &clientOptions{ BrokerEndpoints: o.BrokerEndpoints, ClientID: o.ClientID, diff --git a/pkg/sink/kafka/kafka_factory_test.go b/pkg/sink/kafka/kafka_factory_test.go index a5a23924dc..24c74a695d 100644 --- a/pkg/sink/kafka/kafka_factory_test.go +++ b/pkg/sink/kafka/kafka_factory_test.go @@ -19,14 +19,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewKafkaOptionsNilUsesWaitForAll(t *testing.T) { - t.Parallel() - - options := newKafkaOptions(nil) - require.Equal(t, int16(WaitForAll), options.RequiredAcks) -} - -func TestNewKafkaOptionsMapsRequiredAcks(t *testing.T) { +func TestNewClientOptionMapsRequiredAcks(t *testing.T) { t.Parallel() testCases := []struct { @@ -45,18 +38,18 @@ func TestNewKafkaOptionsMapsRequiredAcks(t *testing.T) { options := NewOptions() options.RequiredAcks = tc.requiredAcks - kafkaOptions := newKafkaOptions(options) + kafkaOptions := newClientOption(options) require.Equal(t, int16(tc.requiredAcks), kafkaOptions.RequiredAcks) }) } } -func TestNewKafkaOptionsMapsMaxRetry(t *testing.T) { +func TestNewClientOptionMapsMaxRetry(t *testing.T) { t.Parallel() options := NewOptions() options.MaxRetry = 7 - kafkaOptions := newKafkaOptions(options) + kafkaOptions := newClientOption(options) require.Equal(t, 7, kafkaOptions.MaxRetry) } diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index 9cb60cbcc0..373a444297 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -41,10 +41,6 @@ func newSyncProducer( o *clientOptions, hook kgo.Hook, ) (*kafkaSyncProducer, error) { - if o == nil { - o = &clientOptions{} - } - opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) From 42de85fc0c08a01586b6ae7adb189e6dd73c2762 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 19:48:48 +0800 Subject: [PATCH 27/61] fix it --- pkg/sink/kafka/admin_client_test.go | 13 --- pkg/sink/kafka/client_options.go | 50 ++++----- pkg/sink/kafka/client_options_test.go | 14 +-- pkg/sink/kafka/gssapi.go | 60 +++++------ pkg/sink/kafka/kafka_factory.go | 2 +- pkg/sink/kafka/options.go | 54 +++++----- pkg/sink/kafka/options_test.go | 40 +++---- pkg/sink/kafka/sasl_config.go | 144 +++++++++++++------------- pkg/sink/kafka/sasl_config_test.go | 6 +- pkg/sink/kafka/sasl_test.go | 36 +++---- 10 files changed, 203 insertions(+), 216 deletions(-) diff --git a/pkg/sink/kafka/admin_client_test.go b/pkg/sink/kafka/admin_client_test.go index 9154f34e0f..61a061f65e 100644 --- a/pkg/sink/kafka/admin_client_test.go +++ b/pkg/sink/kafka/admin_client_test.go @@ -14,24 +14,11 @@ package kafka import ( - "context" "testing" - "github.com/pingcap/ticdc/pkg/common" "github.com/stretchr/testify/require" ) -func TestNewAdminClientEmptyOptionsReturnsError(t *testing.T) { - t.Parallel() - - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "kafka-admin-empty-options") - client, err := newAdminClient(context.Background(), changefeedID, &clientOptions{}, nil) - if client != nil { - client.Close() - } - require.Error(t, err) -} - func TestAdminClientCreateTopicNilDetailReturnsError(t *testing.T) { t.Parallel() diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index aab0e5d3e3..82fe4210cf 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -50,7 +50,7 @@ type clientOptions struct { EnableTLS bool Credential *security.Credential InsecureSkipVerify bool - SASL *SASL + sasl *saslConfig DialTimeout time.Duration WriteTimeout time.Duration @@ -103,7 +103,7 @@ func newOptions( opts = append(opts, kgo.DialTLSConfig(tlsConfig)) } - if o.SASL != nil && o.SASL.SASLMechanism != "" { + if o.sasl != nil && o.sasl.mechanism != "" { mechanism, err := buildSaslMechanism(ctx, o) if err != nil { return nil, errors.Trace(err) @@ -139,30 +139,30 @@ func newTLSConfig(o *clientOptions) (*tls.Config, error) { } func buildSaslMechanism(ctx context.Context, o *clientOptions) (sasl.Mechanism, error) { - if o.SASL == nil { + if o.sasl == nil { return nil, nil } - switch SASLMechanism(strings.ToUpper(string(o.SASL.SASLMechanism))) { - case PlainMechanism: + switch saslMechanism(strings.ToUpper(string(o.sasl.mechanism))) { + case plainMechanism: auth := plain.Auth{ - User: o.SASL.SASLUser, - Pass: o.SASL.SASLPassword, + User: o.sasl.user, + Pass: o.sasl.password, } return auth.AsMechanism(), nil - case SCRAM256Mechanism: + case scram256Mechanism: auth := scram.Auth{ - User: o.SASL.SASLUser, - Pass: o.SASL.SASLPassword, + User: o.sasl.user, + Pass: o.sasl.password, } return auth.AsSha256Mechanism(), nil - case SCRAM512Mechanism: + case scram512Mechanism: auth := scram.Auth{ - User: o.SASL.SASLUser, - Pass: o.SASL.SASLPassword, + User: o.sasl.user, + Pass: o.sasl.password, } return auth.AsSha512Mechanism(), nil - case OAuthMechanism: + case oauthMechanism: tokenSource, err := newOauthTokenSource(ctx, o) if err != nil { return nil, errors.Trace(err) @@ -174,33 +174,33 @@ func buildSaslMechanism(ctx context.Context, o *clientOptions) (sasl.Mechanism, } return oauth.Auth{Token: token.AccessToken}, nil }), nil - case GSSAPIMechanism: - return buildGSSAPIMechanism(o.SASL.GSSAPI) + case gssapiMechanismName: + return buildGSSAPIMechanism(o.sasl.gssapi) default: } - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.SASL.SASLMechanism) + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.sasl.mechanism) } func newOauthTokenSource(ctx context.Context, o *clientOptions) (oauth2.TokenSource, error) { endpointParams := url.Values{} - if o.SASL.OAuth2.GrantType != "" { - endpointParams.Set("grant_type", o.SASL.OAuth2.GrantType) + if o.sasl.oauth2.grantType != "" { + endpointParams.Set("grant_type", o.sasl.oauth2.grantType) } - if o.SASL.OAuth2.Audience != "" { - endpointParams.Set("audience", o.SASL.OAuth2.Audience) + if o.sasl.oauth2.audience != "" { + endpointParams.Set("audience", o.sasl.oauth2.audience) } - tokenURL, err := url.Parse(o.SASL.OAuth2.TokenURL) + tokenURL, err := url.Parse(o.sasl.oauth2.tokenURL) if err != nil { return nil, errors.Trace(err) } cfg := &clientcredentials.Config{ - ClientID: o.SASL.OAuth2.ClientID, - ClientSecret: o.SASL.OAuth2.ClientSecret, + ClientID: o.sasl.oauth2.clientID, + ClientSecret: o.sasl.oauth2.clientSecret, TokenURL: tokenURL.String(), EndpointParams: endpointParams, - Scopes: o.SASL.OAuth2.Scopes, + Scopes: o.sasl.oauth2.scopes, } return cfg.TokenSource(ctx), nil } diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index e0d4e6f96f..8e9b2512d7 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -136,13 +136,13 @@ func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { t.Parallel() _, err := newOauthTokenSource(context.Background(), &clientOptions{ - SASL: &SASL{ - OAuth2: OAuth2{ - ClientID: "client-id", - ClientSecret: "client-secret", - TokenURL: "http://test.com/Segment%%2815197306101420000%29", - Scopes: []string{"scope1", "scope2"}, - GrantType: "client_credentials", + sasl: &saslConfig{ + oauth2: oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: "http://test.com/Segment%%2815197306101420000%29", + scopes: []string{"scope1", "scope2"}, + grantType: "client_credentials", }, }, }) diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/gssapi.go index 1c3c6b7c41..fa0d8fa87d 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -22,8 +22,8 @@ import ( "github.com/jcmturner/gofork/encoding/asn1" "github.com/jcmturner/gokrb5/v8/asn1tools" - krb5client "github.com/jcmturner/gokrb5/v8/client" - krb5config "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/config" "github.com/jcmturner/gokrb5/v8/gssapi" "github.com/jcmturner/gokrb5/v8/iana/chksumtype" "github.com/jcmturner/gokrb5/v8/iana/keyusage" @@ -51,11 +51,11 @@ type kerborosClient interface { } type gssapiMechanism struct { - config GSSAPI + config gssapiConfig } func (m *gssapiMechanism) Name() string { - return "GSSAPI" + return string(gssapiMechanismName) } func (m *gssapiMechanism) Authenticate( @@ -72,7 +72,7 @@ func (m *gssapiMechanism) Authenticate( } serverHost := strings.SplitN(host, ":", 2)[0] - spn := fmt.Sprintf("%s/%s", m.config.ServiceName, serverHost) + spn := fmt.Sprintf("%s/%s", m.config.serviceName, serverHost) ticket, encKey, err := client.GetServiceTicket(spn) if err != nil { client.Destroy() @@ -164,45 +164,45 @@ func (s *gssapiSession) nextMessage(challenge []byte) ([]byte, error) { } } -func buildGSSAPIMechanism(g GSSAPI) (sasl.Mechanism, error) { - if g.ServiceName == "" { +func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { + if g.serviceName == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") } - if g.KerberosConfigPath == "" { + if g.kerberosConfigPath == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") } - if g.Username == "" { + if g.username == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") } - if g.Realm == "" { + if g.realm == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") } - switch g.AuthType { - case UserAuth: - if g.Password == "" { + switch g.authType { + case userAuth: + if g.password == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") } - case KeyTabAuth: - if g.KeyTabPath == "" { + case keyTabAuth: + if g.keyTabPath == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") } default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.AuthType) + "unsupported sasl-gssapi-auth-type %d", g.authType) } return &gssapiMechanism{config: g}, nil } type krb5Client struct { - krb5client.Client + client.Client } func (c *krb5Client) Domain() string { @@ -213,29 +213,29 @@ func (c *krb5Client) CName() types.PrincipalName { return c.Credentials.CName() } -func newKerborosClient(g GSSAPI) (kerborosClient, error) { - cfg, err := krb5config.Load(g.KerberosConfigPath) +func newKerborosClient(g gssapiConfig) (kerborosClient, error) { + cfg, err := config.Load(g.kerberosConfigPath) if err != nil { return nil, errors.Trace(err) } - var client *krb5client.Client - switch g.AuthType { - case KeyTabAuth: - kt, err := keytab.Load(g.KeyTabPath) + var krbClient *client.Client + switch g.authType { + case keyTabAuth: + kt, err := keytab.Load(g.keyTabPath) if err != nil { return nil, errors.Trace(err) } - client = krb5client.NewWithKeytab( - g.Username, g.Realm, kt, cfg, krb5client.DisablePAFXFAST(g.DisablePAFXFAST)) - case UserAuth: - client = krb5client.NewWithPassword( - g.Username, g.Realm, g.Password, cfg, krb5client.DisablePAFXFAST(g.DisablePAFXFAST)) + krbClient = client.NewWithKeytab( + g.username, g.realm, kt, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)) + case userAuth: + krbClient = client.NewWithPassword( + g.username, g.realm, g.password, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)) default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.AuthType) + "unsupported sasl-gssapi-auth-type %d", g.authType) } - return &krb5Client{*client}, nil + return &krb5Client{*krbClient}, nil } func newKrb5Token( diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go index 733c99f1ca..14e31fe333 100644 --- a/pkg/sink/kafka/kafka_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -124,7 +124,7 @@ func newClientOption(o *options) *clientOptions { EnableTLS: o.EnableTLS, Credential: o.Credential, InsecureSkipVerify: o.InsecureSkipVerify, - SASL: o.SASL, + sasl: o.sasl, DialTimeout: o.DialTimeout, WriteTimeout: o.WriteTimeout, diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index d932edd425..dedf9bb809 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -66,7 +66,7 @@ const ( SASLTypeSCRAMSHA256 = "SCRAM-SHA-256" // SASLTypeSCRAMSHA512 represents the SCRAM-SHA-512 mechanism. SASLTypeSCRAMSHA512 = "SCRAM-SHA-512" - // SASLTypeGSSAPI represents the gssapi mechanism. + // SASLTypeGSSAPI represents the GSSAPI mechanism. SASLTypeGSSAPI = "GSSAPI" // SASLTypeOAuth represents the SASL/OAUTHBEARER mechanism (Kafka 2.0.0+) SASLTypeOAuth = "OAUTHBEARER" @@ -166,7 +166,7 @@ type options struct { EnableTLS bool Credential *security.Credential InsecureSkipVerify bool - SASL *SASL + sasl *saslConfig // Timeout for network configurations, default to `10s` DialTimeout time.Duration @@ -186,7 +186,7 @@ func NewOptions() *options { RequiredAcks: WaitForAll, Credential: &security.Credential{}, InsecureSkipVerify: false, - SASL: &SASL{}, + sasl: &saslConfig{}, AutoCreate: true, DialTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, @@ -418,56 +418,56 @@ func (o *options) applyTLS(params *urlConfig) error { func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConfig) error { if urlParameter.SASLUser != nil && *urlParameter.SASLUser != "" { - o.SASL.SASLUser = *urlParameter.SASLUser + o.sasl.user = *urlParameter.SASLUser } if urlParameter.SASLPassword != nil && *urlParameter.SASLPassword != "" { - o.SASL.SASLPassword = *urlParameter.SASLPassword + o.sasl.password = *urlParameter.SASLPassword } if urlParameter.SASLMechanism != nil && *urlParameter.SASLMechanism != "" { - mechanism, err := SASLMechanismFromString(*urlParameter.SASLMechanism) + mechanism, err := saslMechanismFromString(*urlParameter.SASLMechanism) if err != nil { return cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) } - o.SASL.SASLMechanism = mechanism + o.sasl.mechanism = mechanism } if urlParameter.SASLGssAPIAuthType != nil && *urlParameter.SASLGssAPIAuthType != "" { - authType, err := AuthTypeFromString(*urlParameter.SASLGssAPIAuthType) + authType, err := gssapiAuthTypeFromString(*urlParameter.SASLGssAPIAuthType) if err != nil { return cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) } - o.SASL.GSSAPI.AuthType = authType + o.sasl.gssapi.authType = authType } if urlParameter.SASLGssAPIKeytabPath != nil && *urlParameter.SASLGssAPIKeytabPath != "" { - o.SASL.GSSAPI.KeyTabPath = *urlParameter.SASLGssAPIKeytabPath + o.sasl.gssapi.keyTabPath = *urlParameter.SASLGssAPIKeytabPath } if urlParameter.SASLGssAPIKerberosConfigPath != nil && *urlParameter.SASLGssAPIKerberosConfigPath != "" { - o.SASL.GSSAPI.KerberosConfigPath = *urlParameter.SASLGssAPIKerberosConfigPath + o.sasl.gssapi.kerberosConfigPath = *urlParameter.SASLGssAPIKerberosConfigPath } if urlParameter.SASLGssAPIServiceName != nil && *urlParameter.SASLGssAPIServiceName != "" { - o.SASL.GSSAPI.ServiceName = *urlParameter.SASLGssAPIServiceName + o.sasl.gssapi.serviceName = *urlParameter.SASLGssAPIServiceName } if urlParameter.SASLGssAPIUser != nil && *urlParameter.SASLGssAPIUser != "" { - o.SASL.GSSAPI.Username = *urlParameter.SASLGssAPIUser + o.sasl.gssapi.username = *urlParameter.SASLGssAPIUser } if urlParameter.SASLGssAPIPassword != nil && *urlParameter.SASLGssAPIPassword != "" { - o.SASL.GSSAPI.Password = *urlParameter.SASLGssAPIPassword + o.sasl.gssapi.password = *urlParameter.SASLGssAPIPassword } if urlParameter.SASLGssAPIRealm != nil && *urlParameter.SASLGssAPIRealm != "" { - o.SASL.GSSAPI.Realm = *urlParameter.SASLGssAPIRealm + o.sasl.gssapi.realm = *urlParameter.SASLGssAPIRealm } if urlParameter.SASLGssAPIDisablePafxfast != nil { - o.SASL.GSSAPI.DisablePAFXFAST = *urlParameter.SASLGssAPIDisablePafxfast + o.sasl.gssapi.disablePAFXFAST = *urlParameter.SASLGssAPIDisablePafxfast } if sinkConfig != nil && sinkConfig.KafkaConfig != nil { @@ -476,7 +476,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if clientID == "" { return cerror.ErrKafkaInvalidConfig.GenWithStack("OAuth2 client ID cannot be empty") } - o.SASL.OAuth2.ClientID = clientID + o.sasl.oauth2.clientID = clientID } if sinkConfig.KafkaConfig.SASLOAuthClientSecret != nil { @@ -493,7 +493,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf return cerror.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 client secret is not base64 encoded") } - o.SASL.OAuth2.ClientSecret = string(decodedClientSecret) + o.sasl.oauth2.clientSecret = string(decodedClientSecret) } if sinkConfig.KafkaConfig.SASLOAuthTokenURL != nil { @@ -502,32 +502,32 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf return cerror.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 token URL cannot be empty") } - o.SASL.OAuth2.TokenURL = tokenURL + o.sasl.oauth2.tokenURL = tokenURL } - if o.SASL.OAuth2.IsEnable() { - if o.SASL.SASLMechanism != OAuthMechanism { + if o.sasl.oauth2.isEnabled() { + if o.sasl.mechanism != oauthMechanism { return cerror.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 is only supported with SASL mechanism type OAUTHBEARER, but got %s", - o.SASL.SASLMechanism) + o.sasl.mechanism) } - if err := o.SASL.OAuth2.Validate(); err != nil { + if err := o.sasl.oauth2.validate(); err != nil { return cerror.ErrKafkaInvalidConfig.Wrap(err) } - o.SASL.OAuth2.SetDefault() + o.sasl.oauth2.setDefault() } if sinkConfig.KafkaConfig.SASLOAuthScopes != nil { - o.SASL.OAuth2.Scopes = sinkConfig.KafkaConfig.SASLOAuthScopes + o.sasl.oauth2.scopes = sinkConfig.KafkaConfig.SASLOAuthScopes } if sinkConfig.KafkaConfig.SASLOAuthGrantType != nil { - o.SASL.OAuth2.GrantType = *sinkConfig.KafkaConfig.SASLOAuthGrantType + o.sasl.oauth2.grantType = *sinkConfig.KafkaConfig.SASLOAuthGrantType } if sinkConfig.KafkaConfig.SASLOAuthAudience != nil { - o.SASL.OAuth2.Audience = *sinkConfig.KafkaConfig.SASLOAuthAudience + o.sasl.oauth2.audience = *sinkConfig.KafkaConfig.SASLOAuthAudience } } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index c3cba1edfc..b0b20383ba 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -757,16 +757,16 @@ func TestMerge(t *testing.T) { require.Equal(t, time.Minute+time.Second, c.DialTimeout) require.Equal(t, 2*time.Minute+time.Second, c.WriteTimeout) require.Equal(t, 1, int(c.RequiredAcks)) - require.Equal(t, "abc", c.SASL.SASLUser) - require.Equal(t, "123", c.SASL.SASLPassword) - require.Equal(t, "plain", strings.ToLower(string(c.SASL.SASLMechanism))) - require.Equal(t, 2, int(c.SASL.GSSAPI.AuthType)) - require.Equal(t, "SASLGssAPIKeytabPath", c.SASL.GSSAPI.KeyTabPath) - require.Equal(t, "service", c.SASL.GSSAPI.ServiceName) - require.Equal(t, "user", c.SASL.GSSAPI.Username) - require.Equal(t, "pass", c.SASL.GSSAPI.Password) - require.Equal(t, "realm", c.SASL.GSSAPI.Realm) - require.Equal(t, true, c.SASL.GSSAPI.DisablePAFXFAST) + require.Equal(t, "abc", c.sasl.user) + require.Equal(t, "123", c.sasl.password) + require.Equal(t, "plain", strings.ToLower(string(c.sasl.mechanism))) + require.Equal(t, 2, int(c.sasl.gssapi.authType)) + require.Equal(t, "SASLGssAPIKeytabPath", c.sasl.gssapi.keyTabPath) + require.Equal(t, "service", c.sasl.gssapi.serviceName) + require.Equal(t, "user", c.sasl.gssapi.username) + require.Equal(t, "pass", c.sasl.gssapi.password) + require.Equal(t, "realm", c.sasl.gssapi.realm) + require.Equal(t, true, c.sasl.gssapi.disablePAFXFAST) require.Equal(t, true, c.EnableTLS) require.Equal(t, "ca.pem", c.Credential.CAPath) require.Equal(t, "cert.pem", c.Credential.CertPath) @@ -838,16 +838,16 @@ func TestMerge(t *testing.T) { require.Equal(t, time.Minute+time.Second, c.DialTimeout) require.Equal(t, 2*time.Minute+time.Second, c.WriteTimeout) require.Equal(t, 1, int(c.RequiredAcks)) - require.Equal(t, "abc", c.SASL.SASLUser) - require.Equal(t, "123", c.SASL.SASLPassword) - require.Equal(t, "plain", strings.ToLower(string(c.SASL.SASLMechanism))) - require.Equal(t, 2, int(c.SASL.GSSAPI.AuthType)) - require.Equal(t, "SASLGssAPIKeytabPath", c.SASL.GSSAPI.KeyTabPath) - require.Equal(t, "service", c.SASL.GSSAPI.ServiceName) - require.Equal(t, "user", c.SASL.GSSAPI.Username) - require.Equal(t, "pass", c.SASL.GSSAPI.Password) - require.Equal(t, "realm", c.SASL.GSSAPI.Realm) - require.Equal(t, true, c.SASL.GSSAPI.DisablePAFXFAST) + require.Equal(t, "abc", c.sasl.user) + require.Equal(t, "123", c.sasl.password) + require.Equal(t, "plain", strings.ToLower(string(c.sasl.mechanism))) + require.Equal(t, 2, int(c.sasl.gssapi.authType)) + require.Equal(t, "SASLGssAPIKeytabPath", c.sasl.gssapi.keyTabPath) + require.Equal(t, "service", c.sasl.gssapi.serviceName) + require.Equal(t, "user", c.sasl.gssapi.username) + require.Equal(t, "pass", c.sasl.gssapi.password) + require.Equal(t, "realm", c.sasl.gssapi.realm) + require.Equal(t, true, c.sasl.gssapi.disablePAFXFAST) require.Equal(t, true, c.EnableTLS) require.Equal(t, "ca.pem", c.Credential.CAPath) require.Equal(t, "cert.pem", c.Credential.CertPath) diff --git a/pkg/sink/kafka/sasl_config.go b/pkg/sink/kafka/sasl_config.go index 6236432ffb..e5d8402b94 100644 --- a/pkg/sink/kafka/sasl_config.go +++ b/pkg/sink/kafka/sasl_config.go @@ -19,120 +19,120 @@ import ( "github.com/pingcap/errors" ) -// SASLMechanism defines SASL mechanism. -type SASLMechanism string +// saslMechanism defines SASL mechanism. +type saslMechanism string // The mechanisms we currently support. const ( - // UnknownMechanism means the SASL mechanism is unknown. - UnknownMechanism SASLMechanism = "" - // PlainMechanism means the SASL mechanism is plain. - PlainMechanism SASLMechanism = "PLAIN" - // SCRAM256Mechanism means the SASL mechanism is SCRAM-SHA-256. - SCRAM256Mechanism SASLMechanism = "SCRAM-SHA-256" - // SCRAM512Mechanism means the SASL mechanism is SCRAM-SHA-512. - SCRAM512Mechanism SASLMechanism = "SCRAM-SHA-512" - // GSSAPIMechanism means the SASL mechanism is GSSAPI. - GSSAPIMechanism SASLMechanism = "GSSAPI" - // OAuthMechanism means the SASL mechanism is OAuth2. - OAuthMechanism SASLMechanism = "OAUTHBEARER" + // unknownMechanism means the SASL mechanism is unknown. + unknownMechanism saslMechanism = "" + // plainMechanism means the SASL mechanism is plain. + plainMechanism saslMechanism = "PLAIN" + // scram256Mechanism means the SASL mechanism is SCRAM-SHA-256. + scram256Mechanism saslMechanism = "SCRAM-SHA-256" + // scram512Mechanism means the SASL mechanism is SCRAM-SHA-512. + scram512Mechanism saslMechanism = "SCRAM-SHA-512" + // gssapiMechanismName means the SASL mechanism is GSSAPI. + gssapiMechanismName saslMechanism = "GSSAPI" + // oauthMechanism means the SASL mechanism is OAUTHBEARER. + oauthMechanism saslMechanism = "OAUTHBEARER" ) -// SASLMechanismFromString converts the string to SASL mechanism. -func SASLMechanismFromString(s string) (SASLMechanism, error) { +// saslMechanismFromString converts the string to a SASL mechanism. +func saslMechanismFromString(s string) (saslMechanism, error) { switch strings.ToLower(s) { case "plain": - return PlainMechanism, nil + return plainMechanism, nil case "scram-sha-256": - return SCRAM256Mechanism, nil + return scram256Mechanism, nil case "scram-sha-512": - return SCRAM512Mechanism, nil + return scram512Mechanism, nil case "gssapi": - return GSSAPIMechanism, nil + return gssapiMechanismName, nil case "oauthbearer": - return OAuthMechanism, nil + return oauthMechanism, nil default: - return UnknownMechanism, errors.Errorf("unknown %s SASL mechanism", s) + return unknownMechanism, errors.Errorf("unknown %s SASL mechanism", s) } } -// SASL holds necessary path parameter to support sasl-scram -type SASL struct { - SASLUser string - SASLPassword string - SASLMechanism SASLMechanism - GSSAPI GSSAPI - OAuth2 OAuth2 +// saslConfig holds necessary path parameter to support sasl-scram +type saslConfig struct { + user string + password string + mechanism saslMechanism + gssapi gssapiConfig + oauth2 oauth2Config } -// OAuth2 holds necessary parameters to support sasl-oauth2. -type OAuth2 struct { - ClientID string - ClientSecret string - TokenURL string - Scopes []string - GrantType string - Audience string +// oauth2Config holds necessary parameters to support sasl-oauth2. +type oauth2Config struct { + clientID string + clientSecret string + tokenURL string + scopes []string + grantType string + audience string } -// Validate validates the parameters of OAuth2. +// validate validates the parameters of oauth2Config. // Some parameters are required, some are optional. -func (o *OAuth2) Validate() error { - if len(o.ClientID) == 0 { +func (o *oauth2Config) validate() error { + if len(o.clientID) == 0 { return errors.New("OAuth2 client id is empty") } - if len(o.ClientSecret) == 0 { + if len(o.clientSecret) == 0 { return errors.New("OAuth2 client secret is empty") } - if len(o.TokenURL) == 0 { + if len(o.tokenURL) == 0 { return errors.New("OAuth2 token url is empty") } return nil } -// SetDefault sets the default value of OAuth2. -func (o *OAuth2) SetDefault() { - o.GrantType = "client_credentials" +// setDefault sets the default value of oauth2Config. +func (o *oauth2Config) setDefault() { + o.grantType = "client_credentials" } -// IsEnable checks whether the OAuth2 is enabled. -// One of values of ClientID, ClientSecret and TokenURL is not empty means enabled. -func (o *OAuth2) IsEnable() bool { - return len(o.ClientID) > 0 || len(o.ClientSecret) > 0 || len(o.TokenURL) > 0 +// isEnabled checks whether the oauth2Config is enabled. +// One of values of clientID, clientSecret and tokenURL is not empty means enabled. +func (o *oauth2Config) isEnabled() bool { + return len(o.clientID) > 0 || len(o.clientSecret) > 0 || len(o.tokenURL) > 0 } -// GSSAPIAuthType defines the type of GSSAPI authentication. -type GSSAPIAuthType int +// gssapiAuthType defines the type of GSSAPI authentication. +type gssapiAuthType int const ( - // UnknownAuth means the auth type is unknown. - UnknownAuth GSSAPIAuthType = 0 - // UserAuth means the auth type is user. - UserAuth GSSAPIAuthType = 1 - // KeyTabAuth means the auth type is keytab. - KeyTabAuth GSSAPIAuthType = 2 + // unknownAuth means the auth type is unknown. + unknownAuth gssapiAuthType = 0 + // userAuth means the auth type is user. + userAuth gssapiAuthType = 1 + // keyTabAuth means the auth type is keytab. + keyTabAuth gssapiAuthType = 2 ) -// AuthTypeFromString convent the string to GSSAPIAuthType. -func AuthTypeFromString(s string) (GSSAPIAuthType, error) { +// gssapiAuthTypeFromString convent the string to gssapiAuthType. +func gssapiAuthTypeFromString(s string) (gssapiAuthType, error) { switch strings.ToLower(s) { case "user": - return UserAuth, nil + return userAuth, nil case "keytab": - return KeyTabAuth, nil + return keyTabAuth, nil default: - return UnknownAuth, errors.Errorf("unknown %s auth type", s) + return unknownAuth, errors.Errorf("unknown %s auth type", s) } } -// GSSAPI holds necessary path parameter to support sasl-gssapi. -type GSSAPI struct { - AuthType GSSAPIAuthType `toml:"sasl-gssapi-auth-type" json:"sasl-gssapi-auth-type"` - KeyTabPath string `toml:"sasl-gssapi-keytab-path" json:"sasl-gssapi-keytab-path"` - KerberosConfigPath string `toml:"sasl-gssapi-kerberos-config-path" json:"sasl-gssapi-kerberos-config-path"` - ServiceName string `toml:"sasl-gssapi-service-name" json:"sasl-gssapi-service-name"` - Username string `toml:"sasl-gssapi-user" json:"sasl-gssapi-user"` - Password string `toml:"sasl-gssapi-password" json:"sasl-gssapi-password"` - Realm string `toml:"sasl-gssapi-realm" json:"sasl-gssapi-realm"` - DisablePAFXFAST bool `toml:"sasl-gssapi-disable-pafxfast" json:"sasl-gssapi-disable-pafxfast"` +// gssapiConfig holds necessary path parameter to support sasl-gssapi. +type gssapiConfig struct { + authType gssapiAuthType + keyTabPath string + kerberosConfigPath string + serviceName string + username string + password string + realm string + disablePAFXFAST bool } diff --git a/pkg/sink/kafka/sasl_config_test.go b/pkg/sink/kafka/sasl_config_test.go index 73ff7b61ae..7bb7200fa7 100644 --- a/pkg/sink/kafka/sasl_config_test.go +++ b/pkg/sink/kafka/sasl_config_test.go @@ -70,7 +70,7 @@ func TestSASLMechanismFromString(t *testing.T) { expectedMechanism: "GSSAPI", }, { - name: "upper case GSSAPI mechanism", + name: "upper case gssapi mechanism", s: "GSSAPI", expectedMechanism: "GSSAPI", }, @@ -78,7 +78,7 @@ func TestSASLMechanismFromString(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - mechanism, err := SASLMechanismFromString(test.s) + mechanism, err := saslMechanismFromString(test.s) if test.expectErr != "" { require.Error(t, err) require.Regexp(t, test.expectErr, err.Error()) @@ -131,7 +131,7 @@ func TestAuthTypeFromString(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() - authType, err := AuthTypeFromString(test.s) + authType, err := gssapiAuthTypeFromString(test.s) if test.expectErr != "" { require.Error(t, err) require.Regexp(t, test.expectErr, err.Error()) diff --git a/pkg/sink/kafka/sasl_test.go b/pkg/sink/kafka/sasl_test.go index a3f71ab1a9..63a0e914eb 100644 --- a/pkg/sink/kafka/sasl_test.go +++ b/pkg/sink/kafka/sasl_test.go @@ -24,15 +24,15 @@ func TestBuildSaslMechanismGSSAPIUserAuth(t *testing.T) { t.Parallel() o := &clientOptions{ - SASL: &SASL{ - SASLMechanism: GSSAPIMechanism, - GSSAPI: GSSAPI{ - AuthType: UserAuth, - KerberosConfigPath: "/etc/krb5.conf", - ServiceName: "kafka", - Username: "alice", - Password: "pwd", - Realm: "EXAMPLE.COM", + sasl: &saslConfig{ + mechanism: gssapiMechanismName, + gssapi: gssapiConfig{ + authType: userAuth, + kerberosConfigPath: "/etc/krb5.conf", + serviceName: "kafka", + username: "alice", + password: "pwd", + realm: "EXAMPLE.COM", }, }, } @@ -46,15 +46,15 @@ func TestBuildSaslMechanismGSSAPIKeytabAuth(t *testing.T) { t.Parallel() o := &clientOptions{ - SASL: &SASL{ - SASLMechanism: GSSAPIMechanism, - GSSAPI: GSSAPI{ - AuthType: KeyTabAuth, - KerberosConfigPath: "/etc/krb5.conf", - ServiceName: "kafka", - Username: "alice", - KeyTabPath: "/tmp/a.keytab", - Realm: "EXAMPLE.COM", + sasl: &saslConfig{ + mechanism: gssapiMechanismName, + gssapi: gssapiConfig{ + authType: keyTabAuth, + kerberosConfigPath: "/etc/krb5.conf", + serviceName: "kafka", + username: "alice", + keyTabPath: "/tmp/a.keytab", + realm: "EXAMPLE.COM", }, }, } From 3bdb07bcaef3b52d47192ba18948fac162b619cc Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 21:58:26 +0800 Subject: [PATCH 28/61] simplify more code --- pkg/sink/kafka/admin_client.go | 14 ++-- pkg/sink/kafka/gssapi.go | 19 +---- pkg/sink/kafka/metrics_hook.go | 121 ++++++++++----------------- pkg/sink/kafka/metrics_hook_test.go | 8 +- pkg/sink/kafka/options.go | 6 +- pkg/sink/kafka/sasl_config.go | 11 --- pkg/sink/kafka/sync_producer.go | 63 +++++++------- pkg/sink/kafka/sync_producer_test.go | 75 +---------------- 8 files changed, 95 insertions(+), 222 deletions(-) diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index b795c36372..b9616fab90 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -61,17 +61,13 @@ func newAdminClient( }, nil } -func (a *kafkaAdminClient) newRequestContext() (context.Context, context.CancelFunc) { - return context.WithTimeout(a.client.Context(), a.timeout) -} - func (a *kafkaAdminClient) GetBrokerConfig(configName string) (value string, err error) { startTime := time.Now() defer func() { observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetBrokerConfig, err, time.Since(startTime)) }() - ctx, cancel := a.newRequestContext() + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() meta, err := a.admin.BrokerMetadata(ctx) @@ -116,7 +112,7 @@ func (a *kafkaAdminClient) GetTopicConfig(topicName string, configName string) ( observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicConfig, err, time.Since(startTime)) }() - ctx, cancel := a.newRequestContext() + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) @@ -164,7 +160,7 @@ func (a *kafkaAdminClient) GetTopicsMeta( return make(map[string]TopicDetail), nil } - ctx, cancel := a.newRequestContext() + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() meta, err := a.admin.Metadata(ctx, topics...) @@ -208,7 +204,7 @@ func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (result map[s return make(map[string]int32), nil } - ctx, cancel := a.newRequestContext() + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() meta, err := a.admin.Metadata(ctx, topics...) @@ -241,7 +237,7 @@ func (a *kafkaAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) ( return err } - ctx, cancel := a.newRequestContext() + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() var responses kadm.CreateTopicResponses diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/gssapi.go index fa0d8fa87d..662a4f441f 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -18,7 +18,6 @@ import ( "encoding/binary" "fmt" "strings" - "sync" "github.com/jcmturner/gofork/encoding/asn1" "github.com/jcmturner/gokrb5/v8/asn1tools" @@ -87,7 +86,7 @@ func (m *gssapiMechanism) Authenticate( } firstMessage, err := session.nextMessage(nil) if err != nil { - session.close() + client.Destroy() return nil, nil, errors.Trace(err) } return session, firstMessage, nil @@ -98,39 +97,27 @@ type gssapiSession struct { ticket messages.Ticket encKey types.EncryptionKey step int - - closeOnce sync.Once } func (s *gssapiSession) Challenge(challenge []byte) (bool, []byte, error) { + defer s.client.Destroy() + switch s.step { case gssAPIVerify: msg, err := s.nextMessage(challenge) if err != nil { - s.close() return false, nil, errors.Trace(err) } // Return a final payload while marking done=true. // The Kafka client writes this message and finishes the auth flow. - s.close() return true, msg, nil case gssAPIFinished: - s.close() return true, nil, nil default: - s.close() return false, nil, errors.New("invalid gssapi session state") } } -func (s *gssapiSession) close() { - s.closeOnce.Do(func() { - if s.client != nil { - s.client.Destroy() - } - }) -} - func (s *gssapiSession) nextMessage(challenge []byte) ([]byte, error) { switch s.step { case gssAPIInitial: diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go index 9653bef9ae..7690529859 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -59,34 +59,18 @@ func (h *metricsHook) cleanupMetrics() { "namespace": h.keyspace, "changefeed": h.changefeed, } - deleteGaugeVecPartialMatch(h.metrics.OutgoingByteRate, labels) - deleteGaugeVecPartialMatch(h.metrics.RequestRate, labels) - deleteGaugeVecPartialMatch(h.metrics.ResponseRate, labels) - deleteGaugeVecPartialMatch(h.metrics.RequestsInFlight, labels) - deleteGaugeVecPartialMatch(h.metrics.RequestLatency, labels) - deleteGaugeVecPartialMatch(h.metrics.CompressionRatio, labels) - deleteGaugeVecPartialMatch(h.metrics.RecordsPerRequest, labels) -} - -func (h *metricsHook) RecordBrokerWrite(nodeID int32, bytesWritten int, err error) { - if nodeID < 0 { - return - } - - ctx, ok := h.loadMetricsContext() - if !ok { - return - } - brokerID := strconv.Itoa(int(nodeID)) - - if ctx.metrics.OutgoingByteRate != nil && bytesWritten > 0 { - ctx.metrics.OutgoingByteRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(float64(bytesWritten)) - } - if ctx.metrics.RequestRate != nil { - ctx.metrics.RequestRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) - } - if err == nil && ctx.metrics.RequestsInFlight != nil { - ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + for _, gaugeVec := range []*prometheus.GaugeVec{ + h.metrics.OutgoingByteRate, + h.metrics.RequestRate, + h.metrics.ResponseRate, + h.metrics.RequestsInFlight, + h.metrics.RequestLatency, + h.metrics.CompressionRatio, + h.metrics.RecordsPerRequest, + } { + if gaugeVec != nil { + gaugeVec.DeletePartialMatch(labels) + } } } @@ -98,7 +82,20 @@ func (h *metricsHook) OnBrokerWrite( _ time.Duration, err error, ) { - h.RecordBrokerWrite(meta.NodeID, bytesWritten, err) + if meta.NodeID < 0 { + return + } + brokerID := strconv.Itoa(int(meta.NodeID)) + + if h.metrics.OutgoingByteRate != nil && bytesWritten > 0 { + h.metrics.OutgoingByteRate.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(float64(bytesWritten)) + } + if h.metrics.RequestRate != nil { + h.metrics.RequestRate.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + } + if err == nil && h.metrics.RequestsInFlight != nil { + h.metrics.RequestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + } } func (h *metricsHook) OnBrokerE2E( @@ -109,23 +106,18 @@ func (h *metricsHook) OnBrokerE2E( if meta.NodeID < 0 { return } - - ctx, ok := h.loadMetricsContext() - if !ok { - return - } brokerID := strconv.Itoa(int(meta.NodeID)) - if e2e.WriteErr == nil && ctx.metrics.RequestsInFlight != nil { - ctx.metrics.RequestsInFlight.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(-1) + if e2e.WriteErr == nil && h.metrics.RequestsInFlight != nil { + h.metrics.RequestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(-1) } - if e2e.BytesRead > 0 && e2e.ReadErr == nil && ctx.metrics.ResponseRate != nil { - ctx.metrics.ResponseRate.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID).Add(1) + if e2e.BytesRead > 0 && e2e.ReadErr == nil && h.metrics.ResponseRate != nil { + h.metrics.ResponseRate.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) } - if e2e.Err() == nil && ctx.metrics.RequestLatency != nil { + if e2e.Err() == nil && h.metrics.RequestLatency != nil { latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 - ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricAvg).Set(latencyMs) - ctx.metrics.RequestLatency.WithLabelValues(ctx.keyspace, ctx.changefeed, brokerID, legacyMetricP99).Set(latencyMs) + h.metrics.RequestLatency.WithLabelValues(h.keyspace, h.changefeed, brokerID, legacyMetricAvg).Set(latencyMs) + h.metrics.RequestLatency.WithLabelValues(h.keyspace, h.changefeed, brokerID, legacyMetricP99).Set(latencyMs) } } @@ -135,43 +127,14 @@ func (h *metricsHook) OnProduceBatchWritten( _ int32, m kgo.ProduceBatchMetrics, ) { - h.RecordProduceBatchWritten(m.NumRecords, m.UncompressedBytes, m.CompressedBytes) -} - -func (h *metricsHook) RecordProduceBatchWritten(numRecords int, uncompressedBytes int, compressedBytes int) { - ctx, ok := h.loadMetricsContext() - if !ok { - return - } - - if ctx.metrics.RecordsPerRequest != nil && numRecords > 0 { - records := float64(numRecords) - ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(records) - ctx.metrics.RecordsPerRequest.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(records) - } - if ctx.metrics.CompressionRatio != nil && uncompressedBytes > 0 && compressedBytes > 0 { - ratio := float64(uncompressedBytes) / float64(compressedBytes) * 100 - ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricAvg).Set(ratio) - ctx.metrics.CompressionRatio.WithLabelValues(ctx.keyspace, ctx.changefeed, legacyMetricP99).Set(ratio) - } -} - -type metricsContext struct { - keyspace string - changefeed string - metrics metricVectors -} - -func (h *metricsHook) loadMetricsContext() (metricsContext, bool) { - return metricsContext{ - keyspace: h.keyspace, - changefeed: h.changefeed, - metrics: h.metrics, - }, true -} - -func deleteGaugeVecPartialMatch(gaugeVec *prometheus.GaugeVec, labels prometheus.Labels) { - if gaugeVec != nil { - gaugeVec.DeletePartialMatch(labels) + if h.metrics.RecordsPerRequest != nil && m.NumRecords > 0 { + records := float64(m.NumRecords) + h.metrics.RecordsPerRequest.WithLabelValues(h.keyspace, h.changefeed, legacyMetricAvg).Set(records) + h.metrics.RecordsPerRequest.WithLabelValues(h.keyspace, h.changefeed, legacyMetricP99).Set(records) + } + if h.metrics.CompressionRatio != nil && m.UncompressedBytes > 0 && m.CompressedBytes > 0 { + ratio := float64(m.UncompressedBytes) / float64(m.CompressedBytes) * 100 + h.metrics.CompressionRatio.WithLabelValues(h.keyspace, h.changefeed, legacyMetricAvg).Set(ratio) + h.metrics.CompressionRatio.WithLabelValues(h.keyspace, h.changefeed, legacyMetricP99).Set(ratio) } } diff --git a/pkg/sink/kafka/metrics_hook_test.go b/pkg/sink/kafka/metrics_hook_test.go index 726eb80518..941af214ae 100644 --- a/pkg/sink/kafka/metrics_hook_test.go +++ b/pkg/sink/kafka/metrics_hook_test.go @@ -63,14 +63,18 @@ func TestMetricsHookRecordsMetricsAndCleanup(t *testing.T) { CompressionRatio: compressionRatio, }) - hook.RecordBrokerWrite(1, 42, nil) + hook.OnBrokerWrite(kgo.BrokerMetadata{NodeID: 1}, 0, 42, 0, 0, nil) hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: 1}, 0, kgo.BrokerE2E{ BytesRead: 42, TimeToWrite: 10 * time.Millisecond, ReadWait: 20 * time.Millisecond, TimeToRead: 30 * time.Millisecond, }) - hook.RecordProduceBatchWritten(3, 100, 50) + hook.OnProduceBatchWritten(kgo.BrokerMetadata{}, "", 0, kgo.ProduceBatchMetrics{ + NumRecords: 3, + UncompressedBytes: 100, + CompressedBytes: 50, + }) require.Equal(t, float64(42), testutil.ToFloat64(outgoingByteRate.WithLabelValues("default", "cf", "1"))) require.Equal(t, float64(1), testutil.ToFloat64(requestRate.WithLabelValues("default", "cf", "1"))) diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index dedf9bb809..53955a9ca3 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -505,7 +505,9 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf o.sasl.oauth2.tokenURL = tokenURL } - if o.sasl.oauth2.isEnabled() { + if o.sasl.oauth2.clientID != "" || + o.sasl.oauth2.clientSecret != "" || + o.sasl.oauth2.tokenURL != "" { if o.sasl.mechanism != oauthMechanism { return cerror.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 is only supported with SASL mechanism type OAUTHBEARER, but got %s", @@ -515,7 +517,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if err := o.sasl.oauth2.validate(); err != nil { return cerror.ErrKafkaInvalidConfig.Wrap(err) } - o.sasl.oauth2.setDefault() + o.sasl.oauth2.grantType = "client_credentials" } if sinkConfig.KafkaConfig.SASLOAuthScopes != nil { diff --git a/pkg/sink/kafka/sasl_config.go b/pkg/sink/kafka/sasl_config.go index e5d8402b94..a6fd90577f 100644 --- a/pkg/sink/kafka/sasl_config.go +++ b/pkg/sink/kafka/sasl_config.go @@ -90,17 +90,6 @@ func (o *oauth2Config) validate() error { return nil } -// setDefault sets the default value of oauth2Config. -func (o *oauth2Config) setDefault() { - o.grantType = "client_credentials" -} - -// isEnabled checks whether the oauth2Config is enabled. -// One of values of clientID, clientSecret and tokenURL is not empty means enabled. -func (o *oauth2Config) isEnabled() bool { - return len(o.clientID) > 0 || len(o.clientSecret) > 0 || len(o.tokenURL) > 0 -} - // gssapiAuthType defines the type of GSSAPI authentication. type gssapiAuthType int diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index 373a444297..cde9b52555 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -62,19 +62,20 @@ func newSyncProducer( }, nil } -func (p *kafkaSyncProducer) newRequestContext() (context.Context, context.CancelFunc) { - return context.WithTimeout(p.client.Context(), p.timeout) -} - func (p *kafkaSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { if p.closed.Load() { return errors.ErrKafkaProducerClosed.GenWithStackByArgs() } - ctx, cancel := p.newRequestContext() + ctx, cancel := context.WithTimeout(p.client.Context(), p.timeout) defer cancel() - record := buildRecord(topic, partitionNum, message) + record := &kgo.Record{ + Topic: topic, + Partition: partitionNum, + Key: message.Key, + Value: message.Value, + } err := p.client.ProduceSync(ctx, record).FirstErr() if legacyKafkaSinkFailpointEnabled(kafkaSinkSyncSendMessageErrorFailpoint) { @@ -85,7 +86,15 @@ func (p *kafkaSyncProducer) SendMessage(topic string, partitionNum int32, messag err = errors.New("kafka sink sync send message injected error") }) - return p.wrapSendError(message, err) + if err != nil { + err = AnnotateEventError( + p.id.Keyspace(), + p.id.Name(), + message.LogInfo, + err, + ) + } + return errors.WrapError(errors.ErrKafkaSendMessage, err) } func (p *kafkaSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { @@ -95,10 +104,15 @@ func (p *kafkaSyncProducer) SendMessages(topic string, partitionNum int32, messa records := make([]*kgo.Record, 0, partitionNum) for i := 0; i < int(partitionNum); i++ { - records = append(records, buildRecord(topic, int32(i), message)) + records = append(records, &kgo.Record{ + Topic: topic, + Partition: int32(i), + Key: message.Key, + Value: message.Value, + }) } - ctx, cancel := p.newRequestContext() + ctx, cancel := context.WithTimeout(p.client.Context(), p.timeout) defer cancel() err := p.client.ProduceSync(ctx, records...).FirstErr() @@ -111,7 +125,15 @@ func (p *kafkaSyncProducer) SendMessages(topic string, partitionNum int32, messa err = errors.New("kafka sink sync send messages injected error") }) - return p.wrapSendError(message, err) + if err != nil { + err = AnnotateEventError( + p.id.Keyspace(), + p.id.Name(), + message.LogInfo, + err, + ) + } + return errors.WrapError(errors.ErrKafkaSendMessage, err) } func (p *kafkaSyncProducer) Heartbeat() {} @@ -131,24 +153,3 @@ func (p *kafkaSyncProducer) Close() { zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start))) } - -func buildRecord(topic string, partition int32, message *common.Message) *kgo.Record { - return &kgo.Record{ - Topic: topic, - Partition: partition, - Key: message.Key, - Value: message.Value, - } -} - -func (p *kafkaSyncProducer) wrapSendError(message *common.Message, err error) error { - if err != nil { - err = AnnotateEventError( - p.id.Keyspace(), - p.id.Name(), - message.LogInfo, - err, - ) - } - return errors.WrapError(errors.ErrKafkaSendMessage, err) -} diff --git a/pkg/sink/kafka/sync_producer_test.go b/pkg/sink/kafka/sync_producer_test.go index 47dbdd79e9..a101b019b9 100644 --- a/pkg/sink/kafka/sync_producer_test.go +++ b/pkg/sink/kafka/sync_producer_test.go @@ -16,9 +16,8 @@ package kafka import ( "testing" - "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" "go.uber.org/atomic" ) @@ -26,77 +25,9 @@ import ( func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { producer := &kafkaSyncProducer{closed: atomic.NewBool(true)} - err := producer.SendMessage("topic", 1, &codecCommon.Message{}) + err := producer.SendMessage("topic", 1, &common.Message{}) require.ErrorIs(t, err, errors.ErrKafkaProducerClosed) - err = producer.SendMessages("topic", 1, &codecCommon.Message{}) + err = producer.SendMessages("topic", 1, &common.Message{}) require.ErrorIs(t, err, errors.ErrKafkaProducerClosed) } - -func TestBuildRecord(t *testing.T) { - message := &codecCommon.Message{ - Key: []byte("key"), - Value: []byte("value"), - } - - record := buildRecord("topic", 3, message) - - require.Equal(t, "topic", record.Topic) - require.Equal(t, int32(3), record.Partition) - require.Equal(t, []byte("key"), record.Key) - require.Equal(t, []byte("value"), record.Value) -} - -func TestSyncProducerWrapSendErrorAnnotatesEventContext(t *testing.T) { - producer := &kafkaSyncProducer{ - id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync-error"), - } - - testCases := []struct { - name string - message *codecCommon.Message - contains []string - }{ - { - name: "ddl", - message: &codecCommon.Message{LogInfo: &codecCommon.MessageLogInfo{ - DDL: &codecCommon.DDLLogInfo{ - Query: "create table t(id int primary key)", - StartTs: 10, - CommitTs: 20, - }, - }}, - contains: []string{ - "keyspace=default", - "changefeed=sync-error", - "eventType=ddl", - `ddlQuery="create table t(id int primary key)"`, - "ddlStartTs=10", - "ddlCommitTs=20", - }, - }, - { - name: "checkpoint", - message: &codecCommon.Message{LogInfo: &codecCommon.MessageLogInfo{ - Checkpoint: &codecCommon.CheckpointLogInfo{CommitTs: 30}, - }}, - contains: []string{ - "keyspace=default", - "changefeed=sync-error", - "eventType=checkpoint", - "checkpointTs=30", - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := producer.wrapSendError(tc.message, errors.New("send failed")) - require.ErrorIs(t, err, errors.ErrKafkaSendMessage) - require.ErrorContains(t, err, "send failed") - for _, expected := range tc.contains { - require.ErrorContains(t, err, expected) - } - }) - } -} From f681f09cf2c6f313d3a95349131c98a8d3ffbec5 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 22:01:56 +0800 Subject: [PATCH 29/61] simplify more code --- pkg/sink/kafka/admin_client.go | 43 ++++-------------- pkg/sink/kafka/admin_metrics.go | 77 --------------------------------- pkg/sink/kafka/kafka_factory.go | 1 - pkg/sink/kafka/metrics.go | 1 - 4 files changed, 9 insertions(+), 113 deletions(-) delete mode 100644 pkg/sink/kafka/admin_metrics.go diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index b9616fab90..ce18e29158 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -61,12 +61,7 @@ func newAdminClient( }, nil } -func (a *kafkaAdminClient) GetBrokerConfig(configName string) (value string, err error) { - startTime := time.Now() - defer func() { - observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetBrokerConfig, err, time.Since(startTime)) - }() - +func (a *kafkaAdminClient) GetBrokerConfig(configName string) (string, error) { ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() @@ -106,12 +101,7 @@ func (a *kafkaAdminClient) GetBrokerConfig(configName string) (value string, err "cannot find the `%s` from the broker's configuration", configName) } -func (a *kafkaAdminClient) GetTopicConfig(topicName string, configName string) (value string, err error) { - startTime := time.Now() - defer func() { - observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicConfig, err, time.Since(startTime)) - }() - +func (a *kafkaAdminClient) GetTopicConfig(topicName string, configName string) (string, error) { ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() @@ -150,12 +140,7 @@ func (a *kafkaAdminClient) GetTopicConfig(topicName string, configName string) ( func (a *kafkaAdminClient) GetTopicsMeta( topics []string, ignoreTopicError bool, -) (result map[string]TopicDetail, err error) { - startTime := time.Now() - defer func() { - observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicsMeta, err, time.Since(startTime)) - }() - +) (map[string]TopicDetail, error) { if len(topics) == 0 { return make(map[string]TopicDetail), nil } @@ -168,7 +153,7 @@ func (a *kafkaAdminClient) GetTopicsMeta( return nil, errors.Trace(err) } - result = make(map[string]TopicDetail, len(topics)) + result := make(map[string]TopicDetail, len(topics)) for _, topic := range topics { detail, ok := meta.Topics[topic] if !ok { @@ -194,12 +179,7 @@ func (a *kafkaAdminClient) GetTopicsMeta( return result, nil } -func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (result map[string]int32, err error) { - startTime := time.Now() - defer func() { - observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodGetTopicsPartitions, err, time.Since(startTime)) - }() - +func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { if len(topics) == 0 { return make(map[string]int32), nil } @@ -212,7 +192,7 @@ func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (result map[s return nil, errors.Trace(err) } - result = make(map[string]int32, len(topics)) + result := make(map[string]int32, len(topics)) for _, topic := range topics { detail, ok := meta.Topics[topic] if !ok { @@ -226,21 +206,16 @@ func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (result map[s return result, nil } -func (a *kafkaAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) (err error) { - startTime := time.Now() - defer func() { - observeAdminCall(a.changefeed.Keyspace(), a.changefeed.Name(), adminMethodCreateTopic, err, time.Since(startTime)) - }() - +func (a *kafkaAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { if detail == nil { - err = errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") - return err + return errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") } ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() var responses kadm.CreateTopicResponses + var err error if validateOnly { responses, err = a.admin.ValidateCreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) } else { diff --git a/pkg/sink/kafka/admin_metrics.go b/pkg/sink/kafka/admin_metrics.go deleted file mode 100644 index 1b4f763b66..0000000000 --- a/pkg/sink/kafka/admin_metrics.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" -) - -const ( - adminMethodGetBrokerConfig = "get_broker_config" - adminMethodGetTopicConfig = "get_topic_config" - adminMethodGetTopicsMeta = "get_topics_meta" - adminMethodGetTopicsPartitions = "get_topics_partitions_num" - adminMethodCreateTopic = "create_topic" - adminCallStatusOK = "ok" - adminCallStatusError = "error" -) - -var ( - adminCallCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_admin_call_total", - Help: "Total kafka admin calls by method and result.", - }, []string{"namespace", "changefeed", "method", "result"}) - adminCallLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_client_admin_call_duration_seconds", - Help: "Latency of kafka admin calls by method and result.", - Buckets: prometheus.DefBuckets, - }, []string{"namespace", "changefeed", "method", "result"}) -) - -func initAdminMetrics(registry *prometheus.Registry) { - registry.MustRegister(adminCallCount) - registry.MustRegister(adminCallLatency) -} - -func cleanupAdminMetrics(keyspace string, changefeed string) { - labels := prometheus.Labels{ - "namespace": keyspace, - "changefeed": changefeed, - } - adminCallCount.DeletePartialMatch(labels) - adminCallLatency.DeletePartialMatch(labels) -} - -func observeAdminCall( - keyspace string, - changefeed string, - method string, - callErr error, - duration time.Duration, -) { - status := adminCallStatusOK - if callErr != nil { - status = adminCallStatusError - } - adminCallCount.WithLabelValues(keyspace, changefeed, method, status).Inc() - adminCallLatency.WithLabelValues(keyspace, changefeed, method, status).Observe(duration.Seconds()) -} diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go index 14e31fe333..6048e209cc 100644 --- a/pkg/sink/kafka/kafka_factory.go +++ b/pkg/sink/kafka/kafka_factory.go @@ -37,7 +37,6 @@ func (c *kafkaMetricsCollector) Run(ctx context.Context) { if c.hook != nil { c.hook.cleanupMetrics() } - cleanupAdminMetrics(c.changefeedID.Keyspace(), c.changefeedID.Name()) } func newKafkaMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index 6d33c678c5..d3f88055e2 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -90,7 +90,6 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(requestsInFlightGauge) registry.MustRegister(responseRateGauge) - initAdminMetrics(registry) claimcheck.InitMetrics(registry) codec.InitMetrics(registry) } From 1edbc65ac8cf4a57ed8797638cc1959164ef5f9f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 22:11:02 +0800 Subject: [PATCH 30/61] fix code --- downstreamadapter/sink/kafka/helper.go | 14 +- pkg/sink/kafka/async_producer.go | 16 +++ pkg/sink/kafka/factory.go | 105 ++++++++++---- ...{kafka_factory_test.go => factory_test.go} | 0 pkg/sink/kafka/kafka_factory.go | 132 ------------------ pkg/sink/kafka/metrics_collector.go | 18 ++- pkg/sink/kafka/metrics_hook.go | 17 +++ pkg/sink/kafka/sync_producer.go | 17 +++ 8 files changed, 145 insertions(+), 174 deletions(-) rename pkg/sink/kafka/{kafka_factory_test.go => factory_test.go} (100%) delete mode 100644 pkg/sink/kafka/kafka_factory.go diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index 9dbf4194bc..27c0ba28ca 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -49,11 +49,10 @@ func (c components) close() { } } -func newKafkaSinkComponentWithFactory(ctx context.Context, +func newKafkaSinkComponent(ctx context.Context, changefeedID commonType.ChangeFeedID, sinkURI *url.URL, sinkConfig *config.SinkConfig, - factoryCreator kafka.FactoryCreator, ) (components, config.Protocol, error) { kafkaComponent := components{} protocol, err := helper.GetProtocol(utils.GetOrZero(sinkConfig.Protocol)) @@ -72,7 +71,7 @@ func newKafkaSinkComponentWithFactory(ctx context.Context, } options.Topic = topic - kafkaComponent.factory, err = factoryCreator(ctx, options, changefeedID) + kafkaComponent.factory, err = kafka.NewFactory(ctx, options, changefeedID) if err != nil { return kafkaComponent, protocol, errors.WrapError(errors.ErrKafkaNewProducer, err) } @@ -129,12 +128,3 @@ func newKafkaSinkComponentWithFactory(ctx context.Context, } return kafkaComponent, protocol, nil } - -func newKafkaSinkComponent( - ctx context.Context, - changefeedID commonType.ChangeFeedID, - sinkURI *url.URL, - sinkConfig *config.SinkConfig, -) (components, config.Protocol, error) { - return newKafkaSinkComponentWithFactory(ctx, changefeedID, sinkURI, sinkConfig, kafka.NewFactory) -} diff --git a/pkg/sink/kafka/async_producer.go b/pkg/sink/kafka/async_producer.go index 869b92f5f1..88df9948ff 100644 --- a/pkg/sink/kafka/async_producer.go +++ b/pkg/sink/kafka/async_producer.go @@ -27,6 +27,22 @@ import ( "go.uber.org/zap" ) +// AsyncProducer is the kafka async producer +type AsyncProducer interface { + // Close shuts down the producer asynchronously and releases its Kafka client + // resources. It does not wait for buffered messages to be flushed. + Close() + + // AsyncSend is the input channel for the user to write messages to that they + // wish to send. + AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error + + // AsyncRunCallback process the messages that has sent to kafka, + // and run tha attached callback. the caller should call this + // method in a background goroutine + AsyncRunCallback(ctx context.Context) error +} + type kafkaAsyncProducer struct { client *kgo.Client changefeedID commonType.ChangeFeedID diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index d80e6b0cce..3d26aa84e6 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -16,8 +16,8 @@ package kafka import ( "context" - commonType "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" ) // Factory is used to produce all kafka components. @@ -32,38 +32,85 @@ type Factory interface { MetricsCollector() MetricsCollector } -// FactoryCreator defines the type of factory creator. -type FactoryCreator func(context.Context, *options, commonType.ChangeFeedID) (Factory, error) +type factory struct { + changefeedID common.ChangeFeedID + clientOption *clientOptions -// SyncProducer is the kafka sync producer -type SyncProducer interface { - // SendMessage produces a given message, and returns only when it either has - // succeeded or failed to produce. It will return the partition and the offset - // of the produced message, or an error if the message failed to produce. - SendMessage(topic string, partitionNum int32, message *common.Message) error + metricsHook *metricsHook +} + +// NewFactory constructs a Factory. +func NewFactory( + ctx context.Context, + o *options, + changefeedID common.ChangeFeedID, +) (Factory, error) { + admin, err := newAdminClient(ctx, changefeedID, newClientOption(o), nil) + if err != nil { + return nil, errors.Trace(err) + } + defer admin.Close() + + if err := adjustOptions(ctx, admin, o, o.Topic); err != nil { + return nil, errors.Trace(err) + } + + return &factory{ + changefeedID: changefeedID, + clientOption: newClientOption(o), + metricsHook: newKafkaMetricsHook(changefeedID), + }, nil +} + +func (f *factory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { + admin, err := newAdminClient(ctx, f.changefeedID, f.clientOption, f.metricsHook) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + } + return admin, nil +} + +func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { + producer, err := newSyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + } + return producer, nil +} - // SendMessages produces a given set of messages, and returns only when all - // messages in the set have either succeeded or failed. Note that messages - // can succeed and fail individually; if some succeed and some fail, - // SendMessages will return an error. - SendMessages(topic string, partitionNum int32, message *common.Message) error +func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { + producer, err := newAsyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + } + return producer, nil +} - // Close shuts down the producer and releases its Kafka client resources. - Close() +func (f *factory) MetricsCollector() MetricsCollector { + return &kafkaMetricsCollector{changefeedID: f.changefeedID, hook: f.metricsHook} } -// AsyncProducer is the kafka async producer -type AsyncProducer interface { - // Close shuts down the producer asynchronously and releases its Kafka client - // resources. It does not wait for buffered messages to be flushed. - Close() +func newClientOption(o *options) *clientOptions { + return &clientOptions{ + BrokerEndpoints: o.BrokerEndpoints, + ClientID: o.ClientID, + + Version: o.Version, + IsAssignedVersion: o.IsAssignedVersion, + + MaxMessageBytes: o.MaxMessageBytes, + ProducerBatchMaxBytes: o.ProducerBatchMaxBytes, + MaxRetry: o.MaxRetry, + Compression: o.Compression, + RequiredAcks: int16(o.RequiredAcks), - // AsyncSend is the input channel for the user to write messages to that they - // wish to send. - AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error + EnableTLS: o.EnableTLS, + Credential: o.Credential, + InsecureSkipVerify: o.InsecureSkipVerify, + sasl: o.sasl, - // AsyncRunCallback process the messages that has sent to kafka, - // and run tha attached callback. the caller should call this - // method in a background goroutine - AsyncRunCallback(ctx context.Context) error + DialTimeout: o.DialTimeout, + WriteTimeout: o.WriteTimeout, + ReadTimeout: o.ReadTimeout, + } } diff --git a/pkg/sink/kafka/kafka_factory_test.go b/pkg/sink/kafka/factory_test.go similarity index 100% rename from pkg/sink/kafka/kafka_factory_test.go rename to pkg/sink/kafka/factory_test.go diff --git a/pkg/sink/kafka/kafka_factory.go b/pkg/sink/kafka/kafka_factory.go deleted file mode 100644 index 6048e209cc..0000000000 --- a/pkg/sink/kafka/kafka_factory.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" -) - -type factory struct { - changefeedID common.ChangeFeedID - clientOption *clientOptions - - metricsHook *metricsHook -} - -type kafkaMetricsCollector struct { - changefeedID common.ChangeFeedID - hook *metricsHook -} - -func (c *kafkaMetricsCollector) Run(ctx context.Context) { - <-ctx.Done() - if c.hook != nil { - c.hook.cleanupMetrics() - } -} - -func newKafkaMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { - return newMetricsHook( - changefeedID.Keyspace(), - changefeedID.Name(), - metricVectors{ - RequestsInFlight: requestsInFlightGauge, - OutgoingByteRate: OutgoingByteRateGauge, - RequestRate: RequestRateGauge, - RequestLatency: RequestLatencyGauge, - ResponseRate: responseRateGauge, - CompressionRatio: compressionRatioGauge, - RecordsPerRequest: recordsPerRequestGauge, - }, - ) -} - -// NewFactory constructs a Factory. -func NewFactory( - ctx context.Context, - o *options, - changefeedID common.ChangeFeedID, -) (Factory, error) { - admin, err := newAdminClient(ctx, changefeedID, newClientOption(o), nil) - if err != nil { - return nil, errors.Trace(err) - } - defer admin.Close() - - if err := adjustOptions(ctx, admin, o, o.Topic); err != nil { - return nil, errors.Trace(err) - } - - return &factory{ - changefeedID: changefeedID, - clientOption: newClientOption(o), - metricsHook: newKafkaMetricsHook(changefeedID), - }, nil -} - -func (f *factory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - admin, err := newAdminClient(ctx, f.changefeedID, f.clientOption, f.metricsHook) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - return admin, nil -} - -func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := newSyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - return producer, nil -} - -func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := newAsyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) - } - return producer, nil -} - -func (f *factory) MetricsCollector() MetricsCollector { - return &kafkaMetricsCollector{changefeedID: f.changefeedID, hook: f.metricsHook} -} - -func newClientOption(o *options) *clientOptions { - return &clientOptions{ - BrokerEndpoints: o.BrokerEndpoints, - ClientID: o.ClientID, - - Version: o.Version, - IsAssignedVersion: o.IsAssignedVersion, - - MaxMessageBytes: o.MaxMessageBytes, - ProducerBatchMaxBytes: o.ProducerBatchMaxBytes, - MaxRetry: o.MaxRetry, - Compression: o.Compression, - RequiredAcks: int16(o.RequiredAcks), - - EnableTLS: o.EnableTLS, - Credential: o.Credential, - InsecureSkipVerify: o.InsecureSkipVerify, - sasl: o.sasl, - - DialTimeout: o.DialTimeout, - WriteTimeout: o.WriteTimeout, - ReadTimeout: o.ReadTimeout, - } -} diff --git a/pkg/sink/kafka/metrics_collector.go b/pkg/sink/kafka/metrics_collector.go index 0146d744c3..182b90d896 100644 --- a/pkg/sink/kafka/metrics_collector.go +++ b/pkg/sink/kafka/metrics_collector.go @@ -13,9 +13,25 @@ package kafka -import "context" +import ( + "context" + + "github.com/pingcap/ticdc/pkg/common" +) // MetricsCollector is the interface for kafka metrics collector. type MetricsCollector interface { Run(ctx context.Context) } + +type kafkaMetricsCollector struct { + changefeedID common.ChangeFeedID + hook *metricsHook +} + +func (c *kafkaMetricsCollector) Run(ctx context.Context) { + <-ctx.Done() + if c.hook != nil { + c.hook.cleanupMetrics() + } +} diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go index 7690529859..bd66b9231f 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -17,6 +17,7 @@ import ( "strconv" "time" + "github.com/pingcap/ticdc/pkg/common" "github.com/prometheus/client_golang/prometheus" "github.com/twmb/franz-go/pkg/kgo" ) @@ -42,6 +43,22 @@ const ( legacyMetricP99 = "p99" ) +func newKafkaMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { + return newMetricsHook( + changefeedID.Keyspace(), + changefeedID.Name(), + metricVectors{ + RequestsInFlight: requestsInFlightGauge, + OutgoingByteRate: OutgoingByteRateGauge, + RequestRate: RequestRateGauge, + RequestLatency: RequestLatencyGauge, + ResponseRate: responseRateGauge, + CompressionRatio: compressionRatioGauge, + RecordsPerRequest: recordsPerRequestGauge, + }, + ) +} + func newMetricsHook( keyspace string, changefeed string, diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index cde9b52555..80e96cc526 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -27,6 +27,23 @@ import ( "go.uber.org/zap" ) +// SyncProducer is the kafka sync producer +type SyncProducer interface { + // SendMessage produces a given message, and returns only when it either has + // succeeded or failed to produce. It will return the partition and the offset + // of the produced message, or an error if the message failed to produce. + SendMessage(topic string, partitionNum int32, message *common.Message) error + + // SendMessages produces a given set of messages, and returns only when all + // messages in the set have either succeeded or failed. Note that messages + // can succeed and fail individually; if some succeed and some fail, + // SendMessages will return an error. + SendMessages(topic string, partitionNum int32, message *common.Message) error + + // Close shuts down the producer and releases its Kafka client resources. + Close() +} + type kafkaSyncProducer struct { id commonType.ChangeFeedID From 69601f4452ee31dbf9a59a0ff42987d8bc2647ee Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 7 Jul 2026 22:32:30 +0800 Subject: [PATCH 31/61] simplify more code --- downstreamadapter/sink/kafka/sink.go | 17 ++-- downstreamadapter/sink/kafka/sink_test.go | 4 - pkg/sink/kafka/admin_client.go | 20 ++-- pkg/sink/kafka/admin_client_test.go | 2 +- pkg/sink/kafka/async_producer.go | 16 ++-- pkg/sink/kafka/async_producer_test.go | 6 +- pkg/sink/kafka/factory.go | 14 +-- pkg/sink/kafka/factory_mock.go | 72 ++++++-------- pkg/sink/kafka/metrics_collector.go | 37 -------- pkg/sink/kafka/metrics_collector_mock.go | 47 ---------- pkg/sink/kafka/metrics_hook.go | 109 ++++++++-------------- pkg/sink/kafka/metrics_hook_test.go | 99 -------------------- pkg/sink/kafka/sync_producer.go | 14 +-- pkg/sink/kafka/sync_producer_test.go | 2 +- scripts/generate-mock.sh | 3 +- 15 files changed, 107 insertions(+), 355 deletions(-) delete mode 100644 pkg/sink/kafka/metrics_collector.go delete mode 100644 pkg/sink/kafka/metrics_collector_mock.go delete mode 100644 pkg/sink/kafka/metrics_hook_test.go diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 21fcd2d41b..1100b73a7a 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -42,9 +42,8 @@ const ( type sink struct { changefeedID commonType.ChangeFeedID - dmlProducer kafka.AsyncProducer - ddlProducer kafka.SyncProducer - metricsCollector kafka.MetricsCollector + dmlProducer kafka.AsyncProducer + ddlProducer kafka.SyncProducer comp components statistics *metrics.Statistics @@ -117,10 +116,9 @@ func newWithComponents( return nil, err } return &sink{ - changefeedID: changefeedID, - dmlProducer: asyncProducer, - ddlProducer: syncProducer, - metricsCollector: comp.factory.MetricsCollector(), + changefeedID: changefeedID, + dmlProducer: asyncProducer, + ddlProducer: syncProducer, partitionRule: helper.GetDDLDispatchRule(protocol), protocol: protocol, @@ -147,11 +145,8 @@ func (s *sink) Run(ctx context.Context) error { g.Go(func() error { return s.sendDMLEvent(ctx) }) - g.Go(func() error { - s.metricsCollector.Run(ctx) - return nil - }) err := g.Wait() + kafka.CleanupMetrics(s.changefeedID) s.isNormal.Store(false) return errors.Trace(err) } diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 9557de7a53..88dd9632cc 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -81,13 +81,9 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, }, nil) adminClient.EXPECT().Close().AnyTimes() - metricsCollector := kafka.NewMockMetricsCollector(ctrl) - metricsCollector.EXPECT().Run(gomock.Any()).AnyTimes() - factory := kafka.NewMockFactory(ctrl) factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(syncProducer, nil) - factory.EXPECT().MetricsCollector().Return(metricsCollector) eventRouter, err := eventrouter.NewEventRouter(sinkConfig, topic, false, false) if err != nil { diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index ce18e29158..c76c89323a 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -27,7 +27,7 @@ import ( "go.uber.org/zap" ) -type kafkaAdminClient struct { +type adminClient struct { changefeed common.ChangeFeedID client *kgo.Client @@ -40,7 +40,7 @@ func newAdminClient( changefeedID common.ChangeFeedID, o *clientOptions, hook kgo.Hook, -) (*kafkaAdminClient, error) { +) (*adminClient, error) { opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -53,7 +53,7 @@ func newAdminClient( timeout := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) - return &kafkaAdminClient{ + return &adminClient{ changefeed: changefeedID, client: client, admin: kadm.NewClient(client), @@ -61,7 +61,7 @@ func newAdminClient( }, nil } -func (a *kafkaAdminClient) GetBrokerConfig(configName string) (string, error) { +func (a *adminClient) GetBrokerConfig(configName string) (string, error) { ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() @@ -101,7 +101,7 @@ func (a *kafkaAdminClient) GetBrokerConfig(configName string) (string, error) { "cannot find the `%s` from the broker's configuration", configName) } -func (a *kafkaAdminClient) GetTopicConfig(topicName string, configName string) (string, error) { +func (a *adminClient) GetTopicConfig(topicName string, configName string) (string, error) { ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() @@ -137,7 +137,7 @@ func (a *kafkaAdminClient) GetTopicConfig(topicName string, configName string) ( "cannot find the `%s` from the topic's configuration", configName) } -func (a *kafkaAdminClient) GetTopicsMeta( +func (a *adminClient) GetTopicsMeta( topics []string, ignoreTopicError bool, ) (map[string]TopicDetail, error) { @@ -179,7 +179,7 @@ func (a *kafkaAdminClient) GetTopicsMeta( return result, nil } -func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { +func (a *adminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { if len(topics) == 0 { return make(map[string]int32), nil } @@ -206,7 +206,7 @@ func (a *kafkaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]i return result, nil } -func (a *kafkaAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { +func (a *adminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { if detail == nil { return errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") } @@ -238,9 +238,9 @@ func (a *kafkaAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) e return errors.Trace(resp.Err) } -func (a *kafkaAdminClient) Heartbeat() {} +func (a *adminClient) Heartbeat() {} -func (a *kafkaAdminClient) Close() { +func (a *adminClient) Close() { if a.admin != nil { a.admin.Close() } diff --git a/pkg/sink/kafka/admin_client_test.go b/pkg/sink/kafka/admin_client_test.go index 61a061f65e..fb25490c46 100644 --- a/pkg/sink/kafka/admin_client_test.go +++ b/pkg/sink/kafka/admin_client_test.go @@ -22,7 +22,7 @@ import ( func TestAdminClientCreateTopicNilDetailReturnsError(t *testing.T) { t.Parallel() - client := &kafkaAdminClient{} + client := &adminClient{} err := client.CreateTopic(nil, false) diff --git a/pkg/sink/kafka/async_producer.go b/pkg/sink/kafka/async_producer.go index 88df9948ff..dd4df1e6b6 100644 --- a/pkg/sink/kafka/async_producer.go +++ b/pkg/sink/kafka/async_producer.go @@ -43,7 +43,7 @@ type AsyncProducer interface { AsyncRunCallback(ctx context.Context) error } -type kafkaAsyncProducer struct { +type asyncProducer struct { client *kgo.Client changefeedID commonType.ChangeFeedID @@ -56,7 +56,7 @@ func newAsyncProducer( changefeedID commonType.ChangeFeedID, o *clientOptions, hook kgo.Hook, -) (*kafkaAsyncProducer, error) { +) (*asyncProducer, error) { opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -67,7 +67,7 @@ func newAsyncProducer( return nil, errors.Trace(err) } - return &kafkaAsyncProducer{ + return &asyncProducer{ client: client, changefeedID: changefeedID, closed: atomic.NewBool(false), @@ -75,7 +75,7 @@ func newAsyncProducer( }, nil } -func (p *kafkaAsyncProducer) Close() { +func (p *asyncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { return } @@ -90,7 +90,7 @@ func (p *kafkaAsyncProducer) Close() { }() } -func (p *kafkaAsyncProducer) AsyncSend( +func (p *asyncProducer) AsyncSend( ctx context.Context, topic string, partition int32, @@ -161,7 +161,7 @@ func (p *kafkaAsyncProducer) AsyncSend( return nil } -func (p *kafkaAsyncProducer) enqueueAsyncSendError( +func (p *asyncProducer) enqueueAsyncSendError( keyspace string, changefeed string, logInfo *common.MessageLogInfo, @@ -180,9 +180,9 @@ func (p *kafkaAsyncProducer) enqueueAsyncSendError( } } -func (p *kafkaAsyncProducer) Heartbeat() {} +func (p *asyncProducer) Heartbeat() {} -func (p *kafkaAsyncProducer) AsyncRunCallback(ctx context.Context) error { +func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { defer p.closed.Store(true) for { select { diff --git a/pkg/sink/kafka/async_producer_test.go b/pkg/sink/kafka/async_producer_test.go index 4f4d89aa0e..e22191290a 100644 --- a/pkg/sink/kafka/async_producer_test.go +++ b/pkg/sink/kafka/async_producer_test.go @@ -26,7 +26,7 @@ import ( ) func TestAsyncSendClosedProducer(t *testing.T) { - producer := &kafkaAsyncProducer{closed: atomic.NewBool(true)} + producer := &asyncProducer{closed: atomic.NewBool(true)} err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) @@ -34,7 +34,7 @@ func TestAsyncSendClosedProducer(t *testing.T) { } func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { - producer := &kafkaAsyncProducer{ + producer := &asyncProducer{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), closed: atomic.NewBool(false), errCh: make(chan error, 1), @@ -50,7 +50,7 @@ func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { func TestAsyncSendLegacyFailpointAnnotatesDMLContext(t *testing.T) { enableLegacyKafkaSinkFailpointForTest(t, kafkaSinkAsyncSendErrorFailpoint) - producer := &kafkaAsyncProducer{ + producer := &asyncProducer{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-legacy-failpoint"), closed: atomic.NewBool(false), errCh: make(chan error, 1), diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 3d26aa84e6..5f1daa6f91 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -20,16 +20,14 @@ import ( "github.com/pingcap/ticdc/pkg/errors" ) -// Factory is used to produce all kafka components. +// Factory is used to produce all Kafka components. type Factory interface { - // AdminClient return a kafka cluster admin client + // AdminClient returns a Kafka cluster admin client. AdminClient(ctx context.Context) (ClusterAdminClient, error) - // SyncProducer creates a sync producer to writer message to kafka + // SyncProducer creates a sync producer to write messages to Kafka. SyncProducer(ctx context.Context) (SyncProducer, error) - // AsyncProducer creates an async producer to writer message to kafka + // AsyncProducer creates an async producer to write messages to Kafka. AsyncProducer(ctx context.Context) (AsyncProducer, error) - // MetricsCollector returns the kafka metrics collector - MetricsCollector() MetricsCollector } type factory struct { @@ -86,10 +84,6 @@ func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { return producer, nil } -func (f *factory) MetricsCollector() MetricsCollector { - return &kafkaMetricsCollector{changefeedID: f.changefeedID, hook: f.metricsHook} -} - func newClientOption(o *options) *clientOptions { return &clientOptions{ BrokerEndpoints: o.BrokerEndpoints, diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index 866a85113f..1ce6cf5977 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/factory.go +// Source: github.com/pingcap/ticdc/pkg/sink/kafka (interfaces: Factory,SyncProducer,AsyncProducer) // Package kafka is a generated GoMock package. package kafka @@ -36,62 +36,48 @@ func (m *MockFactory) EXPECT() *MockFactoryMockRecorder { } // AdminClient mocks base method. -func (m *MockFactory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { +func (m *MockFactory) AdminClient(arg0 context.Context) (ClusterAdminClient, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AdminClient", ctx) + ret := m.ctrl.Call(m, "AdminClient", arg0) ret0, _ := ret[0].(ClusterAdminClient) ret1, _ := ret[1].(error) return ret0, ret1 } // AdminClient indicates an expected call of AdminClient. -func (mr *MockFactoryMockRecorder) AdminClient(ctx interface{}) *gomock.Call { +func (mr *MockFactoryMockRecorder) AdminClient(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminClient", reflect.TypeOf((*MockFactory)(nil).AdminClient), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminClient", reflect.TypeOf((*MockFactory)(nil).AdminClient), arg0) } // AsyncProducer mocks base method. -func (m *MockFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { +func (m *MockFactory) AsyncProducer(arg0 context.Context) (AsyncProducer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncProducer", ctx) + ret := m.ctrl.Call(m, "AsyncProducer", arg0) ret0, _ := ret[0].(AsyncProducer) ret1, _ := ret[1].(error) return ret0, ret1 } // AsyncProducer indicates an expected call of AsyncProducer. -func (mr *MockFactoryMockRecorder) AsyncProducer(ctx interface{}) *gomock.Call { +func (mr *MockFactoryMockRecorder) AsyncProducer(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), ctx) -} - -// MetricsCollector mocks base method. -func (m *MockFactory) MetricsCollector() MetricsCollector { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MetricsCollector") - ret0, _ := ret[0].(MetricsCollector) - return ret0 -} - -// MetricsCollector indicates an expected call of MetricsCollector. -func (mr *MockFactoryMockRecorder) MetricsCollector() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MetricsCollector", reflect.TypeOf((*MockFactory)(nil).MetricsCollector)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), arg0) } // SyncProducer mocks base method. -func (m *MockFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { +func (m *MockFactory) SyncProducer(arg0 context.Context) (SyncProducer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SyncProducer", ctx) + ret := m.ctrl.Call(m, "SyncProducer", arg0) ret0, _ := ret[0].(SyncProducer) ret1, _ := ret[1].(error) return ret0, ret1 } // SyncProducer indicates an expected call of SyncProducer. -func (mr *MockFactoryMockRecorder) SyncProducer(ctx interface{}) *gomock.Call { +func (mr *MockFactoryMockRecorder) SyncProducer(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncProducer", reflect.TypeOf((*MockFactory)(nil).SyncProducer), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncProducer", reflect.TypeOf((*MockFactory)(nil).SyncProducer), arg0) } // MockSyncProducer is a mock of SyncProducer interface. @@ -130,31 +116,31 @@ func (mr *MockSyncProducerMockRecorder) Close() *gomock.Call { } // SendMessage mocks base method. -func (m *MockSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { +func (m *MockSyncProducer) SendMessage(arg0 string, arg1 int32, arg2 *common.Message) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessage", topic, partitionNum, message) + ret := m.ctrl.Call(m, "SendMessage", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // SendMessage indicates an expected call of SendMessage. -func (mr *MockSyncProducerMockRecorder) SendMessage(topic, partitionNum, message interface{}) *gomock.Call { +func (mr *MockSyncProducerMockRecorder) SendMessage(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), topic, partitionNum, message) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), arg0, arg1, arg2) } // SendMessages mocks base method. -func (m *MockSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { +func (m *MockSyncProducer) SendMessages(arg0 string, arg1 int32, arg2 *common.Message) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessages", topic, partitionNum, message) + ret := m.ctrl.Call(m, "SendMessages", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // SendMessages indicates an expected call of SendMessages. -func (mr *MockSyncProducerMockRecorder) SendMessages(topic, partitionNum, message interface{}) *gomock.Call { +func (mr *MockSyncProducerMockRecorder) SendMessages(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), topic, partitionNum, message) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), arg0, arg1, arg2) } // MockAsyncProducer is a mock of AsyncProducer interface. @@ -181,31 +167,31 @@ func (m *MockAsyncProducer) EXPECT() *MockAsyncProducerMockRecorder { } // AsyncRunCallback mocks base method. -func (m *MockAsyncProducer) AsyncRunCallback(ctx context.Context) error { +func (m *MockAsyncProducer) AsyncRunCallback(arg0 context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncRunCallback", ctx) + ret := m.ctrl.Call(m, "AsyncRunCallback", arg0) ret0, _ := ret[0].(error) return ret0 } // AsyncRunCallback indicates an expected call of AsyncRunCallback. -func (mr *MockAsyncProducerMockRecorder) AsyncRunCallback(ctx interface{}) *gomock.Call { +func (mr *MockAsyncProducerMockRecorder) AsyncRunCallback(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncRunCallback", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncRunCallback), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncRunCallback", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncRunCallback), arg0) } // AsyncSend mocks base method. -func (m *MockAsyncProducer) AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error { +func (m *MockAsyncProducer) AsyncSend(arg0 context.Context, arg1 string, arg2 int32, arg3 *common.Message) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncSend", ctx, topic, partition, message) + ret := m.ctrl.Call(m, "AsyncSend", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // AsyncSend indicates an expected call of AsyncSend. -func (mr *MockAsyncProducerMockRecorder) AsyncSend(ctx, topic, partition, message interface{}) *gomock.Call { +func (mr *MockAsyncProducerMockRecorder) AsyncSend(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncSend", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncSend), ctx, topic, partition, message) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncSend", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncSend), arg0, arg1, arg2, arg3) } // Close mocks base method. diff --git a/pkg/sink/kafka/metrics_collector.go b/pkg/sink/kafka/metrics_collector.go deleted file mode 100644 index 182b90d896..0000000000 --- a/pkg/sink/kafka/metrics_collector.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - - "github.com/pingcap/ticdc/pkg/common" -) - -// MetricsCollector is the interface for kafka metrics collector. -type MetricsCollector interface { - Run(ctx context.Context) -} - -type kafkaMetricsCollector struct { - changefeedID common.ChangeFeedID - hook *metricsHook -} - -func (c *kafkaMetricsCollector) Run(ctx context.Context) { - <-ctx.Done() - if c.hook != nil { - c.hook.cleanupMetrics() - } -} diff --git a/pkg/sink/kafka/metrics_collector_mock.go b/pkg/sink/kafka/metrics_collector_mock.go deleted file mode 100644 index 9e5bc51448..0000000000 --- a/pkg/sink/kafka/metrics_collector_mock.go +++ /dev/null @@ -1,47 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/metrics_collector.go - -// Package kafka is a generated GoMock package. -package kafka - -import ( - context "context" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockMetricsCollector is a mock of MetricsCollector interface. -type MockMetricsCollector struct { - ctrl *gomock.Controller - recorder *MockMetricsCollectorMockRecorder -} - -// MockMetricsCollectorMockRecorder is the mock recorder for MockMetricsCollector. -type MockMetricsCollectorMockRecorder struct { - mock *MockMetricsCollector -} - -// NewMockMetricsCollector creates a new mock instance. -func NewMockMetricsCollector(ctrl *gomock.Controller) *MockMetricsCollector { - mock := &MockMetricsCollector{ctrl: ctrl} - mock.recorder = &MockMetricsCollectorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockMetricsCollector) EXPECT() *MockMetricsCollectorMockRecorder { - return m.recorder -} - -// Run mocks base method. -func (m *MockMetricsCollector) Run(ctx context.Context) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Run", ctx) -} - -// Run indicates an expected call of Run. -func (mr *MockMetricsCollectorMockRecorder) Run(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockMetricsCollector)(nil).Run), ctx) -} diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go index bd66b9231f..5e07c05757 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -22,73 +22,40 @@ import ( "github.com/twmb/franz-go/pkg/kgo" ) +// metricsHook adapts franz-go client callbacks to TiCDC's Kafka sink metrics. +// franz-go calls these hook methods while writing requests, receiving responses, +// and flushing produce batches. The hook does not poll Kafka; it only converts +// the callback payloads into the existing TiCDC Kafka metric vectors. type metricsHook struct { keyspace string changefeed string - metrics metricVectors -} - -type metricVectors struct { - RequestsInFlight *prometheus.GaugeVec - OutgoingByteRate *prometheus.GaugeVec - RequestRate *prometheus.GaugeVec - RequestLatency *prometheus.GaugeVec - ResponseRate *prometheus.GaugeVec - CompressionRatio *prometheus.GaugeVec - RecordsPerRequest *prometheus.GaugeVec } const ( - legacyMetricAvg = "avg" - legacyMetricP99 = "p99" + metricAvg = "avg" + metricP99 = "p99" ) func newKafkaMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { - return newMetricsHook( - changefeedID.Keyspace(), - changefeedID.Name(), - metricVectors{ - RequestsInFlight: requestsInFlightGauge, - OutgoingByteRate: OutgoingByteRateGauge, - RequestRate: RequestRateGauge, - RequestLatency: RequestLatencyGauge, - ResponseRate: responseRateGauge, - CompressionRatio: compressionRatioGauge, - RecordsPerRequest: recordsPerRequestGauge, - }, - ) -} - -func newMetricsHook( - keyspace string, - changefeed string, - metrics metricVectors, -) *metricsHook { return &metricsHook{ - keyspace: keyspace, - changefeed: changefeed, - metrics: metrics, + keyspace: changefeedID.Keyspace(), + changefeed: changefeedID.Name(), } } -func (h *metricsHook) cleanupMetrics() { +// CleanupMetrics removes Kafka sink metric series for a changefeed when its sink exits. +func CleanupMetrics(changefeedID common.ChangeFeedID) { labels := prometheus.Labels{ - "namespace": h.keyspace, - "changefeed": h.changefeed, - } - for _, gaugeVec := range []*prometheus.GaugeVec{ - h.metrics.OutgoingByteRate, - h.metrics.RequestRate, - h.metrics.ResponseRate, - h.metrics.RequestsInFlight, - h.metrics.RequestLatency, - h.metrics.CompressionRatio, - h.metrics.RecordsPerRequest, - } { - if gaugeVec != nil { - gaugeVec.DeletePartialMatch(labels) - } + "namespace": changefeedID.Keyspace(), + "changefeed": changefeedID.Name(), } + OutgoingByteRateGauge.DeletePartialMatch(labels) + RequestRateGauge.DeletePartialMatch(labels) + responseRateGauge.DeletePartialMatch(labels) + requestsInFlightGauge.DeletePartialMatch(labels) + RequestLatencyGauge.DeletePartialMatch(labels) + compressionRatioGauge.DeletePartialMatch(labels) + recordsPerRequestGauge.DeletePartialMatch(labels) } func (h *metricsHook) OnBrokerWrite( @@ -104,14 +71,12 @@ func (h *metricsHook) OnBrokerWrite( } brokerID := strconv.Itoa(int(meta.NodeID)) - if h.metrics.OutgoingByteRate != nil && bytesWritten > 0 { - h.metrics.OutgoingByteRate.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(float64(bytesWritten)) - } - if h.metrics.RequestRate != nil { - h.metrics.RequestRate.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + if bytesWritten > 0 { + OutgoingByteRateGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(float64(bytesWritten)) } - if err == nil && h.metrics.RequestsInFlight != nil { - h.metrics.RequestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + RequestRateGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + if err == nil { + requestsInFlightGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) } } @@ -125,16 +90,16 @@ func (h *metricsHook) OnBrokerE2E( } brokerID := strconv.Itoa(int(meta.NodeID)) - if e2e.WriteErr == nil && h.metrics.RequestsInFlight != nil { - h.metrics.RequestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(-1) + if e2e.WriteErr == nil { + requestsInFlightGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(-1) } - if e2e.BytesRead > 0 && e2e.ReadErr == nil && h.metrics.ResponseRate != nil { - h.metrics.ResponseRate.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + if e2e.BytesRead > 0 && e2e.ReadErr == nil { + responseRateGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) } - if e2e.Err() == nil && h.metrics.RequestLatency != nil { + if e2e.Err() == nil { latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 - h.metrics.RequestLatency.WithLabelValues(h.keyspace, h.changefeed, brokerID, legacyMetricAvg).Set(latencyMs) - h.metrics.RequestLatency.WithLabelValues(h.keyspace, h.changefeed, brokerID, legacyMetricP99).Set(latencyMs) + RequestLatencyGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricAvg).Set(latencyMs) + RequestLatencyGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricP99).Set(latencyMs) } } @@ -144,14 +109,14 @@ func (h *metricsHook) OnProduceBatchWritten( _ int32, m kgo.ProduceBatchMetrics, ) { - if h.metrics.RecordsPerRequest != nil && m.NumRecords > 0 { + if m.NumRecords > 0 { records := float64(m.NumRecords) - h.metrics.RecordsPerRequest.WithLabelValues(h.keyspace, h.changefeed, legacyMetricAvg).Set(records) - h.metrics.RecordsPerRequest.WithLabelValues(h.keyspace, h.changefeed, legacyMetricP99).Set(records) + recordsPerRequestGauge.WithLabelValues(h.keyspace, h.changefeed, metricAvg).Set(records) + recordsPerRequestGauge.WithLabelValues(h.keyspace, h.changefeed, metricP99).Set(records) } - if h.metrics.CompressionRatio != nil && m.UncompressedBytes > 0 && m.CompressedBytes > 0 { + if m.UncompressedBytes > 0 && m.CompressedBytes > 0 { ratio := float64(m.UncompressedBytes) / float64(m.CompressedBytes) * 100 - h.metrics.CompressionRatio.WithLabelValues(h.keyspace, h.changefeed, legacyMetricAvg).Set(ratio) - h.metrics.CompressionRatio.WithLabelValues(h.keyspace, h.changefeed, legacyMetricP99).Set(ratio) + compressionRatioGauge.WithLabelValues(h.keyspace, h.changefeed, metricAvg).Set(ratio) + compressionRatioGauge.WithLabelValues(h.keyspace, h.changefeed, metricP99).Set(ratio) } } diff --git a/pkg/sink/kafka/metrics_hook_test.go b/pkg/sink/kafka/metrics_hook_test.go deleted file mode 100644 index 941af214ae..0000000000 --- a/pkg/sink/kafka/metrics_hook_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kgo" -) - -func TestMetricsHookRecordsMetricsAndCleanup(t *testing.T) { - outgoingByteRate := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_outgoing_byte_rate"}, - []string{"namespace", "changefeed", "broker"}, - ) - requestRate := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_request_rate"}, - []string{"namespace", "changefeed", "broker"}, - ) - requestsInFlight := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_in_flight_requests"}, - []string{"namespace", "changefeed", "broker"}, - ) - responseRate := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_response_rate"}, - []string{"namespace", "changefeed", "broker"}, - ) - requestLatency := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_request_latency"}, - []string{"namespace", "changefeed", "broker", "type"}, - ) - recordsPerRequest := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_records_per_request"}, - []string{"namespace", "changefeed", "type"}, - ) - compressionRatio := prometheus.NewGaugeVec( - prometheus.GaugeOpts{Name: "kafka_producer_compression_ratio"}, - []string{"namespace", "changefeed", "type"}, - ) - - hook := newMetricsHook("default", "cf", metricVectors{ - OutgoingByteRate: outgoingByteRate, - RequestRate: requestRate, - RequestsInFlight: requestsInFlight, - ResponseRate: responseRate, - RequestLatency: requestLatency, - RecordsPerRequest: recordsPerRequest, - CompressionRatio: compressionRatio, - }) - - hook.OnBrokerWrite(kgo.BrokerMetadata{NodeID: 1}, 0, 42, 0, 0, nil) - hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: 1}, 0, kgo.BrokerE2E{ - BytesRead: 42, - TimeToWrite: 10 * time.Millisecond, - ReadWait: 20 * time.Millisecond, - TimeToRead: 30 * time.Millisecond, - }) - hook.OnProduceBatchWritten(kgo.BrokerMetadata{}, "", 0, kgo.ProduceBatchMetrics{ - NumRecords: 3, - UncompressedBytes: 100, - CompressedBytes: 50, - }) - - require.Equal(t, float64(42), testutil.ToFloat64(outgoingByteRate.WithLabelValues("default", "cf", "1"))) - require.Equal(t, float64(1), testutil.ToFloat64(requestRate.WithLabelValues("default", "cf", "1"))) - require.Equal(t, float64(0), testutil.ToFloat64(requestsInFlight.WithLabelValues("default", "cf", "1"))) - require.Equal(t, float64(1), testutil.ToFloat64(responseRate.WithLabelValues("default", "cf", "1"))) - require.Equal(t, float64(60), testutil.ToFloat64(requestLatency.WithLabelValues("default", "cf", "1", legacyMetricAvg))) - require.Equal(t, float64(60), testutil.ToFloat64(requestLatency.WithLabelValues("default", "cf", "1", legacyMetricP99))) - require.Equal(t, float64(3), testutil.ToFloat64(recordsPerRequest.WithLabelValues("default", "cf", legacyMetricAvg))) - require.Equal(t, float64(3), testutil.ToFloat64(recordsPerRequest.WithLabelValues("default", "cf", legacyMetricP99))) - require.Equal(t, float64(200), testutil.ToFloat64(compressionRatio.WithLabelValues("default", "cf", legacyMetricAvg))) - require.Equal(t, float64(200), testutil.ToFloat64(compressionRatio.WithLabelValues("default", "cf", legacyMetricP99))) - - hook.cleanupMetrics() - - require.Equal(t, 0, testutil.CollectAndCount(outgoingByteRate)) - require.Equal(t, 0, testutil.CollectAndCount(requestRate)) - require.Equal(t, 0, testutil.CollectAndCount(requestsInFlight)) - require.Equal(t, 0, testutil.CollectAndCount(responseRate)) - require.Equal(t, 0, testutil.CollectAndCount(requestLatency)) - require.Equal(t, 0, testutil.CollectAndCount(recordsPerRequest)) - require.Equal(t, 0, testutil.CollectAndCount(compressionRatio)) -} diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index 80e96cc526..5981625ca8 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -44,7 +44,7 @@ type SyncProducer interface { Close() } -type kafkaSyncProducer struct { +type syncProducer struct { id commonType.ChangeFeedID client *kgo.Client @@ -57,7 +57,7 @@ func newSyncProducer( changefeedID commonType.ChangeFeedID, o *clientOptions, hook kgo.Hook, -) (*kafkaSyncProducer, error) { +) (*syncProducer, error) { opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -71,7 +71,7 @@ func newSyncProducer( timeout := maxTimeoutWithDefault(o.ReadTimeout, 0) - return &kafkaSyncProducer{ + return &syncProducer{ id: changefeedID, client: client, closed: atomic.NewBool(false), @@ -79,7 +79,7 @@ func newSyncProducer( }, nil } -func (p *kafkaSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { +func (p *syncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { if p.closed.Load() { return errors.ErrKafkaProducerClosed.GenWithStackByArgs() } @@ -114,7 +114,7 @@ func (p *kafkaSyncProducer) SendMessage(topic string, partitionNum int32, messag return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *kafkaSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { +func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { if p.closed.Load() { return errors.ErrKafkaProducerClosed.GenWithStackByArgs() } @@ -153,9 +153,9 @@ func (p *kafkaSyncProducer) SendMessages(topic string, partitionNum int32, messa return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *kafkaSyncProducer) Heartbeat() {} +func (p *syncProducer) Heartbeat() {} -func (p *kafkaSyncProducer) Close() { +func (p *syncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { log.Warn("kafka DDL producer already closed", zap.String("keyspace", p.id.Keyspace()), diff --git a/pkg/sink/kafka/sync_producer_test.go b/pkg/sink/kafka/sync_producer_test.go index a101b019b9..f87b37d233 100644 --- a/pkg/sink/kafka/sync_producer_test.go +++ b/pkg/sink/kafka/sync_producer_test.go @@ -23,7 +23,7 @@ import ( ) func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { - producer := &kafkaSyncProducer{closed: atomic.NewBool(true)} + producer := &syncProducer{closed: atomic.NewBool(true)} err := producer.SendMessage("topic", 1, &common.Message{}) require.ErrorIs(t, err, errors.ErrKafkaProducerClosed) diff --git a/scripts/generate-mock.sh b/scripts/generate-mock.sh index bf10fdeaa0..dfd53026c7 100755 --- a/scripts/generate-mock.sh +++ b/scripts/generate-mock.sh @@ -35,8 +35,7 @@ fi "$MOCKGEN" -source pkg/api/v2/api_client.go -destination pkg/api/v2/mock/api_client_mock.go -package mock "$MOCKGEN" -source pkg/sink/codec/simple/marshaller.go -destination pkg/sink/codec/simple/mock/marshaller.go "$MOCKGEN" -source pkg/sink/kafka/cluster_admin_client.go -destination pkg/sink/kafka/cluster_admin_client_mock.go -package kafka -"$MOCKGEN" -source pkg/sink/kafka/factory.go -destination pkg/sink/kafka/factory_mock.go -package kafka -"$MOCKGEN" -source pkg/sink/kafka/metrics_collector.go -destination pkg/sink/kafka/metrics_collector_mock.go -package kafka +"$MOCKGEN" -destination pkg/sink/kafka/factory_mock.go -package kafka -self_package github.com/pingcap/ticdc/pkg/sink/kafka github.com/pingcap/ticdc/pkg/sink/kafka Factory,SyncProducer,AsyncProducer "$MOCKGEN" -source pkg/keyspace/keyspace_manager.go -destination pkg/keyspace/keyspace_manager_mock.go -package keyspace "$MOCKGEN" -source pkg/txnutil/gc/gc_manager.go -destination pkg/txnutil/gc/gc_manager_mock.go -package gc "$MOCKGEN" -source pkg/txnutil/gc/gc_client.go -destination pkg/txnutil/gc/gc_client_mock.go -package gc From 6f4796dda26a2bdafe6cf4456ae9b3eb14fc8eb6 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 27 Jul 2026 19:21:44 +0800 Subject: [PATCH 32/61] simplify the timeout --- go.mod | 2 +- go.sum | 4 +-- pkg/sink/kafka/admin_client.go | 4 +-- pkg/sink/kafka/client_options.go | 27 +++--------------- pkg/sink/kafka/client_options_test.go | 25 +---------------- pkg/sink/kafka/factory.go | 5 ++-- pkg/sink/kafka/factory_test.go | 38 +++++++++++++++++++++++++ pkg/sink/kafka/options.go | 17 ++++++++++-- pkg/sink/kafka/options_test.go | 40 +++++++++++++++++++++++++-- pkg/sink/kafka/sync_producer.go | 4 +-- 10 files changed, 101 insertions(+), 65 deletions(-) diff --git a/go.mod b/go.mod index 1dd225ce09..8210cdf063 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/tikv/pd v1.1.0-beta.0.20260604125942-9f1c47b1e851 github.com/tikv/pd/client v0.0.0-20260604125942-9f1c47b1e851 github.com/tinylib/msgp v1.5.0 - github.com/twmb/franz-go v1.21.4 + github.com/twmb/franz-go v1.21.5 github.com/twmb/franz-go/pkg/kadm v1.18.0 github.com/uber-go/atomic v1.4.0 github.com/xdg/scram v1.0.5 diff --git a/go.sum b/go.sum index 9647e35b2c..3680df8330 100644 --- a/go.sum +++ b/go.sum @@ -965,8 +965,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/twmb/franz-go v1.21.4 h1:skglTjGHOHHKxVdUG3A563gynBDhvSWFBBHXKOOMS8M= -github.com/twmb/franz-go v1.21.4/go.mod h1:rfoMTnVk7107fhTGxfEKIHP/e7tPe6oyij/ywzO0czk= +github.com/twmb/franz-go v1.21.5 h1:cVYI2+JTTKSvohhy8bCOleYrS7G79ZBrLVFIJsoHm8M= +github.com/twmb/franz-go v1.21.5/go.mod h1:rfoMTnVk7107fhTGxfEKIHP/e7tPe6oyij/ywzO0czk= github.com/twmb/franz-go/pkg/kadm v1.18.0 h1:WRf/LZmDdcDXwX7WMbtDU++v+b3NzYh2bCGoPMmzirw= github.com/twmb/franz-go/pkg/kadm v1.18.0/go.mod h1:XeLhGoLXLFzK8/ryv5FfpxPxGwj4oFEGpPJMB/x6KDE= github.com/twmb/franz-go/pkg/kmsg v1.13.1 h1:fG5kItwysTk5UXqVwb64EpQEy3TydF3vYYK21nUQ+bI= diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index a3eb89d492..2d9870a9ce 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -51,13 +51,11 @@ func newAdminClient( return nil, errors.Trace(err) } - timeout := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) - return &adminClient{ changefeed: changefeedID, client: client, admin: kadm.NewClient(client), - timeout: timeout, + timeout: o.RequestTimeout, }, nil } diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index 82fe4210cf..af10e2ea37 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -52,21 +52,8 @@ type clientOptions struct { InsecureSkipVerify bool sasl *saslConfig - DialTimeout time.Duration - WriteTimeout time.Duration - ReadTimeout time.Duration -} - -const ( - defaultRequestTimeout = 10 * time.Second -) - -func maxTimeoutWithDefault(readTimeout, writeTimeout time.Duration) time.Duration { - timeout := max(readTimeout, writeTimeout) - if timeout <= 0 { - timeout = defaultRequestTimeout - } - return timeout + DialTimeout time.Duration + RequestTimeout time.Duration } func newOptions( @@ -74,14 +61,12 @@ func newOptions( o *clientOptions, hook kgo.Hook, ) ([]kgo.Opt, error) { - timeoutOverhead := maxTimeoutWithDefault(o.ReadTimeout, o.WriteTimeout) - opts := []kgo.Opt{ kgo.WithContext(ctx), kgo.SeedBrokers(o.BrokerEndpoints...), kgo.ClientID(o.ClientID), kgo.DialTimeout(o.DialTimeout), - kgo.RequestTimeoutOverhead(timeoutOverhead), + kgo.RequestTimeoutOverhead(o.RequestTimeout), } if hook != nil { opts = append(opts, kgo.WithHooks(hook)) @@ -208,10 +193,6 @@ func newOauthTokenSource(ctx context.Context, o *clientOptions) (oauth2.TokenSou func newProducerOptions( o *clientOptions, ) []kgo.Opt { - produceTimeout := o.ReadTimeout - if produceTimeout < 100*time.Millisecond { - produceTimeout = defaultRequestTimeout - } producerBatchMaxBytes := o.ProducerBatchMaxBytes if producerBatchMaxBytes <= 0 { producerBatchMaxBytes = o.MaxMessageBytes @@ -224,7 +205,7 @@ func newProducerOptions( kgo.MaxProduceRequestsInflightPerBroker(1), kgo.RecordRetries(o.MaxRetry), kgo.ProducerBatchMaxBytes(int32(producerBatchMaxBytes)), - kgo.ProduceRequestTimeout(produceTimeout), + kgo.ProduceRequestTimeout(o.RequestTimeout), kgo.ProducerLinger(0), newCompressionOption(o), } diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index 8e9b2512d7..b664d4effd 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -16,7 +16,6 @@ package kafka import ( "context" "testing" - "time" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kgo" @@ -44,29 +43,6 @@ func TestNewRequiredAcks(t *testing.T) { } } -func TestMaxTimeoutWithDefault(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - readTimeout time.Duration - writeTimeout time.Duration - expected time.Duration - }{ - {name: "read timeout is max", readTimeout: 3 * time.Second, writeTimeout: 2 * time.Second, expected: 3 * time.Second}, - {name: "write timeout is max", readTimeout: 2 * time.Second, writeTimeout: 4 * time.Second, expected: 4 * time.Second}, - {name: "both zero use default", readTimeout: 0, writeTimeout: 0, expected: defaultRequestTimeout}, - {name: "both negative use default", readTimeout: -time.Second, writeTimeout: -2 * time.Second, expected: defaultRequestTimeout}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - require.Equal(t, tc.expected, maxTimeoutWithDefault(tc.readTimeout, tc.writeTimeout)) - }) - } -} - func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { t.Parallel() @@ -91,6 +67,7 @@ func TestNewProducerOptionsUsesProducerBatchMaxBytes(t *testing.T) { ProducerBatchMaxBytes: producerBatchMaxBytes, MaxRetry: defaultMaxRetry, RequiredAcks: int16(WaitForAll), + RequestTimeout: defaultTimeout, } opts, err := newOptions(context.Background(), o, nil) diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 994d5b9368..db83959081 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -103,8 +103,7 @@ func newClientOption(o *options) *clientOptions { InsecureSkipVerify: o.InsecureSkipVerify, sasl: o.sasl, - DialTimeout: o.DialTimeout, - WriteTimeout: o.WriteTimeout, - ReadTimeout: o.ReadTimeout, + DialTimeout: o.DialTimeout, + RequestTimeout: max(o.ReadTimeout, o.WriteTimeout), } } diff --git a/pkg/sink/kafka/factory_test.go b/pkg/sink/kafka/factory_test.go index 24c74a695d..4ac588dced 100644 --- a/pkg/sink/kafka/factory_test.go +++ b/pkg/sink/kafka/factory_test.go @@ -15,6 +15,7 @@ package kafka import ( "testing" + "time" "github.com/stretchr/testify/require" ) @@ -53,3 +54,40 @@ func TestNewClientOptionMapsMaxRetry(t *testing.T) { kafkaOptions := newClientOption(options) require.Equal(t, 7, kafkaOptions.MaxRetry) } + +func TestNewClientOptionDerivesRequestTimeout(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + readTimeout time.Duration + writeTimeout time.Duration + expectedRequestTimeout time.Duration + }{ + { + name: "request timeout uses larger write timeout", + readTimeout: time.Second, + writeTimeout: 2 * time.Minute, + expectedRequestTimeout: 2 * time.Minute, + }, + { + name: "request timeout uses larger read timeout", + readTimeout: 5 * time.Second, + writeTimeout: time.Second, + expectedRequestTimeout: 5 * time.Second, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + o := NewOptions() + o.ReadTimeout = tc.readTimeout + o.WriteTimeout = tc.writeTimeout + + clientOption := newClientOption(o) + require.Equal(t, tc.expectedRequestTimeout, clientOption.RequestTimeout) + }) + } +} diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 49e17a12f4..642a920c3b 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -39,6 +39,8 @@ const ( defaultPartitionNum = 3 // defaultMaxRetry is the default retry budget for Kafka producers. defaultMaxRetry = 5 + // defaultTimeout is the default timeout for Kafka connections and requests. + defaultTimeout = 10 * time.Second ) const ( @@ -185,9 +187,9 @@ func NewOptions() *options { InsecureSkipVerify: false, sasl: &saslConfig{}, AutoCreate: true, - DialTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - ReadTimeout: 10 * time.Second, + DialTimeout: defaultTimeout, + WriteTimeout: defaultTimeout, + ReadTimeout: defaultTimeout, } } @@ -290,6 +292,9 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return err } + if a <= 0 { + a = defaultTimeout + } o.DialTimeout = a } @@ -298,6 +303,9 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return err } + if a <= 0 { + a = defaultTimeout + } o.WriteTimeout = a } @@ -306,6 +314,9 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if err != nil { return err } + if a <= 0 { + a = defaultTimeout + } o.ReadTimeout = a } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 6b63e22efc..9c0a2a06e9 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -399,9 +399,9 @@ func TestClientID(t *testing.T) { func TestTimeout(t *testing.T) { options := NewOptions() - require.Equal(t, 10*time.Second, options.DialTimeout) - require.Equal(t, 10*time.Second, options.ReadTimeout) - require.Equal(t, 10*time.Second, options.WriteTimeout) + require.Equal(t, defaultTimeout, options.DialTimeout) + require.Equal(t, defaultTimeout, options.ReadTimeout) + require.Equal(t, defaultTimeout, options.WriteTimeout) uri := "kafka://127.0.0.1:9092/kafka-test?dial-timeout=5s&read-timeout=1000ms" + "&write-timeout=2m" @@ -416,6 +416,40 @@ func TestTimeout(t *testing.T) { require.Equal(t, 2*time.Minute, options.WriteTimeout) } +func TestTimeoutFallsBackForNonPositiveValues(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + parameter string + timeout func(*options) time.Duration + }{ + {name: "dial timeout", parameter: "dial-timeout", timeout: func(o *options) time.Duration { return o.DialTimeout }}, + {name: "read timeout", parameter: "read-timeout", timeout: func(o *options) time.Duration { return o.ReadTimeout }}, + {name: "write timeout", parameter: "write-timeout", timeout: func(o *options) time.Duration { return o.WriteTimeout }}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + for _, value := range []string{"0s", "-1s"} { + sinkURI, err := url.Parse("kafka://127.0.0.1:9092/kafka-test?" + tc.parameter + "=" + value) + require.NoError(t, err) + + o := NewOptions() + err = o.Apply( + commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), + sinkURI, + config.GetDefaultReplicaConfig().Sink, + ) + require.NoError(t, err) + require.Equal(t, defaultTimeout, tc.timeout(o)) + } + }) + } +} + func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *testing.T) { tests := []struct { name string diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index 5981625ca8..816097d140 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -69,13 +69,11 @@ func newSyncProducer( return nil, errors.Trace(err) } - timeout := maxTimeoutWithDefault(o.ReadTimeout, 0) - return &syncProducer{ id: changefeedID, client: client, closed: atomic.NewBool(false), - timeout: timeout, + timeout: o.RequestTimeout, }, nil } From 2f2e965b33fb1e74f89ac3352a780a300dedefe1 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 27 Jul 2026 21:33:18 +0800 Subject: [PATCH 33/61] adjust file layout --- downstreamadapter/sink/kafka/helper.go | 18 +- downstreamadapter/sink/kafka/sink.go | 8 +- downstreamadapter/sink/kafka/sink_test.go | 16 +- .../sink/topicmanager/kafka_topic_manager.go | 16 +- .../topicmanager/kafka_topic_manager_test.go | 80 ++++----- pkg/sink/kafka/{admin_client.go => admin.go} | 51 ++++-- pkg/sink/kafka/admin_mock.go | 120 +++++++++++++ .../{admin_client_test.go => admin_test.go} | 6 +- pkg/sink/kafka/async_producer_mock.go | 76 +++++++++ pkg/sink/kafka/cluster_admin_client.go | 44 ----- pkg/sink/kafka/cluster_admin_client_mock.go | 120 ------------- pkg/sink/kafka/factory.go | 10 +- pkg/sink/kafka/factory_mock.go | 159 ++---------------- pkg/sink/kafka/options.go | 18 +- pkg/sink/kafka/options_test.go | 34 ++-- pkg/sink/kafka/sync_producer_mock.go | 75 +++++++++ scripts/generate-mock.sh | 6 +- 17 files changed, 433 insertions(+), 424 deletions(-) rename pkg/sink/kafka/{admin_client.go => admin.go} (80%) create mode 100644 pkg/sink/kafka/admin_mock.go rename pkg/sink/kafka/{admin_client_test.go => admin_test.go} (83%) create mode 100644 pkg/sink/kafka/async_producer_mock.go delete mode 100644 pkg/sink/kafka/cluster_admin_client.go delete mode 100644 pkg/sink/kafka/cluster_admin_client_mock.go create mode 100644 pkg/sink/kafka/sync_producer_mock.go diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index baf6c71c8d..0afb6c31d1 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -36,13 +36,13 @@ type components struct { columnSelector *columnselector.ColumnSelectors eventRouter *eventrouter.EventRouter topicManager topicmanager.TopicManager - adminClient kafka.ClusterAdminClient + admin kafka.Admin factory kafka.Factory } func (c components) close() { - if c.adminClient != nil { - c.adminClient.Close() + if c.admin != nil { + c.admin.Close() } if c.topicManager != nil { c.topicManager.Close() @@ -107,16 +107,16 @@ func newKafkaSinkComponent( return kafkaComponent, protocol, errors.Trace(err) } - kafkaComponent.adminClient, err = kafkaComponent.factory.AdminClient(ctx) + kafkaComponent.admin, err = kafkaComponent.factory.Admin(ctx) if err != nil { return kafkaComponent, protocol, errors.WrapError(errors.ErrKafkaNewProducer, err) } - // We must close adminClient when this func return cause by an error - // otherwise the adminClient will never be closed and lead to a goroutine leak. + // We must close admin when this func return cause by an error + // otherwise the admin will never be closed and lead to a goroutine leak. defer func() { - if err != nil && kafkaComponent.adminClient != nil { - kafkaComponent.adminClient.Close() + if err != nil && kafkaComponent.admin != nil { + kafkaComponent.admin.Close() } }() @@ -125,7 +125,7 @@ func newKafkaSinkComponent( changefeedID, topic, options.DeriveTopicConfig(), - kafkaComponent.adminClient, + kafkaComponent.admin, ) if err != nil { return kafkaComponent, protocol, errors.Trace(err) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 65539e0e99..218609c59f 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -108,13 +108,13 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. return errors.WrapError(errors.ErrKafkaNewProducer, err) } - adminClient, err := factory.AdminClient(ctx) + admin, err := factory.Admin(ctx) if err != nil { return errors.WrapError(errors.ErrKafkaNewProducer, err) } - defer adminClient.Close() + defer admin.Close() - topics, err := adminClient.GetTopicsMeta([]string{topic}, false) + topics, err := admin.GetTopicsMeta([]string{topic}, false) if err != nil { return errors.Trace(err) } @@ -128,7 +128,7 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. } // the topic is not created, only validate. - err = adminClient.CreateTopic(&kafka.TopicDetail{ + err = admin.CreateTopic(&kafka.TopicDetail{ Name: topic, NumPartitions: topicConfig.PartitionNum, ReplicationFactor: topicConfig.ReplicationFactor, diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 457b2f1006..e97656bb37 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -84,15 +84,15 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, } options.Topic = topic - adminClient := kafka.NewMockClusterAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{kafkaSinkTestTopic}, true).Return( + admin := kafka.NewMockAdmin(ctrl) + admin.EXPECT().GetTopicsMeta([]string{kafkaSinkTestTopic}, true).Return( map[string]kafka.TopicDetail{ kafkaSinkTestTopic: { Name: kafkaSinkTestTopic, NumPartitions: 1, }, }, nil) - adminClient.EXPECT().Close().AnyTimes() + admin.EXPECT().Close().AnyTimes() factory := kafka.NewMockFactory(ctrl) factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) @@ -126,7 +126,7 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, changefeedID, topic, options.DeriveTopicConfig(), - adminClient, + admin, ) if err != nil { return nil, err @@ -138,14 +138,14 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, columnSelector: columnSelector, eventRouter: eventRouter, topicManager: topicManager, - adminClient: adminClient, + admin: admin, factory: factory, } - // We must close adminClient when this func return cause by an error - // otherwise the adminClient will never be closed and lead to a goroutine leak. + // We must close admin when this func return cause by an error + // otherwise the admin will never be closed and lead to a goroutine leak. defer func() { - if err != nil && comp.adminClient != nil { + if err != nil && comp.admin != nil { comp.close() } }() diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index c80c7d65a1..34ded87120 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -41,7 +41,7 @@ type kafkaTopicManager struct { defaultTopic string - admin kafka.ClusterAdminClient + admin kafka.Admin cfg *kafka.AutoCreateTopicConfig topics sync.Map @@ -55,10 +55,10 @@ func GetTopicManagerAndTryCreateTopic( changefeedID common.ChangeFeedID, topic string, topicCfg *kafka.AutoCreateTopicConfig, - adminClient kafka.ClusterAdminClient, + admin kafka.Admin, ) (TopicManager, error) { topicManager := newKafkaTopicManager( - ctx, topic, changefeedID, adminClient, topicCfg, + ctx, topic, changefeedID, admin, topicCfg, ) if _, err := topicManager.CreateTopicAndWaitUntilVisible(ctx, topic); err != nil { @@ -73,7 +73,7 @@ func newKafkaTopicManager( ctx context.Context, defaultTopic string, changefeedID common.ChangeFeedID, - admin kafka.ClusterAdminClient, + admin kafka.Admin, cfg *kafka.AutoCreateTopicConfig, ) *kafkaTopicManager { mgr := &kafkaTopicManager{ @@ -173,7 +173,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err numPartitions, err := m.admin.GetTopicsPartitionsNum(topics) if err != nil { log.Warn( - "Kafka admin client describe topics failed", + "Kafka admin describe topics failed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.Duration("duration", time.Since(start)), @@ -197,7 +197,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err // can be safely written to. The reason is that it may take several seconds after // CreateTopic returns success for all the brokers to become aware that the // topics have been created. -// See https://kafka.apache.org/23/javadoc/org/apache/kafka/clients/admin/AdminClient.html +// See https://kafka.apache.org/23/javadoc/org/apache/kafka/clients/admin/Admin.html func (m *kafkaTopicManager) waitUntilTopicVisible( ctx context.Context, topicName string, @@ -252,7 +252,7 @@ func (m *kafkaTopicManager) createTopic( }, false) if err != nil { log.Error( - "Kafka admin client create the topic failed", + "Kafka admin create the topic failed", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topicName), @@ -265,7 +265,7 @@ func (m *kafkaTopicManager) createTopic( } log.Info( - "Kafka admin client create the topic success", + "Kafka admin create the topic success", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topicName), diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index 2551cfcc2a..bb739676a2 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -27,13 +27,13 @@ import ( const kafkaTopicManagerTestTopic = "mock_topic" -type mockAdminClientWithDeniedDescribe struct { - *kafka.MockClusterAdminClient +type mockAdminWithDeniedDescribe struct { + *kafka.MockAdmin createTopicCalled bool describeCount int } -func (m *mockAdminClientWithDeniedDescribe) GetTopicsMeta( +func (m *mockAdminWithDeniedDescribe) GetTopicsMeta( topics []string, ignoreTopicError bool, ) (map[string]kafka.TopicDetail, error) { @@ -44,7 +44,7 @@ func (m *mockAdminClientWithDeniedDescribe) GetTopicsMeta( return nil, kerr.TopicAuthorizationFailed } -func (m *mockAdminClientWithDeniedDescribe) CreateTopic( +func (m *mockAdminWithDeniedDescribe) CreateTopic( detail *kafka.TopicDetail, validateOnly bool, ) error { @@ -52,13 +52,13 @@ func (m *mockAdminClientWithDeniedDescribe) CreateTopic( return nil } -type mockAdminClientWithDeniedCreate struct { - *kafka.MockClusterAdminClient +type mockAdminWithDeniedCreate struct { + *kafka.MockAdmin createTopicCalled bool describeCount int } -func (m *mockAdminClientWithDeniedCreate) GetTopicsMeta( +func (m *mockAdminWithDeniedCreate) GetTopicsMeta( topics []string, ignoreTopicError bool, ) (map[string]kafka.TopicDetail, error) { @@ -66,7 +66,7 @@ func (m *mockAdminClientWithDeniedCreate) GetTopicsMeta( return map[string]kafka.TopicDetail{}, nil } -func (m *mockAdminClientWithDeniedCreate) CreateTopic( +func (m *mockAdminWithDeniedCreate) CreateTopic( detail *kafka.TopicDetail, validateOnly bool, ) error { @@ -78,7 +78,7 @@ func TestCreateTopic(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - adminClient := kafka.NewMockClusterAdminClient(ctrl) + admin := kafka.NewMockAdmin(ctrl) cfg := &kafka.AutoCreateTopicConfig{ AutoCreate: true, PartitionNum: 2, @@ -92,39 +92,39 @@ func TestCreateTopic(t *testing.T) { var gotFailedTopicDetail *kafka.TopicDetail var gotFailedTopicValidateOnly bool gomock.InOrder( - adminClient.EXPECT().GetTopicsMeta([]string{kafkaTopicManagerTestTopic}, true).Return( + admin.EXPECT().GetTopicsMeta([]string{kafkaTopicManagerTestTopic}, true).Return( map[string]kafka.TopicDetail{ kafkaTopicManagerTestTopic: { Name: kafkaTopicManagerTestTopic, NumPartitions: 2, }, }, nil), - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return( + admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( + admin.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( func(detail *kafka.TopicDetail, validateOnly bool) error { gotNewTopicDetail = detail gotNewTopicValidateOnly = validateOnly return nil }), - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return( map[string]kafka.TopicDetail{ "new-topic": { Name: "new-topic", NumPartitions: 2, }, }, nil), - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic2"}, true).Return( + admin.EXPECT().GetTopicsMeta([]string{"new-topic2"}, true).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic2"}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{"new-topic2"}, false).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, true).Return( + admin.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, true).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, false).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( + admin.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( func(detail *kafka.TopicDetail, validateOnly bool) error { gotFailedTopicDetail = detail gotFailedTopicValidateOnly = validateOnly @@ -132,7 +132,7 @@ func TestCreateTopic(t *testing.T) { }), ) - manager := newKafkaTopicManager(ctx, kafkaTopicManagerTestTopic, changefeedID, adminClient, cfg) + manager := newKafkaTopicManager(ctx, kafkaTopicManagerTestTopic, changefeedID, admin, cfg) defer manager.Close() partitionNum, err := manager.CreateTopicAndWaitUntilVisible(ctx, kafkaTopicManagerTestTopic) require.NoError(t, err) @@ -153,7 +153,7 @@ func TestCreateTopic(t *testing.T) { // Try to create a topic without auto create. cfg.AutoCreate = false - manager = newKafkaTopicManager(ctx, "new-topic2", changefeedID, adminClient, cfg) + manager = newKafkaTopicManager(ctx, "new-topic2", changefeedID, admin, cfg) defer manager.Close() _, err = manager.CreateTopicAndWaitUntilVisible(ctx, "new-topic2") require.Regexp( @@ -170,7 +170,7 @@ func TestCreateTopic(t *testing.T) { PartitionNum: 2, ReplicationFactor: 4, } - manager = newKafkaTopicManager(ctx, topic, changefeedID, adminClient, cfg) + manager = newKafkaTopicManager(ctx, topic, changefeedID, admin, cfg) defer manager.Close() _, err = manager.CreateTopicAndWaitUntilVisible(ctx, topic) require.Regexp( @@ -187,7 +187,7 @@ func TestCreateTopicWaitsUntilVisible(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - adminClient := kafka.NewMockClusterAdminClient(ctrl) + admin := kafka.NewMockAdmin(ctrl) cfg := &kafka.AutoCreateTopicConfig{ AutoCreate: true, PartitionNum: 2, @@ -196,11 +196,11 @@ func TestCreateTopicWaitsUntilVisible(t *testing.T) { topic := "delayed-topic" gomock.InOrder( - adminClient.EXPECT().GetTopicsMeta([]string{topic}, true).Return( + admin.EXPECT().GetTopicsMeta([]string{topic}, true).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( map[string]kafka.TopicDetail{}, nil), - adminClient.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( + admin.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( func(detail *kafka.TopicDetail, validateOnly bool) error { require.Equal(t, &kafka.TopicDetail{ Name: topic, @@ -210,11 +210,11 @@ func TestCreateTopicWaitsUntilVisible(t *testing.T) { require.False(t, validateOnly) return nil }), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( nil, errors.New("unknown topic or partition")), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( nil, errors.New("unknown topic or partition")), - adminClient.EXPECT().GetTopicsMeta([]string{topic}, false).Return( + admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( map[string]kafka.TopicDetail{ topic: { Name: topic, @@ -225,7 +225,7 @@ func TestCreateTopicWaitsUntilVisible(t *testing.T) { ctx := context.Background() changefeedID := common.NewChangefeedID4Test("test", "test") - manager := newKafkaTopicManager(ctx, topic, changefeedID, adminClient, cfg) + manager := newKafkaTopicManager(ctx, topic, changefeedID, admin, cfg) defer manager.Close() partitionNum, err := manager.CreateTopicAndWaitUntilVisible(ctx, topic) @@ -237,8 +237,8 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - adminClient := &mockAdminClientWithDeniedDescribe{ - MockClusterAdminClient: kafka.NewMockClusterAdminClient(ctrl), + admin := &mockAdminWithDeniedDescribe{ + MockAdmin: kafka.NewMockAdmin(ctrl), } cfg := &kafka.AutoCreateTopicConfig{ AutoCreate: true, @@ -248,14 +248,14 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { changefeedID := common.NewChangefeedID4Test("test", "test") ctx := context.Background() - manager := newKafkaTopicManager(ctx, "precreated-topic", changefeedID, adminClient, cfg) + manager := newKafkaTopicManager(ctx, "precreated-topic", changefeedID, admin, cfg) defer manager.Close() partitionNum, err := manager.CreateTopicAndWaitUntilVisible(ctx, "precreated-topic") require.NoError(t, err) require.Equal(t, int32(2), partitionNum) - require.False(t, adminClient.createTopicCalled) - require.Equal(t, 2, adminClient.describeCount) + require.False(t, admin.createTopicCalled) + require.Equal(t, 2, admin.describeCount) partitions, ok := manager.topics.Load("precreated-topic") require.True(t, ok) @@ -266,8 +266,8 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - adminClient := &mockAdminClientWithDeniedCreate{ - MockClusterAdminClient: kafka.NewMockClusterAdminClient(ctrl), + admin := &mockAdminWithDeniedCreate{ + MockAdmin: kafka.NewMockAdmin(ctrl), } cfg := &kafka.AutoCreateTopicConfig{ AutoCreate: true, @@ -277,14 +277,14 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { changefeedID := common.NewChangefeedID4Test("test", "test") ctx := context.Background() - manager := newKafkaTopicManager(ctx, "precreated-topic", changefeedID, adminClient, cfg) + manager := newKafkaTopicManager(ctx, "precreated-topic", changefeedID, admin, cfg) defer manager.Close() partitionNum, err := manager.CreateTopicAndWaitUntilVisible(ctx, "precreated-topic") require.NoError(t, err) require.Equal(t, int32(2), partitionNum) - require.True(t, adminClient.createTopicCalled) - require.Equal(t, 2, adminClient.describeCount) + require.True(t, admin.createTopicCalled) + require.Equal(t, 2, admin.describeCount) partitions, ok := manager.topics.Load("precreated-topic") require.True(t, ok) diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin.go similarity index 80% rename from pkg/sink/kafka/admin_client.go rename to pkg/sink/kafka/admin.go index 2d9870a9ce..be3129e7b3 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin.go @@ -27,7 +27,36 @@ import ( "go.uber.org/zap" ) -type adminClient struct { +// TopicDetail represent a topic's detail information. +type TopicDetail struct { + Name string + NumPartitions int32 + ReplicationFactor int16 +} + +// Admin manages and inspects Kafka topics, brokers, configurations, and ACLs. +type Admin interface { + // GetBrokerConfig return the broker level configuration with the `configName` + GetBrokerConfig(configName string) (string, error) + + // GetTopicConfig return the topic level configuration with the `configName` + GetTopicConfig(topicName string, configName string) (string, error) + + // GetTopicsMeta return all target topics' metadata + // if `ignoreTopicError` is true, ignore the topic error and return the metadata of valid topics + GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) + + // GetTopicsPartitionsNum return the number of partitions of each topic. + GetTopicsPartitionsNum(topics []string) (map[string]int32, error) + + // CreateTopic creates a new topic. + CreateTopic(detail *TopicDetail, validateOnly bool) error + + // Close shuts down the admin. + Close() +} + +type admin struct { changefeed common.ChangeFeedID client *kgo.Client @@ -35,12 +64,12 @@ type adminClient struct { timeout time.Duration } -func newAdminClient( +func newAdmin( ctx context.Context, changefeedID common.ChangeFeedID, o *clientOptions, hook kgo.Hook, -) (*adminClient, error) { +) (*admin, error) { opts, err := newOptions(ctx, o, hook) if err != nil { return nil, errors.Trace(err) @@ -51,7 +80,7 @@ func newAdminClient( return nil, errors.Trace(err) } - return &adminClient{ + return &admin{ changefeed: changefeedID, client: client, admin: kadm.NewClient(client), @@ -59,7 +88,7 @@ func newAdminClient( }, nil } -func (a *adminClient) GetBrokerConfig(configName string) (string, error) { +func (a *admin) GetBrokerConfig(configName string) (string, error) { ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() @@ -99,7 +128,7 @@ func (a *adminClient) GetBrokerConfig(configName string) (string, error) { "cannot find the `%s` from the broker's configuration", configName) } -func (a *adminClient) GetTopicConfig(topicName string, configName string) (string, error) { +func (a *admin) GetTopicConfig(topicName string, configName string) (string, error) { ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() @@ -135,7 +164,7 @@ func (a *adminClient) GetTopicConfig(topicName string, configName string) (strin "cannot find the `%s` from the topic's configuration", configName) } -func (a *adminClient) GetTopicsMeta( +func (a *admin) GetTopicsMeta( topics []string, ignoreTopicError bool, ) (map[string]TopicDetail, error) { @@ -183,7 +212,7 @@ func IsAdminAuthorizationFailed(err error) bool { errors.Is(err, kerr.ClusterAuthorizationFailed) } -func (a *adminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { +func (a *admin) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { if len(topics) == 0 { return make(map[string]int32), nil } @@ -210,7 +239,7 @@ func (a *adminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, return result, nil } -func (a *adminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { +func (a *admin) CreateTopic(detail *TopicDetail, validateOnly bool) error { if detail == nil { return errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") } @@ -242,9 +271,7 @@ func (a *adminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error return errors.Trace(resp.Err) } -func (a *adminClient) Heartbeat() {} - -func (a *adminClient) Close() { +func (a *admin) Close() { if a.admin != nil { a.admin.Close() } diff --git a/pkg/sink/kafka/admin_mock.go b/pkg/sink/kafka/admin_mock.go new file mode 100644 index 0000000000..86f3ba41f4 --- /dev/null +++ b/pkg/sink/kafka/admin_mock.go @@ -0,0 +1,120 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/sink/kafka/admin.go + +// Package kafka is a generated GoMock package. +package kafka + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockAdmin is a mock of Admin interface. +type MockAdmin struct { + ctrl *gomock.Controller + recorder *MockAdminMockRecorder +} + +// MockAdminMockRecorder is the mock recorder for MockAdmin. +type MockAdminMockRecorder struct { + mock *MockAdmin +} + +// NewMockAdmin creates a new mock instance. +func NewMockAdmin(ctrl *gomock.Controller) *MockAdmin { + mock := &MockAdmin{ctrl: ctrl} + mock.recorder = &MockAdminMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAdmin) EXPECT() *MockAdminMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockAdmin) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockAdminMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockAdmin)(nil).Close)) +} + +// CreateTopic mocks base method. +func (m *MockAdmin) CreateTopic(detail *TopicDetail, validateOnly bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateTopic", detail, validateOnly) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateTopic indicates an expected call of CreateTopic. +func (mr *MockAdminMockRecorder) CreateTopic(detail, validateOnly interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockAdmin)(nil).CreateTopic), detail, validateOnly) +} + +// GetBrokerConfig mocks base method. +func (m *MockAdmin) GetBrokerConfig(configName string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBrokerConfig", configName) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBrokerConfig indicates an expected call of GetBrokerConfig. +func (mr *MockAdminMockRecorder) GetBrokerConfig(configName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockAdmin)(nil).GetBrokerConfig), configName) +} + +// GetTopicConfig mocks base method. +func (m *MockAdmin) GetTopicConfig(topicName, configName string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopicConfig", topicName, configName) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTopicConfig indicates an expected call of GetTopicConfig. +func (mr *MockAdminMockRecorder) GetTopicConfig(topicName, configName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockAdmin)(nil).GetTopicConfig), topicName, configName) +} + +// GetTopicsMeta mocks base method. +func (m *MockAdmin) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopicsMeta", topics, ignoreTopicError) + ret0, _ := ret[0].(map[string]TopicDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTopicsMeta indicates an expected call of GetTopicsMeta. +func (mr *MockAdminMockRecorder) GetTopicsMeta(topics, ignoreTopicError interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdmin)(nil).GetTopicsMeta), topics, ignoreTopicError) +} + +// GetTopicsPartitionsNum mocks base method. +func (m *MockAdmin) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", topics) + ret0, _ := ret[0].(map[string]int32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTopicsPartitionsNum indicates an expected call of GetTopicsPartitionsNum. +func (mr *MockAdminMockRecorder) GetTopicsPartitionsNum(topics interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockAdmin)(nil).GetTopicsPartitionsNum), topics) +} diff --git a/pkg/sink/kafka/admin_client_test.go b/pkg/sink/kafka/admin_test.go similarity index 83% rename from pkg/sink/kafka/admin_client_test.go rename to pkg/sink/kafka/admin_test.go index fb25490c46..85848836d0 100644 --- a/pkg/sink/kafka/admin_client_test.go +++ b/pkg/sink/kafka/admin_test.go @@ -19,12 +19,12 @@ import ( "github.com/stretchr/testify/require" ) -func TestAdminClientCreateTopicNilDetailReturnsError(t *testing.T) { +func TestAdminCreateTopicNilDetailReturnsError(t *testing.T) { t.Parallel() - client := &adminClient{} + a := &admin{} - err := client.CreateTopic(nil, false) + err := a.CreateTopic(nil, false) require.Error(t, err) require.Contains(t, err.Error(), "topic detail must not be nil") diff --git a/pkg/sink/kafka/async_producer_mock.go b/pkg/sink/kafka/async_producer_mock.go new file mode 100644 index 0000000000..8a91530ab0 --- /dev/null +++ b/pkg/sink/kafka/async_producer_mock.go @@ -0,0 +1,76 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/sink/kafka/async_producer.go + +// Package kafka is a generated GoMock package. +package kafka + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + common "github.com/pingcap/ticdc/pkg/sink/codec/common" +) + +// MockAsyncProducer is a mock of AsyncProducer interface. +type MockAsyncProducer struct { + ctrl *gomock.Controller + recorder *MockAsyncProducerMockRecorder +} + +// MockAsyncProducerMockRecorder is the mock recorder for MockAsyncProducer. +type MockAsyncProducerMockRecorder struct { + mock *MockAsyncProducer +} + +// NewMockAsyncProducer creates a new mock instance. +func NewMockAsyncProducer(ctrl *gomock.Controller) *MockAsyncProducer { + mock := &MockAsyncProducer{ctrl: ctrl} + mock.recorder = &MockAsyncProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAsyncProducer) EXPECT() *MockAsyncProducerMockRecorder { + return m.recorder +} + +// AsyncRunCallback mocks base method. +func (m *MockAsyncProducer) AsyncRunCallback(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AsyncRunCallback", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// AsyncRunCallback indicates an expected call of AsyncRunCallback. +func (mr *MockAsyncProducerMockRecorder) AsyncRunCallback(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncRunCallback", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncRunCallback), ctx) +} + +// AsyncSend mocks base method. +func (m *MockAsyncProducer) AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AsyncSend", ctx, topic, partition, message) + ret0, _ := ret[0].(error) + return ret0 +} + +// AsyncSend indicates an expected call of AsyncSend. +func (mr *MockAsyncProducerMockRecorder) AsyncSend(ctx, topic, partition, message interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncSend", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncSend), ctx, topic, partition, message) +} + +// Close mocks base method. +func (m *MockAsyncProducer) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockAsyncProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockAsyncProducer)(nil).Close)) +} diff --git a/pkg/sink/kafka/cluster_admin_client.go b/pkg/sink/kafka/cluster_admin_client.go deleted file mode 100644 index d559b53931..0000000000 --- a/pkg/sink/kafka/cluster_admin_client.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -// TopicDetail represent a topic's detail information. -type TopicDetail struct { - Name string - NumPartitions int32 - ReplicationFactor int16 -} - -// ClusterAdminClient is the administrative client for Kafka, -// which supports managing and inspecting topics, brokers, configurations and ACLs. -type ClusterAdminClient interface { - // GetBrokerConfig return the broker level configuration with the `configName` - GetBrokerConfig(configName string) (string, error) - - // GetTopicConfig return the topic level configuration with the `configName` - GetTopicConfig(topicName string, configName string) (string, error) - - // GetTopicsMeta return all target topics' metadata - // if `ignoreTopicError` is true, ignore the topic error and return the metadata of valid topics - GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) - - // GetTopicsPartitionsNum return the number of partitions of each topic. - GetTopicsPartitionsNum(topics []string) (map[string]int32, error) - - // CreateTopic creates a new topic. - CreateTopic(detail *TopicDetail, validateOnly bool) error - - // Close shuts down the admin client. - Close() -} diff --git a/pkg/sink/kafka/cluster_admin_client_mock.go b/pkg/sink/kafka/cluster_admin_client_mock.go deleted file mode 100644 index 72536d5084..0000000000 --- a/pkg/sink/kafka/cluster_admin_client_mock.go +++ /dev/null @@ -1,120 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/cluster_admin_client.go - -// Package kafka is a generated GoMock package. -package kafka - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockClusterAdminClient is a mock of ClusterAdminClient interface. -type MockClusterAdminClient struct { - ctrl *gomock.Controller - recorder *MockClusterAdminClientMockRecorder -} - -// MockClusterAdminClientMockRecorder is the mock recorder for MockClusterAdminClient. -type MockClusterAdminClientMockRecorder struct { - mock *MockClusterAdminClient -} - -// NewMockClusterAdminClient creates a new mock instance. -func NewMockClusterAdminClient(ctrl *gomock.Controller) *MockClusterAdminClient { - mock := &MockClusterAdminClient{ctrl: ctrl} - mock.recorder = &MockClusterAdminClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClusterAdminClient) EXPECT() *MockClusterAdminClientMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockClusterAdminClient) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockClusterAdminClientMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClusterAdminClient)(nil).Close)) -} - -// CreateTopic mocks base method. -func (m *MockClusterAdminClient) CreateTopic(detail *TopicDetail, validateOnly bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTopic", detail, validateOnly) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateTopic indicates an expected call of CreateTopic. -func (mr *MockClusterAdminClientMockRecorder) CreateTopic(detail, validateOnly interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockClusterAdminClient)(nil).CreateTopic), detail, validateOnly) -} - -// GetBrokerConfig mocks base method. -func (m *MockClusterAdminClient) GetBrokerConfig(configName string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBrokerConfig", configName) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBrokerConfig indicates an expected call of GetBrokerConfig. -func (mr *MockClusterAdminClientMockRecorder) GetBrokerConfig(configName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockClusterAdminClient)(nil).GetBrokerConfig), configName) -} - -// GetTopicConfig mocks base method. -func (m *MockClusterAdminClient) GetTopicConfig(topicName, configName string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicConfig", topicName, configName) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTopicConfig indicates an expected call of GetTopicConfig. -func (mr *MockClusterAdminClientMockRecorder) GetTopicConfig(topicName, configName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockClusterAdminClient)(nil).GetTopicConfig), topicName, configName) -} - -// GetTopicsMeta mocks base method. -func (m *MockClusterAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsMeta", topics, ignoreTopicError) - ret0, _ := ret[0].(map[string]TopicDetail) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTopicsMeta indicates an expected call of GetTopicsMeta. -func (mr *MockClusterAdminClientMockRecorder) GetTopicsMeta(topics, ignoreTopicError interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockClusterAdminClient)(nil).GetTopicsMeta), topics, ignoreTopicError) -} - -// GetTopicsPartitionsNum mocks base method. -func (m *MockClusterAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", topics) - ret0, _ := ret[0].(map[string]int32) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTopicsPartitionsNum indicates an expected call of GetTopicsPartitionsNum. -func (mr *MockClusterAdminClientMockRecorder) GetTopicsPartitionsNum(topics interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockClusterAdminClient)(nil).GetTopicsPartitionsNum), topics) -} diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index db83959081..7caf389e0b 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -22,8 +22,8 @@ import ( // Factory is used to produce all Kafka components. type Factory interface { - // AdminClient returns a Kafka cluster admin client. - AdminClient(ctx context.Context) (ClusterAdminClient, error) + // Admin returns a Kafka admin. + Admin(ctx context.Context) (Admin, error) // SyncProducer creates a sync producer to write messages to Kafka. SyncProducer(ctx context.Context) (SyncProducer, error) // AsyncProducer creates an async producer to write messages to Kafka. @@ -43,7 +43,7 @@ func NewFactory( o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - admin, err := newAdminClient(ctx, changefeedID, newClientOption(o), nil) + admin, err := newAdmin(ctx, changefeedID, newClientOption(o), nil) if err != nil { return nil, errors.Trace(err) } @@ -60,8 +60,8 @@ func NewFactory( }, nil } -func (f *factory) AdminClient(ctx context.Context) (ClusterAdminClient, error) { - admin, err := newAdminClient(ctx, f.changefeedID, f.clientOption, f.metricsHook) +func (f *factory) Admin(ctx context.Context) (Admin, error) { + admin, err := newAdmin(ctx, f.changefeedID, f.clientOption, f.metricsHook) if err != nil { return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) } diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index 1ce6cf5977..26e523b6fb 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/pingcap/ticdc/pkg/sink/kafka (interfaces: Factory,SyncProducer,AsyncProducer) +// Source: pkg/sink/kafka/factory.go // Package kafka is a generated GoMock package. package kafka @@ -9,7 +9,6 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" - common "github.com/pingcap/ticdc/pkg/sink/codec/common" ) // MockFactory is a mock of Factory interface. @@ -35,173 +34,47 @@ func (m *MockFactory) EXPECT() *MockFactoryMockRecorder { return m.recorder } -// AdminClient mocks base method. -func (m *MockFactory) AdminClient(arg0 context.Context) (ClusterAdminClient, error) { +// Admin mocks base method. +func (m *MockFactory) Admin(ctx context.Context) (Admin, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AdminClient", arg0) - ret0, _ := ret[0].(ClusterAdminClient) + ret := m.ctrl.Call(m, "Admin", ctx) + ret0, _ := ret[0].(Admin) ret1, _ := ret[1].(error) return ret0, ret1 } -// AdminClient indicates an expected call of AdminClient. -func (mr *MockFactoryMockRecorder) AdminClient(arg0 interface{}) *gomock.Call { +// Admin indicates an expected call of Admin. +func (mr *MockFactoryMockRecorder) Admin(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminClient", reflect.TypeOf((*MockFactory)(nil).AdminClient), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Admin", reflect.TypeOf((*MockFactory)(nil).Admin), ctx) } // AsyncProducer mocks base method. -func (m *MockFactory) AsyncProducer(arg0 context.Context) (AsyncProducer, error) { +func (m *MockFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncProducer", arg0) + ret := m.ctrl.Call(m, "AsyncProducer", ctx) ret0, _ := ret[0].(AsyncProducer) ret1, _ := ret[1].(error) return ret0, ret1 } // AsyncProducer indicates an expected call of AsyncProducer. -func (mr *MockFactoryMockRecorder) AsyncProducer(arg0 interface{}) *gomock.Call { +func (mr *MockFactoryMockRecorder) AsyncProducer(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), ctx) } // SyncProducer mocks base method. -func (m *MockFactory) SyncProducer(arg0 context.Context) (SyncProducer, error) { +func (m *MockFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SyncProducer", arg0) + ret := m.ctrl.Call(m, "SyncProducer", ctx) ret0, _ := ret[0].(SyncProducer) ret1, _ := ret[1].(error) return ret0, ret1 } // SyncProducer indicates an expected call of SyncProducer. -func (mr *MockFactoryMockRecorder) SyncProducer(arg0 interface{}) *gomock.Call { +func (mr *MockFactoryMockRecorder) SyncProducer(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncProducer", reflect.TypeOf((*MockFactory)(nil).SyncProducer), arg0) -} - -// MockSyncProducer is a mock of SyncProducer interface. -type MockSyncProducer struct { - ctrl *gomock.Controller - recorder *MockSyncProducerMockRecorder -} - -// MockSyncProducerMockRecorder is the mock recorder for MockSyncProducer. -type MockSyncProducerMockRecorder struct { - mock *MockSyncProducer -} - -// NewMockSyncProducer creates a new mock instance. -func NewMockSyncProducer(ctrl *gomock.Controller) *MockSyncProducer { - mock := &MockSyncProducer{ctrl: ctrl} - mock.recorder = &MockSyncProducerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSyncProducer) EXPECT() *MockSyncProducerMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockSyncProducer) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockSyncProducerMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSyncProducer)(nil).Close)) -} - -// SendMessage mocks base method. -func (m *MockSyncProducer) SendMessage(arg0 string, arg1 int32, arg2 *common.Message) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessage", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMessage indicates an expected call of SendMessage. -func (mr *MockSyncProducerMockRecorder) SendMessage(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), arg0, arg1, arg2) -} - -// SendMessages mocks base method. -func (m *MockSyncProducer) SendMessages(arg0 string, arg1 int32, arg2 *common.Message) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessages", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMessages indicates an expected call of SendMessages. -func (mr *MockSyncProducerMockRecorder) SendMessages(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), arg0, arg1, arg2) -} - -// MockAsyncProducer is a mock of AsyncProducer interface. -type MockAsyncProducer struct { - ctrl *gomock.Controller - recorder *MockAsyncProducerMockRecorder -} - -// MockAsyncProducerMockRecorder is the mock recorder for MockAsyncProducer. -type MockAsyncProducerMockRecorder struct { - mock *MockAsyncProducer -} - -// NewMockAsyncProducer creates a new mock instance. -func NewMockAsyncProducer(ctrl *gomock.Controller) *MockAsyncProducer { - mock := &MockAsyncProducer{ctrl: ctrl} - mock.recorder = &MockAsyncProducerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAsyncProducer) EXPECT() *MockAsyncProducerMockRecorder { - return m.recorder -} - -// AsyncRunCallback mocks base method. -func (m *MockAsyncProducer) AsyncRunCallback(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncRunCallback", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// AsyncRunCallback indicates an expected call of AsyncRunCallback. -func (mr *MockAsyncProducerMockRecorder) AsyncRunCallback(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncRunCallback", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncRunCallback), arg0) -} - -// AsyncSend mocks base method. -func (m *MockAsyncProducer) AsyncSend(arg0 context.Context, arg1 string, arg2 int32, arg3 *common.Message) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncSend", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// AsyncSend indicates an expected call of AsyncSend. -func (mr *MockAsyncProducerMockRecorder) AsyncSend(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncSend", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncSend), arg0, arg1, arg2, arg3) -} - -// Close mocks base method. -func (m *MockAsyncProducer) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockAsyncProducerMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockAsyncProducer)(nil).Close)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncProducer", reflect.TypeOf((*MockFactory)(nil).SyncProducer), ctx) } diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 642a920c3b..95dac3134e 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -593,7 +593,7 @@ func NewKafkaClientID(captureAddr string, func adjustOptions( ctx context.Context, changefeedID common.ChangeFeedID, - admin ClusterAdminClient, + admin Admin, options *options, topic string, ) error { @@ -611,7 +611,7 @@ func adjustOptions( func adjustTopicOptions( ctx context.Context, changefeedID common.ChangeFeedID, - admin ClusterAdminClient, + admin Admin, options *options, topic string, topics map[string]TopicDetail, @@ -635,7 +635,7 @@ func adjustTopicOptions( func validateRequiredAcks( ctx context.Context, - admin ClusterAdminClient, + admin Admin, topics map[string]TopicDetail, topic string, options *options, @@ -652,7 +652,7 @@ func validateRequiredAcks( func adjustExistingTopicOption( ctx context.Context, changefeedID common.ChangeFeedID, - admin ClusterAdminClient, + admin Admin, options *options, topic string, info TopicDetail, @@ -681,7 +681,7 @@ func adjustExistingTopicOption( } func adjustNewTopicOptions( - admin ClusterAdminClient, + admin Admin, changefeedID common.ChangeFeedID, options *options, topic string, @@ -708,7 +708,7 @@ func adjustNewTopicOptions( func getTopicMaxMessageBytes( ctx context.Context, - admin ClusterAdminClient, + admin Admin, topic string, ) (int, error) { raw, err := getTopicConfig( @@ -726,7 +726,7 @@ func getTopicMaxMessageBytes( return maxMessageBytes, nil } -func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, error) { +func getBrokerMaxMessageBytes(admin Admin) (int, error) { raw, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) if err != nil { return 0, errors.Trace(err) @@ -740,7 +740,7 @@ func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, error) { func validateMinInsyncReplicas( ctx context.Context, - admin ClusterAdminClient, + admin Admin, topics map[string]TopicDetail, topic string, replicationFactor int, @@ -811,7 +811,7 @@ func validateMinInsyncReplicas( // NOTICE: The configuration names of topic and broker may be different for the same configuration. func getTopicConfig( ctx context.Context, - admin ClusterAdminClient, + admin Admin, topicName string, topicConfigName string, brokerConfigName string, diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 9c0a2a06e9..86bd184edf 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -41,7 +41,7 @@ const ( ) type kafkaAdminFixture struct { - admin *MockClusterAdminClient + admin *MockAdmin topics map[string]TopicDetail brokerConfig map[string]string topicConfig map[string]map[string]string @@ -52,7 +52,7 @@ func newKafkaAdminFixture(t *testing.T) *kafkaAdminFixture { ctrl := gomock.NewController(t) fixture := &kafkaAdminFixture{ - admin: NewMockClusterAdminClient(ctrl), + admin: NewMockAdmin(ctrl), topics: make(map[string]TopicDetail), brokerConfig: map[string]string{ BrokerMessageMaxBytesConfigName: mockBrokerMessageMaxBytes, @@ -480,13 +480,13 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t for _, test := range tests { t.Run(test.name, func(t *testing.T) { adminFixture := newKafkaAdminFixture(t) - adminClient := adminFixture.admin + admin := adminFixture.admin detail := &TopicDetail{ Name: topicName, NumPartitions: 3, } - err := adminClient.CreateTopic(detail, false) + err := admin.CreateTopic(detail, false) require.NoError(t, err) configuredMaxMessageBytes := test.configuredMaxMessageBytes(adminFixture) @@ -504,7 +504,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t expectedProducerLimit := adminFixture.brokerMessageMaxBytes() ctx := context.Background() - err = adjustOptions(ctx, changefeedID, adminClient, options, topicName) + err = adjustOptions(ctx, changefeedID, admin, options, topicName) require.NoError(t, err) require.NotEqual(t, configuredMaxMessageBytes, options.MaxMessageBytes) @@ -521,7 +521,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t func TestAdjustConfigMinInsyncReplicas(t *testing.T) { adminFixture := newKafkaAdminFixture(t) - adminClient := adminFixture.admin + admin := adminFixture.admin options := NewOptions() options.BrokerEndpoints = []string{"127.0.0.1:9092"} @@ -535,7 +535,7 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { err := adjustOptions( ctx, changefeedID, - adminClient, + admin, options, "create-new-fail-invalid-min-insync-replicas", ) @@ -548,9 +548,9 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { // topic not exist, and `min.insync.replicas` not found in broker's configuration adminFixture.dropBrokerConfig(MinInsyncReplicasConfigName) topicName := "no-topic-no-min-insync-replicas" - err = adjustOptions(ctx, changefeedID, adminClient, options, "no-topic-no-min-insync-replicas") + err = adjustOptions(ctx, changefeedID, admin, options, "no-topic-no-min-insync-replicas") require.Nil(t, err) - err = adminClient.CreateTopic(&TopicDetail{ + err = admin.CreateTopic(&TopicDetail{ Name: topicName, ReplicationFactor: 1, }, false) @@ -561,18 +561,18 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { // topic exist, but `min.insync.replicas` not found in topic and broker configuration topicName = "topic-no-options-entry" - err = adminClient.CreateTopic(&TopicDetail{ + err = admin.CreateTopic(&TopicDetail{ Name: topicName, ReplicationFactor: 3, NumPartitions: 3, }, false) require.Nil(t, err) - err = adjustOptions(ctx, changefeedID, adminClient, options, topicName) + err = adjustOptions(ctx, changefeedID, admin, options, topicName) require.Nil(t, err) // topic found, and have `min.insync.replicas`, but set to 2, larger than `replication-factor`. adminFixture.setMinInsyncReplicas("2") - err = adjustOptions(ctx, changefeedID, adminClient, options, defaultMockTopicName) + err = adjustOptions(ctx, changefeedID, admin, options, defaultMockTopicName) require.Regexp(t, ".*`replication-factor` 1 is smaller than the `min.insync.replicas` 2 of topic.*", errors.Cause(err), @@ -581,7 +581,7 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { func TestSkipAdjustConfigMinInsyncReplicasWhenRequiredAcksIsNotWailAll(t *testing.T) { adminFixture := newKafkaAdminFixture(t) - adminClient := adminFixture.admin + admin := adminFixture.admin options := NewOptions() options.BrokerEndpoints = []string{"127.0.0.1:9092"} @@ -594,7 +594,7 @@ func TestSkipAdjustConfigMinInsyncReplicasWhenRequiredAcksIsNotWailAll(t *testin err := adjustOptions( context.Background(), changefeedID, - adminClient, + admin, options, "skip-check-min-insync-replicas", ) @@ -767,7 +767,7 @@ func TestConfigurationCombinations(t *testing.T) { t.Run(a.name, func(t *testing.T) { adminFixture := newKafkaAdminFixture(t) adminFixture.setMessageMaxBytes(a.brokerMessageMaxBytes, a.topicMaxMessageBytes) - adminClient := adminFixture.admin + admin := adminFixture.admin uri := fmt.Sprintf(a.uriTemplate, a.uriParams...) sinkURI, err := url.Parse(uri) @@ -789,7 +789,7 @@ func TestConfigurationCombinations(t *testing.T) { } changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") - err = adjustOptions(ctx, changefeedID, adminClient, options, topic) + err = adjustOptions(ctx, changefeedID, admin, options, topic) require.Nil(t, err) require.Equal(t, sourceMaxMessageBytes, options.MaxMessageBytes) require.Equal( @@ -799,7 +799,7 @@ func TestConfigurationCombinations(t *testing.T) { ) require.Equal(t, sourceMaxMessageBytes, newClientOption(options).ProducerBatchMaxBytes) - adminClient.Close() + admin.Close() }) } } diff --git a/pkg/sink/kafka/sync_producer_mock.go b/pkg/sink/kafka/sync_producer_mock.go new file mode 100644 index 0000000000..58429b08be --- /dev/null +++ b/pkg/sink/kafka/sync_producer_mock.go @@ -0,0 +1,75 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/sink/kafka/sync_producer.go + +// Package kafka is a generated GoMock package. +package kafka + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + common "github.com/pingcap/ticdc/pkg/sink/codec/common" +) + +// MockSyncProducer is a mock of SyncProducer interface. +type MockSyncProducer struct { + ctrl *gomock.Controller + recorder *MockSyncProducerMockRecorder +} + +// MockSyncProducerMockRecorder is the mock recorder for MockSyncProducer. +type MockSyncProducerMockRecorder struct { + mock *MockSyncProducer +} + +// NewMockSyncProducer creates a new mock instance. +func NewMockSyncProducer(ctrl *gomock.Controller) *MockSyncProducer { + mock := &MockSyncProducer{ctrl: ctrl} + mock.recorder = &MockSyncProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyncProducer) EXPECT() *MockSyncProducerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockSyncProducer) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockSyncProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSyncProducer)(nil).Close)) +} + +// SendMessage mocks base method. +func (m *MockSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessage", topic, partitionNum, message) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessage indicates an expected call of SendMessage. +func (mr *MockSyncProducerMockRecorder) SendMessage(topic, partitionNum, message interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), topic, partitionNum, message) +} + +// SendMessages mocks base method. +func (m *MockSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessages", topic, partitionNum, message) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessages indicates an expected call of SendMessages. +func (mr *MockSyncProducerMockRecorder) SendMessages(topic, partitionNum, message interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), topic, partitionNum, message) +} diff --git a/scripts/generate-mock.sh b/scripts/generate-mock.sh index dfd53026c7..e72b8bfd6f 100755 --- a/scripts/generate-mock.sh +++ b/scripts/generate-mock.sh @@ -34,8 +34,10 @@ fi "$MOCKGEN" -source pkg/api/v2/changefeed.go -destination pkg/api/v2/mock/changefeed_mock.go -package mock "$MOCKGEN" -source pkg/api/v2/api_client.go -destination pkg/api/v2/mock/api_client_mock.go -package mock "$MOCKGEN" -source pkg/sink/codec/simple/marshaller.go -destination pkg/sink/codec/simple/mock/marshaller.go -"$MOCKGEN" -source pkg/sink/kafka/cluster_admin_client.go -destination pkg/sink/kafka/cluster_admin_client_mock.go -package kafka -"$MOCKGEN" -destination pkg/sink/kafka/factory_mock.go -package kafka -self_package github.com/pingcap/ticdc/pkg/sink/kafka github.com/pingcap/ticdc/pkg/sink/kafka Factory,SyncProducer,AsyncProducer +"$MOCKGEN" -source pkg/sink/kafka/admin.go -destination pkg/sink/kafka/admin_mock.go -package kafka +"$MOCKGEN" -source pkg/sink/kafka/factory.go -destination pkg/sink/kafka/factory_mock.go -package kafka +"$MOCKGEN" -source pkg/sink/kafka/sync_producer.go -destination pkg/sink/kafka/sync_producer_mock.go -package kafka +"$MOCKGEN" -source pkg/sink/kafka/async_producer.go -destination pkg/sink/kafka/async_producer_mock.go -package kafka "$MOCKGEN" -source pkg/keyspace/keyspace_manager.go -destination pkg/keyspace/keyspace_manager_mock.go -package keyspace "$MOCKGEN" -source pkg/txnutil/gc/gc_manager.go -destination pkg/txnutil/gc/gc_manager_mock.go -package gc "$MOCKGEN" -source pkg/txnutil/gc/gc_client.go -destination pkg/txnutil/gc/gc_client_mock.go -package gc From 7a2451c892f6b572a21af493de93f5fa7d5c54bf Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 28 Jul 2026 10:59:26 +0800 Subject: [PATCH 34/61] fix more code --- downstreamadapter/sink/kafka/helper.go | 4 +- downstreamadapter/sink/kafka/sink.go | 4 +- downstreamadapter/sink/pulsar/helper.go | 2 +- pkg/errors/error.go | 6 +- pkg/sink/kafka/async_producer.go | 2 - pkg/sink/kafka/client_options.go | 8 +- pkg/sink/kafka/client_options_test.go | 6 +- pkg/sink/kafka/factory.go | 11 ++- pkg/sink/kafka/factory_test.go | 73 +++++++++++++++++++ pkg/sink/kafka/gssapi.go | 8 +- pkg/sink/kafka/sync_producer.go | 2 - .../http_api/util/test_case.py | 2 +- 12 files changed, 93 insertions(+), 35 deletions(-) diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index 877d0c54a9..63503ff19d 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -88,7 +88,7 @@ func newKafkaSinkComponent( comp.factory, err = kafka.NewFactory(ctx, options, changefeedID) if err != nil { - return comp, protocol, errors.WrapError(errors.ErrKafkaNewProducer, err) + return comp, protocol, err } isAvroLike := protocol == config.ProtocolAvro || protocol == config.ProtocolDebeziumAvro @@ -128,7 +128,7 @@ func newKafkaSinkComponent( comp.admin, err = comp.factory.Admin(ctx) if err != nil { - return comp, protocol, errors.WrapError(errors.ErrKafkaNewProducer, err) + return comp, protocol, err } comp.topicManager, err = topicmanager.GetTopicManagerAndTryCreateTopic( diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index d0e70ed4ca..8ce17bada8 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -112,12 +112,12 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. factory, err := kafka.NewFactory(ctx, options, changefeedID) if err != nil { - return errors.WrapError(errors.ErrKafkaNewProducer, err) + return err } admin, err := factory.Admin(ctx) if err != nil { - return errors.WrapError(errors.ErrKafkaNewProducer, err) + return err } defer admin.Close() diff --git a/downstreamadapter/sink/pulsar/helper.go b/downstreamadapter/sink/pulsar/helper.go index 12e1beda52..59e592d0de 100644 --- a/downstreamadapter/sink/pulsar/helper.go +++ b/downstreamadapter/sink/pulsar/helper.go @@ -98,7 +98,7 @@ func newPulsarSinkComponentWithFactory(ctx context.Context, pulsarComponent.client, err = factoryCreator(pulsarComponent.config, changefeedID, sinkConfig) if err != nil { - return pulsarComponent, protocol, errors.WrapError(errors.ErrKafkaNewProducer, err) + return pulsarComponent, protocol, errors.WrapError(errors.ErrPulsarNewProducer, err) } topic, err := helper.GetTopic(sinkURI) diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 2cfa46fea7..e02a907577 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -144,9 +144,9 @@ var ( "only support these values: 0(NoResponse),1(WaitForLocal) and -1(WaitForAll)", errors.RFCCodeText("CDC:ErrKafkaInvalidRequiredAcks"), ) - ErrKafkaNewProducer = errors.Normalize( - "new kafka producer", - errors.RFCCodeText("CDC:ErrKafkaNewProducer"), + ErrNewKafkaSink = errors.Normalize( + "create kafka sink failed", + errors.RFCCodeText("CDC:ErrNewKafkaSink"), ) ErrKafkaInvalidClientID = errors.Normalize( "invalid kafka client ID '%s'", diff --git a/pkg/sink/kafka/async_producer.go b/pkg/sink/kafka/async_producer.go index dd4df1e6b6..d674064ada 100644 --- a/pkg/sink/kafka/async_producer.go +++ b/pkg/sink/kafka/async_producer.go @@ -180,8 +180,6 @@ func (p *asyncProducer) enqueueAsyncSendError( } } -func (p *asyncProducer) Heartbeat() {} - func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { defer p.closed.Store(true) for { diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index af10e2ea37..87b1850bde 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -41,7 +41,6 @@ type clientOptions struct { Version string IsAssignedVersion bool - MaxMessageBytes int ProducerBatchMaxBytes int MaxRetry int Compression string @@ -193,18 +192,13 @@ func newOauthTokenSource(ctx context.Context, o *clientOptions) (oauth2.TokenSou func newProducerOptions( o *clientOptions, ) []kgo.Opt { - producerBatchMaxBytes := o.ProducerBatchMaxBytes - if producerBatchMaxBytes <= 0 { - producerBatchMaxBytes = o.MaxMessageBytes - } - return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), kgo.RequiredAcks(newRequiredAcks(o)), kgo.DisableIdempotentWrite(), kgo.MaxProduceRequestsInflightPerBroker(1), kgo.RecordRetries(o.MaxRetry), - kgo.ProducerBatchMaxBytes(int32(producerBatchMaxBytes)), + kgo.ProducerBatchMaxBytes(int32(o.ProducerBatchMaxBytes)), kgo.ProduceRequestTimeout(o.RequestTimeout), kgo.ProducerLinger(0), newCompressionOption(o), diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index b664d4effd..cbeaf25fb8 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -57,13 +57,9 @@ func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { func TestNewProducerOptionsUsesProducerBatchMaxBytes(t *testing.T) { t.Parallel() - const ( - encoderMaxMessageBytes = 800 - producerBatchMaxBytes = 1048588 - ) + const producerBatchMaxBytes = 1048588 o := &clientOptions{ BrokerEndpoints: []string{"127.0.0.1:9092"}, - MaxMessageBytes: encoderMaxMessageBytes, ProducerBatchMaxBytes: producerBatchMaxBytes, MaxRetry: defaultMaxRetry, RequiredAcks: int16(WaitForAll), diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 47ea77ce0f..5042b4af64 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -45,12 +45,12 @@ func NewFactory( ) (Factory, error) { admin, err := newAdmin(ctx, changefeedID, newClientOption(o), nil) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } defer admin.Close() if err := adjustOptions(changefeedID, admin, o, o.Topic); err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } return &factory{ @@ -63,7 +63,7 @@ func NewFactory( func (f *factory) Admin(ctx context.Context) (Admin, error) { admin, err := newAdmin(ctx, f.changefeedID, f.clientOption, f.metricsHook) if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } return admin, nil } @@ -71,7 +71,7 @@ func (f *factory) Admin(ctx context.Context) (Admin, error) { func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { producer, err := newSyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } return producer, nil } @@ -79,7 +79,7 @@ func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { producer, err := newAsyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) if err != nil { - return nil, errors.WrapError(errors.ErrKafkaNewProducer, err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } return producer, nil } @@ -92,7 +92,6 @@ func newClientOption(o *options) *clientOptions { Version: o.Version, IsAssignedVersion: o.IsAssignedVersion, - MaxMessageBytes: o.MaxMessageBytes, ProducerBatchMaxBytes: o.MaxMessageBytes, MaxRetry: o.MaxRetry, Compression: o.Compression, diff --git a/pkg/sink/kafka/factory_test.go b/pkg/sink/kafka/factory_test.go index 4ac588dced..b4e3c6929c 100644 --- a/pkg/sink/kafka/factory_test.go +++ b/pkg/sink/kafka/factory_test.go @@ -14,9 +14,12 @@ package kafka import ( + "context" "testing" "time" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" ) @@ -91,3 +94,73 @@ func TestNewClientOptionDerivesRequestTimeout(t *testing.T) { }) } } + +func TestNewFactoryAdminCreationReturnsKafkaSinkError(t *testing.T) { + t.Parallel() + + options := NewOptions() + options.Version = "invalid" + options.IsAssignedVersion = true + + factory, err := NewFactory( + context.Background(), + options, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), + ) + require.Nil(t, factory) + requireNewKafkaSinkError(t, err) +} + +func TestFactoryComponentCreationReturnsKafkaSinkError(t *testing.T) { + t.Parallel() + + factory := &factory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), + clientOption: &clientOptions{ + Version: "invalid", + IsAssignedVersion: true, + }, + } + + testCases := []struct { + name string + create func() error + }{ + { + name: "admin", + create: func() error { + _, err := factory.Admin(context.Background()) + return err + }, + }, + { + name: "sync producer", + create: func() error { + _, err := factory.SyncProducer(context.Background()) + return err + }, + }, + { + name: "async producer", + create: func() error { + _, err := factory.AsyncProducer(context.Background()) + return err + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + requireNewKafkaSinkError(t, tc.create()) + }) + } +} + +func requireNewKafkaSinkError(t *testing.T, err error) { + t.Helper() + + errCode, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrNewKafkaSink.RFCCode(), errCode) +} diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/gssapi.go index 662a4f441f..b2b72c386b 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -41,7 +41,7 @@ const ( gssAPIFinished = 3 ) -type kerborosClient interface { +type kerberosClient interface { Login() error GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) Domain() string @@ -61,7 +61,7 @@ func (m *gssapiMechanism) Authenticate( _ context.Context, host string, ) (sasl.Session, []byte, error) { - client, err := newKerborosClient(m.config) + client, err := newKerberosClient(m.config) if err != nil { return nil, nil, errors.Trace(err) } @@ -93,7 +93,7 @@ func (m *gssapiMechanism) Authenticate( } type gssapiSession struct { - client kerborosClient + client kerberosClient ticket messages.Ticket encKey types.EncryptionKey step int @@ -200,7 +200,7 @@ func (c *krb5Client) CName() types.PrincipalName { return c.Credentials.CName() } -func newKerborosClient(g gssapiConfig) (kerborosClient, error) { +func newKerberosClient(g gssapiConfig) (kerberosClient, error) { cfg, err := config.Load(g.kerberosConfigPath) if err != nil { return nil, errors.Trace(err) diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index 816097d140..b41270fc20 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -151,8 +151,6 @@ func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *c return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *syncProducer) Heartbeat() {} - func (p *syncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { log.Warn("kafka DDL producer already closed", diff --git a/tests/integration_tests/http_api/util/test_case.py b/tests/integration_tests/http_api/util/test_case.py index 30c2a2e65e..8a3c8e4dbf 100644 --- a/tests/integration_tests/http_api/util/test_case.py +++ b/tests/integration_tests/http_api/util/test_case.py @@ -175,7 +175,7 @@ def create_changefeed(sink_uri): }) headers = {"Content-Type": "application/json"} resp = rq.post(url, data=data, headers=headers) - assert "CDC:ErrKafkaNewProducer" in resp.text, f"{resp.text}" + assert "CDC:ErrNewKafkaSink" in resp.text, f"{resp.text}" assert "not found, ResolveEndpointV2" not in resp.text, f"{resp.text}" print("pass test: create changefeed") From 6f68f46b9b23135e2c6350f982e1aaf036434a27 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 28 Jul 2026 15:16:45 +0800 Subject: [PATCH 35/61] simplify the code further --- downstreamadapter/sink/kafka/sink.go | 5 +- .../sink/topicmanager/kafka_topic_manager.go | 27 +++-- .../topicmanager/kafka_topic_manager_test.go | 45 +++---- metrics/grafana/ticdc_new_arch.json | 44 +++++-- .../ticdc_new_arch_next_gen.json | 44 +++++-- .../ticdc_new_arch_with_keyspace_name.json | 44 +++++-- pkg/errors/error.go | 8 ++ pkg/sink/kafka/admin.go | 74 +++++------- pkg/sink/kafka/admin_mock.go | 17 +-- pkg/sink/kafka/admin_test.go | 67 ++++++++++- pkg/sink/kafka/async_producer.go | 45 +++---- pkg/sink/kafka/async_producer_test.go | 47 +++----- pkg/sink/kafka/client_options.go | 63 +++------- pkg/sink/kafka/client_options_test.go | 25 ++-- pkg/sink/kafka/factory.go | 42 ++----- pkg/sink/kafka/factory_test.go | 43 +------ pkg/sink/kafka/failpoint.go | 45 ------- pkg/sink/kafka/failpoint_test.go | 41 ------- pkg/sink/kafka/gssapi.go | 98 +++++---------- pkg/sink/kafka/metrics.go | 88 +++++++------- pkg/sink/kafka/metrics_hook.go | 112 +++++++++++++----- pkg/sink/kafka/options.go | 33 +++--- pkg/sink/kafka/options_test.go | 22 +--- pkg/sink/kafka/sasl_config.go | 8 +- pkg/sink/kafka/sasl_test.go | 4 +- pkg/sink/kafka/sync_producer.go | 14 +-- 26 files changed, 486 insertions(+), 619 deletions(-) delete mode 100644 pkg/sink/kafka/failpoint.go delete mode 100644 pkg/sink/kafka/failpoint_test.go diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 8ce17bada8..36491f52a4 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -135,7 +135,7 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. } // the topic is not created, only validate. - err = admin.CreateTopic(&kafka.TopicDetail{ + err = admin.CreateTopic(kafka.TopicDetail{ Name: topic, NumPartitions: topicConfig.PartitionNum, ReplicationFactor: topicConfig.ReplicationFactor, @@ -187,6 +187,7 @@ func newWithComponents( } comp.close() statistics.Close() + kafka.CleanupMetrics(changefeedID) }() asyncProducer, err = comp.factory.AsyncProducer(ctx) @@ -229,7 +230,6 @@ func (s *sink) Run(ctx context.Context) error { return s.sendDMLEvent(ctx) }) err := g.Wait() - kafka.CleanupMetrics(s.changefeedID) s.isNormal.Store(false) return errors.Trace(err) } @@ -605,6 +605,7 @@ func (s *sink) Close() { s.dmlProducer.Close() s.comp.close() s.statistics.Close() + kafka.CleanupMetrics(s.changefeedID) } func (s *sink) BatchCount() int { diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index a3f426ae3b..464b091b53 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -169,7 +169,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err }) start := time.Now() - numPartitions, err := m.admin.GetTopicsPartitionsNum(topics) + topicDetails, err := m.admin.GetTopicsMeta(topics, false) if err != nil { log.Warn( "Kafka admin describe topics failed", @@ -180,6 +180,14 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err ) return nil, err } + numPartitions := make(map[string]int32, len(topicDetails)) + for _, topic := range topics { + detail, ok := topicDetails[topic] + if !ok { + return nil, errors.ErrKafkaTopicNotFound.GenWithStackByArgs(topic) + } + numPartitions[topic] = detail.NumPartitions + } // it may happen the following case: // 1. user create the default topic with partition number set as 3 manually @@ -216,11 +224,15 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( ) return err } + detail, ok := meta[topicName] + if !ok { + return errors.ErrKafkaTopicNotFound.GenWithStackByArgs(topicName) + } log.Info("topic found", zap.String("keyspace", m.changefeedID.Keyspace()), zap.String("changefeed", m.changefeedID.Name()), zap.String("topic", topicName), - zap.Int32("partitionNumber", meta[topicName].NumPartitions), + zap.Int32("partitionNumber", detail.NumPartitions), zap.Duration("duration", time.Since(start))) return nil }, retry.WithBackoffBaseDelay(500), @@ -246,7 +258,7 @@ func (m *kafkaTopicManager) createTopic( } start := time.Now() - err := m.admin.CreateTopic(&kafka.TopicDetail{ + err := m.admin.CreateTopic(kafka.TopicDetail{ Name: topicName, NumPartitions: m.cfg.PartitionNum, ReplicationFactor: m.cfg.ReplicationFactor, @@ -297,15 +309,6 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( return numPartition, nil } - topicDetails, err = m.admin.GetTopicsMeta([]string{topicName}, false) - if err != nil { - if kafka.IsAdminAuthorizationFailed(err) { - return m.useConfiguredPartitionNum(topicName, err), nil - } - } else if numPartition, ok := m.tryStoreTopicMeta(topicName, topicDetails); ok { - return numPartition, nil - } - partitionNum, err := m.createTopic(ctx, topicName) if err != nil { if kafka.IsAdminAuthorizationFailed(err) { diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index 5b79294c8c..99d646b2e2 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kerr" ) const kafkaTopicManagerTestTopic = "mock_topic" @@ -38,14 +37,11 @@ func (m *mockAdminWithDeniedDescribe) GetTopicsMeta( ignoreTopicError bool, ) (map[string]kafka.TopicDetail, error) { m.describeCount++ - if ignoreTopicError { - return map[string]kafka.TopicDetail{}, nil - } - return nil, kerr.TopicAuthorizationFailed + return nil, errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs() } func (m *mockAdminWithDeniedDescribe) CreateTopic( - detail *kafka.TopicDetail, + detail kafka.TopicDetail, validateOnly bool, ) error { m.createTopicCalled = true @@ -67,11 +63,11 @@ func (m *mockAdminWithDeniedCreate) GetTopicsMeta( } func (m *mockAdminWithDeniedCreate) CreateTopic( - detail *kafka.TopicDetail, + detail kafka.TopicDetail, validateOnly bool, ) error { m.createTopicCalled = true - return kerr.ClusterAuthorizationFailed + return errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs() } func TestCreateTopic(t *testing.T) { @@ -88,9 +84,9 @@ func TestCreateTopic(t *testing.T) { changefeedID := common.NewChangefeedID4Test("test", "test") ctx := context.Background() - var gotNewTopicDetail *kafka.TopicDetail + var gotNewTopicDetail kafka.TopicDetail var gotNewTopicValidateOnly bool - var gotFailedTopicDetail *kafka.TopicDetail + var gotFailedTopicDetail kafka.TopicDetail var gotFailedTopicValidateOnly bool gomock.InOrder( admin.EXPECT().GetTopicsMeta([]string{kafkaTopicManagerTestTopic}, true).Return( @@ -102,10 +98,8 @@ func TestCreateTopic(t *testing.T) { }, nil), admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return( map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return( - map[string]kafka.TopicDetail{}, nil), admin.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( - func(detail *kafka.TopicDetail, validateOnly bool) error { + func(detail kafka.TopicDetail, validateOnly bool) error { gotNewTopicDetail = detail gotNewTopicValidateOnly = validateOnly return nil @@ -119,14 +113,10 @@ func TestCreateTopic(t *testing.T) { }, nil), admin.EXPECT().GetTopicsMeta([]string{"new-topic2"}, true).Return( map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic2"}, false).Return( - map[string]kafka.TopicDetail{}, nil), admin.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, true).Return( map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, false).Return( - map[string]kafka.TopicDetail{}, nil), admin.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( - func(detail *kafka.TopicDetail, validateOnly bool) error { + func(detail kafka.TopicDetail, validateOnly bool) error { gotFailedTopicDetail = detail gotFailedTopicValidateOnly = validateOnly return errors.New("invalid replication factor") @@ -143,7 +133,7 @@ func TestCreateTopic(t *testing.T) { partitionNum, err = manager.CreateTopicAndWaitUntilVisible(ctx, "new-topic") require.NoError(t, err) require.Equal(t, int32(2), partitionNum) - require.Equal(t, &kafka.TopicDetail{ + require.Equal(t, kafka.TopicDetail{ Name: "new-topic", NumPartitions: 2, ReplicationFactor: 1, @@ -185,7 +175,6 @@ func TestCreateTopic(t *testing.T) { "kafka create topic failed: invalid replication factor", err, ) - require.NotNil(t, gotFailedTopicDetail) require.Equal(t, "new-topic-failed", gotFailedTopicDetail.Name) require.False(t, gotFailedTopicValidateOnly) } @@ -199,8 +188,6 @@ func TestCreateTopicValidatesReplicationFactor(t *testing.T) { gomock.InOrder( admin.EXPECT().GetTopicsMeta([]string{topic}, true). Return(map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{topic}, false). - Return(map[string]kafka.TopicDetail{}, nil), admin.EXPECT().GetBrokerConfig(kafka.MinInsyncReplicasConfigName). Return("2", nil), ) @@ -238,11 +225,9 @@ func TestCreateTopicWaitsUntilVisible(t *testing.T) { gomock.InOrder( admin.EXPECT().GetTopicsMeta([]string{topic}, true).Return( map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - map[string]kafka.TopicDetail{}, nil), admin.EXPECT().CreateTopic(gomock.Any(), false).DoAndReturn( - func(detail *kafka.TopicDetail, validateOnly bool) error { - require.Equal(t, &kafka.TopicDetail{ + func(detail kafka.TopicDetail, validateOnly bool) error { + require.Equal(t, kafka.TopicDetail{ Name: topic, NumPartitions: 2, ReplicationFactor: 1, @@ -251,9 +236,9 @@ func TestCreateTopicWaitsUntilVisible(t *testing.T) { return nil }), admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, errors.New("unknown topic or partition")), + nil, errors.ErrKafkaTopicNotFound.GenWithStackByArgs(topic)), admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, errors.New("unknown topic or partition")), + nil, errors.ErrKafkaTopicNotFound.GenWithStackByArgs(topic)), admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( map[string]kafka.TopicDetail{ topic: { @@ -295,7 +280,7 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { require.NoError(t, err) require.Equal(t, int32(2), partitionNum) require.False(t, admin.createTopicCalled) - require.Equal(t, 2, admin.describeCount) + require.Equal(t, 1, admin.describeCount) partitions, ok := manager.topics.Load("precreated-topic") require.True(t, ok) @@ -324,7 +309,7 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { require.NoError(t, err) require.Equal(t, int32(2), partitionNum) require.True(t, admin.createTopicCalled) - require.Equal(t, 2, admin.describeCount) + require.Equal(t, 1, admin.describeCount) partitions, ok := manager.topics.Load("precreated-topic") require.True(t, ok) diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 0a3f4bbd37..3cb0b73ed0 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -17788,7 +17788,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", + "expr": "sum(rate(ticdc_sink_kafka_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -17946,7 +17946,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean", + "description": "Kafka request end-to-end duration in seconds. Average and p99 are calculated from one-minute histogram rates.", "fieldConfig": { "defaults": { "links": [] @@ -17994,12 +17994,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker, type)", + "expr": "sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-avg", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace, changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" } ], "thresholds": [], @@ -18097,11 +18106,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", + "expr": "sum(rate(ticdc_sink_kafka_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", "refId": "A" } ], @@ -18152,7 +18161,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate", + "description": "Records in each successfully written topic-partition batch. Average and p99 are calculated from one-minute histogram rates.", "fieldConfig": { "defaults": { "links": [] @@ -18200,19 +18209,28 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)", + "expr": "sum(rate(ticdc_sink_kafka_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace, changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Request", + "title": "Kafka Records Per Batch", "tooltip": { "shared": true, "sort": 0, @@ -18255,7 +18273,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The compression ratio times 100 of record batches for all topics. Compression ratio = Size of original data / Size of compressed data * 100", + "description": "Compression ratio of successfully written batches. Compression ratio = uncompressed byte rate / compressed byte rate * 100.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -18301,11 +18319,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)", + "expr": "100 * sum(rate(ticdc_sink_kafka_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", "refId": "A" } ], diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index f920ac2ccd..7383ed064f 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -17788,7 +17788,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(rate(ticdc_sink_kafka_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -17946,7 +17946,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean", + "description": "Kafka request end-to-end duration in seconds. Average and p99 are calculated from one-minute histogram rates.", "fieldConfig": { "defaults": { "links": [] @@ -17994,12 +17994,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker, type)", + "expr": "sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" } ], "thresholds": [], @@ -18097,11 +18106,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(rate(ticdc_sink_kafka_producer_requests_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", "refId": "A" } ], @@ -18152,7 +18161,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate", + "description": "Records in each successfully written topic-partition batch. Average and p99 are calculated from one-minute histogram rates.", "fieldConfig": { "defaults": { "links": [] @@ -18200,19 +18209,28 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", + "expr": "sum(rate(ticdc_sink_kafka_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Request", + "title": "Kafka Records Per Batch", "tooltip": { "shared": true, "sort": 0, @@ -18255,7 +18273,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The compression ratio times 100 of record batches for all topics. Compression ratio = Size of original data / Size of compressed data * 100", + "description": "Compression ratio of successfully written batches. Compression ratio = uncompressed byte rate / compressed byte rate * 100.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -18301,11 +18319,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", + "expr": "100 * sum(rate(ticdc_sink_kafka_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", "refId": "A" } ], diff --git a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json index f71e45145a..5b7bb7ea09 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json +++ b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json @@ -6313,7 +6313,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(rate(ticdc_sink_kafka_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -6471,7 +6471,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean", + "description": "Kafka request end-to-end duration in seconds. Average and p99 are calculated from one-minute histogram rates.", "fieldConfig": { "defaults": { "links": [] @@ -6519,12 +6519,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker, type)", + "expr": "sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" } ], "thresholds": [], @@ -6622,11 +6631,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(rate(ticdc_sink_kafka_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", "refId": "A" } ], @@ -6677,7 +6686,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate", + "description": "Records in each successfully written topic-partition batch. Average and p99 are calculated from one-minute histogram rates.", "fieldConfig": { "defaults": { "links": [] @@ -6725,19 +6734,28 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", + "expr": "sum(rate(ticdc_sink_kafka_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Request", + "title": "Kafka Records Per Batch", "tooltip": { "shared": true, "sort": 0, @@ -6780,7 +6798,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The compression ratio times 100 of record batches for all topics. Compression ratio = Size of original data / Size of compressed data * 100", + "description": "Compression ratio of successfully written batches. Compression ratio = uncompressed byte rate / compressed byte rate * 100.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -6826,11 +6844,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", + "expr": "100 * sum(rate(ticdc_sink_kafka_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", "refId": "A" } ], diff --git a/pkg/errors/error.go b/pkg/errors/error.go index e02a907577..6a62296938 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -164,6 +164,14 @@ var ( "kafka create topic failed", errors.RFCCodeText("CDC:ErrKafkaCreateTopic"), ) + ErrKafkaTopicNotFound = errors.Normalize( + "kafka topic %s not found", + errors.RFCCodeText("CDC:ErrKafkaTopicNotFound"), + ) + ErrKafkaAuthorizationFailed = errors.Normalize( + "kafka authorization failed", + errors.RFCCodeText("CDC:ErrKafkaAuthorizationFailed"), + ) ErrKafkaInvalidTopicExpression = errors.Normalize( "invalid topic expression: %s ", errors.RFCCodeText("CDC:ErrKafkaTopicExprInvalid"), diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index be3129e7b3..aa85aa388c 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -46,11 +46,8 @@ type Admin interface { // if `ignoreTopicError` is true, ignore the topic error and return the metadata of valid topics GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) - // GetTopicsPartitionsNum return the number of partitions of each topic. - GetTopicsPartitionsNum(topics []string) (map[string]int32, error) - // CreateTopic creates a new topic. - CreateTopic(detail *TopicDetail, validateOnly bool) error + CreateTopic(detail TopicDetail, validateOnly bool) error // Close shuts down the admin. Close() @@ -67,7 +64,7 @@ type admin struct { func newAdmin( ctx context.Context, changefeedID common.ChangeFeedID, - o *clientOptions, + o *options, hook kgo.Hook, ) (*admin, error) { opts, err := newOptions(ctx, o, hook) @@ -84,7 +81,7 @@ func newAdmin( changefeed: changefeedID, client: client, admin: kadm.NewClient(client), - timeout: o.RequestTimeout, + timeout: o.requestTimeout(), }, nil } @@ -177,13 +174,27 @@ func (a *admin) GetTopicsMeta( meta, err := a.admin.Metadata(ctx, topics...) if err != nil { + if isKafkaAuthorizationFailed(err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err) + } return nil, errors.Trace(err) } + return topicDetailsFromMetadata(meta, topics, ignoreTopicError) +} + +func topicDetailsFromMetadata( + meta kadm.Metadata, + topics []string, + ignoreTopicError bool, +) (map[string]TopicDetail, error) { result := make(map[string]TopicDetail, len(topics)) for _, topic := range topics { detail, ok := meta.Topics[topic] if !ok { + if !ignoreTopicError { + return nil, errors.ErrKafkaTopicNotFound.GenWithStackByArgs(topic) + } continue } if detail.Err == nil { @@ -194,56 +205,30 @@ func (a *admin) GetTopicsMeta( continue } if errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { + if !ignoreTopicError { + return nil, errors.WrapError(errors.ErrKafkaTopicNotFound, detail.Err, topic) + } continue } - if !ignoreTopicError { - return nil, errors.Trace(detail.Err) + if isKafkaAuthorizationFailed(detail.Err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, detail.Err) } - log.Warn("fetch topic meta failed", - zap.String("keyspace", a.changefeed.Keyspace()), zap.String("changefeed", a.changefeed.Name()), - zap.String("topic", topic), zap.Error(detail.Err)) + return nil, errors.Trace(detail.Err) } return result, nil } // IsAdminAuthorizationFailed checks whether err is an authorization failure from Kafka admin APIs. func IsAdminAuthorizationFailed(err error) bool { - return errors.Is(err, kerr.TopicAuthorizationFailed) || - errors.Is(err, kerr.ClusterAuthorizationFailed) + return errors.Is(err, errors.ErrKafkaAuthorizationFailed) || isKafkaAuthorizationFailed(err) } -func (a *admin) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - if len(topics) == 0 { - return make(map[string]int32), nil - } - - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) - defer cancel() - - meta, err := a.admin.Metadata(ctx, topics...) - if err != nil { - return nil, errors.Trace(err) - } - - result := make(map[string]int32, len(topics)) - for _, topic := range topics { - detail, ok := meta.Topics[topic] - if !ok { - return nil, errors.Trace(kerr.UnknownTopicOrPartition) - } - if detail.Err != nil { - return nil, errors.Trace(detail.Err) - } - result[topic] = int32(len(detail.Partitions)) - } - return result, nil +func isKafkaAuthorizationFailed(err error) bool { + return errors.Is(err, kerr.TopicAuthorizationFailed) || + errors.Is(err, kerr.ClusterAuthorizationFailed) } -func (a *admin) CreateTopic(detail *TopicDetail, validateOnly bool) error { - if detail == nil { - return errors.ErrKafkaInvalidConfig.GenWithStack("topic detail must not be nil") - } - +func (a *admin) CreateTopic(detail TopicDetail, validateOnly bool) error { ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) defer cancel() @@ -268,6 +253,9 @@ func (a *admin) CreateTopic(detail *TopicDetail, validateOnly bool) error { if errors.Is(resp.Err, kerr.TopicAlreadyExists) { return nil } + if isKafkaAuthorizationFailed(resp.Err) { + return errors.WrapError(errors.ErrKafkaAuthorizationFailed, resp.Err) + } return errors.Trace(resp.Err) } diff --git a/pkg/sink/kafka/admin_mock.go b/pkg/sink/kafka/admin_mock.go index 86f3ba41f4..2c113b1bdb 100644 --- a/pkg/sink/kafka/admin_mock.go +++ b/pkg/sink/kafka/admin_mock.go @@ -46,7 +46,7 @@ func (mr *MockAdminMockRecorder) Close() *gomock.Call { } // CreateTopic mocks base method. -func (m *MockAdmin) CreateTopic(detail *TopicDetail, validateOnly bool) error { +func (m *MockAdmin) CreateTopic(detail TopicDetail, validateOnly bool) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateTopic", detail, validateOnly) ret0, _ := ret[0].(error) @@ -103,18 +103,3 @@ func (mr *MockAdminMockRecorder) GetTopicsMeta(topics, ignoreTopicError interfac mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdmin)(nil).GetTopicsMeta), topics, ignoreTopicError) } - -// GetTopicsPartitionsNum mocks base method. -func (m *MockAdmin) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", topics) - ret0, _ := ret[0].(map[string]int32) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTopicsPartitionsNum indicates an expected call of GetTopicsPartitionsNum. -func (mr *MockAdminMockRecorder) GetTopicsPartitionsNum(topics interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockAdmin)(nil).GetTopicsPartitionsNum), topics) -} diff --git a/pkg/sink/kafka/admin_test.go b/pkg/sink/kafka/admin_test.go index 85848836d0..512849a59d 100644 --- a/pkg/sink/kafka/admin_test.go +++ b/pkg/sink/kafka/admin_test.go @@ -16,16 +16,73 @@ package kafka import ( "testing" + "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kerr" ) -func TestAdminCreateTopicNilDetailReturnsError(t *testing.T) { +func TestTopicDetailsFromMetadata(t *testing.T) { t.Parallel() - a := &admin{} + const topic = "topic" + testCases := []struct { + name string + metadata kadm.Metadata + ignoreTopicError bool + expected map[string]TopicDetail + expectedError error + }{ + { + name: "success", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Partitions: kadm.PartitionDetails{0: {}, 1: {}}}, + }}, + expected: map[string]TopicDetail{ + topic: {Name: topic, NumPartitions: 2}, + }, + }, + { + name: "ignore unknown topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, + }}, + ignoreTopicError: true, + expected: map[string]TopicDetail{}, + }, + { + name: "return unknown topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, + }}, + expectedError: errors.ErrKafkaTopicNotFound, + }, + { + name: "return missing topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{}}, + expectedError: errors.ErrKafkaTopicNotFound, + }, + { + name: "do not ignore authorization failure", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.TopicAuthorizationFailed}, + }}, + ignoreTopicError: true, + expectedError: errors.ErrKafkaAuthorizationFailed, + }, + } - err := a.CreateTopic(nil, false) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - require.Error(t, err) - require.Contains(t, err.Error(), "topic detail must not be nil") + actual, err := topicDetailsFromMetadata(tc.metadata, []string{topic}, tc.ignoreTopicError) + if tc.expectedError != nil { + require.ErrorIs(t, err, tc.expectedError) + return + } + require.NoError(t, err) + require.Equal(t, tc.expected, actual) + }) + } } diff --git a/pkg/sink/kafka/async_producer.go b/pkg/sink/kafka/async_producer.go index d674064ada..457c538377 100644 --- a/pkg/sink/kafka/async_producer.go +++ b/pkg/sink/kafka/async_producer.go @@ -29,8 +29,8 @@ import ( // AsyncProducer is the kafka async producer type AsyncProducer interface { - // Close shuts down the producer asynchronously and releases its Kafka client - // resources. It does not wait for buffered messages to be flushed. + // Close shuts down the producer and releases its Kafka client resources. + // Buffered messages fail instead of being flushed. Close() // AsyncSend is the input channel for the user to write messages to that they @@ -47,15 +47,16 @@ type asyncProducer struct { client *kgo.Client changefeedID commonType.ChangeFeedID - closed *atomic.Bool - errCh chan error + closeStarted *atomic.Bool + closed *atomic.Bool + errCh chan error } func newAsyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, - o *clientOptions, - hook kgo.Hook, + o *options, + hook *metricsHook, ) (*asyncProducer, error) { opts, err := newOptions(ctx, o, hook) if err != nil { @@ -70,24 +71,24 @@ func newAsyncProducer( return &asyncProducer{ client: client, changefeedID: changefeedID, + closeStarted: atomic.NewBool(false), closed: atomic.NewBool(false), errCh: make(chan error, 1), }, nil } func (p *asyncProducer) Close() { - if !p.closed.CompareAndSwap(false, true) { + if !p.closeStarted.CompareAndSwap(false, true) { return } - - go func() { - start := time.Now() - p.client.Close() - log.Info("Close kafka async producer success", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.Duration("duration", time.Since(start))) - }() + p.closed.Store(true) + + start := time.Now() + p.client.Close() + log.Info("Close kafka async producer success", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.Duration("duration", time.Since(start))) } func (p *asyncProducer) AsyncSend( @@ -111,18 +112,6 @@ func (p *asyncProducer) AsyncSend( changefeed = p.changefeedID.Name() ) - if legacyKafkaSinkFailpointEnabled(kafkaSinkAsyncSendErrorFailpoint) { - log.Info("KafkaSinkAsyncSendError error injected", - zap.String("keyspace", keyspace), zap.String("changefeed", changefeed)) - p.enqueueAsyncSendError( - keyspace, - changefeed, - message.LogInfo, - errors.New("kafka sink injected error"), - ) - return nil - } - failpoint.Inject("KafkaSinkAsyncSendError", func() { log.Info("KafkaSinkAsyncSendError error injected", zap.String("keyspace", keyspace), zap.String("changefeed", changefeed)) diff --git a/pkg/sink/kafka/async_producer_test.go b/pkg/sink/kafka/async_producer_test.go index e22191290a..1dc5bbf24a 100644 --- a/pkg/sink/kafka/async_producer_test.go +++ b/pkg/sink/kafka/async_producer_test.go @@ -16,12 +16,12 @@ package kafka import ( "context" "testing" - "time" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/atomic" ) @@ -47,41 +47,28 @@ func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { require.True(t, producer.closed.Load()) } -func TestAsyncSendLegacyFailpointAnnotatesDMLContext(t *testing.T) { - enableLegacyKafkaSinkFailpointForTest(t, kafkaSinkAsyncSendErrorFailpoint) +func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { + client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) + require.NoError(t, err) + callbackCalled := atomic.NewBool(false) producer := &asyncProducer{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-legacy-failpoint"), + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), + closeStarted: atomic.NewBool(false), closed: atomic.NewBool(false), errCh: make(chan error, 1), } - message := &codeccommon.Message{ - Key: []byte("key"), - Value: []byte("value"), - LogInfo: &codeccommon.MessageLogInfo{Rows: []codeccommon.RowLogInfo{ - { - Type: "insert", - Database: "db", - Table: "t", - StartTs: 1, - CommitTs: 2, - PrimaryKeys: []codeccommon.ColumnLogInfo{ - {Name: "id", Value: 1}, - }, - }, - }}, - } - - err := producer.AsyncSend(context.Background(), "topic", 0, message) + err = producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{ + Callback: func() { + callbackCalled.Store(true) + }, + }) require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - err = producer.AsyncRunCallback(ctx) + producer.Close() - require.ErrorContains(t, err, "kafka sink injected error") - require.ErrorContains(t, err, "keyspace=default") - require.ErrorContains(t, err, "changefeed=async-legacy-failpoint") - require.ErrorContains(t, err, "eventType=dml") - require.ErrorContains(t, err, `"Table":"t"`) + require.False(t, callbackCalled.Load()) + err = producer.AsyncRunCallback(context.Background()) + require.ErrorIs(t, err, kgo.ErrClientClosed) } diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index 87b1850bde..ded1b2fdc0 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -18,11 +18,9 @@ import ( "crypto/tls" "net/url" "strings" - "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/security" "github.com/twmb/franz-go/pkg/kgo" "github.com/twmb/franz-go/pkg/kversion" "github.com/twmb/franz-go/pkg/sasl" @@ -34,30 +32,9 @@ import ( "golang.org/x/oauth2/clientcredentials" ) -type clientOptions struct { - BrokerEndpoints []string - ClientID string - - Version string - IsAssignedVersion bool - - ProducerBatchMaxBytes int - MaxRetry int - Compression string - RequiredAcks int16 - - EnableTLS bool - Credential *security.Credential - InsecureSkipVerify bool - sasl *saslConfig - - DialTimeout time.Duration - RequestTimeout time.Duration -} - func newOptions( ctx context.Context, - o *clientOptions, + o *options, hook kgo.Hook, ) ([]kgo.Opt, error) { opts := []kgo.Opt{ @@ -65,7 +42,7 @@ func newOptions( kgo.SeedBrokers(o.BrokerEndpoints...), kgo.ClientID(o.ClientID), kgo.DialTimeout(o.DialTimeout), - kgo.RequestTimeoutOverhead(o.RequestTimeout), + kgo.RequestTimeoutOverhead(o.requestTimeout()), } if hook != nil { opts = append(opts, kgo.WithHooks(hook)) @@ -98,7 +75,7 @@ func newOptions( return opts, nil } -func newTLSConfig(o *clientOptions) (*tls.Config, error) { +func newTLSConfig(o *options) (*tls.Config, error) { tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, NextProtos: []string{"h2", "http/1.1"}, @@ -110,24 +87,14 @@ func newTLSConfig(o *clientOptions) (*tls.Config, error) { return nil, errors.Trace(err) } tlsConfig = credentialTlsConfig - if tlsConfig.MinVersion == 0 { - tlsConfig.MinVersion = tls.VersionTLS12 - } - if len(tlsConfig.NextProtos) == 0 { - tlsConfig.NextProtos = []string{"h2", "http/1.1"} - } } tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify return tlsConfig, nil } -func buildSaslMechanism(ctx context.Context, o *clientOptions) (sasl.Mechanism, error) { - if o.sasl == nil { - return nil, nil - } - - switch saslMechanism(strings.ToUpper(string(o.sasl.mechanism))) { +func buildSaslMechanism(ctx context.Context, o *options) (sasl.Mechanism, error) { + switch o.sasl.mechanism { case plainMechanism: auth := plain.Auth{ User: o.sasl.user, @@ -165,7 +132,7 @@ func buildSaslMechanism(ctx context.Context, o *clientOptions) (sasl.Mechanism, return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.sasl.mechanism) } -func newOauthTokenSource(ctx context.Context, o *clientOptions) (oauth2.TokenSource, error) { +func newOauthTokenSource(ctx context.Context, o *options) (oauth2.TokenSource, error) { endpointParams := url.Values{} if o.sasl.oauth2.grantType != "" { endpointParams.Set("grant_type", o.sasl.oauth2.grantType) @@ -190,7 +157,7 @@ func newOauthTokenSource(ctx context.Context, o *clientOptions) (oauth2.TokenSou } func newProducerOptions( - o *clientOptions, + o *options, ) []kgo.Opt { return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), @@ -198,28 +165,28 @@ func newProducerOptions( kgo.DisableIdempotentWrite(), kgo.MaxProduceRequestsInflightPerBroker(1), kgo.RecordRetries(o.MaxRetry), - kgo.ProducerBatchMaxBytes(int32(o.ProducerBatchMaxBytes)), - kgo.ProduceRequestTimeout(o.RequestTimeout), + kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), + kgo.ProduceRequestTimeout(o.requestTimeout()), kgo.ProducerLinger(0), newCompressionOption(o), } } -func newRequiredAcks(o *clientOptions) kgo.Acks { +func newRequiredAcks(o *options) kgo.Acks { switch o.RequiredAcks { - case -1: + case WaitForAll: return kgo.AllISRAcks() - case 1: + case WaitForLocal: return kgo.LeaderAck() - case 0: + case NoResponse: return kgo.NoAck() default: - log.Warn("unsupported required acks", zap.Int16("requiredAcks", o.RequiredAcks)) + log.Warn("unsupported required acks", zap.Int16("requiredAcks", int16(o.RequiredAcks))) return kgo.AllISRAcks() } } -func newCompressionOption(o *clientOptions) kgo.Opt { +func newCompressionOption(o *options) kgo.Opt { compression := strings.ToLower(strings.TrimSpace(o.Compression)) var codec kgo.CompressionCodec switch compression { diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go index cbeaf25fb8..52062cf38e 100644 --- a/pkg/sink/kafka/client_options_test.go +++ b/pkg/sink/kafka/client_options_test.go @@ -26,7 +26,7 @@ func TestNewRequiredAcks(t *testing.T) { testCases := []struct { name string - requiredAcks int16 + requiredAcks RequiredAcks expected kgo.Acks }{ {name: "all", requiredAcks: -1, expected: kgo.AllISRAcks()}, @@ -38,7 +38,7 @@ func TestNewRequiredAcks(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { t.Parallel() - require.Equal(t, tc.expected, newRequiredAcks(&clientOptions{RequiredAcks: tc.requiredAcks})) + require.Equal(t, tc.expected, newRequiredAcks(&options{RequiredAcks: tc.requiredAcks})) }) } } @@ -46,9 +46,10 @@ func TestNewRequiredAcks(t *testing.T) { func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { t.Parallel() - opts, err := newOptions(context.Background(), &clientOptions{ + opts, err := newOptions(context.Background(), &options{ Version: "invalid", IsAssignedVersion: true, + sasl: &saslConfig{}, }, nil) require.Nil(t, opts) require.ErrorContains(t, err, "invalid kafka version invalid") @@ -58,12 +59,14 @@ func TestNewProducerOptionsUsesProducerBatchMaxBytes(t *testing.T) { t.Parallel() const producerBatchMaxBytes = 1048588 - o := &clientOptions{ - BrokerEndpoints: []string{"127.0.0.1:9092"}, - ProducerBatchMaxBytes: producerBatchMaxBytes, - MaxRetry: defaultMaxRetry, - RequiredAcks: int16(WaitForAll), - RequestTimeout: defaultTimeout, + o := &options{ + BrokerEndpoints: []string{"127.0.0.1:9092"}, + MaxMessageBytes: producerBatchMaxBytes, + MaxRetry: defaultMaxRetry, + RequiredAcks: WaitForAll, + ReadTimeout: defaultTimeout, + WriteTimeout: defaultTimeout, + sasl: &saslConfig{}, } opts, err := newOptions(context.Background(), o, nil) @@ -95,7 +98,7 @@ func TestNewCompressionOptionMapsToProducerBatchCompression(t *testing.T) { client, err := kgo.NewClient( kgo.SeedBrokers("127.0.0.1:9092"), - newCompressionOption(&clientOptions{Compression: tc.compression}), + newCompressionOption(&options{Compression: tc.compression}), ) require.NoError(t, err) defer client.Close() @@ -108,7 +111,7 @@ func TestNewCompressionOptionMapsToProducerBatchCompression(t *testing.T) { func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { t.Parallel() - _, err := newOauthTokenSource(context.Background(), &clientOptions{ + _, err := newOauthTokenSource(context.Background(), &options{ sasl: &saslConfig{ oauth2: oauth2Config{ clientID: "client-id", diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 5042b4af64..d092a2ecb2 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -32,9 +32,7 @@ type Factory interface { type factory struct { changefeedID common.ChangeFeedID - clientOption *clientOptions - - metricsHook *metricsHook + options options } // NewFactory constructs a Factory. @@ -43,7 +41,7 @@ func NewFactory( o *options, changefeedID common.ChangeFeedID, ) (Factory, error) { - admin, err := newAdmin(ctx, changefeedID, newClientOption(o), nil) + admin, err := newAdmin(ctx, changefeedID, o, nil) if err != nil { return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } @@ -55,13 +53,12 @@ func NewFactory( return &factory{ changefeedID: changefeedID, - clientOption: newClientOption(o), - metricsHook: newKafkaMetricsHook(changefeedID), + options: *o, }, nil } func (f *factory) Admin(ctx context.Context) (Admin, error) { - admin, err := newAdmin(ctx, f.changefeedID, f.clientOption, f.metricsHook) + admin, err := newAdmin(ctx, f.changefeedID, &f.options, nil) if err != nil { return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } @@ -69,40 +66,21 @@ func (f *factory) Admin(ctx context.Context) (Admin, error) { } func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := newSyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) + hook := newKafkaMetricsHook(f.changefeedID) + producer, err := newSyncProducer(ctx, f.changefeedID, &f.options, hook) if err != nil { + CleanupMetrics(f.changefeedID) return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } return producer, nil } func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := newAsyncProducer(ctx, f.changefeedID, f.clientOption, f.metricsHook) + hook := newKafkaMetricsHook(f.changefeedID) + producer, err := newAsyncProducer(ctx, f.changefeedID, &f.options, hook) if err != nil { + CleanupMetrics(f.changefeedID) return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } return producer, nil } - -func newClientOption(o *options) *clientOptions { - return &clientOptions{ - BrokerEndpoints: o.BrokerEndpoints, - ClientID: o.ClientID, - - Version: o.Version, - IsAssignedVersion: o.IsAssignedVersion, - - ProducerBatchMaxBytes: o.MaxMessageBytes, - MaxRetry: o.MaxRetry, - Compression: o.Compression, - RequiredAcks: int16(o.RequiredAcks), - - EnableTLS: o.EnableTLS, - Credential: o.Credential, - InsecureSkipVerify: o.InsecureSkipVerify, - sasl: o.sasl, - - DialTimeout: o.DialTimeout, - RequestTimeout: max(o.ReadTimeout, o.WriteTimeout), - } -} diff --git a/pkg/sink/kafka/factory_test.go b/pkg/sink/kafka/factory_test.go index b4e3c6929c..0928fbd00a 100644 --- a/pkg/sink/kafka/factory_test.go +++ b/pkg/sink/kafka/factory_test.go @@ -23,42 +23,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewClientOptionMapsRequiredAcks(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - requiredAcks RequiredAcks - }{ - {name: "wait for all", requiredAcks: WaitForAll}, - {name: "wait for local", requiredAcks: WaitForLocal}, - {name: "no response", requiredAcks: NoResponse}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - options := NewOptions() - options.RequiredAcks = tc.requiredAcks - - kafkaOptions := newClientOption(options) - require.Equal(t, int16(tc.requiredAcks), kafkaOptions.RequiredAcks) - }) - } -} - -func TestNewClientOptionMapsMaxRetry(t *testing.T) { - t.Parallel() - - options := NewOptions() - options.MaxRetry = 7 - - kafkaOptions := newClientOption(options) - require.Equal(t, 7, kafkaOptions.MaxRetry) -} - -func TestNewClientOptionDerivesRequestTimeout(t *testing.T) { +func TestOptionsDerivesRequestTimeout(t *testing.T) { t.Parallel() testCases := []struct { @@ -89,8 +54,7 @@ func TestNewClientOptionDerivesRequestTimeout(t *testing.T) { o.ReadTimeout = tc.readTimeout o.WriteTimeout = tc.writeTimeout - clientOption := newClientOption(o) - require.Equal(t, tc.expectedRequestTimeout, clientOption.RequestTimeout) + require.Equal(t, tc.expectedRequestTimeout, o.requestTimeout()) }) } } @@ -116,9 +80,10 @@ func TestFactoryComponentCreationReturnsKafkaSinkError(t *testing.T) { factory := &factory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), - clientOption: &clientOptions{ + options: options{ Version: "invalid", IsAssignedVersion: true, + sasl: &saslConfig{}, }, } diff --git a/pkg/sink/kafka/failpoint.go b/pkg/sink/kafka/failpoint.go deleted file mode 100644 index faca9a64b3..0000000000 --- a/pkg/sink/kafka/failpoint.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "os" - "sync/atomic" - - "github.com/pingcap/failpoint" -) - -const ( - legacyKafkaSinkFailpointPrefix = "github.com/pingcap/ticdc/pkg/sink/kafka/" - - kafkaSinkAsyncSendErrorFailpoint = "KafkaSinkAsyncSendError" - kafkaSinkSyncSendMessageErrorFailpoint = "KafkaSinkSyncSendMessageError" - kafkaSinkSyncSendMessagesErrorFailpoint = "KafkaSinkSyncSendMessagesError" -) - -var legacyKafkaSinkFailpointsRuntimeEnabled atomic.Bool - -func init() { - legacyKafkaSinkFailpointsRuntimeEnabled.Store( - os.Getenv("GO_FAILPOINTS") != "" || os.Getenv("GO_FAILPOINTS_HTTP") != "", - ) -} - -func legacyKafkaSinkFailpointEnabled(name string) bool { - if !legacyKafkaSinkFailpointsRuntimeEnabled.Load() { - return false - } - _, err := failpoint.Eval(legacyKafkaSinkFailpointPrefix + name) - return err == nil -} diff --git a/pkg/sink/kafka/failpoint_test.go b/pkg/sink/kafka/failpoint_test.go deleted file mode 100644 index 7de5934351..0000000000 --- a/pkg/sink/kafka/failpoint_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "testing" - - "github.com/pingcap/failpoint" - "github.com/stretchr/testify/require" -) - -func enableLegacyKafkaSinkFailpointForTest(t *testing.T, name string) { - t.Helper() - - previous := legacyKafkaSinkFailpointsRuntimeEnabled.Load() - legacyKafkaSinkFailpointsRuntimeEnabled.Store(true) - - failpointPath := legacyKafkaSinkFailpointPrefix + name - require.NoError(t, failpoint.Enable(failpointPath, "return(true)")) - t.Cleanup(func() { - _ = failpoint.Disable(failpointPath) - legacyKafkaSinkFailpointsRuntimeEnabled.Store(previous) - }) -} - -func TestLegacyKafkaSinkFailpointEnabled(t *testing.T) { - enableLegacyKafkaSinkFailpointForTest(t, kafkaSinkSyncSendMessageErrorFailpoint) - - require.True(t, legacyKafkaSinkFailpointEnabled(kafkaSinkSyncSendMessageErrorFailpoint)) -} diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/gssapi.go index b2b72c386b..7af48d5137 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -34,21 +34,10 @@ import ( ) const ( - tokIDKrbAPReq = 256 - gssAPIGeneric = 0x60 - gssAPIInitial = 1 - gssAPIVerify = 2 - gssAPIFinished = 3 + tokIDKrbAPReq = 256 + gssAPIGeneric = 0x60 ) -type kerberosClient interface { - Login() error - GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) - Domain() string - CName() types.PrincipalName - Destroy() -} - type gssapiMechanism struct { config gssapiConfig } @@ -78,77 +67,48 @@ func (m *gssapiMechanism) Authenticate( return nil, nil, errors.Trace(err) } - session := &gssapiSession{ - client: client, - ticket: ticket, - encKey: encKey, - step: gssAPIInitial, + token, err := newKrb5Token(client.Domain(), client.CName(), ticket, encKey) + if err != nil { + client.Destroy() + return nil, nil, errors.Trace(err) } - firstMessage, err := session.nextMessage(nil) + firstMessage, err := appendGSSAPIHeader(token) if err != nil { client.Destroy() return nil, nil, errors.Trace(err) } - return session, firstMessage, nil + return &gssapiSession{client: client, encKey: encKey}, firstMessage, nil } type gssapiSession struct { - client kerberosClient - ticket messages.Ticket + client *krb5Client encKey types.EncryptionKey - step int } func (s *gssapiSession) Challenge(challenge []byte) (bool, []byte, error) { defer s.client.Destroy() - switch s.step { - case gssAPIVerify: - msg, err := s.nextMessage(challenge) + wrapTokenReq := gssapi.WrapToken{} + if err := wrapTokenReq.Unmarshal(challenge, true); err != nil { + return false, nil, errors.Trace(err) + } + isValid, err := wrapTokenReq.Verify(s.encKey, keyusage.GSSAPI_ACCEPTOR_SEAL) + if !isValid { if err != nil { return false, nil, errors.Trace(err) } - // Return a final payload while marking done=true. - // The Kafka client writes this message and finishes the auth flow. - return true, msg, nil - case gssAPIFinished: - return true, nil, nil - default: - return false, nil, errors.New("invalid gssapi session state") + return false, nil, errors.New("invalid gssapi wrap token") } -} -func (s *gssapiSession) nextMessage(challenge []byte) ([]byte, error) { - switch s.step { - case gssAPIInitial: - token, err := newKrb5Token(s.client.Domain(), s.client.CName(), s.ticket, s.encKey) - if err != nil { - return nil, errors.Trace(err) - } - s.step = gssAPIVerify - return appendGSSAPIHeader(token) - case gssAPIVerify: - wrapTokenReq := gssapi.WrapToken{} - if err := wrapTokenReq.Unmarshal(challenge, true); err != nil { - return nil, errors.Trace(err) - } - isValid, err := wrapTokenReq.Verify(s.encKey, keyusage.GSSAPI_ACCEPTOR_SEAL) - if !isValid { - if err != nil { - return nil, errors.Trace(err) - } - return nil, errors.New("invalid gssapi wrap token") - } - - wrapTokenResp, err := gssapi.NewInitiatorWrapToken(wrapTokenReq.Payload, s.encKey) - if err != nil { - return nil, errors.Trace(err) - } - s.step = gssAPIFinished - return wrapTokenResp.Marshal() - default: - return nil, errors.New("invalid gssapi session state") + wrapTokenResp, err := gssapi.NewInitiatorWrapToken(wrapTokenReq.Payload, s.encKey) + if err != nil { + return false, nil, errors.Trace(err) } + msg, err := wrapTokenResp.Marshal() + if err != nil { + return false, nil, errors.Trace(err) + } + return true, msg, nil } func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { @@ -200,7 +160,7 @@ func (c *krb5Client) CName() types.PrincipalName { return c.Credentials.CName() } -func newKerberosClient(g gssapiConfig) (kerberosClient, error) { +func newKerberosClient(g gssapiConfig) (*krb5Client, error) { cfg, err := config.Load(g.kerberosConfigPath) if err != nil { return nil, errors.Trace(err) @@ -256,13 +216,9 @@ func newKrb5Token( func newAuthenticatorChecksum() []byte { sum := make([]byte, 24) - flags := []int{gssapi.ContextFlagInteg, gssapi.ContextFlagConf} binary.LittleEndian.PutUint32(sum[:4], 16) - for _, flag := range flags { - current := binary.LittleEndian.Uint32(sum[20:24]) - current |= uint32(flag) - binary.LittleEndian.PutUint32(sum[20:24], current) - } + flags := uint32(gssapi.ContextFlagInteg | gssapi.ContextFlagConf) + binary.LittleEndian.PutUint32(sum[20:24], flags) return sum } diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index d3f88055e2..029d7643e2 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -28,67 +28,69 @@ var ( Help: "The current number of in-flight requests" + " awaiting a response for all brokers.", }, []string{"namespace", "changefeed", "broker"}) - // OutgoingByteRateGauge for outgoing events. - // Meter mark for each request's size in bytes. - OutgoingByteRateGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + outgoingBytesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_outgoing_byte_rate", - Help: "Bytes/second written off all brokers.", + Name: "kafka_producer_outgoing_bytes_total", + Help: "Total bytes written to Kafka brokers, excluding TLS overhead.", }, []string{"namespace", "changefeed", "broker"}) - // RequestRateGauge Meter mark by 1 for each request. - RequestRateGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + requestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_request_rate", - Help: "Requests/second sent to all brokers.", - }, []string{"namespace", "changefeed", "broker"}) - // RequestLatencyGauge Histogram update by `requestLatency`. - RequestLatencyGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + Name: "kafka_producer_requests_total", + Help: "Total Kafka requests by broker and write result.", + }, []string{"namespace", "changefeed", "broker", "result"}) + responsesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_request_latency", - Help: "The request latency for all brokers.", - }, []string{"namespace", "changefeed", "broker", "type"}) - // Histogram update by `compression-ratio`. - compressionRatioGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + Name: "kafka_producer_responses_total", + Help: "Total Kafka responses by broker and read result.", + }, []string{"namespace", "changefeed", "broker", "result"}) + requestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_compression_ratio", - Help: "The compression ratio times 100 of record batches for all topics.", - }, []string{"namespace", "changefeed", "type"}) - // updated by `records-per-request`. - recordsPerRequestGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + Name: "kafka_producer_request_duration_seconds", + Help: "Kafka request end-to-end duration in seconds.", + Buckets: prometheus.DefBuckets, + }, []string{"namespace", "changefeed", "broker"}) + recordsPerBatch = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_records_per_request", - Help: "The number of records per request for all topics.", - }, []string{"namespace", "changefeed", "type"}) - - // Meter mark by 1 once a response received. - responseRateGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ + Name: "kafka_producer_records_per_batch", + Help: "Number of records in each successfully written topic-partition batch.", + Buckets: prometheus.ExponentialBuckets(1, 2, 15), + }, []string{"namespace", "changefeed"}) + uncompressedBytesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_response_rate", - Help: "Responses/second received from all brokers.", - }, []string{"namespace", "changefeed", "broker"}) + Name: "kafka_producer_uncompressed_bytes_total", + Help: "Total serialized record bytes before compression in successfully written batches.", + }, []string{"namespace", "changefeed"}) + compressedBytesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_producer_compressed_bytes_total", + Help: "Total serialized record bytes after compression in successfully written batches.", + }, []string{"namespace", "changefeed"}) ) // InitMetrics registers all metrics in this file. func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(compressionRatioGauge) - registry.MustRegister(recordsPerRequestGauge) - registry.MustRegister(OutgoingByteRateGauge) - registry.MustRegister(RequestRateGauge) - registry.MustRegister(RequestLatencyGauge) + registry.MustRegister(outgoingBytesTotal) + registry.MustRegister(requestsTotal) + registry.MustRegister(responsesTotal) + registry.MustRegister(requestDuration) + registry.MustRegister(recordsPerBatch) + registry.MustRegister(uncompressedBytesTotal) + registry.MustRegister(compressedBytesTotal) registry.MustRegister(requestsInFlightGauge) - registry.MustRegister(responseRateGauge) claimcheck.InitMetrics(registry) codec.InitMetrics(registry) diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/metrics_hook.go index 5e07c05757..5adb06305d 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/metrics_hook.go @@ -15,6 +15,7 @@ package kafka import ( "strconv" + "sync" "time" "github.com/pingcap/ticdc/pkg/common" @@ -29,33 +30,79 @@ import ( type metricsHook struct { keyspace string changefeed string + + brokers sync.Map + + recordsPerBatch prometheus.Observer + uncompressedBytesTotal prometheus.Counter + compressedBytesTotal prometheus.Counter +} + +type brokerMetrics struct { + outgoingBytesTotal prometheus.Counter + requestsSuccess prometheus.Counter + requestsWriteError prometheus.Counter + responsesSuccess prometheus.Counter + responsesReadError prometheus.Counter + requestsInFlight prometheus.Gauge + requestDuration prometheus.Observer } const ( - metricAvg = "avg" - metricP99 = "p99" + metricResultSuccess = "success" + metricResultWriteError = "write_error" + metricResultReadError = "read_error" ) func newKafkaMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { + keyspace := changefeedID.Keyspace() + changefeed := changefeedID.Name() return &metricsHook{ - keyspace: changefeedID.Keyspace(), - changefeed: changefeedID.Name(), + keyspace: keyspace, + changefeed: changefeed, + recordsPerBatch: recordsPerBatch.WithLabelValues(keyspace, changefeed), + uncompressedBytesTotal: uncompressedBytesTotal.WithLabelValues(keyspace, changefeed), + compressedBytesTotal: compressedBytesTotal.WithLabelValues(keyspace, changefeed), } } -// CleanupMetrics removes Kafka sink metric series for a changefeed when its sink exits. +func (h *metricsHook) broker(nodeID int32) *brokerMetrics { + if cached, ok := h.brokers.Load(nodeID); ok { + return cached.(*brokerMetrics) + } + + brokerID := strconv.Itoa(int(nodeID)) + metrics := &brokerMetrics{ + outgoingBytesTotal: outgoingBytesTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID), + requestsSuccess: requestsTotal.WithLabelValues( + h.keyspace, h.changefeed, brokerID, metricResultSuccess), + requestsWriteError: requestsTotal.WithLabelValues( + h.keyspace, h.changefeed, brokerID, metricResultWriteError), + responsesSuccess: responsesTotal.WithLabelValues( + h.keyspace, h.changefeed, brokerID, metricResultSuccess), + responsesReadError: responsesTotal.WithLabelValues( + h.keyspace, h.changefeed, brokerID, metricResultReadError), + requestsInFlight: requestsInFlightGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID), + requestDuration: requestDuration.WithLabelValues(h.keyspace, h.changefeed, brokerID), + } + actual, _ := h.brokers.LoadOrStore(nodeID, metrics) + return actual.(*brokerMetrics) +} + +// CleanupMetrics removes Kafka sink metric series after all of its clients are closed. func CleanupMetrics(changefeedID common.ChangeFeedID) { labels := prometheus.Labels{ "namespace": changefeedID.Keyspace(), "changefeed": changefeedID.Name(), } - OutgoingByteRateGauge.DeletePartialMatch(labels) - RequestRateGauge.DeletePartialMatch(labels) - responseRateGauge.DeletePartialMatch(labels) + outgoingBytesTotal.DeletePartialMatch(labels) + requestsTotal.DeletePartialMatch(labels) + responsesTotal.DeletePartialMatch(labels) requestsInFlightGauge.DeletePartialMatch(labels) - RequestLatencyGauge.DeletePartialMatch(labels) - compressionRatioGauge.DeletePartialMatch(labels) - recordsPerRequestGauge.DeletePartialMatch(labels) + requestDuration.DeletePartialMatch(labels) + recordsPerBatch.DeletePartialMatch(labels) + uncompressedBytesTotal.DeletePartialMatch(labels) + compressedBytesTotal.DeletePartialMatch(labels) } func (h *metricsHook) OnBrokerWrite( @@ -69,14 +116,16 @@ func (h *metricsHook) OnBrokerWrite( if meta.NodeID < 0 { return } - brokerID := strconv.Itoa(int(meta.NodeID)) + metrics := h.broker(meta.NodeID) if bytesWritten > 0 { - OutgoingByteRateGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(float64(bytesWritten)) + metrics.outgoingBytesTotal.Add(float64(bytesWritten)) } - RequestRateGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) - if err == nil { - requestsInFlightGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + if err != nil { + metrics.requestsWriteError.Inc() + } else { + metrics.requestsSuccess.Inc() + metrics.requestsInFlight.Inc() } } @@ -88,18 +137,20 @@ func (h *metricsHook) OnBrokerE2E( if meta.NodeID < 0 { return } - brokerID := strconv.Itoa(int(meta.NodeID)) + metrics := h.broker(meta.NodeID) if e2e.WriteErr == nil { - requestsInFlightGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(-1) - } - if e2e.BytesRead > 0 && e2e.ReadErr == nil { - responseRateGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID).Add(1) + metrics.requestsInFlight.Dec() + if e2e.BytesRead > 0 || e2e.ReadErr != nil { + if e2e.ReadErr != nil { + metrics.responsesReadError.Inc() + } else { + metrics.responsesSuccess.Inc() + } + } } if e2e.Err() == nil { - latencyMs := float64(e2e.DurationE2E().Microseconds()) / 1000 - RequestLatencyGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricAvg).Set(latencyMs) - RequestLatencyGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricP99).Set(latencyMs) + metrics.requestDuration.Observe(e2e.DurationE2E().Seconds()) } } @@ -110,13 +161,12 @@ func (h *metricsHook) OnProduceBatchWritten( m kgo.ProduceBatchMetrics, ) { if m.NumRecords > 0 { - records := float64(m.NumRecords) - recordsPerRequestGauge.WithLabelValues(h.keyspace, h.changefeed, metricAvg).Set(records) - recordsPerRequestGauge.WithLabelValues(h.keyspace, h.changefeed, metricP99).Set(records) + h.recordsPerBatch.Observe(float64(m.NumRecords)) + } + if m.UncompressedBytes > 0 { + h.uncompressedBytesTotal.Add(float64(m.UncompressedBytes)) } - if m.UncompressedBytes > 0 && m.CompressedBytes > 0 { - ratio := float64(m.UncompressedBytes) / float64(m.CompressedBytes) * 100 - compressionRatioGauge.WithLabelValues(h.keyspace, h.changefeed, metricAvg).Set(ratio) - compressionRatioGauge.WithLabelValues(h.keyspace, h.changefeed, metricP99).Set(ratio) + if m.CompressedBytes > 0 { + h.compressedBytesTotal.Add(float64(m.CompressedBytes)) } } diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 5a89a335cb..53334c7919 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -172,6 +172,10 @@ type options struct { ReadTimeout time.Duration } +func (o *options) requestTimeout() time.Duration { + return max(o.ReadTimeout, o.WriteTimeout) +} + // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ @@ -290,36 +294,24 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, } if urlParameter.DialTimeout != nil && *urlParameter.DialTimeout != "" { - a, err := time.ParseDuration(*urlParameter.DialTimeout) + o.DialTimeout, err = parseTimeout(*urlParameter.DialTimeout) if err != nil { return err } - if a <= 0 { - a = defaultTimeout - } - o.DialTimeout = a } if urlParameter.WriteTimeout != nil && *urlParameter.WriteTimeout != "" { - a, err := time.ParseDuration(*urlParameter.WriteTimeout) + o.WriteTimeout, err = parseTimeout(*urlParameter.WriteTimeout) if err != nil { return err } - if a <= 0 { - a = defaultTimeout - } - o.WriteTimeout = a } if urlParameter.ReadTimeout != nil && *urlParameter.ReadTimeout != "" { - a, err := time.ParseDuration(*urlParameter.ReadTimeout) + o.ReadTimeout, err = parseTimeout(*urlParameter.ReadTimeout) if err != nil { return err } - if a <= 0 { - a = defaultTimeout - } - o.ReadTimeout = a } if urlParameter.RequiredAcks != nil { @@ -343,6 +335,17 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, return nil } +func parseTimeout(value string) (time.Duration, error) { + timeout, err := time.ParseDuration(value) + if err != nil { + return 0, err + } + if timeout <= 0 { + return defaultTimeout, nil + } + return timeout, nil +} + func mergeConfig( sinkConfig *config.SinkConfig, urlParameters *urlConfig, diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index b98d3c59db..81d4db8f25 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -68,8 +68,6 @@ func newKafkaAdminFixture(t *testing.T) *kafkaAdminFixture { fixture.admin.EXPECT().Close().AnyTimes() fixture.admin.EXPECT().GetTopicsMeta(gomock.Any(), gomock.Any()). DoAndReturn(fixture.getTopicsMeta).AnyTimes() - fixture.admin.EXPECT().GetTopicsPartitionsNum(gomock.Any()). - DoAndReturn(fixture.getTopicsPartitionsNum).AnyTimes() fixture.admin.EXPECT().GetBrokerConfig(gomock.Any()). DoAndReturn(fixture.getBrokerConfig).AnyTimes() fixture.admin.EXPECT().GetTopicConfig(gomock.Any(), gomock.Any()). @@ -96,18 +94,6 @@ func (f *kafkaAdminFixture) getTopicsMeta( return result, nil } -func (f *kafkaAdminFixture) getTopicsPartitionsNum( - topics []string, -) (map[string]int32, error) { - result := make(map[string]int32, len(topics)) - for _, topic := range topics { - if detail, ok := f.topics[topic]; ok { - result[topic] = detail.NumPartitions - } - } - return result, nil -} - func (f *kafkaAdminFixture) getBrokerConfig(configName string) (string, error) { if value, ok := f.brokerConfig[configName]; ok { return value, nil @@ -128,7 +114,7 @@ func (f *kafkaAdminFixture) getTopicConfig(topicName string, configName string) "cannot find the `%s` from the topic's configuration", configName) } -func (f *kafkaAdminFixture) createTopic(detail *TopicDetail, _ bool) error { +func (f *kafkaAdminFixture) createTopic(detail TopicDetail, _ bool) error { if detail.ReplicationFactor > mockClusterReplicationFactor { return errors.New("invalid replication factor") } @@ -136,7 +122,7 @@ func (f *kafkaAdminFixture) createTopic(detail *TopicDetail, _ bool) error { detail.ReplicationFactor != mockClusterReplicationFactor { return errors.New("policy violation") } - f.topics[detail.Name] = *detail + f.topics[detail.Name] = detail return nil } @@ -493,7 +479,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t adminFixture := newKafkaAdminFixture(t) admin := adminFixture.admin - detail := &TopicDetail{ + detail := TopicDetail{ Name: topicName, NumPartitions: 3, } @@ -524,7 +510,6 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t min(configuredMaxMessageBytes, expectedProducerLimit), options.MaxBatchedBytes, ) - require.Equal(t, expectedProducerLimit, newClientOption(options).ProducerBatchMaxBytes) }) } } @@ -759,7 +744,6 @@ func TestConfigurationCombinations(t *testing.T) { min(configuredMaxMessageBytes, sourceMaxMessageBytes), options.MaxBatchedBytes, ) - require.Equal(t, sourceMaxMessageBytes, newClientOption(options).ProducerBatchMaxBytes) admin.Close() }) diff --git a/pkg/sink/kafka/sasl_config.go b/pkg/sink/kafka/sasl_config.go index a6fd90577f..b75857b9df 100644 --- a/pkg/sink/kafka/sasl_config.go +++ b/pkg/sink/kafka/sasl_config.go @@ -24,8 +24,6 @@ type saslMechanism string // The mechanisms we currently support. const ( - // unknownMechanism means the SASL mechanism is unknown. - unknownMechanism saslMechanism = "" // plainMechanism means the SASL mechanism is plain. plainMechanism saslMechanism = "PLAIN" // scram256Mechanism means the SASL mechanism is SCRAM-SHA-256. @@ -52,7 +50,7 @@ func saslMechanismFromString(s string) (saslMechanism, error) { case "oauthbearer": return oauthMechanism, nil default: - return unknownMechanism, errors.Errorf("unknown %s SASL mechanism", s) + return "", errors.Errorf("unknown %s SASL mechanism", s) } } @@ -94,8 +92,6 @@ func (o *oauth2Config) validate() error { type gssapiAuthType int const ( - // unknownAuth means the auth type is unknown. - unknownAuth gssapiAuthType = 0 // userAuth means the auth type is user. userAuth gssapiAuthType = 1 // keyTabAuth means the auth type is keytab. @@ -110,7 +106,7 @@ func gssapiAuthTypeFromString(s string) (gssapiAuthType, error) { case "keytab": return keyTabAuth, nil default: - return unknownAuth, errors.Errorf("unknown %s auth type", s) + return 0, errors.Errorf("unknown %s auth type", s) } } diff --git a/pkg/sink/kafka/sasl_test.go b/pkg/sink/kafka/sasl_test.go index 63a0e914eb..152199080c 100644 --- a/pkg/sink/kafka/sasl_test.go +++ b/pkg/sink/kafka/sasl_test.go @@ -23,7 +23,7 @@ import ( func TestBuildSaslMechanismGSSAPIUserAuth(t *testing.T) { t.Parallel() - o := &clientOptions{ + o := &options{ sasl: &saslConfig{ mechanism: gssapiMechanismName, gssapi: gssapiConfig{ @@ -45,7 +45,7 @@ func TestBuildSaslMechanismGSSAPIUserAuth(t *testing.T) { func TestBuildSaslMechanismGSSAPIKeytabAuth(t *testing.T) { t.Parallel() - o := &clientOptions{ + o := &options{ sasl: &saslConfig{ mechanism: gssapiMechanismName, gssapi: gssapiConfig{ diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index b41270fc20..09e6225153 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -55,8 +55,8 @@ type syncProducer struct { func newSyncProducer( ctx context.Context, changefeedID commonType.ChangeFeedID, - o *clientOptions, - hook kgo.Hook, + o *options, + hook *metricsHook, ) (*syncProducer, error) { opts, err := newOptions(ctx, o, hook) if err != nil { @@ -73,7 +73,7 @@ func newSyncProducer( id: changefeedID, client: client, closed: atomic.NewBool(false), - timeout: o.RequestTimeout, + timeout: o.requestTimeout(), }, nil } @@ -93,10 +93,6 @@ func (p *syncProducer) SendMessage(topic string, partitionNum int32, message *co } err := p.client.ProduceSync(ctx, record).FirstErr() - if legacyKafkaSinkFailpointEnabled(kafkaSinkSyncSendMessageErrorFailpoint) { - err = errors.New("kafka sink sync send message injected error") - } - failpoint.Inject("KafkaSinkSyncSendMessageError", func() { err = errors.New("kafka sink sync send message injected error") }) @@ -132,10 +128,6 @@ func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *c err := p.client.ProduceSync(ctx, records...).FirstErr() - if legacyKafkaSinkFailpointEnabled(kafkaSinkSyncSendMessagesErrorFailpoint) { - err = errors.New("kafka sink sync send messages injected error") - } - failpoint.Inject("KafkaSinkSyncSendMessagesError", func() { err = errors.New("kafka sink sync send messages injected error") }) From f0b31922f3a9f3bc447ad90a3c70912ee28f0bfb Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 28 Jul 2026 18:06:17 +0800 Subject: [PATCH 36/61] simplify the code further --- downstreamadapter/sink/kafka/sink_test.go | 22 +----- .../topicmanager/kafka_topic_manager_test.go | 77 ++++++------------- go.mod | 1 + go.sum | 2 + pkg/sink/kafka/admin.go | 8 +- pkg/sink/kafka/async_producer.go | 11 +-- pkg/sink/kafka/async_producer_test.go | 10 +-- pkg/sink/kafka/client_options.go | 4 +- pkg/sink/kafka/factory_test.go | 39 ++-------- pkg/sink/kafka/gssapi.go | 21 ++--- pkg/sink/kafka/sasl_test.go | 66 +++++++--------- pkg/sink/kafka/sync_producer.go | 5 +- pkg/sink/kafka/sync_producer_test.go | 4 +- tests/utils/kafka_topic/main.go | 50 ++++++++---- 14 files changed, 116 insertions(+), 204 deletions(-) diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index fbef63e55c..2d98ae09a4 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -22,7 +22,6 @@ import ( "testing" "time" - "github.com/IBM/sarama" "github.com/golang/mock/gomock" "github.com/pingcap/ticdc/downstreamadapter/sink/columnselector" "github.com/pingcap/ticdc/downstreamadapter/sink/eventrouter" @@ -36,6 +35,7 @@ import ( codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" "go.uber.org/atomic" ) @@ -88,22 +88,8 @@ func TestSinkWorkersReturnContextError(t *testing.T) { } func TestVerifyInvalidConfig(t *testing.T) { - broker := sarama.NewMockBroker(t, 1) - defer broker.Close() - broker.SetHandlerByMap(map[string]sarama.MockResponse{ - "ApiVersionsRequest": sarama.NewMockApiVersionsResponse(t).SetApiKeys( - []sarama.ApiVersionsResponseKey{ - {ApiKey: 0}, - {ApiKey: 1}, - {ApiKey: 2}, - {ApiKey: 3, MaxVersion: 9}, - }), - "MetadataRequest": sarama.NewMockMetadataResponse(t). - SetController(broker.BrokerID()). - SetBroker(broker.Addr(), broker.BrokerID()). - SetLeader(kafkaSinkTestTopic, 0, broker.BrokerID()), - "DescribeConfigsRequest": sarama.NewMockDescribeConfigsResponse(t), - }) + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, kafkaSinkTestTopic)) + defer cluster.Close() schemaRegistry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "invalid response", http.StatusInternalServerError) @@ -115,7 +101,7 @@ func TestVerifyInvalidConfig(t *testing.T) { Protocol: &avroProtocol, SchemaRegistry: &schemaRegistry.URL, } - sinkURI, err := url.Parse("kafka://" + broker.Addr() + "/" + kafkaSinkTestTopic + + sinkURI, err := url.Parse("kafka://" + cluster.ListenAddrs()[0] + "/" + kafkaSinkTestTopic + "?required-acks=1&kafka-version=2.4.0") require.NoError(t, err) diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index 57863f11ea..57784994f5 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -27,50 +27,6 @@ import ( const kafkaTopicManagerTestTopic = "mock_topic" -type mockAdminWithDeniedDescribe struct { - *kafka.MockAdmin - createTopicCalled bool - describeCount int -} - -func (m *mockAdminWithDeniedDescribe) GetTopicsMeta( - topics []string, - ignoreTopicError bool, -) (map[string]kafka.TopicDetail, error) { - m.describeCount++ - return nil, errors.WrapError(errors.ErrKafkaAdminAPI, kerr.TopicAuthorizationFailed, "describe-topic", topics[0]) -} - -func (m *mockAdminWithDeniedDescribe) CreateTopic( - detail kafka.TopicDetail, - validateOnly bool, -) error { - m.createTopicCalled = true - return nil -} - -type mockAdminWithDeniedCreate struct { - *kafka.MockAdmin - createTopicCalled bool - describeCount int -} - -func (m *mockAdminWithDeniedCreate) GetTopicsMeta( - topics []string, - ignoreTopicError bool, -) (map[string]kafka.TopicDetail, error) { - m.describeCount++ - return map[string]kafka.TopicDetail{}, nil -} - -func (m *mockAdminWithDeniedCreate) CreateTopic( - detail kafka.TopicDetail, - validateOnly bool, -) error { - m.createTopicCalled = true - return errors.WrapError(errors.ErrKafkaAdminAPI, kerr.ClusterAuthorizationFailed, "create-topic", detail.Name) -} - func TestCreateTopic(t *testing.T) { t.Parallel() @@ -260,9 +216,16 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - admin := &mockAdminWithDeniedDescribe{ - MockAdmin: kafka.NewMockAdmin(ctrl), - } + admin := kafka.NewMockAdmin(ctrl) + admin.EXPECT().GetTopicsMeta([]string{"precreated-topic"}, true).Return( + nil, + errors.WrapError( + errors.ErrKafkaAdminAPI, + kerr.TopicAuthorizationFailed, + "describe-topic", + "precreated-topic", + ), + ) cfg := &kafka.AutoCreateTopicConfig{ AutoCreate: true, PartitionNum: 2, @@ -277,8 +240,6 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { partitionNum, err := manager.CreateTopicAndWaitUntilVisible(ctx, "precreated-topic") require.NoError(t, err) require.Equal(t, int32(2), partitionNum) - require.False(t, admin.createTopicCalled) - require.Equal(t, 1, admin.describeCount) partitions, ok := manager.topics.Load("precreated-topic") require.True(t, ok) @@ -289,9 +250,19 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - admin := &mockAdminWithDeniedCreate{ - MockAdmin: kafka.NewMockAdmin(ctrl), - } + admin := kafka.NewMockAdmin(ctrl) + gomock.InOrder( + admin.EXPECT().GetTopicsMeta([]string{"precreated-topic"}, true).Return( + map[string]kafka.TopicDetail{}, nil), + admin.EXPECT().CreateTopic(gomock.Any(), false).Return( + errors.WrapError( + errors.ErrKafkaAdminAPI, + kerr.ClusterAuthorizationFailed, + "create-topic", + "precreated-topic", + ), + ), + ) cfg := &kafka.AutoCreateTopicConfig{ AutoCreate: true, PartitionNum: 2, @@ -306,8 +277,6 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { partitionNum, err := manager.CreateTopicAndWaitUntilVisible(ctx, "precreated-topic") require.NoError(t, err) require.Equal(t, int32(2), partitionNum) - require.True(t, admin.createTopicCalled) - require.Equal(t, 1, admin.describeCount) partitions, ok := manager.topics.Load("precreated-topic") require.True(t, ok) diff --git a/go.mod b/go.mod index 3aa474a4d8..5360ee2158 100644 --- a/go.mod +++ b/go.mod @@ -76,6 +76,7 @@ require ( github.com/tinylib/msgp v1.5.0 github.com/twmb/franz-go v1.21.5 github.com/twmb/franz-go/pkg/kadm v1.18.0 + github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 github.com/uber-go/atomic v1.4.0 github.com/xdg/scram v1.0.5 github.com/zeebo/assert v1.3.0 diff --git a/go.sum b/go.sum index 9a62a8f0e9..8737ce9e33 100644 --- a/go.sum +++ b/go.sum @@ -969,6 +969,8 @@ github.com/twmb/franz-go v1.21.5 h1:cVYI2+JTTKSvohhy8bCOleYrS7G79ZBrLVFIJsoHm8M= github.com/twmb/franz-go v1.21.5/go.mod h1:rfoMTnVk7107fhTGxfEKIHP/e7tPe6oyij/ywzO0czk= github.com/twmb/franz-go/pkg/kadm v1.18.0 h1:WRf/LZmDdcDXwX7WMbtDU++v+b3NzYh2bCGoPMmzirw= github.com/twmb/franz-go/pkg/kadm v1.18.0/go.mod h1:XeLhGoLXLFzK8/ryv5FfpxPxGwj4oFEGpPJMB/x6KDE= +github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 h1:OLL/h1pOsMuCnJEp3IW0L2W3jHATeTUMAmE6PxxbhtU= +github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7/go.mod h1:9j4VxU2ng6tHgD4lIkNJ5OJ3D6vgPhhIp3tBa7dJgLA= github.com/twmb/franz-go/pkg/kmsg v1.13.1 h1:fG5kItwysTk5UXqVwb64EpQEy3TydF3vYYK21nUQ+bI= github.com/twmb/franz-go/pkg/kmsg v1.13.1/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index 845471168c..64decaa1f7 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -214,10 +214,6 @@ func topicDetailsFromMetadata( // IsAdminAuthorizationFailed checks whether err is an authorization failure from Kafka admin APIs. func IsAdminAuthorizationFailed(err error) bool { - return isKafkaAuthorizationFailed(err) -} - -func isKafkaAuthorizationFailed(err error) bool { return errors.Is(err, kerr.TopicAuthorizationFailed) || errors.Is(err, kerr.ClusterAuthorizationFailed) } @@ -254,7 +250,5 @@ func (a *admin) CreateTopic(detail TopicDetail, validateOnly bool) error { } func (a *admin) Close() { - if a.admin != nil { - a.admin.Close() - } + a.admin.Close() } diff --git a/pkg/sink/kafka/async_producer.go b/pkg/sink/kafka/async_producer.go index 1c2e11d182..5c144497cd 100644 --- a/pkg/sink/kafka/async_producer.go +++ b/pkg/sink/kafka/async_producer.go @@ -15,6 +15,7 @@ package kafka import ( "context" + "sync/atomic" "time" "github.com/pingcap/log" @@ -22,7 +23,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/twmb/franz-go/pkg/kgo" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -46,8 +46,8 @@ type asyncProducer struct { client *kgo.Client changefeedID commonType.ChangeFeedID - closeStarted *atomic.Bool - closed *atomic.Bool + closeStarted atomic.Bool + closed atomic.Bool errCh chan error } @@ -70,8 +70,6 @@ func newAsyncProducer( return &asyncProducer{ client: client, changefeedID: changefeedID, - closeStarted: atomic.NewBool(false), - closed: atomic.NewBool(false), errCh: make(chan error, 1), }, nil } @@ -155,9 +153,6 @@ func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { zap.String("changefeed", p.changefeedID.Name())) return context.Cause(ctx) case err := <-p.errCh: - if err == nil { - return nil - } return err } } diff --git a/pkg/sink/kafka/async_producer_test.go b/pkg/sink/kafka/async_producer_test.go index 79e30d1e5c..e1961a89ac 100644 --- a/pkg/sink/kafka/async_producer_test.go +++ b/pkg/sink/kafka/async_producer_test.go @@ -15,6 +15,7 @@ package kafka import ( "context" + "sync/atomic" "testing" "github.com/pingcap/ticdc/pkg/common" @@ -22,11 +23,11 @@ import ( codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kgo" - "go.uber.org/atomic" ) func TestAsyncSendClosedProducer(t *testing.T) { - producer := &asyncProducer{closed: atomic.NewBool(true)} + producer := &asyncProducer{} + producer.closed.Store(true) err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) @@ -36,7 +37,6 @@ func TestAsyncSendClosedProducer(t *testing.T) { func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { producer := &asyncProducer{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), - closed: atomic.NewBool(false), errCh: make(chan error, 1), } producer.errCh <- errors.New("queued async error") @@ -51,12 +51,10 @@ func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) require.NoError(t, err) - callbackCalled := atomic.NewBool(false) + var callbackCalled atomic.Bool producer := &asyncProducer{ client: client, changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), - closeStarted: atomic.NewBool(false), - closed: atomic.NewBool(false), errCh: make(chan error, 1), } err = producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{ diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go index c20d4396f0..d5349d7f46 100644 --- a/pkg/sink/kafka/client_options.go +++ b/pkg/sink/kafka/client_options.go @@ -190,7 +190,7 @@ func newCompressionOption(o *options) kgo.Opt { compression := strings.ToLower(strings.TrimSpace(o.Compression)) var codec kgo.CompressionCodec switch compression { - case "none": + case "", "none": codec = kgo.NoCompression() case "gzip": codec = kgo.GzipCompression() @@ -200,8 +200,6 @@ func newCompressionOption(o *options) kgo.Opt { codec = kgo.Lz4Compression() case "zstd": codec = kgo.ZstdCompression() - case "": - codec = kgo.NoCompression() default: log.Warn("unsupported compression algorithm", zap.String("compression", o.Compression)) codec = kgo.NoCompression() diff --git a/pkg/sink/kafka/factory_test.go b/pkg/sink/kafka/factory_test.go index 0928fbd00a..b418e3c493 100644 --- a/pkg/sink/kafka/factory_test.go +++ b/pkg/sink/kafka/factory_test.go @@ -87,39 +87,12 @@ func TestFactoryComponentCreationReturnsKafkaSinkError(t *testing.T) { }, } - testCases := []struct { - name string - create func() error - }{ - { - name: "admin", - create: func() error { - _, err := factory.Admin(context.Background()) - return err - }, - }, - { - name: "sync producer", - create: func() error { - _, err := factory.SyncProducer(context.Background()) - return err - }, - }, - { - name: "async producer", - create: func() error { - _, err := factory.AsyncProducer(context.Background()) - return err - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - requireNewKafkaSinkError(t, tc.create()) - }) - } + _, err := factory.Admin(context.Background()) + requireNewKafkaSinkError(t, err) + _, err = factory.SyncProducer(context.Background()) + requireNewKafkaSinkError(t, err) + _, err = factory.AsyncProducer(context.Background()) + requireNewKafkaSinkError(t, err) } func requireNewKafkaSinkError(t *testing.T, err error) { diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/gssapi.go index 7af48d5137..d03e372ee9 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/gssapi.go @@ -67,7 +67,8 @@ func (m *gssapiMechanism) Authenticate( return nil, nil, errors.Trace(err) } - token, err := newKrb5Token(client.Domain(), client.CName(), ticket, encKey) + token, err := newKrb5Token( + client.Credentials.Domain(), client.Credentials.CName(), ticket, encKey) if err != nil { client.Destroy() return nil, nil, errors.Trace(err) @@ -81,7 +82,7 @@ func (m *gssapiMechanism) Authenticate( } type gssapiSession struct { - client *krb5Client + client *client.Client encKey types.EncryptionKey } @@ -148,19 +149,7 @@ func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { return &gssapiMechanism{config: g}, nil } -type krb5Client struct { - client.Client -} - -func (c *krb5Client) Domain() string { - return c.Credentials.Domain() -} - -func (c *krb5Client) CName() types.PrincipalName { - return c.Credentials.CName() -} - -func newKerberosClient(g gssapiConfig) (*krb5Client, error) { +func newKerberosClient(g gssapiConfig) (*client.Client, error) { cfg, err := config.Load(g.kerberosConfigPath) if err != nil { return nil, errors.Trace(err) @@ -182,7 +171,7 @@ func newKerberosClient(g gssapiConfig) (*krb5Client, error) { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "unsupported sasl-gssapi-auth-type %d", g.authType) } - return &krb5Client{*krbClient}, nil + return krbClient, nil } func newKrb5Token( diff --git a/pkg/sink/kafka/sasl_test.go b/pkg/sink/kafka/sasl_test.go index 152199080c..14b5e80ded 100644 --- a/pkg/sink/kafka/sasl_test.go +++ b/pkg/sink/kafka/sasl_test.go @@ -20,46 +20,38 @@ import ( "github.com/stretchr/testify/require" ) -func TestBuildSaslMechanismGSSAPIUserAuth(t *testing.T) { +func TestBuildSaslMechanismGSSAPI(t *testing.T) { t.Parallel() - o := &options{ - sasl: &saslConfig{ - mechanism: gssapiMechanismName, - gssapi: gssapiConfig{ - authType: userAuth, - kerberosConfigPath: "/etc/krb5.conf", - serviceName: "kafka", - username: "alice", - password: "pwd", - realm: "EXAMPLE.COM", - }, - }, + testCases := []struct { + name string + authType gssapiAuthType + password string + keyTabPath string + }{ + {name: "user", authType: userAuth, password: "pwd"}, + {name: "keytab", authType: keyTabAuth, keyTabPath: "/tmp/a.keytab"}, } - mechanism, err := buildSaslMechanism(context.Background(), o) - require.NoError(t, err) - require.Equal(t, "GSSAPI", mechanism.Name()) -} - -func TestBuildSaslMechanismGSSAPIKeytabAuth(t *testing.T) { - t.Parallel() - - o := &options{ - sasl: &saslConfig{ - mechanism: gssapiMechanismName, - gssapi: gssapiConfig{ - authType: keyTabAuth, - kerberosConfigPath: "/etc/krb5.conf", - serviceName: "kafka", - username: "alice", - keyTabPath: "/tmp/a.keytab", - realm: "EXAMPLE.COM", - }, - }, + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + o := &options{sasl: &saslConfig{ + mechanism: gssapiMechanismName, + gssapi: gssapiConfig{ + authType: tc.authType, + kerberosConfigPath: "/etc/krb5.conf", + serviceName: "kafka", + username: "alice", + password: tc.password, + keyTabPath: tc.keyTabPath, + realm: "EXAMPLE.COM", + }, + }} + + mechanism, err := buildSaslMechanism(context.Background(), o) + require.NoError(t, err) + require.Equal(t, "GSSAPI", mechanism.Name()) + }) } - - mechanism, err := buildSaslMechanism(context.Background(), o) - require.NoError(t, err) - require.Equal(t, "GSSAPI", mechanism.Name()) } diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/sync_producer.go index 400674f870..4eea04cf1e 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/sync_producer.go @@ -15,6 +15,7 @@ package kafka import ( "context" + "sync/atomic" "time" "github.com/pingcap/log" @@ -22,7 +23,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/twmb/franz-go/pkg/kgo" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -47,7 +47,7 @@ type syncProducer struct { id commonType.ChangeFeedID client *kgo.Client - closed *atomic.Bool + closed atomic.Bool timeout time.Duration } @@ -71,7 +71,6 @@ func newSyncProducer( return &syncProducer{ id: changefeedID, client: client, - closed: atomic.NewBool(false), timeout: o.requestTimeout(), }, nil } diff --git a/pkg/sink/kafka/sync_producer_test.go b/pkg/sink/kafka/sync_producer_test.go index 4bc0f06407..8d3abbb704 100644 --- a/pkg/sink/kafka/sync_producer_test.go +++ b/pkg/sink/kafka/sync_producer_test.go @@ -19,11 +19,11 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { - producer := &syncProducer{closed: atomic.NewBool(true)} + producer := &syncProducer{} + producer.closed.Store(true) err := producer.SendMessage("topic", 1, &common.Message{}) require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) diff --git a/tests/utils/kafka_topic/main.go b/tests/utils/kafka_topic/main.go index 6227492cf9..5a55f33ec1 100644 --- a/tests/utils/kafka_topic/main.go +++ b/tests/utils/kafka_topic/main.go @@ -14,12 +14,14 @@ package main import ( + "context" "flag" "log" "strconv" "strings" - "github.com/IBM/sarama" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kgo" ) func main() { @@ -36,33 +38,47 @@ func main() { log.Fatal("max-message-bytes must be greater than zero") } + ctx := context.Background() value := strconv.Itoa(*maxMessageBytes) - config := sarama.NewConfig() - config.ClientID = "ticdc-integration-test-kafka-topic" - admin, err := sarama.NewClusterAdmin(strings.Split(*brokers, ","), config) + client, err := kgo.NewClient( + kgo.SeedBrokers(strings.Split(*brokers, ",")...), + kgo.ClientID("ticdc-integration-test-kafka-topic"), + ) if err != nil { log.Fatalf("create Kafka admin client: %v", err) } - defer func() { - if err := admin.Close(); err != nil { - log.Printf("close Kafka admin client: %v", err) - } - }() + defer client.Close() + admin := kadm.NewClient(client) - configEntries := map[string]*string{"max.message.bytes": &value} if *alter { - if err := admin.AlterConfig(sarama.TopicResource, *topic, configEntries, false); err != nil { + responses, err := admin.AlterTopicConfigsState(ctx, []kadm.AlterConfig{{ + Name: "max.message.bytes", + Value: &value, + }}, *topic) + if err != nil { log.Fatalf("alter Kafka topic %s: %v", *topic, err) } + response, err := responses.On(*topic, nil) + if err != nil { + log.Fatalf("find altered Kafka topic %s response: %v", *topic, err) + } + if response.Err != nil { + log.Fatalf("alter Kafka topic %s: %v", *topic, response.Err) + } return } - detail := &sarama.TopicDetail{ - NumPartitions: 1, - ReplicationFactor: 1, - ConfigEntries: configEntries, - } - if err := admin.CreateTopic(*topic, detail, false); err != nil { + responses, err := admin.CreateTopics(ctx, 1, 1, map[string]*string{ + "max.message.bytes": &value, + }, *topic) + if err != nil { log.Fatalf("create Kafka topic %s: %v", *topic, err) } + response, err := responses.On(*topic, nil) + if err != nil { + log.Fatalf("find created Kafka topic %s response: %v", *topic, err) + } + if response.Err != nil { + log.Fatalf("create Kafka topic %s: %v", *topic, response.Err) + } } From 1a2b4f082a2e20a502f9a90ec30ffb946de106c7 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 29 Jul 2026 17:17:09 +0800 Subject: [PATCH 37/61] revert some changes --- tests/integration_tests/http_api/util/test_case.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/http_api/util/test_case.py b/tests/integration_tests/http_api/util/test_case.py index 8a3c8e4dbf..c959682b7e 100644 --- a/tests/integration_tests/http_api/util/test_case.py +++ b/tests/integration_tests/http_api/util/test_case.py @@ -1,4 +1,5 @@ import sys +import os import requests as rq from requests.exceptions import RequestException import time @@ -175,7 +176,9 @@ def create_changefeed(sink_uri): }) headers = {"Content-Type": "application/json"} resp = rq.post(url, data=data, headers=headers) - assert "CDC:ErrNewKafkaSink" in resp.text, f"{resp.text}" + expected_error = "CDC:ErrKafkaNewProducer" if os.getenv( + "TICDC_NEWARCH") == "false" else "CDC:ErrNewKafkaSink" + assert expected_error in resp.text, f"{resp.text}" assert "not found, ResolveEndpointV2" not in resp.text, f"{resp.text}" print("pass test: create changefeed") From cfc6a3f4f747722c0beb8aa3184b7bc4227258f2 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 14 Aug 2026 17:04:47 +0800 Subject: [PATCH 38/61] restore the sarama --- .golangci.yml | 1 + downstreamadapter/sink/kafka/helper.go | 10 +- downstreamadapter/sink/kafka/sink.go | 34 +- downstreamadapter/sink/kafka/sink_test.go | 94 +++-- .../sink/topicmanager/kafka_topic_manager.go | 28 +- .../topicmanager/kafka_topic_manager_test.go | 338 ++++++++------- go.mod | 6 +- metrics/grafana/ticdc_new_arch.json | 97 ++++- .../ticdc_new_arch_next_gen.json | 97 ++++- .../ticdc_new_arch_with_keyspace_name.json | 97 ++++- pkg/leakutil/leak_helper.go | 7 + pkg/logger/log.go | 40 +- .../codec/canal/canal_json_txn_encoder.go | 2 +- pkg/sink/codec/common/message.go | 18 +- pkg/sink/codec/common/message_test.go | 29 -- pkg/sink/codec/open/codec.go | 2 +- pkg/sink/codec/open/encoder_test.go | 4 +- pkg/sink/kafka/admin.go | 282 ++++++------- pkg/sink/kafka/admin_client.go | 52 +++ pkg/sink/kafka/admin_client_mock.go | 136 ++++++ pkg/sink/kafka/admin_mock.go | 107 ----- pkg/sink/kafka/admin_test.go | 150 ------- pkg/sink/kafka/async_producer_mock.go | 76 ---- pkg/sink/kafka/async_producer_test.go | 72 ---- pkg/sink/kafka/client_options.go | 208 --------- pkg/sink/kafka/client_options_test.go | 126 ------ pkg/sink/kafka/factory.go | 112 ++--- pkg/sink/kafka/factory_mock.go | 155 ++++++- pkg/sink/kafka/factory_test.go | 104 ----- pkg/sink/kafka/franz/admin.go | 291 +++++++++++++ pkg/sink/kafka/franz/admin_test.go | 247 +++++++++++ pkg/sink/kafka/{ => franz}/async_producer.go | 72 ++-- pkg/sink/kafka/franz/async_producer_test.go | 196 +++++++++ pkg/sink/kafka/franz/config.go | 249 +++++++++++ pkg/sink/kafka/franz/config_test.go | 249 +++++++++++ pkg/sink/kafka/franz/factory.go | 52 +++ pkg/sink/kafka/franz/factory_test.go | 63 +++ pkg/sink/kafka/{ => franz}/gssapi.go | 141 +++++-- pkg/sink/kafka/franz/gssapi_test.go | 233 ++++++++++ pkg/sink/kafka/franz/logger.go | 124 ++++++ pkg/sink/kafka/franz/logger_test.go | 107 +++++ pkg/sink/kafka/franz/logutil.go | 65 +++ pkg/sink/kafka/franz/logutil_test.go | 75 ++++ pkg/sink/kafka/franz/metrics.go | 89 ++++ pkg/sink/kafka/{ => franz}/metrics_hook.go | 24 +- pkg/sink/kafka/franz/metrics_hook_test.go | 121 ++++++ pkg/sink/kafka/{ => franz}/sync_producer.go | 68 ++- pkg/sink/kafka/franz/sync_producer_test.go | 116 +++++ pkg/sink/kafka/franz_adapter.go | 218 ++++++++++ pkg/sink/kafka/logutil_test.go | 5 +- pkg/sink/kafka/main_test.go | 6 +- pkg/sink/kafka/metrics.go | 90 ++-- pkg/sink/kafka/metrics_collector.go | 214 ++++++++++ pkg/sink/kafka/options.go | 63 +-- pkg/sink/kafka/options_test.go | 399 +++++++++--------- pkg/sink/kafka/sarama_admin_mock.go | 175 ++++++++ pkg/sink/kafka/sarama_admin_test.go | 377 +++++++++++++++++ pkg/sink/kafka/sarama_async_producer.go | 173 ++++++++ pkg/sink/kafka/sarama_config.go | 269 ++++++++++++ pkg/sink/kafka/sarama_config_test.go | 316 ++++++++++++++ pkg/sink/kafka/sarama_factory.go | 194 +++++++++ .../kafka/sarama_oauth2_token_provider.go | 84 ++++ .../sarama_oauth2_token_provider_test.go | 134 ++++++ pkg/sink/kafka/sarama_sync_producer.go | 130 ++++++ pkg/sink/kafka/sarama_sync_producer_mock.go | 130 ++++++ pkg/sink/kafka/sarama_sync_producer_test.go | 141 +++++++ pkg/sink/kafka/sasl_config.go | 6 +- pkg/sink/kafka/sasl_test.go | 57 --- pkg/sink/kafka/selector.go | 36 ++ pkg/sink/kafka/selector_test.go | 118 ++++++ pkg/sink/kafka/sync_producer_mock.go | 75 ---- pkg/sink/kafka/sync_producer_test.go | 33 -- scripts/generate-mock.sh | 6 +- .../kafka_compression/data/gzip_data.sql | 21 + .../kafka_compression/data/lz4_data.sql | 21 + .../kafka_compression/data/snappy_data.sql | 21 + .../kafka_compression/data/zstd_data.sql | 21 + .../kafka_compression/run.sh | 28 +- tests/utils/kafka_topic/main.go | 50 +-- 79 files changed, 6619 insertions(+), 2058 deletions(-) delete mode 100644 pkg/sink/codec/common/message_test.go create mode 100644 pkg/sink/kafka/admin_client.go create mode 100644 pkg/sink/kafka/admin_client_mock.go delete mode 100644 pkg/sink/kafka/admin_mock.go delete mode 100644 pkg/sink/kafka/admin_test.go delete mode 100644 pkg/sink/kafka/async_producer_mock.go delete mode 100644 pkg/sink/kafka/async_producer_test.go delete mode 100644 pkg/sink/kafka/client_options.go delete mode 100644 pkg/sink/kafka/client_options_test.go delete mode 100644 pkg/sink/kafka/factory_test.go create mode 100644 pkg/sink/kafka/franz/admin.go create mode 100644 pkg/sink/kafka/franz/admin_test.go rename pkg/sink/kafka/{ => franz}/async_producer.go (63%) create mode 100644 pkg/sink/kafka/franz/async_producer_test.go create mode 100644 pkg/sink/kafka/franz/config.go create mode 100644 pkg/sink/kafka/franz/config_test.go create mode 100644 pkg/sink/kafka/franz/factory.go create mode 100644 pkg/sink/kafka/franz/factory_test.go rename pkg/sink/kafka/{ => franz}/gssapi.go (62%) create mode 100644 pkg/sink/kafka/franz/gssapi_test.go create mode 100644 pkg/sink/kafka/franz/logger.go create mode 100644 pkg/sink/kafka/franz/logger_test.go create mode 100644 pkg/sink/kafka/franz/logutil.go create mode 100644 pkg/sink/kafka/franz/logutil_test.go create mode 100644 pkg/sink/kafka/franz/metrics.go rename pkg/sink/kafka/{ => franz}/metrics_hook.go (92%) create mode 100644 pkg/sink/kafka/franz/metrics_hook_test.go rename pkg/sink/kafka/{ => franz}/sync_producer.go (62%) create mode 100644 pkg/sink/kafka/franz/sync_producer_test.go create mode 100644 pkg/sink/kafka/franz_adapter.go create mode 100644 pkg/sink/kafka/metrics_collector.go create mode 100644 pkg/sink/kafka/sarama_admin_mock.go create mode 100644 pkg/sink/kafka/sarama_admin_test.go create mode 100644 pkg/sink/kafka/sarama_async_producer.go create mode 100644 pkg/sink/kafka/sarama_config.go create mode 100644 pkg/sink/kafka/sarama_config_test.go create mode 100644 pkg/sink/kafka/sarama_factory.go create mode 100644 pkg/sink/kafka/sarama_oauth2_token_provider.go create mode 100644 pkg/sink/kafka/sarama_oauth2_token_provider_test.go create mode 100644 pkg/sink/kafka/sarama_sync_producer.go create mode 100644 pkg/sink/kafka/sarama_sync_producer_mock.go create mode 100644 pkg/sink/kafka/sarama_sync_producer_test.go delete mode 100644 pkg/sink/kafka/sasl_test.go create mode 100644 pkg/sink/kafka/selector.go create mode 100644 pkg/sink/kafka/selector_test.go delete mode 100644 pkg/sink/kafka/sync_producer_mock.go delete mode 100644 pkg/sink/kafka/sync_producer_test.go create mode 100644 tests/integration_tests/kafka_compression/data/gzip_data.sql create mode 100644 tests/integration_tests/kafka_compression/data/lz4_data.sql create mode 100644 tests/integration_tests/kafka_compression/data/snappy_data.sql create mode 100644 tests/integration_tests/kafka_compression/data/zstd_data.sql diff --git a/.golangci.yml b/.golangci.yml index 8a89fd0a80..547c742d89 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -81,6 +81,7 @@ linters: - name: unreachable-code - name: unused-parameter - name: var-declaration + - name: var-naming # G104: Audit errors not checked (duplicates errcheck). # G115: integer overflow conversions are too noisy for this codebase. diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index 6d3a53c9be..a6fa184a15 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -36,14 +36,14 @@ type components struct { columnSelector *columnselector.ColumnSelectors eventRouter *eventrouter.EventRouter topicManager topicmanager.TopicManager - admin kafka.Admin + adminClient kafka.AdminClient factory kafka.Factory claimCheck *claimcheck.ClaimCheck } func (c components) close() { - if c.admin != nil { - c.admin.Close() + if c.adminClient != nil { + c.adminClient.Close() } if c.topicManager != nil { c.topicManager.Close() @@ -125,7 +125,7 @@ func newKafkaSinkComponent( return comp, protocol, err } - comp.admin, err = comp.factory.Admin(ctx) + comp.adminClient, err = comp.factory.AdminClient(ctx) if err != nil { return comp, protocol, err } @@ -135,7 +135,7 @@ func newKafkaSinkComponent( changefeedID, topic, options.DeriveTopicConfig(), - comp.admin, + comp.adminClient, ) if err != nil { return comp, protocol, err diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 8d61a2a03f..88e40aef8d 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -47,8 +47,9 @@ const ( type sink struct { changefeedID common.ChangeFeedID - dmlProducer kafka.AsyncProducer - ddlProducer kafka.SyncProducer + dmlProducer kafka.AsyncProducer + ddlProducer kafka.SyncProducer + metricsCollector kafka.MetricsCollector comp components statistics *metrics.Statistics @@ -71,10 +72,6 @@ func (s *sink) SinkType() common.SinkType { return common.KafkaSinkType } -var createKafkaFactory = func(createFactory func() (kafka.Factory, error)) (kafka.Factory, error) { - return createFactory() -} - func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, sinkConfig *config.SinkConfig) error { protocol, err := helper.GetProtocol(util.GetOrZero(sinkConfig.Protocol)) if err != nil { @@ -115,20 +112,18 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, return err } - factory, err := createKafkaFactory(func() (kafka.Factory, error) { - return kafka.NewFactory(ctx, options, changefeedID) - }) + factory, err := kafka.NewFactory(ctx, options, changefeedID) if err != nil { return err } - admin, err := factory.Admin(ctx) + adminClient, err := factory.AdminClient(ctx) if err != nil { return err } - defer admin.Close() + defer adminClient.Close() - err = topicmanager.EnsureTopic(ctx, changefeedID, topic, options.DeriveTopicConfig(), admin) + err = topicmanager.EnsureTopic(ctx, changefeedID, topic, options.DeriveTopicConfig(), adminClient) if err != nil { return err } @@ -175,7 +170,7 @@ func newWithComponents( } comp.close() statistics.Close() - kafka.CleanupMetrics(changefeedID) + kafka.CleanupFactoryMetrics(comp.factory) }() asyncProducer, err = comp.factory.AsyncProducer(ctx) @@ -188,9 +183,10 @@ func newWithComponents( return nil, err } return &sink{ - changefeedID: changefeedID, - dmlProducer: asyncProducer, - ddlProducer: syncProducer, + changefeedID: changefeedID, + dmlProducer: asyncProducer, + ddlProducer: syncProducer, + metricsCollector: comp.factory.MetricsCollector(comp.adminClient), partitionRule: helper.GetDDLDispatchRule(protocol), protocol: protocol, @@ -217,6 +213,10 @@ func (s *sink) Run(ctx context.Context) error { g.Go(func() error { return s.sendDMLEvent(ctx) }) + g.Go(func() error { + s.metricsCollector.Run(ctx) + return nil + }) err := g.Wait() s.isNormal.Store(false) return err @@ -566,7 +566,7 @@ func (s *sink) Close() { s.dmlProducer.Close() s.comp.close() s.statistics.Close() - kafka.CleanupMetrics(s.changefeedID) + kafka.CleanupFactoryMetrics(s.comp.factory) } func (s *sink) BatchCount() int { diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 8e47e06035..daf408414a 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -35,11 +35,16 @@ import ( "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/pingcap/tidb/pkg/meta/model" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" "go.uber.org/atomic" ) const kafkaSinkTestTopic = "mock_topic" +type noopMetricsCollector struct{} + +func (noopMetricsCollector) Run(context.Context) {} + func TestSinkWorkersReturnContextError(t *testing.T) { contexts := []struct { name string @@ -87,6 +92,9 @@ func TestSinkWorkersReturnContextError(t *testing.T) { } func TestVerifyInvalidConfig(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, kafkaSinkTestTopic)) + defer cluster.Close() + schemaRegistry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "invalid response", http.StatusInternalServerError) })) @@ -97,28 +105,10 @@ func TestVerifyInvalidConfig(t *testing.T) { Protocol: &avroProtocol, SchemaRegistry: &schemaRegistry.URL, } - sinkURI, err := url.Parse("kafka://127.0.0.1:9092/" + kafkaSinkTestTopic + + sinkURI, err := url.Parse("kafka://" + cluster.ListenAddrs()[0] + "/" + kafkaSinkTestTopic + "?required-acks=1&kafka-version=2.4.0") require.NoError(t, err) - ctrl := gomock.NewController(t) - admin := kafka.NewMockAdmin(ctrl) - factory := kafka.NewMockFactory(ctrl) - gomock.InOrder( - factory.EXPECT().Admin(gomock.Any()).Return(admin, nil), - admin.EXPECT().GetTopicsMeta([]string{kafkaSinkTestTopic}, true).Return( - map[string]kafka.TopicDetail{kafkaSinkTestTopic: {Name: kafkaSinkTestTopic}}, nil), - admin.EXPECT().Close(), - ) - - originalCreateKafkaFactory := createKafkaFactory - createKafkaFactory = func(_ func() (kafka.Factory, error)) (kafka.Factory, error) { - return factory, nil - } - t.Cleanup(func() { - createKafkaFactory = originalCreateKafkaFactory - }) - changefeedID := common.NewChangefeedID4Test("test", "verify-invalid-config") err = Verify(context.Background(), changefeedID, sinkURI, sinkConfig) require.ErrorContains(t, err, "ErrAvroSchemaAPIError") @@ -188,7 +178,8 @@ func TestKafkaSinkBasicFunctionality(t *testing.T) { dmlEvent.CommitTs = 2 ctx, cancel := context.WithCancel(context.Background()) - kafkaSink, topicManager, asyncProducer, syncProducer := newKafkaSinkForTest(t, ctx, config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, asyncProducer, syncProducer := newKafkaSinkForTest( + t, ctx, config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(1), nil).AnyTimes() asyncProducer.EXPECT().AsyncRunCallback(gomock.Any()).Return(nil).AnyTimes() asyncProducer.EXPECT().AsyncSend(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). @@ -235,12 +226,12 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { t.Run("async producer creation fails", func(t *testing.T) { ctrl := gomock.NewController(t) factory := kafka.NewMockFactory(ctrl) - admin := kafka.NewMockAdmin(ctrl) + adminClient := kafka.NewMockAdminClient(ctrl) topicManager := topicmanager.NewMockTopicManager(ctrl) cause := errors.ErrKafkaSendMessage.GenWithStackByArgs() factory.EXPECT().AsyncProducer(gomock.Any()).Return(nil, cause) - admin.EXPECT().Close() + adminClient.EXPECT().Close() topicManager.EXPECT().Close() kafkaSink, err := newWithComponents( @@ -248,7 +239,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { common.NewChangefeedID4Test("test", "async-creation-fails"), common.DefaultKeyspaceID, config.ProtocolOpen, - components{factory: factory, admin: admin, topicManager: topicManager}, + components{factory: factory, adminClient: adminClient, topicManager: topicManager}, ) require.Nil(t, kafkaSink) @@ -258,7 +249,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { t.Run("sync producer creation fails", func(t *testing.T) { ctrl := gomock.NewController(t) factory := kafka.NewMockFactory(ctrl) - admin := kafka.NewMockAdmin(ctrl) + adminClient := kafka.NewMockAdminClient(ctrl) topicManager := topicmanager.NewMockTopicManager(ctrl) asyncProducer := kafka.NewMockAsyncProducer(ctrl) cause := errors.ErrKafkaSendMessage.GenWithStackByArgs() @@ -266,7 +257,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(nil, cause) asyncProducer.EXPECT().Close() - admin.EXPECT().Close() + adminClient.EXPECT().Close() topicManager.EXPECT().Close() kafkaSink, err := newWithComponents( @@ -274,7 +265,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { common.NewChangefeedID4Test("test", "sync-creation-fails"), common.DefaultKeyspaceID, config.ProtocolOpen, - components{factory: factory, admin: admin, topicManager: topicManager}, + components{factory: factory, adminClient: adminClient, topicManager: topicManager}, ) require.Nil(t, kafkaSink) @@ -284,7 +275,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { t.Run("successful construction owns resources until close", func(t *testing.T) { ctrl := gomock.NewController(t) factory := kafka.NewMockFactory(ctrl) - admin := kafka.NewMockAdmin(ctrl) + adminClient := kafka.NewMockAdminClient(ctrl) topicManager := topicmanager.NewMockTopicManager(ctrl) asyncProducer := kafka.NewMockAsyncProducer(ctrl) syncProducer := kafka.NewMockSyncProducer(ctrl) @@ -292,9 +283,10 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(syncProducer, nil) + factory.EXPECT().MetricsCollector(adminClient).Return(noopMetricsCollector{}) asyncProducer.EXPECT().Close().Do(func() { closeCount.Add(1) }) syncProducer.EXPECT().Close().Do(func() { closeCount.Add(1) }) - admin.EXPECT().Close().Do(func() { closeCount.Add(1) }) + adminClient.EXPECT().Close().Do(func() { closeCount.Add(1) }) topicManager.EXPECT().Close().Do(func() { closeCount.Add(1) }) kafkaSink, err := newWithComponents( @@ -302,7 +294,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { common.NewChangefeedID4Test("test", "successful-construction"), common.DefaultKeyspaceID, config.ProtocolOpen, - components{factory: factory, admin: admin, topicManager: topicManager}, + components{factory: factory, adminClient: adminClient, topicManager: topicManager}, ) require.NoError(t, err) @@ -325,7 +317,8 @@ func TestKafkaSinkDML(t *testing.T) { sent := make(chan *codecCommon.Message, 1) ctx, cancel := context.WithCancelCause(t.Context()) - kafkaSink, topicManager, asyncProducer, _ := newKafkaSinkForTest(t, ctx, config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, asyncProducer, _ := newKafkaSinkForTest( + t, ctx, config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(1), nil) asyncProducer.EXPECT().AsyncSend(gomock.Any(), kafkaSinkTestTopic, int32(0), gomock.Any()). DoAndReturn(func(_ context.Context, _ string, _ int32, message *codecCommon.Message) error { @@ -366,7 +359,8 @@ func TestKafkaSinkDML(t *testing.T) { cause := errors.ErrKafkaSendMessage.GenWithStackByArgs() ctx, cancel := context.WithCancel(t.Context()) defer cancel() - kafkaSink, topicManager, asyncProducer, _ := newKafkaSinkForTest(t, ctx, config.ProtocolCanalJSON, &config.SinkConfig{}) + kafkaSink, topicManager, asyncProducer, _ := newKafkaSinkForTest( + t, ctx, config.ProtocolCanalJSON, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(1), nil) asyncProducer.EXPECT().AsyncSend(gomock.Any(), kafkaSinkTestTopic, int32(0), gomock.Any()).Return(cause) @@ -378,7 +372,8 @@ func TestKafkaSinkDML(t *testing.T) { t.Run("returns topic manager error unchanged", func(t *testing.T) { dmlEvent := eventHelper.DML2Event("test", "t", "insert into t values (3, 'three')") - kafkaSink, topicManager, _, _ := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, _, _ := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) cause := context.DeadlineExceeded topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(0), cause) kafkaSink.AddDMLEvent(dmlEvent) @@ -397,7 +392,8 @@ func TestKafkaSinkDDL(t *testing.T) { } t.Run("all partitions", func(t *testing.T) { - kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(4), nil) syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(4), gomock.Any()). DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { @@ -410,7 +406,8 @@ func TestKafkaSinkDDL(t *testing.T) { }) t.Run("partition zero", func(t *testing.T) { - kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest(t, t.Context(), config.ProtocolCanalJSON, &config.SinkConfig{}) + kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( + t, t.Context(), config.ProtocolCanalJSON, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(4), nil) syncProducer.EXPECT().SendMessage(kafkaSinkTestTopic, int32(0), gomock.Any()). DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { @@ -422,7 +419,8 @@ func TestKafkaSinkDDL(t *testing.T) { }) t.Run("topic manager error", func(t *testing.T) { - kafkaSink, topicManager, _, _ := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, _, _ := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) cause := context.DeadlineExceeded topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(0), cause) @@ -430,7 +428,8 @@ func TestKafkaSinkDDL(t *testing.T) { }) t.Run("producer error marks sink abnormal", func(t *testing.T) { - kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) cause := errors.ErrKafkaSendMessage.GenWithStackByArgs() topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(2), nil) syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(2), gomock.Any()).Return(cause) @@ -440,14 +439,16 @@ func TestKafkaSinkDDL(t *testing.T) { }) t.Run("nil encoded message", func(t *testing.T) { - kafkaSink, _, _, _ := newKafkaSinkForTest(t, t.Context(), config.ProtocolDebezium, &config.SinkConfig{}) + kafkaSink, _, _, _ := newKafkaSinkForTest( + t, t.Context(), config.ProtocolDebezium, &config.SinkConfig{}) unsupportedDDL := &commonEvent.DDLEvent{Type: byte(model.ActionNone), Query: "unsupported"} require.NoError(t, kafkaSink.sendDDLEvent(unsupportedDDL)) }) t.Run("unsupported block event", func(t *testing.T) { - kafkaSink, _, _, _ := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, _, _, _ := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) syncPoint := commonEvent.NewSyncPointEvent(common.NewDispatcherID(), 1, 1, 1) require.ErrorIs(t, kafkaSink.WriteBlockEvent(syncPoint), errors.ErrInvalidEventType) @@ -456,7 +457,8 @@ func TestKafkaSinkDDL(t *testing.T) { func TestKafkaSinkCheckpoint(t *testing.T) { t.Run("default topic without tables", func(t *testing.T) { - kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(3), nil) syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(3), gomock.Any()). DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { @@ -474,7 +476,8 @@ func TestKafkaSinkCheckpoint(t *testing.T) { {Matcher: []string{"db1.t1"}, PartitionRule: "table", TopicRule: "topic-a"}, {Matcher: []string{"db2.t2"}, PartitionRule: "table", TopicRule: "topic-b"}, }} - kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, sinkConfig) + kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, sinkConfig) kafkaSink.SetTableSchemaStore(commonEvent.NewTableSchemaStore([]*heartbeatpb.SchemaInfo{ {SchemaName: "db1", Tables: []*heartbeatpb.TableInfo{{TableName: "t1"}}}, {SchemaName: "db2", Tables: []*heartbeatpb.TableInfo{{TableName: "t2"}}}, @@ -493,7 +496,8 @@ func TestKafkaSinkCheckpoint(t *testing.T) { }) t.Run("topic manager error", func(t *testing.T) { - kafkaSink, topicManager, _, _ := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) + kafkaSink, topicManager, _, _ := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) cause := context.DeadlineExceeded topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(0), cause) kafkaSink.checkpointChan <- 100 @@ -505,7 +509,8 @@ func TestKafkaSinkCheckpoint(t *testing.T) { sinkConfig := &config.SinkConfig{DispatchRules: []*config.DispatchRule{ {Matcher: []string{"db1.t1"}, PartitionRule: "table", TopicRule: "topic-a"}, }} - kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest(t, t.Context(), config.ProtocolOpen, sinkConfig) + kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( + t, t.Context(), config.ProtocolOpen, sinkConfig) kafkaSink.SetTableSchemaStore(commonEvent.NewTableSchemaStore([]*heartbeatpb.SchemaInfo{ {SchemaName: "db1", Tables: []*heartbeatpb.TableInfo{{TableName: "t1"}}}, }, common.KafkaSinkType, false)) @@ -521,7 +526,8 @@ func TestKafkaSinkCheckpoint(t *testing.T) { }) t.Run("nil encoded message", func(t *testing.T) { - kafkaSink, _, _, _ := newKafkaSinkForTest(t, t.Context(), config.ProtocolCanalJSON, &config.SinkConfig{}) + kafkaSink, _, _, _ := newKafkaSinkForTest( + t, t.Context(), config.ProtocolCanalJSON, &config.SinkConfig{}) kafkaSink.checkpointChan <- 100 close(kafkaSink.checkpointChan) @@ -568,6 +574,8 @@ func newKafkaSinkForTest( factory := kafka.NewMockFactory(ctrl) factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(syncProducer, nil) + factory.EXPECT().MetricsCollector(nil).Return(noopMetricsCollector{}) + kafkaSink, err := newWithComponents(ctx, changefeedID, common.DefaultKeyspaceID, protocol, components{ encoderGroup: encoderGroup, encoder: encoder, diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index 9c4186f910..a0110217f3 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -40,7 +40,7 @@ type kafkaTopicManager struct { defaultTopic string - admin kafka.Admin + admin kafka.AdminClient cfg *kafka.AutoCreateTopicConfig topics sync.Map @@ -52,7 +52,7 @@ type kafkaTopicManager struct { func newKafkaTopicManager( defaultTopic string, changefeedID common.ChangeFeedID, - admin kafka.Admin, + admin kafka.AdminClient, cfg *kafka.AutoCreateTopicConfig, ) *kafkaTopicManager { return &kafkaTopicManager{ @@ -69,9 +69,9 @@ func EnsureTopic( changefeedID common.ChangeFeedID, topic string, topicCfg *kafka.AutoCreateTopicConfig, - admin kafka.Admin, + adminClient kafka.AdminClient, ) error { - topicManager := newKafkaTopicManager(topic, changefeedID, admin, topicCfg) + topicManager := newKafkaTopicManager(topic, changefeedID, adminClient, topicCfg) _, err := topicManager.CreateTopicAndWaitUntilVisible(ctx, topic) return err } @@ -82,9 +82,9 @@ func GetTopicManagerAndTryCreateTopic( changefeedID common.ChangeFeedID, topic string, topicCfg *kafka.AutoCreateTopicConfig, - admin kafka.Admin, + adminClient kafka.AdminClient, ) (TopicManager, error) { - topicManager := newKafkaTopicManager(topic, changefeedID, admin, topicCfg) + topicManager := newKafkaTopicManager(topic, changefeedID, adminClient, topicCfg) if _, err := topicManager.CreateTopicAndWaitUntilVisible(ctx, topic); err != nil { return nil, err @@ -165,7 +165,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err }) start := time.Now() - topicDetails, err := m.admin.GetTopicsMeta(topics, false) + numPartitions, err := m.admin.GetTopicsPartitionsNum(topics) if err != nil { log.Warn( "kafka topic metadata refresh failed", @@ -176,14 +176,6 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err ) return nil, err } - numPartitions := make(map[string]int32, len(topicDetails)) - for _, topic := range topics { - detail, ok := topicDetails[topic] - if !ok { - return nil, errors.ErrKafkaAdminAPI.GenWithStackByArgs("describe-topic", topic) - } - numPartitions[topic] = detail.NumPartitions - } // it may happen the following case: // 1. user create the default topic with partition number set as 3 manually @@ -200,7 +192,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err // can be safely written to. The reason is that it may take several seconds after // CreateTopic returns success for all the brokers to become aware that the // topics have been created. -// See https://kafka.apache.org/23/javadoc/org/apache/kafka/clients/admin/Admin.html +// See https://kafka.apache.org/23/javadoc/org/apache/kafka/clients/admin/AdminClient.html func (m *kafkaTopicManager) waitUntilTopicVisible( ctx context.Context, topicName string, @@ -249,7 +241,7 @@ func (m *kafkaTopicManager) createTopic( } start := time.Now() - err := m.admin.CreateTopic(kafka.TopicDetail{ + err := m.admin.CreateTopic(&kafka.TopicDetail{ Name: topicName, NumPartitions: m.cfg.PartitionNum, ReplicationFactor: m.cfg.ReplicationFactor, @@ -267,6 +259,7 @@ func (m *kafkaTopicManager) createTopic( ) return 0, err } + m.tryUpdatePartitionsAndLogging(topicName, m.cfg.PartitionNum) return m.cfg.PartitionNum, nil @@ -292,6 +285,7 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( if numPartition, ok := m.tryStoreTopicMeta(topicName, topicDetails); ok { return numPartition, nil } + topicDetails, err = m.admin.GetTopicsMeta([]string{topicName}, false) if err != nil { if kafka.IsAuthorizationFailed(err) { diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index a58b51a3d8..e4dc2612eb 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kerr" ) const kafkaTopicManagerTestTopic = "mock_topic" @@ -30,128 +29,152 @@ const kafkaTopicManagerTestTopic = "mock_topic" func TestCreateTopic(t *testing.T) { t.Parallel() - ctrl := gomock.NewController(t) - admin := kafka.NewMockAdmin(ctrl) - cfg := &kafka.AutoCreateTopicConfig{ - AutoCreate: true, - PartitionNum: 2, - ReplicationFactor: 1, - RequiredAcks: kafka.WaitForAll, - } - changefeedID := common.NewChangefeedID4Test("test", "test") - ctx := context.Background() - var gotNewTopicDetail kafka.TopicDetail - var gotFailedTopicDetail kafka.TopicDetail - gomock.InOrder( - admin.EXPECT().GetTopicsMeta([]string{kafkaTopicManagerTestTopic}, true).Return( + + t.Run("existing topic", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{kafkaTopicManagerTestTopic}, true).Return( map[string]kafka.TopicDetail{ kafkaTopicManagerTestTopic: { Name: kafkaTopicManagerTestTopic, NumPartitions: 2, }, - }, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return( - map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return( - nil, errors.WrapError( - errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", "new-topic")), - admin.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail kafka.TopicDetail) error { - gotNewTopicDetail = detail + }, nil) + manager := newKafkaTopicManager( + kafkaTopicManagerTestTopic, + changefeedID, + adminClient, + &kafka.AutoCreateTopicConfig{PartitionNum: 2}, + ) + + partitionNum, err := manager.CreateTopicAndWaitUntilVisible(context.Background(), kafkaTopicManagerTestTopic) + + require.NoError(t, err) + require.Equal(t, int32(2), partitionNum) + }) + + t.Run("create missing topic", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + adminClient := kafka.NewMockAdminClient(ctrl) + var createdTopic *kafka.TopicDetail + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).DoAndReturn( + func([]string, bool) (map[string]kafka.TopicDetail, error) { + if createdTopic == nil { + return map[string]kafka.TopicDetail{}, nil + } + return map[string]kafka.TopicDetail{ + createdTopic.Name: { + Name: createdTopic.Name, + NumPartitions: createdTopic.NumPartitions, + }, + }, nil + }).Times(2) + adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( + func(detail *kafka.TopicDetail) error { + copy := *detail + createdTopic = © return nil - }), - admin.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return( - map[string]kafka.TopicDetail{ - "new-topic": { - Name: "new-topic", - NumPartitions: 2, - }, - }, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic2"}, true).Return( - map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic2"}, false).Return( - nil, errors.WrapError( - errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", "new-topic2")), - admin.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, true).Return( - map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{"new-topic-failed"}, false).Return( - nil, errors.WrapError( - errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", "new-topic-failed")), - admin.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail kafka.TopicDetail) error { - gotFailedTopicDetail = detail - return errors.ErrKafkaInvalidConfig.GenWithStack("invalid replication factor %d", detail.ReplicationFactor) - }), - ) + }) + manager := newKafkaTopicManager( + kafkaTopicManagerTestTopic, + changefeedID, + adminClient, + &kafka.AutoCreateTopicConfig{ + AutoCreate: true, + PartitionNum: 2, + ReplicationFactor: 1, + RequiredAcks: kafka.WaitForLocal, + }, + ) - manager := newKafkaTopicManager(kafkaTopicManagerTestTopic, changefeedID, admin, cfg) - partitionNum, err := manager.CreateTopicAndWaitUntilVisible(ctx, kafkaTopicManagerTestTopic) - require.NoError(t, err) - require.Equal(t, int32(2), partitionNum) + partitionNum, err := manager.CreateTopicAndWaitUntilVisible(context.Background(), "new-topic") - cfg.RequiredAcks = kafka.WaitForLocal - partitionNum, err = manager.CreateTopicAndWaitUntilVisible(ctx, "new-topic") - require.NoError(t, err) - require.Equal(t, int32(2), partitionNum) - require.Equal(t, kafka.TopicDetail{ - Name: "new-topic", - NumPartitions: 2, - ReplicationFactor: 1, - }, gotNewTopicDetail) - partitionsNum, err := manager.GetPartitionNum(ctx, "new-topic") - require.NoError(t, err) - require.Equal(t, int32(2), partitionsNum) + require.NoError(t, err) + require.Equal(t, int32(2), partitionNum) + require.Equal(t, &kafka.TopicDetail{ + Name: "new-topic", + NumPartitions: 2, + ReplicationFactor: 1, + }, createdTopic) + partitionsNum, err := manager.GetPartitionNum(context.Background(), "new-topic") + require.NoError(t, err) + require.Equal(t, int32(2), partitionsNum) + }) + + t.Run("auto create disabled", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + manager := newKafkaTopicManager( + "new-topic", + changefeedID, + adminClient, + &kafka.AutoCreateTopicConfig{ + AutoCreate: false, + PartitionNum: 2, + ReplicationFactor: 1, + RequiredAcks: kafka.WaitForAll, + }, + ) + + _, err := manager.CreateTopicAndWaitUntilVisible(context.Background(), "new-topic") + + require.ErrorContains(t, err, "`auto-create-topic` is false, and new-topic not found") + }) + + t.Run("create error", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + var createdTopic *kafka.TopicDetail + adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( + func(detail *kafka.TopicDetail) error { + copy := *detail + createdTopic = © + return errors.ErrKafkaAdminAPI.GenWithStackByArgs("create-topic", detail.Name) + }) + manager := newKafkaTopicManager( + "new-topic", + changefeedID, + adminClient, + &kafka.AutoCreateTopicConfig{ + AutoCreate: true, + PartitionNum: 2, + ReplicationFactor: 4, + }, + ) - // Try to create a topic without auto create. - cfg = &kafka.AutoCreateTopicConfig{ - AutoCreate: false, - PartitionNum: 2, - ReplicationFactor: 1, - RequiredAcks: kafka.WaitForAll, - } - manager = newKafkaTopicManager("new-topic2", changefeedID, admin, cfg) - _, err = manager.CreateTopicAndWaitUntilVisible(ctx, "new-topic2") - require.Regexp( - t, - "`auto-create-topic` is false, and new-topic2 not found", - err, - ) + _, err := manager.CreateTopicAndWaitUntilVisible(context.Background(), "new-topic") - topic := "new-topic-failed" - // Invalid replication factor. - // It happens when replication-factor is greater than the number of brokers. - cfg = &kafka.AutoCreateTopicConfig{ - AutoCreate: true, - PartitionNum: 2, - ReplicationFactor: 4, - } - manager = newKafkaTopicManager(topic, changefeedID, admin, cfg) - _, err = manager.CreateTopicAndWaitUntilVisible(ctx, topic) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) - require.Equal(t, "new-topic-failed", gotFailedTopicDetail.Name) + require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.Equal(t, "new-topic", createdTopic.Name) + }) } func TestCreateTopicValidatesReplicationFactor(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - admin := kafka.NewMockAdmin(ctrl) - topic := "new-topic" - gomock.InOrder( - admin.EXPECT().GetTopicsMeta([]string{topic}, true). - Return(map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{topic}, false). - Return(nil, errors.WrapError( - errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", topic)), - admin.EXPECT().GetBrokerConfig(kafka.MinInsyncReplicasConfigName). - Return("2", true, nil), - ) - + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetBrokerConfig(kafka.MinInsyncReplicasConfigName).Return("2", true, nil) manager := newKafkaTopicManager( "new-topic", common.NewChangefeedID4Test("test", "test"), - admin, + adminClient, &kafka.AutoCreateTopicConfig{ AutoCreate: true, PartitionNum: 2, @@ -169,69 +192,72 @@ func TestEnsureTopicExistsWaitsUntilVisible(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - admin := kafka.NewMockAdmin(ctrl) - cfg := &kafka.AutoCreateTopicConfig{ - AutoCreate: true, - PartitionNum: 2, - ReplicationFactor: 1, - } - - topic := "delayed-topic" - gomock.InOrder( - admin.EXPECT().GetTopicsMeta([]string{topic}, true).Return( - map[string]kafka.TopicDetail{}, nil), - admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, errors.WrapError( - errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", topic)), - admin.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail kafka.TopicDetail) error { - require.Equal(t, kafka.TopicDetail{ - Name: topic, - NumPartitions: 2, - ReplicationFactor: 1, - }, detail) - return nil - }), - admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, errors.WrapError(errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", topic)), - admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - nil, errors.WrapError(errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", topic)), - admin.EXPECT().GetTopicsMeta([]string{topic}, false).Return( - map[string]kafka.TopicDetail{ - topic: { - Name: topic, + adminClient := kafka.NewMockAdminClient(ctrl) + created := false + postCreateDescribeCount := 0 + adminClient.EXPECT().GetTopicsMeta([]string{"delayed-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"delayed-topic"}, false).DoAndReturn( + func([]string, bool) (map[string]kafka.TopicDetail, error) { + if !created { + return map[string]kafka.TopicDetail{}, nil + } + postCreateDescribeCount++ + if postCreateDescribeCount == 1 { + return map[string]kafka.TopicDetail{}, nil + } + return map[string]kafka.TopicDetail{ + "delayed-topic": { + Name: "delayed-topic", NumPartitions: 2, }, - }, nil), + }, nil + }).Times(3) + adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( + func(detail *kafka.TopicDetail) error { + require.Equal(t, &kafka.TopicDetail{ + Name: "delayed-topic", + NumPartitions: 2, + ReplicationFactor: 1, + }, detail) + created = true + return nil + }) + + err := EnsureTopic( + context.Background(), + common.NewChangefeedID4Test("test", "test"), + "delayed-topic", + &kafka.AutoCreateTopicConfig{ + AutoCreate: true, + PartitionNum: 2, + ReplicationFactor: 1, + }, + adminClient, ) - ctx := context.Background() - changefeedID := common.NewChangefeedID4Test("test", "test") - err := EnsureTopic(ctx, changefeedID, topic, cfg, admin) require.NoError(t, err) + require.Equal(t, 2, postCreateDescribeCount) } func TestGetTopicManagerStartsBackgroundRefreshAfterTopicReady(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - admin := kafka.NewMockAdmin(ctrl) - topic := "existing-topic" - admin.EXPECT().GetTopicsMeta([]string{topic}, true).Return( + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{"existing-topic"}, true).Return( map[string]kafka.TopicDetail{ - topic: { - Name: topic, + "existing-topic": { + Name: "existing-topic", NumPartitions: 2, }, - }, nil, - ) + }, nil) manager, err := GetTopicManagerAndTryCreateTopic( t.Context(), common.NewChangefeedID4Test("test", "test"), - topic, + "existing-topic", &kafka.AutoCreateTopicConfig{PartitionNum: 2}, - admin, + adminClient, ) require.NoError(t, err) @@ -243,14 +269,14 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - admin := kafka.NewMockAdmin(ctrl) - admin.EXPECT().GetTopicsMeta([]string{"default-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) - admin.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return( + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return( nil, errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs("describe-topic", "default-topic")) manager := newKafkaTopicManager( "default-topic", common.NewChangefeedID4Test("test", "test"), - admin, + adminClient, &kafka.AutoCreateTopicConfig{ AutoCreate: true, PartitionNum: 2, @@ -271,10 +297,10 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) - admin := kafka.NewMockAdmin(ctrl) - admin.EXPECT().GetTopicsMeta([]string{"default-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) - admin.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) - admin.EXPECT().CreateTopic(kafka.TopicDetail{ + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, true).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().CreateTopic(&kafka.TopicDetail{ Name: "default-topic", NumPartitions: 2, ReplicationFactor: 1, @@ -282,7 +308,7 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { manager := newKafkaTopicManager( "default-topic", common.NewChangefeedID4Test("test", "test"), - admin, + adminClient, &kafka.AutoCreateTopicConfig{ AutoCreate: true, PartitionNum: 2, diff --git a/go.mod b/go.mod index 578448a63c..57e7997fd4 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 github.com/BurntSushi/toml v1.6.0 github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/IBM/sarama v1.41.2 github.com/KimMachineGun/automemlimit v0.2.4 github.com/agiledragon/gomonkey/v2 v2.11.0 github.com/apache/pulsar-client-go v0.13.0 @@ -63,6 +64,7 @@ require ( github.com/prometheus/client_golang v1.23.0 github.com/prometheus/client_model v0.6.2 github.com/r3labs/diff v1.1.0 + github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 github.com/robfig/cron v1.2.0 github.com/shirou/gopsutil/v3 v3.24.5 github.com/soheilhy/cmux v0.1.5 @@ -77,6 +79,7 @@ require ( github.com/twmb/franz-go v1.21.5 github.com/twmb/franz-go/pkg/kadm v1.18.0 github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 + github.com/twmb/franz-go/pkg/kmsg v1.13.1 github.com/uber-go/atomic v1.4.0 github.com/xdg/scram v1.0.5 github.com/zeebo/assert v1.3.0 @@ -126,7 +129,6 @@ require ( github.com/Azure/go-ntlmssp v0.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/DataDog/zstd v1.5.5 // indirect - github.com/IBM/sarama v1.41.2 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect @@ -307,7 +309,6 @@ require ( github.com/qri-io/jsonschema v0.2.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.55.0 // indirect - github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect @@ -341,7 +342,6 @@ require ( github.com/tklauser/numcpus v0.11.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/twmb/franz-go/pkg/kmsg v1.13.1 // indirect github.com/twmb/murmur3 v1.1.6 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 3cb0b73ed0..a30616fa7b 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -17788,12 +17788,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "B" } ], "thresholds": [], @@ -17897,6 +17906,15 @@ "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "B" } ], "thresholds": [], @@ -17946,7 +17964,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Kafka request end-to-end duration in seconds. Average and p99 are calculated from one-minute histogram rates.", + "description": "The request latency in seconds for all brokers. Franz values show the one-minute average and p99.", "fieldConfig": { "defaults": { "links": [] @@ -17994,21 +18012,30 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace, changefeed, instance, broker))", + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "C" } ], "thresholds": [], @@ -18106,12 +18133,30 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-request-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-response-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "C" } ], "thresholds": [], @@ -18161,7 +18206,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records in each successfully written topic-partition batch. Average and p99 are calculated from one-minute histogram rates.", + "description": "Records per producer batch for franz-go and per request for Sarama.", "fieldConfig": { "defaults": { "links": [] @@ -18209,28 +18254,37 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", + "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace, changefeed, instance))", + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p99", + "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-p99", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Batch", + "title": "Kafka Records Per Batch or Request", "tooltip": { "shared": true, "sort": 0, @@ -18273,7 +18327,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Compression ratio of successfully written batches. Compression ratio = uncompressed byte rate / compressed byte rate * 100.", + "description": "The compression ratio times 100 of record batches for all topics. Compression ratio = Size of original data / Size of compressed data * 100", "fieldConfig": { "defaults": {}, "overrides": [] @@ -18319,12 +18373,21 @@ "targets": [ { "exemplar": true, - "expr": "100 * sum(rate(ticdc_sink_kafka_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", + "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}", + "refId": "B" } ], "thresholds": [], diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index 7383ed064f..a230576074 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -17788,12 +17788,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "B" } ], "thresholds": [], @@ -17897,6 +17906,15 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "B" } ], "thresholds": [], @@ -17946,7 +17964,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Kafka request end-to-end duration in seconds. Average and p99 are calculated from one-minute histogram rates.", + "description": "The request latency in seconds for all brokers. Franz values show the one-minute average and p99.", "fieldConfig": { "defaults": { "links": [] @@ -17994,21 +18012,30 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance, broker))", + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "C" } ], "thresholds": [], @@ -18106,12 +18133,30 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_requests_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "C" } ], "thresholds": [], @@ -18161,7 +18206,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records in each successfully written topic-partition batch. Average and p99 are calculated from one-minute histogram rates.", + "description": "Records per producer batch for franz-go and per request for Sarama.", "fieldConfig": { "defaults": { "links": [] @@ -18209,28 +18254,37 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance))", + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Batch", + "title": "Kafka Records Per Batch or Request", "tooltip": { "shared": true, "sort": 0, @@ -18273,7 +18327,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Compression ratio of successfully written batches. Compression ratio = uncompressed byte rate / compressed byte rate * 100.", + "description": "The compression ratio times 100 of record batches for all topics. Compression ratio = Size of original data / Size of compressed data * 100", "fieldConfig": { "defaults": {}, "overrides": [] @@ -18319,12 +18373,21 @@ "targets": [ { "exemplar": true, - "expr": "100 * sum(rate(ticdc_sink_kafka_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}", + "refId": "B" } ], "thresholds": [], diff --git a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json index 5b7bb7ea09..6cab1ef756 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json +++ b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json @@ -6313,12 +6313,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "B" } ], "thresholds": [], @@ -6422,6 +6431,15 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "B" } ], "thresholds": [], @@ -6471,7 +6489,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Kafka request end-to-end duration in seconds. Average and p99 are calculated from one-minute histogram rates.", + "description": "The request latency in seconds for all brokers. Franz values show the one-minute average and p99.", "fieldConfig": { "defaults": { "links": [] @@ -6519,21 +6537,30 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance, broker))", + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "C" } ], "thresholds": [], @@ -6631,12 +6658,30 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "C" } ], "thresholds": [], @@ -6686,7 +6731,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records in each successfully written topic-partition batch. Average and p99 are calculated from one-minute histogram rates.", + "description": "Records per producer batch for franz-go and per request for Sarama.", "fieldConfig": { "defaults": { "links": [] @@ -6734,28 +6779,37 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name, changefeed, instance))", + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Batch", + "title": "Kafka Records Per Batch or Request", "tooltip": { "shared": true, "sort": 0, @@ -6798,7 +6852,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Compression ratio of successfully written batches. Compression ratio = uncompressed byte rate / compressed byte rate * 100.", + "description": "The compression ratio times 100 of record batches for all topics. Compression ratio = Size of original data / Size of compressed data * 100", "fieldConfig": { "defaults": {}, "overrides": [] @@ -6844,12 +6898,21 @@ "targets": [ { "exemplar": true, - "expr": "100 * sum(rate(ticdc_sink_kafka_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}", + "refId": "B" } ], "thresholds": [], diff --git a/pkg/leakutil/leak_helper.go b/pkg/leakutil/leak_helper.go index a6e68c7291..5e70e16786 100644 --- a/pkg/leakutil/leak_helper.go +++ b/pkg/leakutil/leak_helper.go @@ -28,6 +28,13 @@ var defaultOpts = []goleak.Option{ // The stack top is usually runtime_pollWait, so match by any-frame. goleak.IgnoreAnyFunction("github.com/godbus/dbus.(*Conn).inWorker"), goleak.IgnoreAnyFunction("github.com/godbus/dbus/v5.(*Conn).inWorker"), + // library used by sarama, ref: https://github.com/rcrowley/go-metrics/pull/266 + goleak.IgnoreTopFunction("github.com/rcrowley/go-metrics.(*meterArbiter).tick"), + // Because we close the sarama producer asynchronously, so we have to ignore these funcs. + goleak.IgnoreTopFunction("github.com/Shopify/sarama.(*client).backgroundMetadataUpdater"), + goleak.IgnoreTopFunction("github.com/Shopify/sarama.(*Broker).responseReceiver"), + goleak.IgnoreTopFunction("github.com/IBM/sarama.(*client).backgroundMetadataUpdater"), + goleak.IgnoreTopFunction("github.com/IBM/sarama.(*Broker).responseReceiver"), goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), } diff --git a/pkg/logger/log.go b/pkg/logger/log.go index 73ca4d129d..480e627c60 100644 --- a/pkg/logger/log.go +++ b/pkg/logger/log.go @@ -17,10 +17,12 @@ import ( "bytes" "context" "io" + stdlog "log" "os" "strconv" "strings" + "github.com/IBM/sarama" "github.com/gin-gonic/gin" "github.com/go-sql-driver/mysql" "github.com/pingcap/log" @@ -93,9 +95,10 @@ func IsDebugEnabled() bool { // loggerOp is the op for logger control type loggerOp struct { - isInitGRPCLogger bool - isInitMySQLLogger bool - output zapcore.WriteSyncer + isInitGRPCLogger bool + isInitSaramaLogger bool + isInitMySQLLogger bool + output zapcore.WriteSyncer } func (op *loggerOp) applyOpts(opts []LoggerOpt) { @@ -114,6 +117,13 @@ func WithInitGRPCLogger() LoggerOpt { } } +// WithInitSaramaLogger enables sarama logger initialization when initializes global logger +func WithInitSaramaLogger() LoggerOpt { + return func(op *loggerOp) { + op.isInitSaramaLogger = true + } +} + // WithInitMySQLLogger enables mysql logger initialization when initializes global logger func WithInitMySQLLogger() LoggerOpt { return func(op *loggerOp) { @@ -134,6 +144,7 @@ func InitLogger(cfg *Config, opts ...LoggerOpt) error { var op loggerOp opts = []LoggerOpt{ WithInitGRPCLogger(), + WithInitSaramaLogger(), WithInitMySQLLogger(), } op.applyOpts(opts) @@ -195,7 +206,7 @@ func InitLogger(cfg *Config, opts ...LoggerOpt) error { // initOptionalComponent initializes some optional components func initOptionalComponent(op *loggerOp, cfg *Config) error { var level zapcore.Level - if op.isInitGRPCLogger { + if op.isInitGRPCLogger || op.isInitSaramaLogger { err := level.UnmarshalText([]byte(cfg.Level)) if err != nil { return errors.Trace(err) @@ -208,6 +219,12 @@ func initOptionalComponent(op *loggerOp, cfg *Config) error { } } + if op.isInitSaramaLogger { + if err := initSaramaLogger(level); err != nil { + return err + } + } + if op.isInitMySQLLogger { if err := initMySQLLogger(); err != nil { return err @@ -239,6 +256,21 @@ func initMySQLLogger() error { return mysql.SetLogger(logger) } +// initSaramaLogger hacks logger used in sarama lib +func initSaramaLogger(level zapcore.Level) error { + if zapcore.InfoLevel.Enabled(level) { + sarama.Logger = stdlog.New(io.Discard, "[Sarama] ", stdlog.LstdFlags) + return nil + } + + logger, err := zap.NewStdLogAt(log.L().With(zap.String("component", "sarama")), level) + if err != nil { + return errors.Trace(err) + } + sarama.Logger = logger + return nil +} + type loggerWriter struct { logFunc func(msg string, fields ...zap.Field) } diff --git a/pkg/sink/codec/canal/canal_json_txn_encoder.go b/pkg/sink/codec/canal/canal_json_txn_encoder.go index 9f73f5901f..61f20ab173 100644 --- a/pkg/sink/codec/canal/canal_json_txn_encoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_encoder.go @@ -50,7 +50,7 @@ func (j *JSONTxnEventEncoder) AppendTxnEvent(rowEvents []*commonEvent.RowEvent) if err != nil { return err } - length := len(value) + length := len(value) + common.MaxRecordOverhead if length > j.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json", zap.Int("maxMessageBytes", j.config.MaxMessageBytes), diff --git a/pkg/sink/codec/common/message.go b/pkg/sink/codec/common/message.go index 773c65ed1e..5cfd56b010 100644 --- a/pkg/sink/codec/common/message.go +++ b/pkg/sink/codec/common/message.go @@ -13,7 +13,17 @@ package common -import "encoding/json" +import ( + "encoding/binary" + "encoding/json" +) + +// MaxRecordOverhead is used to calculate message size by sarama kafka client. +// reference: https://github.com/IBM/sarama/blob/ +// 66521126c71c522c15a36663ae9cddc2b024c799/async_producer.go#L233 +// For TiCDC, minimum supported kafka version is `0.11.0.2`, +// which will be treated as `version = 2` by sarama producer. +const MaxRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 // MessageType is the type of message, which is used by MqSink and RedoLog. type MessageType int @@ -77,9 +87,11 @@ type CheckpointLogInfo struct { CommitTs uint64 } -// Length returns the encoded key/value payload size of the sink message. +// Length returns the expected size of the Kafka message +// We didn't append any `Headers` when send the message, so ignore the calculations related to it. +// If `ProducerMessage` Headers fields used, this method should also adjust. func (m *Message) Length() int { - return len(m.Key) + len(m.Value) + return len(m.Key) + len(m.Value) + MaxRecordOverhead } // GetRowsCount returns the number of rows batched in one Message diff --git a/pkg/sink/codec/common/message_test.go b/pkg/sink/codec/common/message_test.go deleted file mode 100644 index d0492bda25..0000000000 --- a/pkg/sink/codec/common/message_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMessageLengthIsPayloadSize(t *testing.T) { - message := &Message{ - Key: []byte("key"), - Value: []byte("value"), - } - - require.Equal(t, len(message.Key)+len(message.Value), message.Length()) -} diff --git a/pkg/sink/codec/open/codec.go b/pkg/sink/codec/open/codec.go index 9a7a7b633f..454cb0d372 100644 --- a/pkg/sink/codec/open/codec.go +++ b/pkg/sink/codec/open/codec.go @@ -114,7 +114,7 @@ func encodeRowChangedEvent( // for single message that is longer than max-message-bytes // 16 is the length of `keyLenByte` and `valueLenByte`, 8 is the length of `versionHead` - length := len(key) + len(valueCompressed) + 16 + 8 + length := len(key) + len(valueCompressed) + common.MaxRecordOverhead + 16 + 8 return key, valueCompressed, length, nil } diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index 5a136b717f..de88347061 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -953,7 +953,7 @@ func TestLargeMessageWithHandleEnableHandleKeyOnly(t *testing.T) { } ctx := context.Background() - codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(130) + codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(168) codecConfig.LargeMessageHandle.LargeMessageHandleOption = config.LargeMessageHandleOptionHandleKeyOnly encoder, err := NewBatchEncoder(codecConfig, nil) require.NoError(t, err) @@ -993,7 +993,7 @@ func TestLargeMessageWithHandleEnableHandleKeyOnly(t *testing.T) { func TestLargeMessageWithoutHandle(t *testing.T) { ctx := context.Background() - codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(100) + codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(150) codecConfig.LargeMessageHandle.LargeMessageHandleOption = config.LargeMessageHandleOptionHandleKeyOnly encoder, err := NewBatchEncoder(codecConfig, nil) require.NoError(t, err) diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index 3d821a9f75..d45ec702d1 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2023 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,92 +14,64 @@ package kafka import ( - "context" "strconv" "strings" - "time" + "github.com/IBM/sarama" + "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - "github.com/twmb/franz-go/pkg/kadm" - "github.com/twmb/franz-go/pkg/kerr" - "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" ) -// TopicDetail represent a topic's detail information. -type TopicDetail struct { - Name string - NumPartitions int32 - ReplicationFactor int16 -} - -// Admin manages and inspects Kafka topics, brokers, configurations, and ACLs. -type Admin interface { - // GetBrokerConfig return the broker level configuration with the `configName` - GetBrokerConfig(configName string) (value string, found bool, err error) - - // GetTopicConfig return the topic level configuration with the `configName` - GetTopicConfig(topicName string, configName string) (value string, found bool, err error) - - // GetTopicsMeta return all target topics' metadata - // if `ignoreTopicError` is true, ignore the topic error and return the metadata of valid topics - GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) - - // CreateTopic creates a new topic. - CreateTopic(detail TopicDetail) error +type saramaAdminClient struct { + changefeed common.ChangeFeedID - // Close shuts down the admin. - Close() + // client is the underlying sarama client created for this admin wrapper. + // It must be closed to stop background goroutines (e.g. metadata updater) and release memory. + client saramaClient + admin saramaClusterAdmin } -type admin struct { - changefeed common.ChangeFeedID - - client *kgo.Client - admin *kadm.Client - timeout time.Duration +type saramaClient interface { + Brokers() []*sarama.Broker + Partitions(topic string) ([]int32, error) + Close() error } -func newAdmin( - ctx context.Context, - changefeedID common.ChangeFeedID, - o *options, - hook kgo.Hook, -) (*admin, error) { - opts, err := newOptions(ctx, o, hook) - if err != nil { - return nil, errors.Trace(err) - } +type saramaClusterAdmin interface { + DescribeCluster() (brokers []*sarama.Broker, controllerID int32, err error) + DescribeConfig(resource sarama.ConfigResource) ([]sarama.ConfigEntry, error) + DescribeTopics(topics []string) (metadata []*sarama.TopicMetadata, err error) + CreateTopic(topic string, detail *sarama.TopicDetail, validateOnly bool) error + Close() error +} - client, err := kgo.NewClient(opts...) - if err != nil { - return nil, errors.Trace(err) +func (a *saramaAdminClient) GetAllBrokers() []Broker { + brokers := a.client.Brokers() + result := make([]Broker, 0, len(brokers)) + for _, broker := range brokers { + result = append(result, Broker{ + ID: broker.ID(), + }) } - - return &admin{ - changefeed: changefeedID, - client: client, - admin: kadm.NewClient(client), - timeout: o.requestTimeout(), - }, nil + return result } -func (a *admin) GetBrokerConfig(configName string) (string, bool, error) { - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) - defer cancel() - - meta, err := a.admin.BrokerMetadata(ctx) +func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, bool, error) { + _, controller, err := a.admin.DescribeCluster() if err != nil { if IsAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-cluster", "cluster") } return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-cluster", "cluster") } - if meta.Controller < 0 { - return "", false, errors.ErrKafkaAdminAPI.GenWithStackByArgs("describe-cluster", "cluster") - } - configs, err := a.admin.DescribeBrokerConfigs(ctx, meta.Controller) + configEntries, err := a.admin.DescribeConfig(sarama.ConfigResource{ + Type: sarama.BrokerResource, + Name: strconv.Itoa(int(controller)), + ConfigNames: []string{configName}, + }) if err != nil { if IsAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) @@ -107,71 +79,45 @@ func (a *admin) GetBrokerConfig(configName string) (string, bool, error) { return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) } - controllerName := strconv.Itoa(int(meta.Controller)) - resource, err := configs.On(controllerName, nil) - if err != nil { - if IsAuthorizationFailed(err) { - return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) - } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) - } - if resource.Err != nil { - if IsAuthorizationFailed(resource.Err) { - return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", configName) - } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", configName) - } - - for _, entry := range resource.Configs { - if entry.Key == configName { - return entry.MaybeValue(), true, nil + // For compatibility with KOP, we checked all return values. + // 1. Kafka only returns requested configs. + // 2. Kop returns all configs. + for _, entry := range configEntries { + if entry.Name == configName { + return entry.Value, true, nil } } return "", false, nil } -func (a *admin) GetTopicConfig(topicName string, configName string) (string, bool, error) { - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) - defer cancel() - - configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) +func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) (string, bool, error) { + configEntries, err := a.admin.DescribeConfig(sarama.ConfigResource{ + Type: sarama.TopicResource, + Name: topicName, + ConfigNames: []string{configName}, + }) if err != nil { if IsAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) } return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) } - resource, err := configs.On(topicName, nil) - if err != nil { - if IsAuthorizationFailed(err) { - return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) - } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) - } - if resource.Err != nil { - if IsAuthorizationFailed(resource.Err) { - return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", topicName) - } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", topicName) - } - for _, entry := range resource.Configs { - if entry.Key == configName { - return entry.MaybeValue(), true, nil + // For compatibility with KOP, we checked all return values. + // 1. Kafka only returns requested configs. + // 2. Kop returns all configs. + for _, entry := range configEntries { + if entry.Name == configName { + return entry.Value, true, nil } } return "", false, nil } -func (a *admin) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { - if len(topics) == 0 { - return make(map[string]TopicDetail), nil - } - - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) - defer cancel() +func (a *saramaAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + result := make(map[string]TopicDetail, len(topics)) - meta, err := a.admin.Metadata(ctx, topics...) + metaList, err := a.admin.DescribeTopics(topics) if err != nil { resource := strings.Join(topics, ",") if IsAuthorizationFailed(err) { @@ -180,33 +126,28 @@ func (a *admin) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[strin return nil, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-topics", resource) } - return topicDetailsFromMetadata(meta, topics, ignoreTopicError) -} - -func topicDetailsFromMetadata(meta kadm.Metadata, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { - result := make(map[string]TopicDetail, len(topics)) - for _, topic := range topics { - detail, ok := meta.Topics[topic] - if !ok { - continue - } - if detail.Err == nil { - result[topic] = TopicDetail{ - Name: topic, - NumPartitions: int32(len(detail.Partitions)), + for _, meta := range metaList { + if meta.Err != sarama.ErrNoError { + if meta.Err == sarama.ErrUnknownTopicOrPartition { + continue } + if !ignoreTopicError { + if IsAuthorizationFailed(meta.Err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, meta.Err, "describe-topic", meta.Name) + } + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, meta.Err, "describe-topic", meta.Name) + } + log.Warn("kafka topic metadata refresh failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("topic", meta.Name), + zap.Error(meta.Err)) continue } - if errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { - continue - } - if ignoreTopicError { - continue - } - if IsAuthorizationFailed(detail.Err) { - return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, detail.Err, "describe-topic", topic) + result[meta.Name] = TopicDetail{ + Name: meta.Name, + NumPartitions: int32(len(meta.Partitions)), } - return nil, errors.WrapError(errors.ErrKafkaAdminAPI, detail.Err, "describe-topic", topic) } return result, nil } @@ -214,41 +155,62 @@ func topicDetailsFromMetadata(meta kadm.Metadata, topics []string, ignoreTopicEr // IsAuthorizationFailed checks whether err is a Kafka authorization failure. func IsAuthorizationFailed(err error) bool { return errors.Is(err, errors.ErrKafkaAuthorizationFailed) || - errors.Is(err, kerr.TopicAuthorizationFailed) || - errors.Is(err, kerr.ClusterAuthorizationFailed) + errors.Is(err, sarama.ErrTopicAuthorizationFailed) || + errors.Is(err, sarama.ErrClusterAuthorizationFailed) +} + +func (a *saramaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + result := make(map[string]int32, len(topics)) + for _, topic := range topics { + partition, err := a.client.Partitions(topic) + if err != nil { + if IsAuthorizationFailed(err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "list-partitions", topic) + } + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, err, "list-partitions", topic) + } + result[topic] = int32(len(partition)) + } + + return result, nil } -func (a *admin) CreateTopic(detail TopicDetail) error { - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) - defer cancel() +func (a *saramaAdminClient) CreateTopic(detail *TopicDetail) error { + request := &sarama.TopicDetail{ + NumPartitions: detail.NumPartitions, + ReplicationFactor: detail.ReplicationFactor, + } - responses, err := a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) - if err != nil { + err := a.admin.CreateTopic(detail.Name, request, false) + // Ignore the already exists error because it's not harmful. + if err != nil && !strings.Contains(err.Error(), sarama.ErrTopicAlreadyExists.Error()) { if IsAuthorizationFailed(err) { return errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "create-topic", detail.Name) } return errors.WrapError(errors.ErrKafkaAdminAPI, err, "create-topic", detail.Name) } + return nil +} - resp, ok := responses[detail.Name] - if !ok { - return errors.ErrKafkaAdminAPI.GenWithStackByArgs("create-topic", detail.Name) - } - if resp.Err == nil { - return nil - } - if errors.Is(resp.Err, kerr.TopicAlreadyExists) { - return nil - } - if errors.Is(resp.Err, kerr.InvalidReplicationFactor) { - return errors.WrapError(errors.ErrKafkaInvalidConfig, resp.Err) +func (a *saramaAdminClient) Close() { + // For admins created via sarama.NewClusterAdminFromClient, admin.Close() takes care + // of closing the underlying client as well. Fall back to closing the client directly + // only when admin is unexpectedly nil. + if a.admin != nil { + if err := a.admin.Close(); err != nil { + log.Warn("kafka admin client close failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.Error(err)) + } + return } - if IsAuthorizationFailed(resp.Err) { - return errors.WrapError(errors.ErrKafkaAuthorizationFailed, resp.Err, "create-topic", detail.Name) + if a.client != nil { + if err := a.client.Close(); err != nil { + log.Warn("kafka client close failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.Error(err)) + } } - return errors.WrapError(errors.ErrKafkaAdminAPI, resp.Err, "create-topic", detail.Name) -} - -func (a *admin) Close() { - a.admin.Close() } diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go new file mode 100644 index 0000000000..cea348c2a2 --- /dev/null +++ b/pkg/sink/kafka/admin_client.go @@ -0,0 +1,52 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +// TopicDetail represent a topic's detail information. +type TopicDetail struct { + Name string + NumPartitions int32 + ReplicationFactor int16 +} + +// Broker represents a Kafka broker. +type Broker struct { + ID int32 +} + +// AdminClient is the administrative client for Kafka, +// which supports managing and inspecting topics, brokers, configurations and ACLs. +type AdminClient interface { + // GetAllBrokers return all brokers among the cluster + GetAllBrokers() []Broker + + // GetBrokerConfig returns the broker-level configuration and whether it exists. + GetBrokerConfig(configName string) (value string, found bool, err error) + + // GetTopicConfig returns the topic-level configuration and whether it exists. + GetTopicConfig(topicName string, configName string) (value string, found bool, err error) + + // GetTopicsMeta return all target topics' metadata + // if `ignoreTopicError` is true, ignore the topic error and return the metadata of valid topics + GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) + + // GetTopicsPartitionsNum return the number of partitions of each topic. + GetTopicsPartitionsNum(topics []string) (map[string]int32, error) + + // CreateTopic creates a new topic. + CreateTopic(detail *TopicDetail) error + + // Close shuts down the admin client. + Close() +} diff --git a/pkg/sink/kafka/admin_client_mock.go b/pkg/sink/kafka/admin_client_mock.go new file mode 100644 index 0000000000..2d2778ea3b --- /dev/null +++ b/pkg/sink/kafka/admin_client_mock.go @@ -0,0 +1,136 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/sink/kafka/admin_client.go + +// Package kafka is a generated GoMock package. +package kafka + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockAdminClient is a mock of AdminClient interface. +type MockAdminClient struct { + ctrl *gomock.Controller + recorder *MockAdminClientMockRecorder +} + +// MockAdminClientMockRecorder is the mock recorder for MockAdminClient. +type MockAdminClientMockRecorder struct { + mock *MockAdminClient +} + +// NewMockAdminClient creates a new mock instance. +func NewMockAdminClient(ctrl *gomock.Controller) *MockAdminClient { + mock := &MockAdminClient{ctrl: ctrl} + mock.recorder = &MockAdminClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAdminClient) EXPECT() *MockAdminClientMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockAdminClient) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockAdminClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockAdminClient)(nil).Close)) +} + +// CreateTopic mocks base method. +func (m *MockAdminClient) CreateTopic(detail *TopicDetail) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateTopic", detail) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateTopic indicates an expected call of CreateTopic. +func (mr *MockAdminClientMockRecorder) CreateTopic(detail interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockAdminClient)(nil).CreateTopic), detail) +} + +// GetAllBrokers mocks base method. +func (m *MockAdminClient) GetAllBrokers() []Broker { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllBrokers") + ret0, _ := ret[0].([]Broker) + return ret0 +} + +// GetAllBrokers indicates an expected call of GetAllBrokers. +func (mr *MockAdminClientMockRecorder) GetAllBrokers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrokers", reflect.TypeOf((*MockAdminClient)(nil).GetAllBrokers)) +} + +// GetBrokerConfig mocks base method. +func (m *MockAdminClient) GetBrokerConfig(configName string) (string, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBrokerConfig", configName) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetBrokerConfig indicates an expected call of GetBrokerConfig. +func (mr *MockAdminClientMockRecorder) GetBrokerConfig(configName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockAdminClient)(nil).GetBrokerConfig), configName) +} + +// GetTopicConfig mocks base method. +func (m *MockAdminClient) GetTopicConfig(topicName, configName string) (string, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopicConfig", topicName, configName) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetTopicConfig indicates an expected call of GetTopicConfig. +func (mr *MockAdminClientMockRecorder) GetTopicConfig(topicName, configName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockAdminClient)(nil).GetTopicConfig), topicName, configName) +} + +// GetTopicsMeta mocks base method. +func (m *MockAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopicsMeta", topics, ignoreTopicError) + ret0, _ := ret[0].(map[string]TopicDetail) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTopicsMeta indicates an expected call of GetTopicsMeta. +func (mr *MockAdminClientMockRecorder) GetTopicsMeta(topics, ignoreTopicError interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsMeta), topics, ignoreTopicError) +} + +// GetTopicsPartitionsNum mocks base method. +func (m *MockAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", topics) + ret0, _ := ret[0].(map[string]int32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTopicsPartitionsNum indicates an expected call of GetTopicsPartitionsNum. +func (mr *MockAdminClientMockRecorder) GetTopicsPartitionsNum(topics interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsPartitionsNum), topics) +} diff --git a/pkg/sink/kafka/admin_mock.go b/pkg/sink/kafka/admin_mock.go deleted file mode 100644 index 5cfcd42754..0000000000 --- a/pkg/sink/kafka/admin_mock.go +++ /dev/null @@ -1,107 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/admin.go - -// Package kafka is a generated GoMock package. -package kafka - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockAdmin is a mock of Admin interface. -type MockAdmin struct { - ctrl *gomock.Controller - recorder *MockAdminMockRecorder -} - -// MockAdminMockRecorder is the mock recorder for MockAdmin. -type MockAdminMockRecorder struct { - mock *MockAdmin -} - -// NewMockAdmin creates a new mock instance. -func NewMockAdmin(ctrl *gomock.Controller) *MockAdmin { - mock := &MockAdmin{ctrl: ctrl} - mock.recorder = &MockAdminMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAdmin) EXPECT() *MockAdminMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockAdmin) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockAdminMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockAdmin)(nil).Close)) -} - -// CreateTopic mocks base method. -func (m *MockAdmin) CreateTopic(detail TopicDetail) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTopic", detail) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateTopic indicates an expected call of CreateTopic. -func (mr *MockAdminMockRecorder) CreateTopic(detail interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockAdmin)(nil).CreateTopic), detail) -} - -// GetBrokerConfig mocks base method. -func (m *MockAdmin) GetBrokerConfig(configName string) (string, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBrokerConfig", configName) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetBrokerConfig indicates an expected call of GetBrokerConfig. -func (mr *MockAdminMockRecorder) GetBrokerConfig(configName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockAdmin)(nil).GetBrokerConfig), configName) -} - -// GetTopicConfig mocks base method. -func (m *MockAdmin) GetTopicConfig(topicName, configName string) (string, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicConfig", topicName, configName) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetTopicConfig indicates an expected call of GetTopicConfig. -func (mr *MockAdminMockRecorder) GetTopicConfig(topicName, configName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockAdmin)(nil).GetTopicConfig), topicName, configName) -} - -// GetTopicsMeta mocks base method. -func (m *MockAdmin) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsMeta", topics, ignoreTopicError) - ret0, _ := ret[0].(map[string]TopicDetail) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTopicsMeta indicates an expected call of GetTopicsMeta. -func (mr *MockAdminMockRecorder) GetTopicsMeta(topics, ignoreTopicError interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdmin)(nil).GetTopicsMeta), topics, ignoreTopicError) -} diff --git a/pkg/sink/kafka/admin_test.go b/pkg/sink/kafka/admin_test.go deleted file mode 100644 index b5ba64aa4f..0000000000 --- a/pkg/sink/kafka/admin_test.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "testing" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kadm" - "github.com/twmb/franz-go/pkg/kerr" - "github.com/twmb/franz-go/pkg/kfake" -) - -func TestTopicDetailsFromMetadata(t *testing.T) { - t.Parallel() - - const topic = "topic" - testCases := []struct { - name string - metadata kadm.Metadata - ignoreTopicError bool - expected map[string]TopicDetail - expectedError error - expectedCause error - }{ - { - name: "success", - metadata: kadm.Metadata{Topics: kadm.TopicDetails{ - topic: {Topic: topic, Partitions: kadm.PartitionDetails{0: {}, 1: {}}}, - }}, - expected: map[string]TopicDetail{ - topic: {Name: topic, NumPartitions: 2}, - }, - }, - { - name: "ignore unknown topic", - metadata: kadm.Metadata{Topics: kadm.TopicDetails{ - topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, - }}, - ignoreTopicError: true, - expected: map[string]TopicDetail{}, - }, - { - name: "skip unknown topic", - metadata: kadm.Metadata{Topics: kadm.TopicDetails{ - topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, - }}, - expected: map[string]TopicDetail{}, - }, - { - name: "skip missing topic", - metadata: kadm.Metadata{Topics: kadm.TopicDetails{}}, - expected: map[string]TopicDetail{}, - }, - { - name: "return authorization failure", - metadata: kadm.Metadata{Topics: kadm.TopicDetails{ - topic: {Topic: topic, Err: kerr.TopicAuthorizationFailed}, - }}, - expectedError: errors.ErrKafkaAuthorizationFailed, - expectedCause: kerr.TopicAuthorizationFailed, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - actual, err := topicDetailsFromMetadata(tc.metadata, []string{topic}, tc.ignoreTopicError) - if tc.expectedError != nil { - require.ErrorIs(t, err, tc.expectedError) - if tc.expectedCause != nil { - require.ErrorIs(t, err, tc.expectedCause) - } - return - } - require.NoError(t, err) - require.Equal(t, tc.expected, actual) - }) - } -} - -func TestIsAuthorizationFailed(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - err error - expected bool - }{ - { - name: "TiCDC authorization error", - err: errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs("describe-topic", "test-topic"), - expected: true, - }, - {name: "topic authorization error", err: kerr.TopicAuthorizationFailed, expected: true}, - {name: "cluster authorization error", err: kerr.ClusterAuthorizationFailed, expected: true}, - {name: "general error", err: kerr.InvalidTopicException}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require.Equal(t, test.expected, IsAuthorizationFailed(test.err)) - }) - } -} - -func TestCreateTopic(t *testing.T) { - t.Parallel() - - cluster := kfake.MustCluster(kfake.NumBrokers(1)) - defer cluster.Close() - - options := NewOptions() - options.BrokerEndpoints = cluster.ListenAddrs() - admin, err := newAdmin( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), - options, - nil, - ) - require.NoError(t, err) - defer admin.Close() - - const topic = "test-topic" - err = admin.CreateTopic(TopicDetail{ - Name: topic, - NumPartitions: 3, - ReplicationFactor: 1, - }) - require.NoError(t, err) - - topics, err := admin.GetTopicsMeta([]string{topic}, false) - require.NoError(t, err) - require.Equal(t, int32(3), topics[topic].NumPartitions) -} diff --git a/pkg/sink/kafka/async_producer_mock.go b/pkg/sink/kafka/async_producer_mock.go deleted file mode 100644 index 8a91530ab0..0000000000 --- a/pkg/sink/kafka/async_producer_mock.go +++ /dev/null @@ -1,76 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/async_producer.go - -// Package kafka is a generated GoMock package. -package kafka - -import ( - context "context" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - common "github.com/pingcap/ticdc/pkg/sink/codec/common" -) - -// MockAsyncProducer is a mock of AsyncProducer interface. -type MockAsyncProducer struct { - ctrl *gomock.Controller - recorder *MockAsyncProducerMockRecorder -} - -// MockAsyncProducerMockRecorder is the mock recorder for MockAsyncProducer. -type MockAsyncProducerMockRecorder struct { - mock *MockAsyncProducer -} - -// NewMockAsyncProducer creates a new mock instance. -func NewMockAsyncProducer(ctrl *gomock.Controller) *MockAsyncProducer { - mock := &MockAsyncProducer{ctrl: ctrl} - mock.recorder = &MockAsyncProducerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAsyncProducer) EXPECT() *MockAsyncProducerMockRecorder { - return m.recorder -} - -// AsyncRunCallback mocks base method. -func (m *MockAsyncProducer) AsyncRunCallback(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncRunCallback", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// AsyncRunCallback indicates an expected call of AsyncRunCallback. -func (mr *MockAsyncProducerMockRecorder) AsyncRunCallback(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncRunCallback", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncRunCallback), ctx) -} - -// AsyncSend mocks base method. -func (m *MockAsyncProducer) AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AsyncSend", ctx, topic, partition, message) - ret0, _ := ret[0].(error) - return ret0 -} - -// AsyncSend indicates an expected call of AsyncSend. -func (mr *MockAsyncProducerMockRecorder) AsyncSend(ctx, topic, partition, message interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncSend", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncSend), ctx, topic, partition, message) -} - -// Close mocks base method. -func (m *MockAsyncProducer) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockAsyncProducerMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockAsyncProducer)(nil).Close)) -} diff --git a/pkg/sink/kafka/async_producer_test.go b/pkg/sink/kafka/async_producer_test.go deleted file mode 100644 index e1961a89ac..0000000000 --- a/pkg/sink/kafka/async_producer_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "sync/atomic" - "testing" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" - "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kgo" -) - -func TestAsyncSendClosedProducer(t *testing.T) { - producer := &asyncProducer{} - producer.closed.Store(true) - - err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) - - require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) -} - -func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { - producer := &asyncProducer{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), - errCh: make(chan error, 1), - } - producer.errCh <- errors.New("queued async error") - - err := producer.AsyncRunCallback(context.Background()) - - require.ErrorContains(t, err, "queued async error") - require.True(t, producer.closed.Load()) -} - -func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { - client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) - require.NoError(t, err) - - var callbackCalled atomic.Bool - producer := &asyncProducer{ - client: client, - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), - errCh: make(chan error, 1), - } - err = producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{ - Callback: func() { - callbackCalled.Store(true) - }, - }) - require.NoError(t, err) - - producer.Close() - - require.False(t, callbackCalled.Load()) - err = producer.AsyncRunCallback(context.Background()) - require.ErrorIs(t, err, kgo.ErrClientClosed) -} diff --git a/pkg/sink/kafka/client_options.go b/pkg/sink/kafka/client_options.go deleted file mode 100644 index a7abb7eb62..0000000000 --- a/pkg/sink/kafka/client_options.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "crypto/tls" - "net/url" - "strings" - - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/twmb/franz-go/pkg/kgo" - "github.com/twmb/franz-go/pkg/kversion" - "github.com/twmb/franz-go/pkg/sasl" - "github.com/twmb/franz-go/pkg/sasl/oauth" - "github.com/twmb/franz-go/pkg/sasl/plain" - "github.com/twmb/franz-go/pkg/sasl/scram" - "go.uber.org/zap" - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" -) - -func newOptions( - ctx context.Context, - o *options, - hook kgo.Hook, -) ([]kgo.Opt, error) { - opts := []kgo.Opt{ - kgo.WithContext(ctx), - kgo.SeedBrokers(o.BrokerEndpoints...), - kgo.ClientID(o.ClientID), - kgo.DialTimeout(o.DialTimeout), - kgo.RequestTimeoutOverhead(o.requestTimeout()), - } - if hook != nil { - opts = append(opts, kgo.WithHooks(hook)) - } - - if o.IsAssignedVersion { - versions := kversion.FromString(o.Version) - if versions == nil { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("invalid kafka version %s", o.Version) - } - opts = append(opts, kgo.MaxVersions(versions)) - } - - if o.EnableTLS { - tlsConfig, err := newTLSConfig(o) - if err != nil { - return nil, errors.Trace(err) - } - opts = append(opts, kgo.DialTLSConfig(tlsConfig)) - } - - if o.sasl != nil && o.sasl.mechanism != "" { - mechanism, err := buildSaslMechanism(ctx, o) - if err != nil { - return nil, errors.Trace(err) - } - opts = append(opts, kgo.SASL(mechanism)) - } - - return opts, nil -} - -func newTLSConfig(o *options) (*tls.Config, error) { - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", "http/1.1"}, - } - - if o.Credential != nil && o.Credential.IsTLSEnabled() { - credentialTlsConfig, err := o.Credential.ToTLSConfig() - if err != nil { - return nil, errors.Trace(err) - } - tlsConfig = credentialTlsConfig - } - - tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify - return tlsConfig, nil -} - -func buildSaslMechanism(ctx context.Context, o *options) (sasl.Mechanism, error) { - switch o.sasl.mechanism { - case plainMechanism: - auth := plain.Auth{ - User: o.sasl.user, - Pass: o.sasl.password, - } - return auth.AsMechanism(), nil - case scram256Mechanism: - auth := scram.Auth{ - User: o.sasl.user, - Pass: o.sasl.password, - } - return auth.AsSha256Mechanism(), nil - case scram512Mechanism: - auth := scram.Auth{ - User: o.sasl.user, - Pass: o.sasl.password, - } - return auth.AsSha512Mechanism(), nil - case oauthMechanism: - tokenSource, err := newOauthTokenSource(ctx, o) - if err != nil { - return nil, errors.Trace(err) - } - return oauth.Oauth(func(context.Context) (oauth.Auth, error) { - token, err := tokenSource.Token() - if err != nil { - return oauth.Auth{}, errors.Trace(err) - } - return oauth.Auth{Token: token.AccessToken}, nil - }), nil - case gssapiMechanismName: - return buildGSSAPIMechanism(o.sasl.gssapi) - default: - } - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", o.sasl.mechanism) -} - -func newOauthTokenSource(ctx context.Context, o *options) (oauth2.TokenSource, error) { - endpointParams := url.Values{} - if o.sasl.oauth2.grantType != "" { - endpointParams.Set("grant_type", o.sasl.oauth2.grantType) - } - if o.sasl.oauth2.audience != "" { - endpointParams.Set("audience", o.sasl.oauth2.audience) - } - - tokenURL, err := url.Parse(o.sasl.oauth2.tokenURL) - if err != nil { - return nil, errors.Trace(err) - } - - cfg := &clientcredentials.Config{ - ClientID: o.sasl.oauth2.clientID, - ClientSecret: o.sasl.oauth2.clientSecret, - TokenURL: tokenURL.String(), - EndpointParams: endpointParams, - Scopes: o.sasl.oauth2.scopes, - } - return cfg.TokenSource(ctx), nil -} - -func newProducerOptions( - o *options, -) []kgo.Opt { - return []kgo.Opt{ - kgo.RecordPartitioner(kgo.ManualPartitioner()), - kgo.RequiredAcks(newRequiredAcks(o)), - kgo.DisableIdempotentWrite(), - kgo.MaxProduceRequestsInflightPerBroker(1), - kgo.RecordRetries(o.MaxRetry), - kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), - kgo.ProduceRequestTimeout(o.requestTimeout()), - kgo.ProducerLinger(0), - newCompressionOption(o), - } -} - -func newRequiredAcks(o *options) kgo.Acks { - switch o.RequiredAcks { - case WaitForAll: - return kgo.AllISRAcks() - case WaitForLocal: - return kgo.LeaderAck() - case NoResponse: - return kgo.NoAck() - default: - log.Warn("unsupported required acks", zap.Int16("requiredAcks", int16(o.RequiredAcks))) - return kgo.AllISRAcks() - } -} - -func newCompressionOption(o *options) kgo.Opt { - compression := strings.ToLower(strings.TrimSpace(o.Compression)) - var codec kgo.CompressionCodec - switch compression { - case "", "none": - codec = kgo.NoCompression() - case "gzip": - codec = kgo.GzipCompression() - case "snappy": - codec = kgo.SnappyCompression() - case "lz4": - codec = kgo.Lz4Compression() - case "zstd": - codec = kgo.ZstdCompression() - default: - log.Warn("unsupported kafka compression algorithm", zap.String("compression", o.Compression)) - codec = kgo.NoCompression() - } - return kgo.ProducerBatchCompression(codec) -} diff --git a/pkg/sink/kafka/client_options_test.go b/pkg/sink/kafka/client_options_test.go deleted file mode 100644 index 52062cf38e..0000000000 --- a/pkg/sink/kafka/client_options_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kgo" -) - -func TestNewRequiredAcks(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - requiredAcks RequiredAcks - expected kgo.Acks - }{ - {name: "all", requiredAcks: -1, expected: kgo.AllISRAcks()}, - {name: "leader", requiredAcks: 1, expected: kgo.LeaderAck()}, - {name: "none", requiredAcks: 0, expected: kgo.NoAck()}, - {name: "invalid fallback all", requiredAcks: 2, expected: kgo.AllISRAcks()}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - require.Equal(t, tc.expected, newRequiredAcks(&options{RequiredAcks: tc.requiredAcks})) - }) - } -} - -func TestNewOptionsRejectsInvalidAssignedVersion(t *testing.T) { - t.Parallel() - - opts, err := newOptions(context.Background(), &options{ - Version: "invalid", - IsAssignedVersion: true, - sasl: &saslConfig{}, - }, nil) - require.Nil(t, opts) - require.ErrorContains(t, err, "invalid kafka version invalid") -} - -func TestNewProducerOptionsUsesProducerBatchMaxBytes(t *testing.T) { - t.Parallel() - - const producerBatchMaxBytes = 1048588 - o := &options{ - BrokerEndpoints: []string{"127.0.0.1:9092"}, - MaxMessageBytes: producerBatchMaxBytes, - MaxRetry: defaultMaxRetry, - RequiredAcks: WaitForAll, - ReadTimeout: defaultTimeout, - WriteTimeout: defaultTimeout, - sasl: &saslConfig{}, - } - - opts, err := newOptions(context.Background(), o, nil) - require.NoError(t, err) - opts = append(opts, newProducerOptions(o)...) - client, err := kgo.NewClient(opts...) - require.NoError(t, err) - defer client.Close() - - require.Equal(t, int32(producerBatchMaxBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) -} - -func TestNewCompressionOptionMapsToProducerBatchCompression(t *testing.T) { - t.Parallel() - - testCases := []struct { - compression string - expected kgo.CompressionCodec - }{ - {compression: "gzip", expected: kgo.GzipCompression()}, - {compression: "snappy", expected: kgo.SnappyCompression()}, - {compression: "lz4", expected: kgo.Lz4Compression()}, - {compression: "zstd", expected: kgo.ZstdCompression()}, - } - - for _, tc := range testCases { - t.Run(tc.compression, func(t *testing.T) { - t.Parallel() - - client, err := kgo.NewClient( - kgo.SeedBrokers("127.0.0.1:9092"), - newCompressionOption(&options{Compression: tc.compression}), - ) - require.NoError(t, err) - defer client.Close() - - require.Equal(t, []kgo.CompressionCodec{tc.expected}, client.OptValue(kgo.ProducerBatchCompression)) - }) - } -} - -func TestNewOauthTokenSourceRejectsInvalidTokenURL(t *testing.T) { - t.Parallel() - - _, err := newOauthTokenSource(context.Background(), &options{ - sasl: &saslConfig{ - oauth2: oauth2Config{ - clientID: "client-id", - clientSecret: "client-secret", - tokenURL: "http://test.com/Segment%%2815197306101420000%29", - scopes: []string{"scope1", "scope2"}, - grantType: "client_credentials", - }, - }, - }) - require.ErrorContains(t, err, "invalid URL escape") -} diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 399c15b226..c19089de4c 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -15,92 +15,56 @@ package kafka import ( "context" - "strings" - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "go.uber.org/zap" + "github.com/pingcap/ticdc/pkg/sink/codec/common" ) -// Factory is used to produce all Kafka components. +// Factory is used to produce all kafka components. type Factory interface { - // Admin returns a Kafka admin. - Admin(ctx context.Context) (Admin, error) - // SyncProducer creates a sync producer to write messages to Kafka. + // AdminClient return a kafka cluster admin client + AdminClient(ctx context.Context) (AdminClient, error) + // SyncProducer creates a sync producer to writer message to kafka SyncProducer(ctx context.Context) (SyncProducer, error) - // AsyncProducer creates an async producer to write messages to Kafka. + // AsyncProducer creates an async producer to writer message to kafka AsyncProducer(ctx context.Context) (AsyncProducer, error) + // MetricsCollector returns the kafka metrics collector + MetricsCollector(adminClient AdminClient) MetricsCollector } -type factory struct { - changefeedID common.ChangeFeedID - options options -} - -// NewFactory constructs a Factory. -func NewFactory( - ctx context.Context, - o *options, - changefeedID common.ChangeFeedID, -) (Factory, error) { - admin, err := newAdmin(ctx, changefeedID, o, nil) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - defer admin.Close() +// SyncProducer is the kafka sync producer +type SyncProducer interface { + // SendMessage produces a given message, and returns only when it either has + // succeeded or failed to produce. It will return the partition and the offset + // of the produced message, or an error if the message failed to produce. + SendMessage(topic string, partitionNum int32, message *common.Message) error - if err := adjustOptions(changefeedID, admin, o, o.Topic); err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - compression := strings.ToLower(strings.TrimSpace(o.Compression)) - if compression == "" { - compression = "none" - } - log.Info("kafka sink configuration resolved", - zap.String("namespace", changefeedID.Keyspace()), - zap.String("changefeed", changefeedID.Name()), - zap.String("topic", o.Topic), - zap.Int32("partitionNum", o.PartitionNum), - zap.Int("maxMessageBytes", o.MaxMessageBytes), - zap.Int("maxBatchedBytes", o.MaxBatchedBytes), - zap.String("compression", compression), - zap.Int16("requiredAcks", int16(o.RequiredAcks)), - zap.Int("maxRetry", o.MaxRetry), - zap.Duration("dialTimeout", o.DialTimeout), - zap.Duration("readTimeout", o.ReadTimeout), - zap.Duration("writeTimeout", o.WriteTimeout)) + // SendMessages produces a given set of messages, and returns only when all + // messages in the set have either succeeded or failed. Note that messages + // can succeed and fail individually; if some succeed and some fail, + // SendMessages will return an error. + SendMessages(topic string, partitionNum int32, message *common.Message) error - return &factory{ - changefeedID: changefeedID, - options: *o, - }, nil + // Close shuts down the producer; you must call this function before a producer + // object passes out of scope, as it may otherwise leak memory. + // You must call this before calling Close on the underlying client. + Close() } -func (f *factory) Admin(ctx context.Context) (Admin, error) { - admin, err := newAdmin(ctx, f.changefeedID, &f.options, nil) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - return admin, nil -} +// AsyncProducer is the kafka async producer +type AsyncProducer interface { + // Close shuts down the producer and waits for any buffered messages to be + // flushed. You must call this function before a producer object passes out of + // scope, as it may otherwise leak memory. You must call this before process + // shutting down, or you may lose messages. You must call this before calling + // Close on the underlying client. + Close() -func (f *factory) SyncProducer(ctx context.Context) (SyncProducer, error) { - hook := newKafkaMetricsHook(f.changefeedID) - producer, err := newSyncProducer(ctx, f.changefeedID, &f.options, hook) - if err != nil { - CleanupMetrics(f.changefeedID) - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - return producer, nil -} + // AsyncSend is the input channel for the user to write messages to that they + // wish to send. + AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error -func (f *factory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - hook := newKafkaMetricsHook(f.changefeedID) - producer, err := newAsyncProducer(ctx, f.changefeedID, &f.options, hook) - if err != nil { - CleanupMetrics(f.changefeedID) - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - return producer, nil + // AsyncRunCallback process the messages that has sent to kafka, + // and run tha attached callback. the caller should call this + // method in a background goroutine + AsyncRunCallback(ctx context.Context) error } diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index 26e523b6fb..ecc8fe131c 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -9,6 +9,7 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" + common "github.com/pingcap/ticdc/pkg/sink/codec/common" ) // MockFactory is a mock of Factory interface. @@ -34,19 +35,19 @@ func (m *MockFactory) EXPECT() *MockFactoryMockRecorder { return m.recorder } -// Admin mocks base method. -func (m *MockFactory) Admin(ctx context.Context) (Admin, error) { +// AdminClient mocks base method. +func (m *MockFactory) AdminClient(ctx context.Context) (AdminClient, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Admin", ctx) - ret0, _ := ret[0].(Admin) + ret := m.ctrl.Call(m, "AdminClient", ctx) + ret0, _ := ret[0].(AdminClient) ret1, _ := ret[1].(error) return ret0, ret1 } -// Admin indicates an expected call of Admin. -func (mr *MockFactoryMockRecorder) Admin(ctx interface{}) *gomock.Call { +// AdminClient indicates an expected call of AdminClient. +func (mr *MockFactoryMockRecorder) AdminClient(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Admin", reflect.TypeOf((*MockFactory)(nil).Admin), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminClient", reflect.TypeOf((*MockFactory)(nil).AdminClient), ctx) } // AsyncProducer mocks base method. @@ -64,6 +65,20 @@ func (mr *MockFactoryMockRecorder) AsyncProducer(ctx interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), ctx) } +// MetricsCollector mocks base method. +func (m *MockFactory) MetricsCollector(adminClient AdminClient) MetricsCollector { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MetricsCollector", adminClient) + ret0, _ := ret[0].(MetricsCollector) + return ret0 +} + +// MetricsCollector indicates an expected call of MetricsCollector. +func (mr *MockFactoryMockRecorder) MetricsCollector(adminClient interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MetricsCollector", reflect.TypeOf((*MockFactory)(nil).MetricsCollector), adminClient) +} + // SyncProducer mocks base method. func (m *MockFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { m.ctrl.T.Helper() @@ -78,3 +93,129 @@ func (mr *MockFactoryMockRecorder) SyncProducer(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncProducer", reflect.TypeOf((*MockFactory)(nil).SyncProducer), ctx) } + +// MockSyncProducer is a mock of SyncProducer interface. +type MockSyncProducer struct { + ctrl *gomock.Controller + recorder *MockSyncProducerMockRecorder +} + +// MockSyncProducerMockRecorder is the mock recorder for MockSyncProducer. +type MockSyncProducerMockRecorder struct { + mock *MockSyncProducer +} + +// NewMockSyncProducer creates a new mock instance. +func NewMockSyncProducer(ctrl *gomock.Controller) *MockSyncProducer { + mock := &MockSyncProducer{ctrl: ctrl} + mock.recorder = &MockSyncProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyncProducer) EXPECT() *MockSyncProducerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockSyncProducer) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockSyncProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSyncProducer)(nil).Close)) +} + +// SendMessage mocks base method. +func (m *MockSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessage", topic, partitionNum, message) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessage indicates an expected call of SendMessage. +func (mr *MockSyncProducerMockRecorder) SendMessage(topic, partitionNum, message interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), topic, partitionNum, message) +} + +// SendMessages mocks base method. +func (m *MockSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessages", topic, partitionNum, message) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessages indicates an expected call of SendMessages. +func (mr *MockSyncProducerMockRecorder) SendMessages(topic, partitionNum, message interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), topic, partitionNum, message) +} + +// MockAsyncProducer is a mock of AsyncProducer interface. +type MockAsyncProducer struct { + ctrl *gomock.Controller + recorder *MockAsyncProducerMockRecorder +} + +// MockAsyncProducerMockRecorder is the mock recorder for MockAsyncProducer. +type MockAsyncProducerMockRecorder struct { + mock *MockAsyncProducer +} + +// NewMockAsyncProducer creates a new mock instance. +func NewMockAsyncProducer(ctrl *gomock.Controller) *MockAsyncProducer { + mock := &MockAsyncProducer{ctrl: ctrl} + mock.recorder = &MockAsyncProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAsyncProducer) EXPECT() *MockAsyncProducerMockRecorder { + return m.recorder +} + +// AsyncRunCallback mocks base method. +func (m *MockAsyncProducer) AsyncRunCallback(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AsyncRunCallback", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// AsyncRunCallback indicates an expected call of AsyncRunCallback. +func (mr *MockAsyncProducerMockRecorder) AsyncRunCallback(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncRunCallback", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncRunCallback), ctx) +} + +// AsyncSend mocks base method. +func (m *MockAsyncProducer) AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AsyncSend", ctx, topic, partition, message) + ret0, _ := ret[0].(error) + return ret0 +} + +// AsyncSend indicates an expected call of AsyncSend. +func (mr *MockAsyncProducerMockRecorder) AsyncSend(ctx, topic, partition, message interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncSend", reflect.TypeOf((*MockAsyncProducer)(nil).AsyncSend), ctx, topic, partition, message) +} + +// Close mocks base method. +func (m *MockAsyncProducer) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockAsyncProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockAsyncProducer)(nil).Close)) +} diff --git a/pkg/sink/kafka/factory_test.go b/pkg/sink/kafka/factory_test.go deleted file mode 100644 index b418e3c493..0000000000 --- a/pkg/sink/kafka/factory_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "testing" - "time" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestOptionsDerivesRequestTimeout(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - readTimeout time.Duration - writeTimeout time.Duration - expectedRequestTimeout time.Duration - }{ - { - name: "request timeout uses larger write timeout", - readTimeout: time.Second, - writeTimeout: 2 * time.Minute, - expectedRequestTimeout: 2 * time.Minute, - }, - { - name: "request timeout uses larger read timeout", - readTimeout: 5 * time.Second, - writeTimeout: time.Second, - expectedRequestTimeout: 5 * time.Second, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - o := NewOptions() - o.ReadTimeout = tc.readTimeout - o.WriteTimeout = tc.writeTimeout - - require.Equal(t, tc.expectedRequestTimeout, o.requestTimeout()) - }) - } -} - -func TestNewFactoryAdminCreationReturnsKafkaSinkError(t *testing.T) { - t.Parallel() - - options := NewOptions() - options.Version = "invalid" - options.IsAssignedVersion = true - - factory, err := NewFactory( - context.Background(), - options, - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), - ) - require.Nil(t, factory) - requireNewKafkaSinkError(t, err) -} - -func TestFactoryComponentCreationReturnsKafkaSinkError(t *testing.T) { - t.Parallel() - - factory := &factory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), - options: options{ - Version: "invalid", - IsAssignedVersion: true, - sasl: &saslConfig{}, - }, - } - - _, err := factory.Admin(context.Background()) - requireNewKafkaSinkError(t, err) - _, err = factory.SyncProducer(context.Background()) - requireNewKafkaSinkError(t, err) - _, err = factory.AsyncProducer(context.Background()) - requireNewKafkaSinkError(t, err) -} - -func requireNewKafkaSinkError(t *testing.T, err error) { - t.Helper() - - errCode, ok := errors.RFCCode(err) - require.True(t, ok) - require.Equal(t, errors.ErrNewKafkaSink.RFCCode(), errCode) -} diff --git a/pkg/sink/kafka/franz/admin.go b/pkg/sink/kafka/franz/admin.go new file mode 100644 index 0000000000..297f695dc5 --- /dev/null +++ b/pkg/sink/kafka/franz/admin.go @@ -0,0 +1,291 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "strconv" + "strings" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo" +) + +type Broker struct{ ID int32 } + +type TopicDetail struct { + Name string + NumPartitions int32 + ReplicationFactor int16 +} + +type Admin struct { + changefeed common.ChangeFeedID + + client *kgo.Client + admin *kadm.Client + timeout time.Duration +} + +func NewAdmin( + ctx context.Context, + changefeedID common.ChangeFeedID, + cfg Config, +) (*Admin, error) { + opts, err := newClientOptions(ctx, changefeedID, "admin", cfg, nil) + if err != nil { + return nil, err + } + + client, err := kgo.NewClient(opts...) + if err != nil { + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + + return &Admin{ + changefeed: changefeedID, + client: client, + admin: kadm.NewClient(client), + timeout: cfg.requestTimeout(), + }, nil +} + +func (a *Admin) GetAllBrokers() []Broker { + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) + defer cancel() + + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + return nil + } + + brokers := make([]Broker, 0, len(meta.Brokers)) + for id := range meta.Brokers { + brokers = append(brokers, Broker{ID: int32(id)}) + } + + return brokers +} + +func (a *Admin) GetBrokerConfig(configName string) (string, bool, error) { + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) + defer cancel() + + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-cluster", "cluster") + } + + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-cluster", "cluster") + } + + if meta.Controller < 0 { + return "", false, errors.ErrKafkaAdminAPI.GenWithStackByArgs("describe-cluster", "cluster") + } + + configs, err := a.admin.DescribeBrokerConfigs(ctx, meta.Controller) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) + } + + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) + } + + controllerName := strconv.Itoa(int(meta.Controller)) + resource, err := configs.On(controllerName, nil) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) + } + + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) + } + + if resource.Err != nil { + if isAuthorizationFailed(resource.Err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", configName) + } + + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", configName) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + return entry.MaybeValue(), true, nil + } + } + + return "", false, nil +} + +func (a *Admin) GetTopicConfig(topicName string, configName string) (string, bool, error) { + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) + defer cancel() + + configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) + } + + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) + } + + resource, err := configs.On(topicName, nil) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) + } + + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) + } + + if resource.Err != nil { + if isAuthorizationFailed(resource.Err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", topicName) + } + + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", topicName) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + return entry.MaybeValue(), true, nil + } + } + + return "", false, nil +} + +func (a *Admin) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + if len(topics) == 0 { + return make(map[string]TopicDetail), nil + } + + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) + defer cancel() + + meta, err := a.admin.Metadata(ctx, topics...) + if err != nil { + resource := strings.Join(topics, ",") + if isAuthorizationFailed(err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-topics", resource) + } + + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-topics", resource) + } + + return topicDetailsFromMetadata(meta, topics, ignoreTopicError) +} + +func topicDetailsFromMetadata(meta kadm.Metadata, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + result := make(map[string]TopicDetail, len(topics)) + for _, topic := range topics { + detail, ok := meta.Topics[topic] + if !ok { + if ignoreTopicError { + continue + } + + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", topic) + } + + if detail.Err == nil { + result[topic] = TopicDetail{ + Name: topic, + NumPartitions: int32(len(detail.Partitions)), + } + continue + } + + if ignoreTopicError && errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { + continue + } + + if isAuthorizationFailed(detail.Err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, detail.Err, "describe-topic", topic) + } + + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, detail.Err, "describe-topic", topic) + } + + return result, nil +} + +func isAuthorizationFailed(err error) bool { + return errors.Is(err, errors.ErrKafkaAuthorizationFailed) || + errors.Is(err, kerr.TopicAuthorizationFailed) || + errors.Is(err, kerr.ClusterAuthorizationFailed) +} + +func (a *Admin) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + details, err := a.GetTopicsMeta(topics, false) + if err != nil { + return nil, err + } + + partitions := make(map[string]int32, len(details)) + for topic, detail := range details { + partitions[topic] = detail.NumPartitions + } + + return partitions, nil +} + +func (a *Admin) CreateTopic(detail *TopicDetail) error { + ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) + defer cancel() + + responses, err := a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) + if err != nil { + if isAuthorizationFailed(err) { + return errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "create-topic", detail.Name) + } + + return errors.WrapError(errors.ErrKafkaAdminAPI, err, "create-topic", detail.Name) + } + + resp, ok := responses[detail.Name] + if !ok { + return errors.ErrKafkaAdminAPI.GenWithStackByArgs("create-topic", detail.Name) + } + + if resp.Err == nil { + return nil + } + + if errors.Is(resp.Err, kerr.TopicAlreadyExists) { + return nil + } + + if errors.Is(resp.Err, kerr.InvalidReplicationFactor) { + return errors.WrapError(errors.ErrKafkaInvalidConfig, resp.Err) + } + + if isAuthorizationFailed(resp.Err) { + return errors.WrapError(errors.ErrKafkaAuthorizationFailed, resp.Err, "create-topic", detail.Name) + } + + return errors.WrapError(errors.ErrKafkaAdminAPI, resp.Err, "create-topic", detail.Name) +} + +func (a *Admin) Close() { + a.admin.Close() +} diff --git a/pkg/sink/kafka/franz/admin_test.go b/pkg/sink/kafka/franz/admin_test.go new file mode 100644 index 0000000000..d80180fc0b --- /dev/null +++ b/pkg/sink/kafka/franz/admin_test.go @@ -0,0 +1,247 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func TestTopicDetailsFromMetadata(t *testing.T) { + t.Parallel() + + const topic = "topic" + testCases := []struct { + name string + metadata kadm.Metadata + ignoreTopicError bool + expected map[string]TopicDetail + expectedError error + expectedCause error + }{ + { + name: "success", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Partitions: kadm.PartitionDetails{0: {}, 1: {}}}, + }}, + expected: map[string]TopicDetail{ + topic: {Name: topic, NumPartitions: 2}, + }, + }, + { + name: "ignore unknown topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, + }}, + ignoreTopicError: true, + expected: map[string]TopicDetail{}, + }, + { + name: "strict unknown topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, + }}, + expectedError: errors.ErrKafkaAdminAPI, + expectedCause: kerr.UnknownTopicOrPartition, + }, + { + name: "strict missing topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{}}, + expectedError: errors.ErrKafkaAdminAPI, + expectedCause: kerr.UnknownTopicOrPartition, + }, + { + name: "return authorization failure", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.TopicAuthorizationFailed}, + }}, + expectedError: errors.ErrKafkaAuthorizationFailed, + expectedCause: kerr.TopicAuthorizationFailed, + }, + { + name: "do not ignore authorization failure", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.TopicAuthorizationFailed}, + }}, + ignoreTopicError: true, + expectedError: errors.ErrKafkaAuthorizationFailed, + expectedCause: kerr.TopicAuthorizationFailed, + }, + { + name: "do not ignore general failure", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.InvalidTopicException}, + }}, + ignoreTopicError: true, + expectedError: errors.ErrKafkaAdminAPI, + expectedCause: kerr.InvalidTopicException, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + actual, err := topicDetailsFromMetadata(tc.metadata, []string{topic}, tc.ignoreTopicError) + if tc.expectedError != nil { + require.ErrorIs(t, err, tc.expectedError) + if tc.expectedCause != nil { + require.ErrorIs(t, err, tc.expectedCause) + } + return + } + require.NoError(t, err) + require.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsAuthorizationFailed(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + expected bool + }{ + { + name: "TiCDC authorization error", + err: errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs("describe-topic", "test-topic"), + expected: true, + }, + {name: "topic authorization error", err: kerr.TopicAuthorizationFailed, expected: true}, + {name: "cluster authorization error", err: kerr.ClusterAuthorizationFailed, expected: true}, + {name: "general error", err: kerr.InvalidTopicException}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.expected, isAuthorizationFailed(test.err)) + }) + } +} + +func TestAdminOperations(t *testing.T) { + const existingTopic = "existing-topic" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, existingTopic)) + defer cluster.Close() + + admin, err := NewAdmin( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), + testConfig(cluster.ListenAddrs()), + ) + require.NoError(t, err) + defer admin.Close() + + require.Len(t, admin.GetAllBrokers(), 1) + + value, found, err := admin.GetBrokerConfig("message.max.bytes") + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "1048588", value) + + _, found, err = admin.GetBrokerConfig("missing") + require.NoError(t, err) + require.False(t, found) + + value, found, err = admin.GetTopicConfig(existingTopic, "max.message.bytes") + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "1048588", value) + + _, found, err = admin.GetTopicConfig(existingTopic, "missing") + require.NoError(t, err) + require.False(t, found) + + partitions, err := admin.GetTopicsPartitionsNum([]string{existingTopic}) + require.NoError(t, err) + require.Equal(t, map[string]int32{existingTopic: 3}, partitions) + + const topic = "test-topic" + err = admin.CreateTopic(&TopicDetail{ + Name: topic, + NumPartitions: 3, + ReplicationFactor: 1, + }) + require.NoError(t, err) + + topics, err := admin.GetTopicsMeta([]string{topic}, false) + require.NoError(t, err) + require.Equal(t, int32(3), topics[topic].NumPartitions) + require.NoError(t, admin.CreateTopic(&TopicDetail{Name: topic, NumPartitions: 3, ReplicationFactor: 1})) +} + +func TestCreateTopicErrors(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1)) + defer cluster.Close() + + admin, err := NewAdmin( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "create-errors"), + testConfig(cluster.ListenAddrs()), + ) + require.NoError(t, err) + defer admin.Close() + + detail := &TopicDetail{Name: "topic", NumPartitions: 1, ReplicationFactor: 1} + + cluster.ControlKey(int16(kmsg.CreateTopics), func(req kmsg.Request) (kmsg.Response, error, bool) { + return req.ResponseKind(), nil, true + }) + require.ErrorIs(t, admin.CreateTopic(detail), errors.ErrKafkaAdminAPI) + + for _, test := range []struct { + name string + code int16 + expected error + }{ + { + name: "invalid replication factor", + code: kerr.InvalidReplicationFactor.Code, + expected: errors.ErrKafkaInvalidConfig, + }, + { + name: "authorization", + code: kerr.TopicAuthorizationFailed.Code, + expected: errors.ErrKafkaAuthorizationFailed, + }, + { + name: "admin API", + code: kerr.InvalidTopicException.Code, + expected: errors.ErrKafkaAdminAPI, + }, + } { + t.Run(test.name, func(t *testing.T) { + cluster.ControlKey(int16(kmsg.CreateTopics), func(req kmsg.Request) (kmsg.Response, error, bool) { + response := req.ResponseKind().(*kmsg.CreateTopicsResponse) + topic := kmsg.NewCreateTopicsResponseTopic() + topic.Topic, topic.ErrorCode = detail.Name, test.code + response.Topics = append(response.Topics, topic) + + return response, nil, true + }) + + require.ErrorIs(t, admin.CreateTopic(detail), test.expected) + }) + } +} diff --git a/pkg/sink/kafka/async_producer.go b/pkg/sink/kafka/franz/async_producer.go similarity index 63% rename from pkg/sink/kafka/async_producer.go rename to pkg/sink/kafka/franz/async_producer.go index 831337e486..89f2787ece 100644 --- a/pkg/sink/kafka/async_producer.go +++ b/pkg/sink/kafka/franz/async_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kafka +package franz import ( "context" @@ -19,62 +19,53 @@ import ( "time" "github.com/pingcap/log" - commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/codec/common" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/zap" ) -// AsyncProducer is the kafka async producer -type AsyncProducer interface { - // Close shuts down the producer and releases its Kafka client resources. - // Buffered messages fail instead of being flushed. - Close() - - // AsyncSend is the input channel for the user to write messages to that they - // wish to send. - AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error - - // AsyncRunCallback process the messages that has sent to kafka, - // and run tha attached callback. the caller should call this - // method in a background goroutine - AsyncRunCallback(ctx context.Context) error -} - -type asyncProducer struct { +type AsyncProducer struct { client *kgo.Client - changefeedID commonType.ChangeFeedID + changefeedID common.ChangeFeedID closeStarted atomic.Bool closed atomic.Bool errCh chan error } -func newAsyncProducer( +func NewAsyncProducer( ctx context.Context, - changefeedID commonType.ChangeFeedID, - o *options, + changefeedID common.ChangeFeedID, + cfg Config, hook *metricsHook, -) (*asyncProducer, error) { - opts, err := newOptions(ctx, o, hook) +) (*AsyncProducer, error) { + opts, err := newClientOptions(ctx, changefeedID, "async-producer", cfg, hook) + if err != nil { + return nil, err + } + + producerOpts, err := producerOptions(cfg) if err != nil { - return nil, errors.Trace(err) + return nil, err } - opts = append(opts, newProducerOptions(o)...) + + opts = append(opts, producerOpts...) + client, err := kgo.NewClient(opts...) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } - return &asyncProducer{ + return &AsyncProducer{ client: client, changefeedID: changefeedID, errCh: make(chan error, 1), }, nil } -func (p *asyncProducer) Close() { +func (p *AsyncProducer) Close() { if !p.closeStarted.CompareAndSwap(false, true) { return } @@ -82,17 +73,18 @@ func (p *asyncProducer) Close() { start := time.Now() p.client.Close() + log.Info("kafka async producer closed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), zap.Duration("duration", time.Since(start))) } -func (p *asyncProducer) AsyncSend( +func (p *AsyncProducer) AsyncSend( ctx context.Context, topic string, partition int32, - message *common.Message, + message *codeccommon.Message, ) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() @@ -118,32 +110,36 @@ func (p *asyncProducer) AsyncSend( p.enqueueAsyncSendError(logInfo, err) return } + if callback != nil { callback() } } + p.client.Produce(ctx, record, promise) + return nil } -func (p *asyncProducer) enqueueAsyncSendError( - logInfo *common.MessageLogInfo, +func (p *AsyncProducer) enqueueAsyncSendError( + logInfo *codeccommon.MessageLogInfo, err error, ) { log.Error("kafka message send failed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), - zap.String("eventContext", BuildEventLogContext( + zap.String("eventContext", buildEventLogContext( p.changefeedID.Keyspace(), p.changefeedID.Name(), logInfo)), zap.Error(err)) + select { case p.errCh <- errors.WrapError(errors.ErrKafkaSendMessage, err): - // todo: remove this default after support dispatcher recover logic. + // Keep the first error until the dispatcher can recover from multiple errors. default: } } -func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { +func (p *AsyncProducer) AsyncRunCallback(ctx context.Context) error { defer p.closed.Store(true) for { select { diff --git a/pkg/sink/kafka/franz/async_producer_test.go b/pkg/sink/kafka/franz/async_producer_test.go new file mode 100644 index 0000000000..9a928e5bc9 --- /dev/null +++ b/pkg/sink/kafka/franz/async_producer_test.go @@ -0,0 +1,196 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func TestAsyncSendClosedProducer(t *testing.T) { + producer := &AsyncProducer{} + producer.closed.Store(true) + + err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) + + require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) +} + +func TestAsyncSendCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + producer := &AsyncProducer{} + require.ErrorIs(t, producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{}), context.Canceled) +} + +func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { + producer := &AsyncProducer{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), + errCh: make(chan error, 1), + } + producer.errCh <- context.DeadlineExceeded + + err := producer.AsyncRunCallback(context.Background()) + + require.ErrorIs(t, err, context.DeadlineExceeded) + require.True(t, producer.closed.Load()) +} + +func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { + client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) + require.NoError(t, err) + + var callbackCalled atomic.Bool + producer := &AsyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), + errCh: make(chan error, 1), + } + err = producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{ + Callback: func() { + callbackCalled.Store(true) + }, + }) + require.NoError(t, err) + + producer.Close() + + require.False(t, callbackCalled.Load()) + err = producer.AsyncRunCallback(context.Background()) + require.ErrorIs(t, err, kgo.ErrClientClosed) +} + +func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { + const topic = "async-topic" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + producer, err := NewAsyncProducer( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), + testConfig(cluster.ListenAddrs()), + nil, + ) + require.NoError(t, err) + defer producer.Close() + + var calls atomic.Int32 + called := make(chan struct{}, 10) + message := &codeccommon.Message{ + Value: []byte("value"), + Callback: func() { + calls.Add(1) + called <- struct{}{} + }, + } + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, message)) + + select { + case <-called: + case <-time.After(time.Second): + t.Fatal("produce callback was not called") + } + + time.Sleep(20 * time.Millisecond) + require.Equal(t, int32(1), calls.Load()) +} + +func TestAsyncProducerReportsProduceFailure(t *testing.T) { + const topic = "async-error" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + return produceResponseWithError(req, 0, kerr.InvalidTopicException.Code) + }) + + producer, err := NewAsyncProducer( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), + testConfig(cluster.ListenAddrs()), + nil, + ) + require.NoError(t, err) + defer producer.Close() + + var callbackCalled atomic.Bool + message := &codeccommon.Message{ + Value: []byte("value"), + Callback: func() { callbackCalled.Store(true) }, + } + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, message)) + + callbackCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + err = producer.AsyncRunCallback(callbackCtx) + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) + require.ErrorIs(t, err, kerr.InvalidTopicException) + require.False(t, callbackCalled.Load()) +} + +func TestBufferBackpressureCanBeCanceled(t *testing.T) { + client, err := kgo.NewClient( + kgo.SeedBrokers("127.0.0.1:1"), + kgo.MaxBufferedBytes(10), + kgo.RecordRetries(100), + ) + require.NoError(t, err) + + producer := &AsyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "backpressure"), + errCh: make(chan error, 1), + } + t.Cleanup(producer.Close) + + require.NoError(t, producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{Value: make([]byte, 10)})) + require.Equal(t, int64(1), client.BufferedProduceRecords()) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{Value: make([]byte, 10)}) + }() + + select { + case <-done: + t.Fatal("second send did not wait for buffer space") + case <-time.After(50 * time.Millisecond): + } + + cancel() + + select { + case err = <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("canceled send remained blocked") + } + + callbackCtx, callbackCancel := context.WithTimeout(context.Background(), time.Second) + defer callbackCancel() + require.ErrorIs(t, producer.AsyncRunCallback(callbackCtx), context.Canceled) +} diff --git a/pkg/sink/kafka/franz/config.go b/pkg/sink/kafka/franz/config.go new file mode 100644 index 0000000000..d1f2f1803a --- /dev/null +++ b/pkg/sink/kafka/franz/config.go @@ -0,0 +1,249 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "crypto/tls" + "net/url" + "strings" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/kversion" + "github.com/twmb/franz-go/pkg/sasl" + "github.com/twmb/franz-go/pkg/sasl/oauth" + "github.com/twmb/franz-go/pkg/sasl/plain" + "github.com/twmb/franz-go/pkg/sasl/scram" + "go.uber.org/zap" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +const ( + defaultMaxBufferedBytes = 64 << 20 + defaultBrokerWriteBytes = 100 << 20 + minProducerBatchBytes = 512 + maxProducerBatchBytes = 1 << 30 + + NoResponse = int16(0) + WaitForLocal = int16(1) + WaitForAll = int16(-1) +) + +type Config struct { + BrokerEndpoints []string + ClientID string + Version string + AssignedVersion bool + MaxMessageBytes int + MaxRetry int + Compression string + RequiredAcks int16 + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + TLSConfig *tls.Config + SASL *SASLConfig +} + +type SASLConfig struct { + Mechanism string + User string + Password string + GSSAPI GSSAPIConfig + OAuth2 OAuth2Config +} + +type GSSAPIConfig struct { + AuthType int + KeyTabPath string + KerberosConfigPath string + ServiceName string + Username string + Password string + Realm string + DisablePAFXFAST bool +} + +type OAuth2Config struct { + ClientID string + ClientSecret string + TokenURL string + Scopes []string + GrantType string + Audience string +} + +func (c Config) requestTimeout() time.Duration { return max(c.ReadTimeout, c.WriteTimeout) } + +func newClientOptions( + ctx context.Context, + changefeedID common.ChangeFeedID, + role string, + cfg Config, + hook *metricsHook, +) ([]kgo.Opt, error) { + opts := []kgo.Opt{ + kgo.WithContext(ctx), + kgo.SeedBrokers(cfg.BrokerEndpoints...), + kgo.ClientID(cfg.ClientID), + kgo.DialTimeout(cfg.DialTimeout), + kgo.RequestTimeoutOverhead(cfg.requestTimeout()), + kgo.WithLogger(newLogger(changefeedID, role)), + } + if hook != nil { + opts = append(opts, kgo.WithHooks(hook)) + } + + if cfg.AssignedVersion { + versions := kversion.FromString(cfg.Version) + if versions == nil { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("invalid kafka version %s", cfg.Version) + } + opts = append(opts, kgo.MaxVersions(versions)) + } + + if cfg.TLSConfig != nil { + opts = append(opts, kgo.DialTLSConfig(cfg.TLSConfig)) + } + + if cfg.SASL != nil && cfg.SASL.Mechanism != "" { + mechanism, err := buildSASLMechanism(ctx, *cfg.SASL) + if err != nil { + return nil, err + } + opts = append(opts, kgo.SASL(mechanism)) + } + + return opts, nil +} + +func buildSASLMechanism(ctx context.Context, cfg SASLConfig) (sasl.Mechanism, error) { + switch strings.ToUpper(cfg.Mechanism) { + case "PLAIN": + return plain.Auth{User: cfg.User, Pass: cfg.Password}.AsMechanism(), nil + case "SCRAM-SHA-256": + return scram.Auth{User: cfg.User, Pass: cfg.Password}.AsSha256Mechanism(), nil + case "SCRAM-SHA-512": + return scram.Auth{User: cfg.User, Pass: cfg.Password}.AsSha512Mechanism(), nil + case "OAUTHBEARER": + tokenSource, err := newOAuthTokenSource(ctx, cfg.OAuth2) + if err != nil { + return nil, err + } + return oauth.Oauth(func(context.Context) (oauth.Auth, error) { + token, err := tokenSource.Token() + if err != nil { + return oauth.Auth{}, errors.WrapError(errors.ErrNewKafkaSink, err) + } + return oauth.Auth{Token: token.AccessToken}, nil + }), nil + case "GSSAPI": + return buildGSSAPIMechanism(cfg.GSSAPI) + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", cfg.Mechanism) + } +} + +func newOAuthTokenSource(ctx context.Context, cfg OAuth2Config) (oauth2.TokenSource, error) { + endpointParams := url.Values{} + if cfg.GrantType != "" { + endpointParams.Set("grant_type", cfg.GrantType) + } + if cfg.Audience != "" { + endpointParams.Set("audience", cfg.Audience) + } + tokenURL, err := url.Parse(cfg.TokenURL) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + config := &clientcredentials.Config{ + ClientID: cfg.ClientID, + ClientSecret: cfg.ClientSecret, + TokenURL: tokenURL.String(), + EndpointParams: endpointParams, + Scopes: cfg.Scopes, + } + return config.TokenSource(ctx), nil +} + +func producerOptions(cfg Config) ([]kgo.Opt, error) { + if cfg.MaxMessageBytes > maxProducerBatchBytes { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "max-message-bytes %d exceeds franz-go limit %d", + cfg.MaxMessageBytes, + maxProducerBatchBytes, + ) + } + + // Use 64 MiB as the default budget, but never make it smaller than the configured message limit. + // Keep franz-go's 10,000-record default as a second bound. + maxBufferedBytes := max(defaultMaxBufferedBytes, cfg.MaxMessageBytes) + maxBatchBytes := max(minProducerBatchBytes, cfg.MaxMessageBytes) + maxBrokerWriteBytes := max(defaultBrokerWriteBytes, maxBatchBytes) + + return []kgo.Opt{ + kgo.RecordPartitioner(kgo.ManualPartitioner()), + kgo.RequiredAcks(requiredAcks(cfg.RequiredAcks)), + kgo.DisableIdempotentWrite(), + kgo.MaxProduceRequestsInflightPerBroker(1), + kgo.RecordRetries(cfg.MaxRetry), + kgo.UnknownTopicRetries(cfg.MaxRetry), + kgo.MaxBufferedBytes(maxBufferedBytes), + kgo.ProducerBatchMaxBytes(int32(maxBatchBytes)), + kgo.BrokerMaxWriteBytes(int32(maxBrokerWriteBytes)), + kgo.ProduceRequestTimeout(cfg.requestTimeout()), + kgo.ProducerLinger(0), + compressionOption(cfg.Compression), + }, nil +} + +func requiredAcks(required int16) kgo.Acks { + switch required { + case WaitForAll: + return kgo.AllISRAcks() + case WaitForLocal: + return kgo.LeaderAck() + case NoResponse: + return kgo.NoAck() + default: + log.Warn("unsupported required acks", zap.Int16("requiredAcks", required)) + return kgo.AllISRAcks() + } +} + +func compressionOption(compression string) kgo.Opt { + var codec kgo.CompressionCodec + switch strings.ToLower(strings.TrimSpace(compression)) { + case "", "none": + codec = kgo.NoCompression() + case "gzip": + codec = kgo.GzipCompression() + case "snappy": + codec = kgo.SnappyCompression() + case "lz4": + codec = kgo.Lz4Compression() + case "zstd": + codec = kgo.ZstdCompression() + default: + log.Warn("unsupported kafka compression algorithm", zap.String("compression", compression)) + codec = kgo.NoCompression() + } + return kgo.ProducerBatchCompression(codec) +} diff --git a/pkg/sink/kafka/franz/config_test.go b/pkg/sink/kafka/franz/config_test.go new file mode 100644 index 0000000000..2b010c586f --- /dev/null +++ b/pkg/sink/kafka/franz/config_test.go @@ -0,0 +1,249 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" +) + +func testConfig(brokers []string) Config { + return Config{ + BrokerEndpoints: brokers, + MaxMessageBytes: 1 << 20, + MaxRetry: 1, + RequiredAcks: WaitForAll, + DialTimeout: time.Second, + ReadTimeout: time.Second, + WriteTimeout: time.Second, + } +} + +func TestRequiredAcks(t *testing.T) { + for _, test := range []struct { + required int16 + expected kgo.Acks + }{ + {required: WaitForAll, expected: kgo.AllISRAcks()}, + {required: WaitForLocal, expected: kgo.LeaderAck()}, + {required: NoResponse, expected: kgo.NoAck()}, + {required: 2, expected: kgo.AllISRAcks()}, + } { + require.Equal(t, test.expected, requiredAcks(test.required)) + } +} + +func TestRequestTimeoutUsesLargerTimeout(t *testing.T) { + cfg := Config{ReadTimeout: time.Second, WriteTimeout: 2 * time.Second} + require.Equal(t, 2*time.Second, cfg.requestTimeout()) + + cfg.ReadTimeout = 3 * time.Second + require.Equal(t, 3*time.Second, cfg.requestTimeout()) +} + +func TestProducerOptionsBoundBufferAndBatch(t *testing.T) { + const batchBytes = 1048588 + cfg := testConfig([]string{"127.0.0.1:9092"}) + cfg.MaxMessageBytes = batchBytes + + opts, err := newClientOptions( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "config"), + "test", + cfg, + nil, + ) + require.NoError(t, err) + + producerOpts, err := producerOptions(cfg) + require.NoError(t, err) + + client, err := kgo.NewClient(append(opts, producerOpts...)...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, int32(batchBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, int64(defaultMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) + require.Equal(t, int64(10000), client.OptValue(kgo.MaxBufferedRecords)) + require.Equal(t, int64(1), client.OptValue(kgo.RecordRetries)) + require.Equal(t, int64(1), client.OptValue(kgo.UnknownTopicRetries)) + require.Equal(t, int32(defaultBrokerWriteBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) +} + +func TestProducerLimitsScaleWithConfiguredMessage(t *testing.T) { + maxMessageBytes := defaultBrokerWriteBytes + 1 + config := testConfig([]string{"127.0.0.1:9092"}) + config.MaxMessageBytes = maxMessageBytes + + producerOpts, err := producerOptions(config) + require.NoError(t, err) + + client, err := kgo.NewClient(producerOpts...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, int64(maxMessageBytes), client.OptValue(kgo.MaxBufferedBytes)) + require.Equal(t, int32(maxMessageBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) +} + +func TestProducerOptionsClampSmallBatch(t *testing.T) { + config := testConfig([]string{"127.0.0.1:9092"}) + config.MaxMessageBytes = minProducerBatchBytes - 1 + + producerOpts, err := producerOptions(config) + require.NoError(t, err) + + client, err := kgo.NewClient(producerOpts...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, int32(minProducerBatchBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) +} + +func TestProducerOptionsRejectOversizedBatch(t *testing.T) { + config := testConfig([]string{"127.0.0.1:9092"}) + config.MaxMessageBytes = maxProducerBatchBytes + 1 + + _, err := producerOptions(config) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} + +func TestCompressionOptions(t *testing.T) { + for _, test := range []struct { + name string + compression string + expected kgo.CompressionCodec + }{ + {name: "none", compression: "none", expected: kgo.NoCompression()}, + {name: "gzip", compression: "gzip", expected: kgo.GzipCompression()}, + {name: "snappy", compression: "snappy", expected: kgo.SnappyCompression()}, + {name: "lz4", compression: "lz4", expected: kgo.Lz4Compression()}, + {name: "zstd", compression: "zstd", expected: kgo.ZstdCompression()}, + {name: "unknown falls back to none", compression: "unknown", expected: kgo.NoCompression()}, + } { + t.Run(test.name, func(t *testing.T) { + cfg := testConfig([]string{"127.0.0.1:9092"}) + cfg.Compression = test.compression + + producerOpts, err := producerOptions(cfg) + require.NoError(t, err) + + client, err := kgo.NewClient(producerOpts...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, []kgo.CompressionCodec{test.expected}, client.OptValue(kgo.ProducerBatchCompression)) + }) + } +} + +func TestInvalidAssignedVersionUsesInvalidConfigError(t *testing.T) { + cfg := testConfig([]string{"127.0.0.1:9092"}) + cfg.Version = "invalid" + cfg.AssignedVersion = true + + _, err := NewAdmin( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "invalid-version"), + cfg, + ) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} + +func TestBuildGSSAPIMechanism(t *testing.T) { + for _, cfg := range []GSSAPIConfig{ + {AuthType: userAuth, Password: "pwd"}, + {AuthType: keyTabAuth, KeyTabPath: "/tmp/a.keytab"}, + } { + cfg.KerberosConfigPath = "/etc/krb5.conf" + cfg.ServiceName = "kafka" + cfg.Username = "alice" + cfg.Realm = "EXAMPLE.COM" + + mechanism, err := buildSASLMechanism(context.Background(), SASLConfig{ + Mechanism: "GSSAPI", + GSSAPI: cfg, + }) + require.NoError(t, err) + require.Equal(t, "GSSAPI", mechanism.Name()) + } +} + +func TestBuildSASLMechanisms(t *testing.T) { + for _, mechanism := range []string{"PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"} { + actual, err := buildSASLMechanism(context.Background(), SASLConfig{ + Mechanism: mechanism, + User: "alice", + Password: "secret", + }) + require.NoError(t, err) + require.Equal(t, mechanism, actual.Name()) + } + + _, err := buildSASLMechanism(context.Background(), SASLConfig{Mechanism: "unknown"}) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} + +func TestOAuthTokenSource(t *testing.T) { + request := make(chan url.Values, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Errorf("parse token request: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + request <- r.PostForm + + w.Header().Set("Content-Type", "application/json") + if _, err := io.WriteString(w, `{"access_token":"token","token_type":"bearer"}`); err != nil { + t.Errorf("write token response: %v", err) + } + })) + defer server.Close() + + source, err := newOAuthTokenSource(context.Background(), OAuth2Config{ + ClientID: "client", + ClientSecret: "secret", + TokenURL: server.URL, + Scopes: []string{"scope-a", "scope-b"}, + GrantType: "custom", + Audience: "audience", + }) + require.NoError(t, err) + + token, err := source.Token() + require.NoError(t, err) + require.Equal(t, "token", token.AccessToken) + + form := <-request + require.Equal(t, "custom", form.Get("grant_type")) + require.Equal(t, "audience", form.Get("audience")) + require.Equal(t, "scope-a scope-b", form.Get("scope")) +} + +func TestOAuthTokenSourceRejectsInvalidURL(t *testing.T) { + _, err := newOAuthTokenSource(context.Background(), OAuth2Config{TokenURL: "http://example.com/%%"}) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/franz/factory.go new file mode 100644 index 0000000000..9e77eda44d --- /dev/null +++ b/pkg/sink/kafka/franz/factory.go @@ -0,0 +1,52 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + + "github.com/pingcap/ticdc/pkg/common" +) + +type Factory struct { + changefeedID common.ChangeFeedID + config Config +} + +func NewFactory(config Config, changefeedID common.ChangeFeedID) *Factory { + return &Factory{changefeedID: changefeedID, config: config} +} + +func (f *Factory) Admin(ctx context.Context) (*Admin, error) { + return NewAdmin(ctx, f.changefeedID, f.config) +} + +func (f *Factory) SyncProducer(ctx context.Context) (*SyncProducer, error) { + producer, err := NewSyncProducer(ctx, f.changefeedID, f.config, newMetricsHook(f.changefeedID)) + if err != nil { + CleanupMetrics(f.changefeedID) + } + return producer, err +} + +func (f *Factory) AsyncProducer(ctx context.Context) (*AsyncProducer, error) { + producer, err := NewAsyncProducer(ctx, f.changefeedID, f.config, newMetricsHook(f.changefeedID)) + if err != nil { + CleanupMetrics(f.changefeedID) + } + return producer, err +} + +func (f *Factory) CleanupMetrics() { CleanupMetrics(f.changefeedID) } diff --git a/pkg/sink/kafka/franz/factory_test.go b/pkg/sink/kafka/franz/factory_test.go new file mode 100644 index 0000000000..477cc37b54 --- /dev/null +++ b/pkg/sink/kafka/franz/factory_test.go @@ -0,0 +1,63 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" +) + +func TestFactoryCreatesAllClients(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1)) + defer cluster.Close() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory") + factory := NewFactory(testConfig(cluster.ListenAddrs()), changefeedID) + + admin, err := factory.Admin(context.Background()) + require.NoError(t, err) + admin.Close() + + syncProducer, err := factory.SyncProducer(context.Background()) + require.NoError(t, err) + syncProducer.Close() + + asyncProducer, err := factory.AsyncProducer(context.Background()) + require.NoError(t, err) + asyncProducer.Close() + + factory.CleanupMetrics() +} + +func TestFactoryCleansMetricsAfterProducerConstructionFailure(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory-error") + config := testConfig([]string{"127.0.0.1:9092"}) + config.Version = "invalid" + config.AssignedVersion = true + factory := NewFactory(config, changefeedID) + + _, err := factory.SyncProducer(context.Background()) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.False(t, recordsPerBatch.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name())) + + _, err = factory.AsyncProducer(context.Background()) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.False(t, recordsPerBatch.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name())) +} diff --git a/pkg/sink/kafka/gssapi.go b/pkg/sink/kafka/franz/gssapi.go similarity index 62% rename from pkg/sink/kafka/gssapi.go rename to pkg/sink/kafka/franz/gssapi.go index d03e372ee9..df2b6ef93d 100644 --- a/pkg/sink/kafka/gssapi.go +++ b/pkg/sink/kafka/franz/gssapi.go @@ -11,13 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kafka +package franz import ( "context" "encoding/binary" - "fmt" - "strings" + "net" "github.com/jcmturner/gofork/encoding/asn1" "github.com/jcmturner/gokrb5/v8/asn1tools" @@ -36,53 +35,85 @@ import ( const ( tokIDKrbAPReq = 256 gssAPIGeneric = 0x60 + userAuth = 1 + keyTabAuth = 2 ) type gssapiMechanism struct { - config gssapiConfig + config GSSAPIConfig + newClient func(GSSAPIConfig) (kerberosClient, error) + newToken func(string, types.PrincipalName, messages.Ticket, types.EncryptionKey) ([]byte, error) +} + +type kerberosClient interface { + Login() error + Destroy() + GetServiceTicket(string) (messages.Ticket, types.EncryptionKey, error) + Domain() string + CName() types.PrincipalName +} + +type gokrb5Client struct { + *client.Client } func (m *gssapiMechanism) Name() string { - return string(gssapiMechanismName) + return "GSSAPI" } func (m *gssapiMechanism) Authenticate( _ context.Context, host string, ) (sasl.Session, []byte, error) { - client, err := newKerberosClient(m.config) + client, err := m.newClient(m.config) if err != nil { - return nil, nil, errors.Trace(err) + return nil, nil, err } + if err = client.Login(); err != nil { client.Destroy() - return nil, nil, errors.Trace(err) + return nil, nil, errors.WrapError(errors.ErrNewKafkaSink, err) } - serverHost := strings.SplitN(host, ":", 2)[0] - spn := fmt.Sprintf("%s/%s", m.config.serviceName, serverHost) + serverHost, err := brokerHost(host) + if err != nil { + client.Destroy() + return nil, nil, err + } + + spn := m.config.ServiceName + "/" + serverHost ticket, encKey, err := client.GetServiceTicket(spn) if err != nil { client.Destroy() - return nil, nil, errors.Trace(err) + return nil, nil, errors.WrapError(errors.ErrNewKafkaSink, err) } - token, err := newKrb5Token( - client.Credentials.Domain(), client.Credentials.CName(), ticket, encKey) + token, err := m.newToken(client.Domain(), client.CName(), ticket, encKey) if err != nil { client.Destroy() - return nil, nil, errors.Trace(err) + return nil, nil, err } + firstMessage, err := appendGSSAPIHeader(token) if err != nil { client.Destroy() - return nil, nil, errors.Trace(err) + return nil, nil, err } + return &gssapiSession{client: client, encKey: encKey}, firstMessage, nil } +func brokerHost(address string) (string, error) { + host, _, err := net.SplitHostPort(address) + if err != nil { + return "", errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + + return host, nil +} + type gssapiSession struct { - client *client.Client + client kerberosClient encKey types.EncryptionKey } @@ -91,89 +122,102 @@ func (s *gssapiSession) Challenge(challenge []byte) (bool, []byte, error) { wrapTokenReq := gssapi.WrapToken{} if err := wrapTokenReq.Unmarshal(challenge, true); err != nil { - return false, nil, errors.Trace(err) + return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) } + isValid, err := wrapTokenReq.Verify(s.encKey, keyusage.GSSAPI_ACCEPTOR_SEAL) if !isValid { if err != nil { - return false, nil, errors.Trace(err) + return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) } - return false, nil, errors.New("invalid gssapi wrap token") + + return false, nil, errors.ErrNewKafkaSink.GenWithStackByArgs() } wrapTokenResp, err := gssapi.NewInitiatorWrapToken(wrapTokenReq.Payload, s.encKey) if err != nil { - return false, nil, errors.Trace(err) + return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) } + msg, err := wrapTokenResp.Marshal() if err != nil { - return false, nil, errors.Trace(err) + return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) } + return true, msg, nil } -func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { - if g.serviceName == "" { +func buildGSSAPIMechanism(g GSSAPIConfig) (sasl.Mechanism, error) { + if g.ServiceName == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") } - if g.kerberosConfigPath == "" { + if g.KerberosConfigPath == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") } - if g.username == "" { + if g.Username == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") } - if g.realm == "" { + if g.Realm == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") } - switch g.authType { + switch g.AuthType { case userAuth: - if g.password == "" { + if g.Password == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") } case keyTabAuth: - if g.keyTabPath == "" { + if g.KeyTabPath == "" { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") } default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.authType) + "unsupported sasl-gssapi-auth-type %d", g.AuthType) } - return &gssapiMechanism{config: g}, nil + return &gssapiMechanism{ + config: g, + newClient: newKerberosClient, + newToken: newKrb5Token, + }, nil } -func newKerberosClient(g gssapiConfig) (*client.Client, error) { - cfg, err := config.Load(g.kerberosConfigPath) +func newKerberosClient(g GSSAPIConfig) (kerberosClient, error) { + cfg, err := config.Load(g.KerberosConfigPath) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } var krbClient *client.Client - switch g.authType { + switch g.AuthType { case keyTabAuth: - kt, err := keytab.Load(g.keyTabPath) + kt, err := keytab.Load(g.KeyTabPath) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } krbClient = client.NewWithKeytab( - g.username, g.realm, kt, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)) + g.Username, g.Realm, kt, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)) case userAuth: krbClient = client.NewWithPassword( - g.username, g.realm, g.password, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)) + g.Username, g.Realm, g.Password, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)) default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.authType) + "unsupported sasl-gssapi-auth-type %d", g.AuthType) } - return krbClient, nil + + return &gokrb5Client{Client: krbClient}, nil } +func (c *gokrb5Client) Domain() string { return c.Credentials.Domain() } + +func (c *gokrb5Client) CName() types.PrincipalName { return c.Credentials.CName() } + func newKrb5Token( domain string, cname types.PrincipalName, @@ -182,42 +226,49 @@ func newKrb5Token( ) ([]byte, error) { authenticator, err := types.NewAuthenticator(domain, cname) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } authenticator.Cksum = types.Checksum{ CksumType: chksumtype.GSSAPI, Checksum: newAuthenticatorChecksum(), } + apReq, err := messages.NewAPReq(ticket, sessionKey, authenticator) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } body, err := apReq.Marshal() if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } + prefix := make([]byte, 2, 2+len(body)) binary.BigEndian.PutUint16(prefix, tokIDKrbAPReq) + return append(prefix, body...), nil } func newAuthenticatorChecksum() []byte { sum := make([]byte, 24) binary.LittleEndian.PutUint32(sum[:4], 16) + flags := uint32(gssapi.ContextFlagInteg | gssapi.ContextFlagConf) binary.LittleEndian.PutUint32(sum[20:24], flags) + return sum } func appendGSSAPIHeader(payload []byte) ([]byte, error) { oidBytes, err := asn1.Marshal(gssapi.OIDKRB5.OID()) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } + tkoLengthBytes := asn1tools.MarshalLengthBytes(len(oidBytes) + len(payload)) header := append([]byte{gssAPIGeneric}, tkoLengthBytes...) header = append(header, oidBytes...) + return append(header, payload...), nil } diff --git a/pkg/sink/kafka/franz/gssapi_test.go b/pkg/sink/kafka/franz/gssapi_test.go new file mode 100644 index 0000000000..1995cf219f --- /dev/null +++ b/pkg/sink/kafka/franz/gssapi_test.go @@ -0,0 +1,233 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "encoding/binary" + "testing" + + "github.com/jcmturner/gokrb5/v8/gssapi" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/iana/nametype" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +type fakeKerberosClient struct { + loginError error + ticketError error + spn string + destroyed bool +} + +func (c *fakeKerberosClient) Login() error { return c.loginError } + +func (c *fakeKerberosClient) Destroy() { c.destroyed = true } + +func (c *fakeKerberosClient) GetServiceTicket(spn string) ( + messages.Ticket, + types.EncryptionKey, + error, +) { + c.spn = spn + + return messages.Ticket{}, testEncryptionKey(), c.ticketError +} + +func (c *fakeKerberosClient) Domain() string { return "EXAMPLE.COM" } + +func (c *fakeKerberosClient) CName() types.PrincipalName { + return types.NewPrincipalName(nametype.KRB_NT_PRINCIPAL, "alice") +} + +func TestBrokerHost(t *testing.T) { + for _, test := range []struct { + address string + expected string + }{ + {address: "broker.example.com:9092", expected: "broker.example.com"}, + {address: "[2001:db8::1]:9092", expected: "2001:db8::1"}, + } { + host, err := brokerHost(test.address) + require.NoError(t, err) + require.Equal(t, test.expected, host) + } + + _, err := brokerHost("2001:db8::1:9092") + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} + +func TestGSSAPIConfigValidation(t *testing.T) { + valid := GSSAPIConfig{ + AuthType: userAuth, + KerberosConfigPath: "/etc/krb5.conf", + ServiceName: "kafka", + Username: "alice", + Password: "secret", + Realm: "EXAMPLE.COM", + } + + for _, mutate := range []func(*GSSAPIConfig){ + func(cfg *GSSAPIConfig) { cfg.ServiceName = "" }, + func(cfg *GSSAPIConfig) { cfg.KerberosConfigPath = "" }, + func(cfg *GSSAPIConfig) { cfg.Username = "" }, + func(cfg *GSSAPIConfig) { cfg.Realm = "" }, + func(cfg *GSSAPIConfig) { cfg.Password = "" }, + func(cfg *GSSAPIConfig) { cfg.AuthType = 0 }, + func(cfg *GSSAPIConfig) { cfg.AuthType, cfg.KeyTabPath = keyTabAuth, "" }, + } { + cfg := valid + mutate(&cfg) + + _, err := buildGSSAPIMechanism(cfg) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + } +} + +func TestGSSAPIEncodingHelpers(t *testing.T) { + checksum := newAuthenticatorChecksum() + require.Len(t, checksum, 24) + require.Equal(t, uint32(16), binary.LittleEndian.Uint32(checksum[:4])) + require.Equal(t, uint32(gssapi.ContextFlagInteg|gssapi.ContextFlagConf), binary.LittleEndian.Uint32(checksum[20:24])) + + payload := []byte{1, 2, 3} + message, err := appendGSSAPIHeader(payload) + require.NoError(t, err) + require.Equal(t, byte(gssAPIGeneric), message[0]) + require.Equal(t, payload, message[len(message)-len(payload):]) +} + +func TestNewKerberosClientRejectsMissingConfig(t *testing.T) { + _, err := newKerberosClient(GSSAPIConfig{ + AuthType: userAuth, + KerberosConfigPath: "/path/that/does/not/exist", + Username: "alice", + Password: "secret", + Realm: "EXAMPLE.COM", + }) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} + +func TestGSSAPIAuthenticate(t *testing.T) { + client := &fakeKerberosClient{} + mechanism := &gssapiMechanism{ + config: GSSAPIConfig{ServiceName: "kafka"}, + newClient: func(GSSAPIConfig) (kerberosClient, error) { + return client, nil + }, + newToken: func( + domain string, + cname types.PrincipalName, + _ messages.Ticket, + _ types.EncryptionKey, + ) ([]byte, error) { + require.Equal(t, "EXAMPLE.COM", domain) + require.Equal(t, "alice", cname.PrincipalNameString()) + + return []byte{1, 2, 3}, nil + }, + } + + session, message, err := mechanism.Authenticate(context.Background(), "[2001:db8::1]:9092") + require.NoError(t, err) + require.NotNil(t, session) + require.Equal(t, "kafka/2001:db8::1", client.spn) + require.Equal(t, []byte{1, 2, 3}, message[len(message)-3:]) + require.False(t, client.destroyed) + + done, _, err := session.Challenge([]byte("invalid")) + require.False(t, done) + require.ErrorIs(t, err, errors.ErrNewKafkaSink) + require.True(t, client.destroyed) +} + +func TestGSSAPIAuthenticateDestroysClientAfterFailure(t *testing.T) { + for _, test := range []struct { + name string + host string + loginError error + ticketError error + tokenError error + }{ + {name: "login", host: "broker:9092", loginError: context.Canceled}, + {name: "broker address", host: "invalid"}, + {name: "service ticket", host: "broker:9092", ticketError: context.DeadlineExceeded}, + {name: "AP request", host: "broker:9092", tokenError: context.Canceled}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeKerberosClient{ + loginError: test.loginError, + ticketError: test.ticketError, + } + mechanism := &gssapiMechanism{ + config: GSSAPIConfig{ServiceName: "kafka"}, + newClient: func(GSSAPIConfig) (kerberosClient, error) { + return client, nil + }, + newToken: func( + string, + types.PrincipalName, + messages.Ticket, + types.EncryptionKey, + ) ([]byte, error) { + return nil, test.tokenError + }, + } + + _, _, err := mechanism.Authenticate(context.Background(), test.host) + require.Error(t, err) + require.True(t, client.destroyed) + }) + } +} + +func TestGSSAPIChallenge(t *testing.T) { + key := testEncryptionKey() + request := gssapi.WrapToken{ + Flags: 1, + EC: 12, + Payload: []byte{1, 2, 3, 4}, + } + require.NoError(t, request.SetCheckSum(key, keyusage.GSSAPI_ACCEPTOR_SEAL)) + + challenge, err := request.Marshal() + require.NoError(t, err) + + client := &fakeKerberosClient{} + session := &gssapiSession{client: client, encKey: key} + done, response, err := session.Challenge(challenge) + require.NoError(t, err) + require.True(t, done) + require.True(t, client.destroyed) + + initiatorToken := gssapi.WrapToken{} + require.NoError(t, initiatorToken.Unmarshal(response, false)) + require.Equal(t, request.Payload, initiatorToken.Payload) + + valid, err := initiatorToken.Verify(key, keyusage.GSSAPI_INITIATOR_SEAL) + require.NoError(t, err) + require.True(t, valid) +} + +func testEncryptionKey() types.EncryptionKey { + return types.EncryptionKey{ + KeyType: etypeID.AES128_CTS_HMAC_SHA1_96, + KeyValue: []byte("0123456789abcdef"), + } +} diff --git a/pkg/sink/kafka/franz/logger.go b/pkg/sink/kafka/franz/logger.go new file mode 100644 index 0000000000..8c8c8d773f --- /dev/null +++ b/pkg/sink/kafka/franz/logger.go @@ -0,0 +1,124 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const logValueLimit = 1024 + +type logger struct { + changefeedID common.ChangeFeedID + role string + now func() time.Time + mu sync.Mutex + windowStart time.Time + counts map[string]uint64 +} + +func newLogger(changefeedID common.ChangeFeedID, role string) kgo.Logger { + return &logger{changefeedID: changefeedID, role: role, now: time.Now, counts: make(map[string]uint64)} +} + +func (l *logger) Level() kgo.LogLevel { + if log.GetLevel() <= zapcore.DebugLevel { + return kgo.LogLevelInfo + } + return kgo.LogLevelWarn +} + +func (l *logger) Log(level kgo.LogLevel, msg string, keyvals ...any) { + if !l.shouldLog(level, msg) { + return + } + fields := []zap.Field{ + zap.String("component", "kafka-client"), + zap.String("keyspace", l.changefeedID.Keyspace()), + zap.String("changefeed", l.changefeedID.Name()), + zap.String("role", l.role), + } + + for i := 0; i < len(keyvals); i += 2 { + key := fmt.Sprint(keyvals[i]) + value := any("") + if i+1 < len(keyvals) { + value = keyvals[i+1] + } + + if isSensitiveLogKey(key) { + value = "[redacted]" + } else if text, ok := value.(string); ok && len(text) > logValueLimit { + value = text[:logValueLimit] + } + fields = append(fields, zap.Any(key, value)) + } + + switch level { + case kgo.LogLevelError: + log.Error(msg, fields...) + case kgo.LogLevelWarn: + log.Warn(msg, fields...) + default: + log.Debug(msg, fields...) + } +} + +func (l *logger) shouldLog(level kgo.LogLevel, msg string) bool { + now := l.now() + key := fmt.Sprintf("%d:%s", level, msg) + + l.mu.Lock() + defer l.mu.Unlock() + + if l.windowStart.IsZero() || now.Sub(l.windowStart) >= time.Minute { + l.windowStart, l.counts = now, make(map[string]uint64) + } + + l.counts[key]++ + + return l.counts[key] <= 5 || l.counts[key]%100 == 0 +} + +func isSensitiveLogKey(key string) bool { + key = strings.ToLower(key) + if key == "key" || key == "value" { + return true + } + + for _, fragment := range []string{ + "password", + "passwd", + "secret", + "token", + "authorization", + "credential", + "sasl", + } { + if strings.Contains(key, fragment) { + return true + } + } + + return false +} diff --git a/pkg/sink/kafka/franz/logger_test.go b/pkg/sink/kafka/franz/logger_test.go new file mode 100644 index 0000000000..2c0fea9164 --- /dev/null +++ b/pkg/sink/kafka/franz/logger_test.go @@ -0,0 +1,107 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "strings" + "sync" + "testing" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestLoggerLevelAndFiltering(t *testing.T) { + oldLevel := log.GetLevel() + defer log.SetLevel(oldLevel) + + clientLogger := newLogger(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "logger"), "producer").(*logger) + + log.SetLevel(zapcore.InfoLevel) + require.Equal(t, kgo.LogLevelWarn, clientLogger.Level()) + + log.SetLevel(zapcore.DebugLevel) + require.Equal(t, kgo.LogLevelInfo, clientLogger.Level()) + + for _, key := range []string{"password", "access_token", "key", "value", "sasl-user"} { + require.True(t, isSensitiveLogKey(key)) + } + + require.NotPanics(t, func() { + clientLogger.Log(kgo.LogLevelWarn, "odd key value", "key-only") + }) +} + +func TestLoggerSamplingIsConcurrent(t *testing.T) { + clientLogger := newLogger(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "logger"), "producer").(*logger) + now := time.Now() + clientLogger.now = func() time.Time { return now } + + var wg sync.WaitGroup + for range 20 { + wg.Go(func() { + clientLogger.shouldLog(kgo.LogLevelWarn, "repeat") + }) + } + + wg.Wait() + require.Equal(t, uint64(20), clientLogger.counts["2:repeat"]) +} + +func TestLoggerPreservesContextAndRedactsValues(t *testing.T) { + core, logs := observer.New(zapcore.DebugLevel) + restore := log.ReplaceGlobals(zap.New(core), nil) + defer restore() + + clientLogger := newLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") + clientLogger.Log( + kgo.LogLevelWarn, + "connection failed", + "password", "secret", + "payload", strings.Repeat("x", logValueLimit+10), + "odd", + ) + + entries := logs.FilterMessage("connection failed").AllUntimed() + require.Len(t, entries, 1) + + fields := entries[0].ContextMap() + require.Equal(t, "kafka-client", fields["component"]) + require.Equal(t, "keyspace", fields["keyspace"]) + require.Equal(t, "changefeed", fields["changefeed"]) + require.Equal(t, "producer", fields["role"]) + require.Equal(t, "[redacted]", fields["password"]) + require.Equal(t, strings.Repeat("x", logValueLimit), fields["payload"]) + require.Equal(t, "", fields["odd"]) +} + +func TestLoggerSamplesRepeatedMessages(t *testing.T) { + core, logs := observer.New(zapcore.DebugLevel) + restore := log.ReplaceGlobals(zap.New(core), nil) + defer restore() + + clientLogger := newLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") + for range 105 { + clientLogger.Log(kgo.LogLevelWarn, "repeated") + } + + require.Len(t, logs.FilterMessage("repeated").AllUntimed(), 6) +} diff --git a/pkg/sink/kafka/franz/logutil.go b/pkg/sink/kafka/franz/logutil.go new file mode 100644 index 0000000000..c7aa49abcd --- /dev/null +++ b/pkg/sink/kafka/franz/logutil.go @@ -0,0 +1,65 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/pingcap/ticdc/pkg/sink/codec/common" +) + +func buildEventLogContext(keyspace, changefeed string, info *common.MessageLogInfo) string { + var text strings.Builder + text.WriteString("keyspace=" + keyspace + ", changefeed=" + changefeed + ", eventType=" + eventType(info)) + if info == nil { + return text.String() + } + if rows, err := json.Marshal(info.Rows); len(info.Rows) > 0 && err == nil { + text.WriteString(", dmlInfo=" + string(rows)) + } + if info.DDL != nil { + if info.DDL.Query != "" { + text.WriteString(", ddlQuery=" + strconv.Quote(info.DDL.Query)) + } + if info.DDL.StartTs != 0 { + text.WriteString(", ddlStartTs=" + strconv.FormatUint(info.DDL.StartTs, 10)) + } + if info.DDL.CommitTs != 0 { + text.WriteString(", ddlCommitTs=" + strconv.FormatUint(info.DDL.CommitTs, 10)) + } + } + if info.Checkpoint != nil && info.Checkpoint.CommitTs != 0 { + text.WriteString(", checkpointTs=" + strconv.FormatUint(info.Checkpoint.CommitTs, 10)) + } + return text.String() +} + +func eventType(info *common.MessageLogInfo) string { + if info == nil { + return "unknown" + } + if info.DDL != nil { + return "ddl" + } + if info.Checkpoint != nil { + return "checkpoint" + } + if len(info.Rows) > 0 { + return "dml" + } + return "unknown" +} diff --git a/pkg/sink/kafka/franz/logutil_test.go b/pkg/sink/kafka/franz/logutil_test.go new file mode 100644 index 0000000000..cdf831f435 --- /dev/null +++ b/pkg/sink/kafka/franz/logutil_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" +) + +func TestEventType(t *testing.T) { + require.Equal(t, "unknown", eventType(nil)) + require.Equal(t, "ddl", eventType(&common.MessageLogInfo{DDL: &common.DDLLogInfo{}})) + require.Equal(t, "checkpoint", eventType(&common.MessageLogInfo{ + Checkpoint: &common.CheckpointLogInfo{}, + })) + require.Equal(t, "dml", eventType(&common.MessageLogInfo{Rows: []common.RowLogInfo{{}}})) + require.Equal(t, "unknown", eventType(&common.MessageLogInfo{})) +} + +func TestBuildEventLogContext(t *testing.T) { + info := &common.MessageLogInfo{ + Rows: []common.RowLogInfo{ + { + Type: "insert", + Database: "database", + Table: "table", + CommitTs: 1, + }, + }, + } + + context := buildEventLogContext("keyspace", "changefeed", info) + require.Contains(t, context, "keyspace=keyspace") + require.Contains(t, context, "changefeed=changefeed") + require.Contains(t, context, "eventType=dml") + require.Contains(t, context, `dmlInfo=[{"Type":"insert"`) + require.Contains(t, context, `"Database":"database"`) + require.Contains(t, context, `"Table":"table"`) + require.Contains(t, context, `"CommitTs":1`) +} + +func TestBuildEventLogContextForBlockEvents(t *testing.T) { + ddlContext := buildEventLogContext("keyspace", "changefeed", &common.MessageLogInfo{ + DDL: &common.DDLLogInfo{ + Query: "CREATE TABLE t(id INT PRIMARY KEY)", + StartTs: 1, + CommitTs: 2, + }, + }) + + require.Contains(t, ddlContext, "eventType=ddl") + require.Contains(t, ddlContext, `ddlQuery="CREATE TABLE t(id INT PRIMARY KEY)"`) + require.Contains(t, ddlContext, "ddlStartTs=1") + require.Contains(t, ddlContext, "ddlCommitTs=2") + + checkpointContext := buildEventLogContext("keyspace", "changefeed", &common.MessageLogInfo{ + Checkpoint: &common.CheckpointLogInfo{CommitTs: 3}, + }) + require.Contains(t, checkpointContext, "eventType=checkpoint") + require.Contains(t, checkpointContext, "checkpointTs=3") +} diff --git a/pkg/sink/kafka/franz/metrics.go b/pkg/sink/kafka/franz/metrics.go new file mode 100644 index 0000000000..db18c28e93 --- /dev/null +++ b/pkg/sink/kafka/franz/metrics.go @@ -0,0 +1,89 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import "github.com/prometheus/client_golang/prometheus" + +var ( + requestsInFlight = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_in_flight_requests", + Help: "Current franz-go requests awaiting a response.", + }, []string{"namespace", "changefeed", "broker"}) + + outgoingBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_outgoing_bytes_total", + Help: "Total bytes written by franz-go, excluding TLS overhead.", + }, []string{"namespace", "changefeed", "broker"}) + + requestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_requests_total", + Help: "Total franz-go requests by broker and write result.", + }, []string{"namespace", "changefeed", "broker", "result"}) + + responsesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_responses_total", + Help: "Total franz-go responses by broker and read result.", + }, []string{"namespace", "changefeed", "broker", "result"}) + + requestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_request_duration_seconds", + Help: "Franz-go request end-to-end duration in seconds.", + Buckets: prometheus.DefBuckets, + }, []string{"namespace", "changefeed", "broker"}) + + recordsPerBatch = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_records_per_batch", + Help: "Records in each successfully written franz-go topic-partition batch.", + Buckets: prometheus.ExponentialBuckets(1, 2, 15), + }, []string{"namespace", "changefeed"}) + + uncompressedBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_uncompressed_bytes_total", + Help: "Total record bytes before compression in successfully written franz-go batches.", + }, []string{"namespace", "changefeed"}) + + compressedBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_compressed_bytes_total", + Help: "Total record bytes after compression in successfully written franz-go batches.", + }, []string{"namespace", "changefeed"}) +) + +func InitMetrics(registry *prometheus.Registry) { + registry.MustRegister( + requestsInFlight, + outgoingBytesTotal, + requestsTotal, + responsesTotal, + requestDuration, + recordsPerBatch, + uncompressedBytesTotal, + compressedBytesTotal, + ) +} diff --git a/pkg/sink/kafka/metrics_hook.go b/pkg/sink/kafka/franz/metrics_hook.go similarity index 92% rename from pkg/sink/kafka/metrics_hook.go rename to pkg/sink/kafka/franz/metrics_hook.go index 5adb06305d..db959a7a10 100644 --- a/pkg/sink/kafka/metrics_hook.go +++ b/pkg/sink/kafka/franz/metrics_hook.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kafka +package franz import ( "strconv" @@ -25,8 +25,8 @@ import ( // metricsHook adapts franz-go client callbacks to TiCDC's Kafka sink metrics. // franz-go calls these hook methods while writing requests, receiving responses, -// and flushing produce batches. The hook does not poll Kafka; it only converts -// the callback payloads into the existing TiCDC Kafka metric vectors. +// and flushing produce batches. The hook does not poll Kafka; it only records +// raw callback values for Prometheus. type metricsHook struct { keyspace string changefeed string @@ -54,9 +54,10 @@ const ( metricResultReadError = "read_error" ) -func newKafkaMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { +func newMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { keyspace := changefeedID.Keyspace() changefeed := changefeedID.Name() + return &metricsHook{ keyspace: keyspace, changefeed: changefeed, @@ -82,23 +83,26 @@ func (h *metricsHook) broker(nodeID int32) *brokerMetrics { h.keyspace, h.changefeed, brokerID, metricResultSuccess), responsesReadError: responsesTotal.WithLabelValues( h.keyspace, h.changefeed, brokerID, metricResultReadError), - requestsInFlight: requestsInFlightGauge.WithLabelValues(h.keyspace, h.changefeed, brokerID), + requestsInFlight: requestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID), requestDuration: requestDuration.WithLabelValues(h.keyspace, h.changefeed, brokerID), } + actual, _ := h.brokers.LoadOrStore(nodeID, metrics) + return actual.(*brokerMetrics) } -// CleanupMetrics removes Kafka sink metric series after all of its clients are closed. +// CleanupMetrics removes producer series after all clients are closed. func CleanupMetrics(changefeedID common.ChangeFeedID) { labels := prometheus.Labels{ "namespace": changefeedID.Keyspace(), "changefeed": changefeedID.Name(), } + outgoingBytesTotal.DeletePartialMatch(labels) requestsTotal.DeletePartialMatch(labels) responsesTotal.DeletePartialMatch(labels) - requestsInFlightGauge.DeletePartialMatch(labels) + requestsInFlight.DeletePartialMatch(labels) requestDuration.DeletePartialMatch(labels) recordsPerBatch.DeletePartialMatch(labels) uncompressedBytesTotal.DeletePartialMatch(labels) @@ -116,11 +120,13 @@ func (h *metricsHook) OnBrokerWrite( if meta.NodeID < 0 { return } + metrics := h.broker(meta.NodeID) if bytesWritten > 0 { metrics.outgoingBytesTotal.Add(float64(bytesWritten)) } + if err != nil { metrics.requestsWriteError.Inc() } else { @@ -137,6 +143,7 @@ func (h *metricsHook) OnBrokerE2E( if meta.NodeID < 0 { return } + metrics := h.broker(meta.NodeID) if e2e.WriteErr == nil { @@ -149,6 +156,7 @@ func (h *metricsHook) OnBrokerE2E( } } } + if e2e.Err() == nil { metrics.requestDuration.Observe(e2e.DurationE2E().Seconds()) } @@ -163,9 +171,11 @@ func (h *metricsHook) OnProduceBatchWritten( if m.NumRecords > 0 { h.recordsPerBatch.Observe(float64(m.NumRecords)) } + if m.UncompressedBytes > 0 { h.uncompressedBytesTotal.Add(float64(m.UncompressedBytes)) } + if m.CompressedBytes > 0 { h.compressedBytesTotal.Add(float64(m.CompressedBytes)) } diff --git a/pkg/sink/kafka/franz/metrics_hook_test.go b/pkg/sink/kafka/franz/metrics_hook_test.go new file mode 100644 index 0000000000..377d28e477 --- /dev/null +++ b/pkg/sink/kafka/franz/metrics_hook_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" +) + +func TestInitMetrics(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "metrics-registration") + CleanupMetrics(changefeedID) + t.Cleanup(func() { CleanupMetrics(changefeedID) }) + + hook := newMetricsHook(changefeedID) + hook.OnProduceBatchWritten( + kgo.BrokerMetadata{}, + "topic", + 0, + kgo.ProduceBatchMetrics{ + NumRecords: 1, + UncompressedBytes: 2, + CompressedBytes: 1, + }, + ) + + registry := prometheus.NewRegistry() + InitMetrics(registry) + + metricFamilies, err := registry.Gather() + require.NoError(t, err) + + names := make([]string, 0, len(metricFamilies)) + for _, family := range metricFamilies { + names = append(names, family.GetName()) + } + + require.Contains(t, names, "ticdc_sink_kafka_franz_producer_records_per_batch") + require.Contains(t, names, "ticdc_sink_kafka_franz_producer_uncompressed_bytes_total") + require.Contains(t, names, "ticdc_sink_kafka_franz_producer_compressed_bytes_total") +} + +func TestMetricsHookRecordsRawValues(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "metrics-hook") + CleanupMetrics(changefeedID) + t.Cleanup(func() { CleanupMetrics(changefeedID) }) + hook := newMetricsHook(changefeedID) + meta := kgo.BrokerMetadata{NodeID: 1} + + hook.OnBrokerWrite(meta, 0, 12, 0, 0, nil) + hook.OnBrokerE2E(meta, 0, kgo.BrokerE2E{ + BytesRead: 8, + TimeToWrite: time.Millisecond, + TimeToRead: time.Millisecond, + }) + hook.OnProduceBatchWritten(meta, "topic", 0, kgo.ProduceBatchMetrics{ + NumRecords: 3, + UncompressedBytes: 10, + CompressedBytes: 5, + }) + + metrics := hook.broker(1) + require.Same(t, metrics, hook.broker(1)) + require.Equal(t, float64(12), testutil.ToFloat64(metrics.outgoingBytesTotal)) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.requestsSuccess)) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.responsesSuccess)) + require.Equal(t, float64(0), testutil.ToFloat64(metrics.requestsInFlight)) + require.Equal(t, float64(10), testutil.ToFloat64( + uncompressedBytesTotal.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name()), + )) + require.Equal(t, float64(5), testutil.ToFloat64( + compressedBytesTotal.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name()), + )) + + hook.OnBrokerWrite(meta, 0, 0, 0, 0, nil) + hook.OnBrokerE2E(meta, 0, kgo.BrokerE2E{ + BytesRead: 8, + TimeToWrite: 3 * time.Millisecond, + ReadWait: time.Millisecond, + TimeToRead: 4 * time.Millisecond, + }) + + metric, ok := metrics.requestDuration.(prometheus.Metric) + require.True(t, ok) + + histogram := &dto.Metric{} + require.NoError(t, metric.Write(histogram)) + require.Equal(t, uint64(2), histogram.GetHistogram().GetSampleCount()) + require.InDelta(t, 0.010, histogram.GetHistogram().GetSampleSum(), 0.000001) + + hook.OnBrokerWrite(meta, 0, 0, 0, 0, context.DeadlineExceeded) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.requestsWriteError)) + + hook.OnBrokerWrite(meta, 0, 0, 0, 0, nil) + hook.OnBrokerE2E(meta, 0, kgo.BrokerE2E{ReadErr: context.Canceled}) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.responsesReadError)) + require.Equal(t, float64(0), testutil.ToFloat64(metrics.requestsInFlight)) + + hook.OnBrokerWrite(kgo.BrokerMetadata{NodeID: -1}, 0, 1, 0, 0, nil) + hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: -1}, 0, kgo.BrokerE2E{}) +} diff --git a/pkg/sink/kafka/sync_producer.go b/pkg/sink/kafka/franz/sync_producer.go similarity index 62% rename from pkg/sink/kafka/sync_producer.go rename to pkg/sink/kafka/franz/sync_producer.go index d94cea621b..59c39c37ba 100644 --- a/pkg/sink/kafka/sync_producer.go +++ b/pkg/sink/kafka/franz/sync_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kafka +package franz import ( "context" @@ -19,63 +19,52 @@ import ( "time" "github.com/pingcap/log" - commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/codec/common" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/zap" ) -// SyncProducer is the kafka sync producer -type SyncProducer interface { - // SendMessage produces a given message, and returns only when it either has - // succeeded or failed to produce. It will return the partition and the offset - // of the produced message, or an error if the message failed to produce. - SendMessage(topic string, partitionNum int32, message *common.Message) error - - // SendMessages produces a given set of messages, and returns only when all - // messages in the set have either succeeded or failed. Note that messages - // can succeed and fail individually; if some succeed and some fail, - // SendMessages will return an error. - SendMessages(topic string, partitionNum int32, message *common.Message) error - - // Close shuts down the producer and releases its Kafka client resources. - Close() -} - -type syncProducer struct { - id commonType.ChangeFeedID +type SyncProducer struct { + id common.ChangeFeedID client *kgo.Client closed atomic.Bool timeout time.Duration } -func newSyncProducer( +func NewSyncProducer( ctx context.Context, - changefeedID commonType.ChangeFeedID, - o *options, + changefeedID common.ChangeFeedID, + cfg Config, hook *metricsHook, -) (*syncProducer, error) { - opts, err := newOptions(ctx, o, hook) +) (*SyncProducer, error) { + opts, err := newClientOptions(ctx, changefeedID, "sync-producer", cfg, hook) + if err != nil { + return nil, err + } + + producerOpts, err := producerOptions(cfg) if err != nil { - return nil, errors.Trace(err) + return nil, err } - opts = append(opts, newProducerOptions(o)...) + + opts = append(opts, producerOpts...) client, err := kgo.NewClient(opts...) if err != nil { - return nil, errors.Trace(err) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } - return &syncProducer{ + return &SyncProducer{ id: changefeedID, client: client, - timeout: o.requestTimeout(), + timeout: cfg.requestTimeout(), }, nil } -func (p *syncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { +func (p *SyncProducer) SendMessage(topic string, partitionNum int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -93,16 +82,18 @@ func (p *syncProducer) SendMessage(topic string, partitionNum int32, message *co if err == nil { return nil } + log.Error("kafka message send failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), - zap.String("eventContext", BuildEventLogContext( + zap.String("eventContext", buildEventLogContext( p.id.Keyspace(), p.id.Name(), message.LogInfo)), zap.Error(err)) + return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { +func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -124,16 +115,18 @@ func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *c if err == nil { return nil } + log.Error("kafka message send failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), - zap.String("eventContext", BuildEventLogContext( + zap.String("eventContext", buildEventLogContext( p.id.Keyspace(), p.id.Name(), message.LogInfo)), zap.Error(err)) + return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *syncProducer) Close() { +func (p *SyncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { log.Warn("kafka ddl producer already closed", zap.String("keyspace", p.id.Keyspace()), @@ -143,6 +136,7 @@ func (p *syncProducer) Close() { start := time.Now() p.client.Close() + log.Info("kafka ddl producer closed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), diff --git a/pkg/sink/kafka/franz/sync_producer_test.go b/pkg/sink/kafka/franz/sync_producer_test.go new file mode 100644 index 0000000000..90e387f988 --- /dev/null +++ b/pkg/sink/kafka/franz/sync_producer_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package franz + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { + producer := &SyncProducer{} + producer.closed.Store(true) + + err := producer.SendMessage("topic", 1, &codeccommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) + + err = producer.SendMessages("topic", 1, &codeccommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) +} + +func TestSyncProducerSendsToRequestedPartitions(t *testing.T) { + const topic = "sync-topic" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) + defer cluster.Close() + + producer, err := NewSyncProducer( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), + testConfig(cluster.ListenAddrs()), + nil, + ) + require.NoError(t, err) + defer producer.Close() + + require.NoError(t, producer.SendMessage(topic, 2, &codeccommon.Message{Key: []byte("key"), Value: []byte("value")})) + require.NoError(t, producer.SendMessages(topic, 3, &codeccommon.Message{Value: []byte("all")})) +} + +func TestSyncProducerReturnsPartialFailure(t *testing.T) { + const topic = "partial-failure" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) + defer cluster.Close() + + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + return produceResponseWithError(req, 1, kerr.InvalidTopicException.Code) + }) + + producer, err := NewSyncProducer( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), + testConfig(cluster.ListenAddrs()), + nil, + ) + require.NoError(t, err) + defer producer.Close() + + err = producer.SendMessages(topic, 3, &codeccommon.Message{Value: []byte("value")}) + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) + require.ErrorIs(t, err, kerr.InvalidTopicException) +} + +func TestSyncProducerCloseIsIdempotent(t *testing.T) { + client, err := NewSyncProducer( + context.Background(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), + testConfig([]string{"127.0.0.1:1"}), + nil, + ) + require.NoError(t, err) + + client.Close() + client.Close() +} + +func produceResponseWithError(req kmsg.Request, failedPartition int32, errorCode int16) (kmsg.Response, error, bool) { + request := req.(*kmsg.ProduceRequest) + response := request.ResponseKind().(*kmsg.ProduceResponse) + + for _, requestTopic := range request.Topics { + responseTopic := kmsg.NewProduceResponseTopic() + responseTopic.Topic = requestTopic.Topic + responseTopic.TopicID = requestTopic.TopicID + + for _, requestPartition := range requestTopic.Partitions { + responsePartition := kmsg.NewProduceResponseTopicPartition() + responsePartition.Partition = requestPartition.Partition + if requestPartition.Partition == failedPartition { + responsePartition.ErrorCode = errorCode + } + responseTopic.Partitions = append(responseTopic.Partitions, responsePartition) + } + + response.Topics = append(response.Topics, responseTopic) + } + + return response, nil, true +} diff --git a/pkg/sink/kafka/franz_adapter.go b/pkg/sink/kafka/franz_adapter.go new file mode 100644 index 0000000000..6ba17e20c6 --- /dev/null +++ b/pkg/sink/kafka/franz_adapter.go @@ -0,0 +1,218 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "crypto/tls" + "strings" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/sink/kafka/franz" + "go.uber.org/zap" +) + +type franzFactoryAdapter struct { + inner *franz.Factory +} + +// NewFranzFactory constructs the additive franz-go implementation while the +// existing Sarama factory remains unchanged. +func NewFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { + config, err := newFranzConfig(o) + if err != nil { + return nil, err + } + + innerAdmin, err := franz.NewAdmin(ctx, changefeedID, config) + if err != nil { + return nil, err + } + + admin := &franzAdminAdapter{inner: innerAdmin} + defer admin.Close() + + if err := adjustOptions(changefeedID, admin, o, o.Topic); err != nil { + return nil, err + } + + config, err = newFranzConfig(o) + if err != nil { + return nil, err + } + + compression := strings.ToLower(strings.TrimSpace(o.Compression)) + if compression == "" { + compression = "none" + } + + log.Info("kafka sink configuration resolved", + zap.String("namespace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.String("client", KafkaClientFranz), + zap.String("topic", o.Topic), + zap.Int32("partitionNum", o.PartitionNum), + zap.Int("maxMessageBytes", o.MaxMessageBytes), + zap.Int("maxBatchedBytes", o.MaxBatchedBytes), + zap.String("compression", compression), + zap.Int16("requiredAcks", int16(o.RequiredAcks)), + zap.Int("maxRetry", o.MaxRetry), + zap.Duration("dialTimeout", o.DialTimeout), + zap.Duration("readTimeout", o.ReadTimeout), + zap.Duration("writeTimeout", o.WriteTimeout)) + + return &franzFactoryAdapter{inner: franz.NewFactory(config, changefeedID)}, nil +} + +func (f *franzFactoryAdapter) AdminClient(ctx context.Context) (AdminClient, error) { + admin, err := f.inner.Admin(ctx) + if err != nil { + return nil, err + } + return &franzAdminAdapter{inner: admin}, nil +} + +func (f *franzFactoryAdapter) SyncProducer(ctx context.Context) (SyncProducer, error) { + return f.inner.SyncProducer(ctx) +} + +func (f *franzFactoryAdapter) AsyncProducer(ctx context.Context) (AsyncProducer, error) { + return f.inner.AsyncProducer(ctx) +} + +func (f *franzFactoryAdapter) MetricsCollector(AdminClient) MetricsCollector { + return franzMetricsCollector{} +} + +func (f *franzFactoryAdapter) CleanupMetrics() { f.inner.CleanupMetrics() } + +type franzMetricsCollector struct{} + +func (franzMetricsCollector) Run(ctx context.Context) { <-ctx.Done() } + +type franzAdminAdapter struct{ inner *franz.Admin } + +func (a *franzAdminAdapter) GetAllBrokers() []Broker { + inner := a.inner.GetAllBrokers() + brokers := make([]Broker, 0, len(inner)) + for _, broker := range inner { + brokers = append(brokers, Broker{ID: broker.ID}) + } + return brokers +} + +func (a *franzAdminAdapter) GetBrokerConfig(name string) (string, bool, error) { + return a.inner.GetBrokerConfig(name) +} + +func (a *franzAdminAdapter) GetTopicConfig(topic, name string) (string, bool, error) { + return a.inner.GetTopicConfig(topic, name) +} + +func (a *franzAdminAdapter) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + inner, err := a.inner.GetTopicsMeta(topics, ignoreTopicError) + if err != nil { + return nil, err + } + + details := make(map[string]TopicDetail, len(inner)) + for topic, detail := range inner { + details[topic] = TopicDetail{ + Name: detail.Name, + NumPartitions: detail.NumPartitions, + ReplicationFactor: detail.ReplicationFactor, + } + } + + return details, nil +} + +func (a *franzAdminAdapter) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { + return a.inner.GetTopicsPartitionsNum(topics) +} + +func (a *franzAdminAdapter) CreateTopic(detail *TopicDetail) error { + return a.inner.CreateTopic(&franz.TopicDetail{ + Name: detail.Name, + NumPartitions: detail.NumPartitions, + ReplicationFactor: detail.ReplicationFactor, + }) +} + +func (a *franzAdminAdapter) Close() { a.inner.Close() } + +func newFranzConfig(o *options) (franz.Config, error) { + config := franz.Config{ + BrokerEndpoints: append([]string(nil), o.BrokerEndpoints...), + ClientID: o.ClientID, + Version: o.Version, + AssignedVersion: o.IsAssignedVersion, + MaxMessageBytes: o.MaxMessageBytes, + MaxRetry: o.MaxRetry, + Compression: o.Compression, + RequiredAcks: int16(o.RequiredAcks), + DialTimeout: o.DialTimeout, + ReadTimeout: o.ReadTimeout, + WriteTimeout: o.WriteTimeout, + } + + if o.EnableTLS { + config.TLSConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + } + + if o.Credential != nil && o.Credential.IsTLSEnabled() { + tlsConfig, err := o.Credential.ToTLSConfig() + if err != nil { + return franz.Config{}, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + + config.TLSConfig = tlsConfig + } + + config.TLSConfig.InsecureSkipVerify = o.InsecureSkipVerify + } + + if o.sasl != nil && o.sasl.mechanism != "" { + config.SASL = &franz.SASLConfig{ + Mechanism: string(o.sasl.mechanism), + User: o.sasl.user, + Password: o.sasl.password, + GSSAPI: franz.GSSAPIConfig{ + AuthType: int(o.sasl.gssapi.authType), + KeyTabPath: o.sasl.gssapi.keyTabPath, + KerberosConfigPath: o.sasl.gssapi.kerberosConfigPath, + ServiceName: o.sasl.gssapi.serviceName, + Username: o.sasl.gssapi.username, + Password: o.sasl.gssapi.password, + Realm: o.sasl.gssapi.realm, + DisablePAFXFAST: o.sasl.gssapi.disablePAFXFAST, + }, + OAuth2: franz.OAuth2Config{ + ClientID: o.sasl.oauth2.clientID, + ClientSecret: o.sasl.oauth2.clientSecret, + TokenURL: o.sasl.oauth2.tokenURL, + Scopes: append([]string(nil), o.sasl.oauth2.scopes...), + GrantType: o.sasl.oauth2.grantType, + Audience: o.sasl.oauth2.audience, + }, + } + } + + return config, nil +} diff --git a/pkg/sink/kafka/logutil_test.go b/pkg/sink/kafka/logutil_test.go index e289375a6a..2af2a0d618 100644 --- a/pkg/sink/kafka/logutil_test.go +++ b/pkg/sink/kafka/logutil_test.go @@ -13,7 +13,6 @@ package kafka import ( - "encoding/json" "strings" "testing" @@ -49,9 +48,7 @@ func TestBuildEventLogContextRowsIncluded(t *testing.T) { } info := &codecCommon.MessageLogInfo{Rows: rows} ctx := BuildEventLogContext("ks", "cf", info) - data, err := json.Marshal(rows) - require.NoError(t, err) - expected := string(data) + expected := formatDMLInfo(rows) require.Contains(t, ctx, "dmlInfo="+expected) require.NotContains(t, ctx, "dmlInfoTruncated") require.NotContains(t, ctx, "truncatedRows") diff --git a/pkg/sink/kafka/main_test.go b/pkg/sink/kafka/main_test.go index 66978ada9e..0e524e68ff 100644 --- a/pkg/sink/kafka/main_test.go +++ b/pkg/sink/kafka/main_test.go @@ -17,12 +17,8 @@ import ( "testing" "github.com/pingcap/ticdc/pkg/leakutil" - "go.uber.org/goleak" ) func TestMain(m *testing.M) { - leakutil.SetUpLeakTest( - m, - goleak.IgnoreAnyFunction("github.com/godbus/dbus.(*Conn).inWorker"), - ) + leakutil.SetUpLeakTest(m) } diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index 029d7643e2..ba3ec0a553 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -16,6 +16,7 @@ package kafka import ( "github.com/pingcap/ticdc/pkg/sink/codec" "github.com/pingcap/ticdc/pkg/sink/kafka/claimcheck" + "github.com/pingcap/ticdc/pkg/sink/kafka/franz" "github.com/prometheus/client_golang/prometheus" ) @@ -28,69 +29,68 @@ var ( Help: "The current number of in-flight requests" + " awaiting a response for all brokers.", }, []string{"namespace", "changefeed", "broker"}) - outgoingBytesTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ + // OutgoingByteRateGauge for outgoing events. + // Meter mark for each request's size in bytes. + OutgoingByteRateGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_outgoing_bytes_total", - Help: "Total bytes written to Kafka brokers, excluding TLS overhead.", + Name: "kafka_producer_outgoing_byte_rate", + Help: "Bytes/second written off all brokers.", }, []string{"namespace", "changefeed", "broker"}) - requestsTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "ticdc", - Subsystem: "sink", - Name: "kafka_producer_requests_total", - Help: "Total Kafka requests by broker and write result.", - }, []string{"namespace", "changefeed", "broker", "result"}) - responsesTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ + // RequestRateGauge Meter mark by 1 for each request. + RequestRateGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_responses_total", - Help: "Total Kafka responses by broker and read result.", - }, []string{"namespace", "changefeed", "broker", "result"}) - requestDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + Name: "kafka_producer_request_rate", + Help: "Requests/second sent to all brokers.", + }, []string{"namespace", "changefeed", "broker"}) + // RequestLatencyGauge Histogram update by `requestLatency`. + RequestLatencyGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_request_duration_seconds", - Help: "Kafka request end-to-end duration in seconds.", - Buckets: prometheus.DefBuckets, - }, []string{"namespace", "changefeed", "broker"}) - recordsPerBatch = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + Name: "kafka_producer_request_latency", + Help: "The request latency for all brokers.", + }, []string{"namespace", "changefeed", "broker", "type"}) + // Histogram update by `compression-ratio`. + compressionRatioGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_records_per_batch", - Help: "Number of records in each successfully written topic-partition batch.", - Buckets: prometheus.ExponentialBuckets(1, 2, 15), - }, []string{"namespace", "changefeed"}) - uncompressedBytesTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ + Name: "kafka_producer_compression_ratio", + Help: "The compression ratio times 100 of record batches for all topics.", + }, []string{"namespace", "changefeed", "type"}) + // updated by `records-per-request`. + recordsPerRequestGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_uncompressed_bytes_total", - Help: "Total serialized record bytes before compression in successfully written batches.", - }, []string{"namespace", "changefeed"}) - compressedBytesTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ + Name: "kafka_producer_records_per_request", + Help: "The number of records per request for all topics.", + }, []string{"namespace", "changefeed", "type"}) + + // Meter mark by 1 once a response received. + responseRateGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "ticdc", Subsystem: "sink", - Name: "kafka_producer_compressed_bytes_total", - Help: "Total serialized record bytes after compression in successfully written batches.", - }, []string{"namespace", "changefeed"}) + Name: "kafka_producer_response_rate", + Help: "Responses/second received from all brokers.", + }, []string{"namespace", "changefeed", "broker"}) ) // InitMetrics registers all metrics in this file. func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(outgoingBytesTotal) - registry.MustRegister(requestsTotal) - registry.MustRegister(responsesTotal) - registry.MustRegister(requestDuration) - registry.MustRegister(recordsPerBatch) - registry.MustRegister(uncompressedBytesTotal) - registry.MustRegister(compressedBytesTotal) + franz.InitMetrics(registry) + registry.MustRegister(compressionRatioGauge) + registry.MustRegister(recordsPerRequestGauge) + registry.MustRegister(OutgoingByteRateGauge) + registry.MustRegister(RequestRateGauge) + registry.MustRegister(RequestLatencyGauge) registry.MustRegister(requestsInFlightGauge) + registry.MustRegister(responseRateGauge) claimcheck.InitMetrics(registry) codec.InitMetrics(registry) diff --git a/pkg/sink/kafka/metrics_collector.go b/pkg/sink/kafka/metrics_collector.go new file mode 100644 index 0000000000..3e54c7d82f --- /dev/null +++ b/pkg/sink/kafka/metrics_collector.go @@ -0,0 +1,214 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strconv" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/rcrowley/go-metrics" + "go.uber.org/zap" +) + +// MetricsCollector is the interface for kafka metrics collector. +type MetricsCollector interface { + Run(ctx context.Context) +} + +const ( + // refreshMetricsInterval specifies the interval of refresh kafka client metrics. + refreshMetricsInterval = 5 * time.Second + // refreshClusterMetaInterval specifies the interval of refresh kafka cluster meta. + // Do not set it too small, because it will cause too many requests to kafka cluster. + // Every request will get all topics and all brokers information. + refreshClusterMetaInterval = 30 * time.Minute +) + +// Sarama metrics names, see https://pkg.go.dev/github.com/IBM/sarama#pkg-overview. +const ( + // Producer level. + compressionRatioMetricName = "compression-ratio" + recordsPerRequestMetricName = "records-per-request" + + // Broker level. + outgoingByteRateMetricNamePrefix = "outgoing-byte-rate-for-broker-" + requestRateMetricNamePrefix = "request-rate-for-broker-" + requestLatencyInMsMetricNamePrefix = "request-latency-in-ms-for-broker-" + requestsInFlightMetricNamePrefix = "requests-in-flight-for-broker-" + responseRateMetricNamePrefix = "response-rate-for-broker-" + + p99 = "p99" + avg = "avg" +) + +type saramaMetricsCollector struct { + changefeedID common.ChangeFeedID + // adminClient is used to get broker infos from broker. + adminClient AdminClient + brokers map[int32]struct{} + registry metrics.Registry +} + +func (m *saramaMetricsCollector) Run(ctx context.Context) { + // Initialize brokers. + m.updateBrokers(ctx) + + refreshMetricsTicker := time.NewTicker(refreshMetricsInterval) + refreshClusterMetaTicker := time.NewTicker(refreshClusterMetaInterval) + defer func() { + refreshMetricsTicker.Stop() + refreshClusterMetaTicker.Stop() + m.cleanupMetrics() + }() + + for { + select { + case <-ctx.Done(): + log.Info("kafka metrics collector stopped", + zap.String("keyspace", m.changefeedID.Keyspace()), + zap.String("changefeed", m.changefeedID.Name())) + return + case <-refreshMetricsTicker.C: + m.collectBrokerMetrics() + m.collectProducerMetrics() + case <-refreshClusterMetaTicker.C: + m.updateBrokers(ctx) + } + } +} + +func (m *saramaMetricsCollector) updateBrokers(ctx context.Context) { + brokers := m.adminClient.GetAllBrokers() + for _, b := range brokers { + m.brokers[b.ID] = struct{}{} + } +} + +func (m *saramaMetricsCollector) collectProducerMetrics() { + keyspace := m.changefeedID.Keyspace() + changefeedID := m.changefeedID.Name() + compressionRatioMetric := m.registry.Get(compressionRatioMetricName) + if histogram, ok := compressionRatioMetric.(metrics.Histogram); ok { + compressionRatioGauge. + WithLabelValues(keyspace, changefeedID, avg). + Set(histogram.Snapshot().Mean()) + compressionRatioGauge.WithLabelValues(keyspace, changefeedID, p99). + Set(histogram.Snapshot().Percentile(0.99)) + } + + recordsPerRequestMetric := m.registry.Get(recordsPerRequestMetricName) + if histogram, ok := recordsPerRequestMetric.(metrics.Histogram); ok { + recordsPerRequestGauge. + WithLabelValues(keyspace, changefeedID, avg). + Set(histogram.Snapshot().Mean()) + recordsPerRequestGauge. + WithLabelValues(keyspace, changefeedID, p99). + Set(histogram.Snapshot().Percentile(0.99)) + } +} + +func (m *saramaMetricsCollector) collectBrokerMetrics() { + keyspace := m.changefeedID.Keyspace() + changefeedID := m.changefeedID.Name() + for id := range m.brokers { + brokerID := strconv.Itoa(int(id)) + outgoingByteRateMetric := m.registry.Get( + getBrokerMetricName(outgoingByteRateMetricNamePrefix, brokerID)) + if meter, ok := outgoingByteRateMetric.(metrics.Meter); ok { + OutgoingByteRateGauge. + WithLabelValues(keyspace, changefeedID, brokerID). + Set(meter.Snapshot().Rate1()) + } + + requestRateMetric := m.registry.Get( + getBrokerMetricName(requestRateMetricNamePrefix, brokerID)) + if meter, ok := requestRateMetric.(metrics.Meter); ok { + RequestRateGauge. + WithLabelValues(keyspace, changefeedID, brokerID). + Set(meter.Snapshot().Rate1()) + } + + requestLatencyMetric := m.registry.Get( + getBrokerMetricName(requestLatencyInMsMetricNamePrefix, brokerID)) + if histogram, ok := requestLatencyMetric.(metrics.Histogram); ok { + RequestLatencyGauge. + WithLabelValues(keyspace, changefeedID, brokerID, avg). + Set(histogram.Snapshot().Mean() / 1000) + RequestLatencyGauge. + WithLabelValues(keyspace, changefeedID, brokerID, p99). + Set(histogram.Snapshot().Percentile(0.99) / 1000) + } + + requestsInFlightMetric := m.registry.Get(getBrokerMetricName( + requestsInFlightMetricNamePrefix, brokerID)) + if counter, ok := requestsInFlightMetric.(metrics.Counter); ok { + requestsInFlightGauge. + WithLabelValues(keyspace, changefeedID, brokerID). + Set(float64(counter.Snapshot().Count())) + } + + responseRateMetric := m.registry.Get(getBrokerMetricName( + responseRateMetricNamePrefix, brokerID)) + if meter, ok := responseRateMetric.(metrics.Meter); ok { + responseRateGauge. + WithLabelValues(keyspace, changefeedID, brokerID). + Set(meter.Snapshot().Rate1()) + } + } +} + +func getBrokerMetricName(prefix, brokerID string) string { + return prefix + brokerID +} + +func (m *saramaMetricsCollector) cleanupProducerMetrics() { + compressionRatioGauge. + DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), avg) + compressionRatioGauge. + DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), p99) + + recordsPerRequestGauge. + DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), avg) + recordsPerRequestGauge. + DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), p99) +} + +func (m *saramaMetricsCollector) cleanupBrokerMetrics() { + keyspace := m.changefeedID.Keyspace() + changefeedID := m.changefeedID.Name() + for id := range m.brokers { + brokerID := strconv.Itoa(int(id)) + OutgoingByteRateGauge. + DeleteLabelValues(keyspace, changefeedID, brokerID) + RequestRateGauge. + DeleteLabelValues(keyspace, changefeedID, brokerID) + RequestLatencyGauge. + DeleteLabelValues(keyspace, changefeedID, brokerID, avg) + RequestLatencyGauge. + DeleteLabelValues(keyspace, changefeedID, brokerID, p99) + requestsInFlightGauge. + DeleteLabelValues(keyspace, changefeedID, brokerID) + responseRateGauge. + DeleteLabelValues(keyspace, changefeedID, brokerID) + + } +} + +func (m *saramaMetricsCollector) cleanupMetrics() { + m.cleanupProducerMetrics() + m.cleanupBrokerMetrics() +} diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 77fdfedbdc..31b6dd8e04 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -38,8 +38,12 @@ const ( defaultPartitionNum = 3 // defaultMaxRetry is the default retry budget for Kafka producers. defaultMaxRetry = 5 - // defaultTimeout is the default timeout for Kafka connections and requests. + // defaultTimeout is the default timeout for Kafka connections. defaultTimeout = 10 * time.Second + // KafkaClientFranz is the default Kafka client implementation. + KafkaClientFranz = "franz" + // KafkaClientSarama keeps the master implementation available as a fallback. + KafkaClientSarama = "sarama" ) const ( @@ -66,7 +70,7 @@ const ( SASLTypeSCRAMSHA256 = "SCRAM-SHA-256" // SASLTypeSCRAMSHA512 represents the SCRAM-SHA-512 mechanism. SASLTypeSCRAMSHA512 = "SCRAM-SHA-512" - // SASLTypeGSSAPI represents the GSSAPI mechanism. + // SASLTypeGSSAPI represents the gssapi mechanism. SASLTypeGSSAPI = "GSSAPI" // SASLTypeOAuth represents the SASL/OAUTHBEARER mechanism (Kafka 2.0.0+) SASLTypeOAuth = "OAUTHBEARER" @@ -108,6 +112,7 @@ func requireAcksFromString(acks int) (RequiredAcks, error) { } type urlConfig struct { + KafkaClient *string `form:"kafka-client"` PartitionNum *int32 `form:"partition-num"` ReplicationFactor *int16 `form:"replication-factor"` KafkaVersion *string `form:"kafka-version"` @@ -140,6 +145,7 @@ type urlConfig struct { // options stores Kafka sink configurations type options struct { + Client string Topic string BrokerEndpoints []string @@ -174,13 +180,10 @@ type options struct { ReadTimeout time.Duration } -func (o *options) requestTimeout() time.Duration { - return max(o.ReadTimeout, o.WriteTimeout) -} - // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ + Client: KafkaClientFranz, Version: "2.4.0", MaxMessageBytes: config.DefaultMaxMessageBytes, MaxBatchedBytes: config.DefaultMaxMessageBytes, @@ -266,6 +269,12 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, o.MaxMessageBytes = *urlParameter.MaxMessageBytes } o.MaxBatchedBytes = o.MaxMessageBytes + if urlParameter.KafkaClient != nil { + o.Client = strings.ToLower(strings.TrimSpace(*urlParameter.KafkaClient)) + } + if o.Client != KafkaClientFranz && o.Client != KafkaClientSarama { + return errors.ErrKafkaInvalidConfig.GenWithStack("invalid kafka-client %q, only support franz and sarama", o.Client) + } if urlParameter.MaxRetry != nil && *urlParameter.MaxRetry >= 0 { o.MaxRetry = *urlParameter.MaxRetry @@ -293,33 +302,36 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, } if urlParameter.DialTimeout != nil && *urlParameter.DialTimeout != "" { - o.DialTimeout, err = time.ParseDuration(*urlParameter.DialTimeout) + a, err := time.ParseDuration(*urlParameter.DialTimeout) if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - if o.DialTimeout <= 0 { + if a <= 0 { return errors.ErrKafkaInvalidConfig.GenWithStack("dial-timeout must be greater than zero") } + o.DialTimeout = a } if urlParameter.WriteTimeout != nil && *urlParameter.WriteTimeout != "" { - o.WriteTimeout, err = time.ParseDuration(*urlParameter.WriteTimeout) + a, err := time.ParseDuration(*urlParameter.WriteTimeout) if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - if o.WriteTimeout <= 0 { + if a <= 0 { return errors.ErrKafkaInvalidConfig.GenWithStack("write-timeout must be greater than zero") } + o.WriteTimeout = a } if urlParameter.ReadTimeout != nil && *urlParameter.ReadTimeout != "" { - o.ReadTimeout, err = time.ParseDuration(*urlParameter.ReadTimeout) + a, err := time.ParseDuration(*urlParameter.ReadTimeout) if err != nil { return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - if o.ReadTimeout <= 0 { + if a <= 0 { return errors.ErrKafkaInvalidConfig.GenWithStack("read-timeout must be greater than zero") } + o.ReadTimeout = a } if urlParameter.RequiredAcks != nil { @@ -399,8 +411,7 @@ func (o *options) applyTLS(params *urlConfig) error { if o.Credential != nil && !o.Credential.IsEmpty() && !o.Credential.IsTLSEnabled() { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "ca, cert and key files should all be supplied") + return errors.ErrKafkaInvalidConfig.GenWithStack("ca, cert and key files should all be supplied") } // if enable-tls is not set, but credential files are set, @@ -413,8 +424,7 @@ func (o *options) applyTLS(params *urlConfig) error { enableTLS := *params.EnableTLS if o.Credential != nil && o.Credential.IsTLSEnabled() && !enableTLS { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "credential files are supplied, but 'enable-tls' is set to false") + return errors.ErrKafkaInvalidConfig.GenWithStack("credential files are supplied, but 'enable-tls' is set to false") } o.EnableTLS = enableTLS } else { @@ -568,7 +578,7 @@ func (o *options) DeriveTopicConfig() *AutoCreateTopicConfig { // ValidateReplicationFactor checks whether a topic created with this config // can satisfy the configured acknowledgment requirement. -func (c *AutoCreateTopicConfig) ValidateReplicationFactor(admin Admin) error { +func (c *AutoCreateTopicConfig) ValidateReplicationFactor(admin AdminClient) error { if c.RequiredAcks != WaitForAll { return nil } @@ -632,7 +642,7 @@ func NewKafkaClientID(captureAddr string, // from the topic or broker configuration. func adjustOptions( changefeedID common.ChangeFeedID, - admin Admin, + admin AdminClient, options *options, topic string, ) error { @@ -640,6 +650,7 @@ func adjustOptions( if err != nil { return err } + info, exists := topics[topic] // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. @@ -658,7 +669,7 @@ func adjustOptions( func adjustExistingTopicOption( changefeedID common.ChangeFeedID, - admin Admin, + admin AdminClient, options *options, info TopicDetail, ) error { @@ -678,7 +689,7 @@ func adjustExistingTopicOption( } func adjustNewTopicOptions( - admin Admin, + admin AdminClient, changefeedID common.ChangeFeedID, options *options, ) { @@ -700,7 +711,7 @@ func adjustNewTopicOptions( } func getTopicMaxMessageBytes( - admin Admin, + admin AdminClient, topic string, ) (int, bool, error) { raw, found, err := getTopicConfig( @@ -716,13 +727,12 @@ func getTopicMaxMessageBytes( } maxMessageBytes, err := strconv.Atoi(raw) if err != nil { - return 0, false, errors.WrapError( - errors.ErrKafkaAdminAPI, err, "parse-config", TopicMaxMessageBytesConfigName) + return 0, false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "parse-config", TopicMaxMessageBytesConfigName) } return maxMessageBytes, true, nil } -func getBrokerMaxMessageBytes(admin Admin) (int, bool, error) { +func getBrokerMaxMessageBytes(admin AdminClient) (int, bool, error) { raw, found, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) if err != nil { return 0, false, err @@ -732,8 +742,7 @@ func getBrokerMaxMessageBytes(admin Admin) (int, bool, error) { } messageMaxBytes, err := strconv.Atoi(raw) if err != nil { - return 0, false, errors.WrapError( - errors.ErrKafkaAdminAPI, err, "parse-config", BrokerMessageMaxBytesConfigName) + return 0, false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "parse-config", BrokerMessageMaxBytesConfigName) } return messageMaxBytes, true, nil } @@ -743,7 +752,7 @@ func getBrokerMaxMessageBytes(admin Admin) (int, bool, error) { // we will try to get it from the broker's configuration. // NOTICE: The configuration names of topic and broker may be different for the same configuration. func getTopicConfig( - admin Admin, + admin AdminClient, topicName string, topicConfigName string, brokerConfigName string, diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 609d62eeae..0cbfdfb884 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -23,131 +23,62 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/golang/mock/gomock" - commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kerr" ) const ( defaultMockTopicName = "mock_topic" // These values model Kafka admin responses, not TiCDC option defaults. - mockClusterReplicationFactor int16 = 3 - mockBrokerMessageMaxBytes = "1048588" - mockTopicMessageMaxBytes = "1048588" - mockMinInsyncReplicas = "1" + mockBrokerMessageMaxBytes = "1048588" + mockTopicMessageMaxBytes = "1048588" ) -type kafkaAdminFixture struct { - admin *MockAdmin - topics map[string]TopicDetail - brokerConfig map[string]string - topicConfig map[string]map[string]string -} - -func newKafkaAdminFixture(t *testing.T) *kafkaAdminFixture { - t.Helper() +func TestKafkaClientSelection(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "client-selection") + require.Equal(t, KafkaClientFranz, NewOptions().Client) - ctrl := gomock.NewController(t) - fixture := &kafkaAdminFixture{ - admin: NewMockAdmin(ctrl), - topics: make(map[string]TopicDetail), - brokerConfig: map[string]string{ - BrokerMessageMaxBytesConfigName: mockBrokerMessageMaxBytes, - MinInsyncReplicasConfigName: mockMinInsyncReplicas, + for _, test := range []struct { + name string + uri string + expected string + wantErr bool + }{ + { + name: "URI selects sarama", + uri: "kafka://127.0.0.1:9092/topic?kafka-client=sarama", + expected: KafkaClientSarama, }, - topicConfig: make(map[string]map[string]string), - } - fixture.addTopic(defaultMockTopicName, defaultPartitionNum) - fixture.topicConfig[defaultMockTopicName] = map[string]string{ - TopicMaxMessageBytesConfigName: mockTopicMessageMaxBytes, - MinInsyncReplicasConfigName: mockMinInsyncReplicas, - } - - fixture.admin.EXPECT().Close().AnyTimes() - fixture.admin.EXPECT().GetTopicsMeta(gomock.Any(), gomock.Any()). - DoAndReturn(fixture.getTopicsMeta).AnyTimes() - fixture.admin.EXPECT().GetBrokerConfig(gomock.Any()). - DoAndReturn(fixture.getBrokerConfig).AnyTimes() - fixture.admin.EXPECT().GetTopicConfig(gomock.Any(), gomock.Any()). - DoAndReturn(fixture.getTopicConfig).AnyTimes() - fixture.admin.EXPECT().CreateTopic(gomock.Any()). - DoAndReturn(fixture.createTopic).AnyTimes() - - return fixture -} - -func (f *kafkaAdminFixture) addTopic(name string, partitionNum int32) { - f.topics[name] = TopicDetail{Name: name, NumPartitions: partitionNum} -} - -func (f *kafkaAdminFixture) getTopicsMeta( - topics []string, _ bool, -) (map[string]TopicDetail, error) { - result := make(map[string]TopicDetail, len(topics)) - for _, topic := range topics { - if detail, ok := f.topics[topic]; ok { - result[topic] = detail - } - } - return result, nil -} - -func (f *kafkaAdminFixture) getBrokerConfig(configName string) (string, bool, error) { - if value, ok := f.brokerConfig[configName]; ok { - return value, true, nil - } - return "", false, nil -} + { + name: "URI value is case insensitive", + uri: "kafka://127.0.0.1:9092/topic?kafka-client=FRANZ", + expected: KafkaClientFranz, + }, + { + name: "invalid client", + uri: "kafka://127.0.0.1:9092/topic?kafka-client=other", + wantErr: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + sinkURI, err := url.Parse(test.uri) + require.NoError(t, err) -func (f *kafkaAdminFixture) getTopicConfig(topicName string, configName string) (string, bool, error) { - if _, ok := f.topics[topicName]; !ok { - return "", false, nil - } - if value, ok := f.topicConfig[topicName][configName]; ok { - return value, true, nil - } - return "", false, nil -} + options := NewOptions() + err = options.Apply(changefeedID, sinkURI, &config.SinkConfig{}) + if test.wantErr { + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + return + } -func (f *kafkaAdminFixture) createTopic(detail TopicDetail) error { - if detail.ReplicationFactor > mockClusterReplicationFactor { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "invalid replication factor %d", detail.ReplicationFactor) - } - if _, ok := f.brokerConfig[MinInsyncReplicasConfigName]; !ok && - detail.ReplicationFactor != mockClusterReplicationFactor { - return errors.WrapError(errors.ErrKafkaAdminAPI, kerr.PolicyViolation, "create-topic", detail.Name) + require.NoError(t, err) + require.Equal(t, test.expected, options.Client) + }) } - f.topics[detail.Name] = detail - return nil -} - -func (f *kafkaAdminFixture) brokerMessageMaxBytes() int { - value, _ := strconv.Atoi(f.brokerConfig[BrokerMessageMaxBytesConfigName]) - return value -} - -func (f *kafkaAdminFixture) topicMaxMessageBytes(topicName string) int { - value, _ := strconv.Atoi(f.topicConfig[topicName][TopicMaxMessageBytesConfigName]) - return value -} - -func (f *kafkaAdminFixture) setMessageMaxBytes(brokerValue, topicValue string) { - f.brokerConfig[BrokerMessageMaxBytesConfigName] = brokerValue - f.topicConfig[defaultMockTopicName][TopicMaxMessageBytesConfigName] = topicValue -} - -func (f *kafkaAdminFixture) setMinInsyncReplicas(minInsyncReplicas string) { - f.topicConfig[defaultMockTopicName][MinInsyncReplicasConfigName] = minInsyncReplicas - f.brokerConfig[MinInsyncReplicasConfigName] = minInsyncReplicas -} - -func (f *kafkaAdminFixture) dropBrokerConfig(configName string) { - delete(f.brokerConfig, configName) } func TestCompleteOptions(t *testing.T) { @@ -162,7 +93,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err := url.Parse(uri) require.NoError(t, err) - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.NoError(t, err) require.Equal(t, int32(1), options.PartitionNum) require.Equal(t, int16(3), options.ReplicationFactor) @@ -177,7 +108,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.NoError(t, err) require.Len(t, options.BrokerEndpoints, 3) @@ -187,7 +118,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Regexp(t, ".*invalid syntax.*", errors.Cause(err)) for _, replicationFactor := range []string{"0", "-1"} { uri = "kafka://127.0.0.1:9092/abc?replication-factor=" + replicationFactor @@ -195,7 +126,7 @@ func TestCompleteOptions(t *testing.T) { require.NoError(t, err) options = NewOptions() err = options.Apply( - commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink, ) @@ -207,7 +138,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Regexp(t, ".*invalid syntax.*", errors.Cause(err)) // Illegal max-retry. @@ -215,7 +146,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Regexp(t, ".*invalid syntax.*", errors.Cause(err)) // Illegal partition-num. @@ -223,7 +154,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Regexp(t, ".*invalid syntax.*", errors.Cause(err)) // Out of range partition-num. @@ -231,7 +162,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Regexp(t, ".*invalid partition num.*", errors.Cause(err)) // Unknown required-acks. @@ -239,7 +170,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Regexp(t, ".*invalid required acks 3.*", errors.Cause(err)) // invalid kafka client id @@ -247,15 +178,15 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + require.True(t, errors.ErrKafkaInvalidConfig.Equal(err)) // max-retry accepts non-negative sink-uri values. uri = "kafka://127.0.0.1:9092/abc?max-retry=7" sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.NoError(t, err) require.Equal(t, 7, options.MaxRetry) @@ -263,7 +194,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.NoError(t, err) require.Equal(t, 0, options.MaxRetry) @@ -272,7 +203,7 @@ func TestCompleteOptions(t *testing.T) { sinkURI, err = url.Parse(uri) require.NoError(t, err) options = NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.NoError(t, err) require.Equal(t, defaultMaxRetry, options.MaxRetry) } @@ -324,7 +255,7 @@ func TestApplySASL(t *testing.T) { "&sasl-gssapi-password=pwd&sasl-gssapi-realm=realm" + "&sasl-gssapi-disable-pafxfast=false", expected: saslConfig{ - mechanism: gssapiMechanismName, + mechanism: gssapiMechanism, gssapi: gssapiConfig{ authType: userAuth, kerberosConfigPath: "/root/config", @@ -343,7 +274,7 @@ func TestApplySASL(t *testing.T) { "&sasl-gssapi-keytab-path=/root/keytab&sasl-gssapi-realm=realm" + "&sasl-gssapi-disable-pafxfast=false", expected: saslConfig{ - mechanism: gssapiMechanismName, + mechanism: gssapiMechanism, gssapi: gssapiConfig{ authType: keyTabAuth, keyTabPath: "/root/keytab", @@ -441,7 +372,7 @@ func TestApplySASL(t *testing.T) { replicaConfig.Sink.KafkaConfig = test.kafkaConfig options := NewOptions() err = options.Apply( - commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, replicaConfig.Sink, ) @@ -513,7 +444,7 @@ func TestApplyTLS(t *testing.T) { require.NoError(t, err) options := NewOptions() err = options.Apply( - commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink, ) @@ -560,7 +491,7 @@ func TestApplyRejectsNonPositiveMaxMessageBytes(t *testing.T) { }, } - changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") for _, test := range tests { t.Run(test.name, func(t *testing.T) { sinkURI, err := url.Parse(test.uri) @@ -586,7 +517,7 @@ func TestApplyRejectsNonPositiveMaxMessageBytes(t *testing.T) { func TestSetPartitionNum(t *testing.T) { options := NewOptions() - changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") err := options.setPartitionNum(changefeedID, 2) require.NoError(t, err) require.Equal(t, int32(2), options.PartitionNum) @@ -598,7 +529,7 @@ func TestSetPartitionNum(t *testing.T) { options.PartitionNum = 3 err = options.setPartitionNum(changefeedID, 2) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.True(t, errors.ErrKafkaInvalidConfig.Equal(err)) } func TestClientID(t *testing.T) { @@ -636,7 +567,7 @@ func TestClientID(t *testing.T) { } for _, tc := range testCases { id, err := NewKafkaClientID(tc.addr, - commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, tc.changefeedID), tc.configuredID) + common.NewChangefeedID4Test(common.DefaultKeyspaceName, tc.changefeedID), tc.configuredID) if tc.hasError { require.Error(t, err) } else { @@ -648,16 +579,16 @@ func TestClientID(t *testing.T) { func TestTimeout(t *testing.T) { options := NewOptions() - require.Equal(t, defaultTimeout, options.DialTimeout) - require.Equal(t, defaultTimeout, options.ReadTimeout) - require.Equal(t, defaultTimeout, options.WriteTimeout) + require.Equal(t, 10*time.Second, options.DialTimeout) + require.Equal(t, 10*time.Second, options.ReadTimeout) + require.Equal(t, 10*time.Second, options.WriteTimeout) uri := "kafka://127.0.0.1:9092/kafka-test?dial-timeout=5s&read-timeout=1000ms" + "&write-timeout=2m" sinkURI, err := url.Parse(uri) require.NoError(t, err) - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.NoError(t, err) require.Equal(t, 5*time.Second, options.DialTimeout) @@ -668,7 +599,7 @@ func TestTimeout(t *testing.T) { func TestApplyRejectsNonPositiveTimeout(t *testing.T) { t.Parallel() - changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") for _, parameter := range []string{"dial-timeout", "read-timeout", "write-timeout"} { for _, value := range []string{"0s", "-1s"} { t.Run(parameter+"="+value, func(t *testing.T) { @@ -690,66 +621,63 @@ func TestApplyRejectsNonPositiveTimeout(t *testing.T) { } func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *testing.T) { + brokerMessageMaxBytes, err := strconv.Atoi(mockBrokerMessageMaxBytes) + require.NoError(t, err) + tests := []struct { name string - configuredMaxMessageBytes func(*kafkaAdminFixture) int + configuredMaxMessageBytes int }{ { - name: "uses broker limit when configured value is below broker", - configuredMaxMessageBytes: func(*kafkaAdminFixture) int { - return 1024 - }, + name: "uses broker limit when configured value is below broker", + configuredMaxMessageBytes: 1024, }, { - name: "uses broker limit when configured value is below broker by one byte", - configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { - return f.brokerMessageMaxBytes() - 1 - }, + name: "uses broker limit when configured value is below broker by one byte", + configuredMaxMessageBytes: brokerMessageMaxBytes - 1, }, { - name: "uses broker limit when configured value is above broker", - configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { - return f.brokerMessageMaxBytes() + 1 - }, + name: "uses broker limit when configured value is above broker", + configuredMaxMessageBytes: brokerMessageMaxBytes + 1, }, } topicName := "test-topic" - changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") for _, test := range tests { t.Run(test.name, func(t *testing.T) { - adminFixture := newKafkaAdminFixture(t) - admin := adminFixture.admin - - detail := TopicDetail{ - Name: topicName, - NumPartitions: 3, - } - err := admin.CreateTopic(detail) - require.NoError(t, err) - - configuredMaxMessageBytes := test.configuredMaxMessageBytes(adminFixture) + ctrl := gomock.NewController(t) + adminClient := NewMockAdminClient(ctrl) + gomock.InOrder( + adminClient.EXPECT().GetTopicsMeta([]string{topicName}, true).Return( + map[string]TopicDetail{ + topicName: {Name: topicName, NumPartitions: 3}, + }, nil), + adminClient.EXPECT().GetTopicConfig(topicName, TopicMaxMessageBytesConfigName). + Return("", false, nil), + adminClient.EXPECT().GetBrokerConfig(BrokerMessageMaxBytesConfigName). + Return(mockBrokerMessageMaxBytes, true, nil), + ) sinkURI, err := url.Parse(fmt.Sprintf( "kafka://127.0.0.1:9092/%s?max-message-bytes=%d", - topicName, configuredMaxMessageBytes, + topicName, test.configuredMaxMessageBytes, )) require.NoError(t, err) options := NewOptions() err = options.Apply(changefeedID, sinkURI, config.GetDefaultReplicaConfig().Sink) require.NoError(t, err) - require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, configuredMaxMessageBytes, options.MaxBatchedBytes) - expectedProducerLimit := adminFixture.brokerMessageMaxBytes() + require.Equal(t, test.configuredMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, test.configuredMaxMessageBytes, options.MaxBatchedBytes) - err = adjustOptions(changefeedID, admin, options, topicName) + err = adjustOptions(changefeedID, adminClient, options, topicName) require.NoError(t, err) - require.NotEqual(t, configuredMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, expectedProducerLimit, options.MaxMessageBytes) + require.NotEqual(t, test.configuredMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, brokerMessageMaxBytes, options.MaxMessageBytes) require.Equal( t, - min(configuredMaxMessageBytes, expectedProducerLimit), + min(test.configuredMaxMessageBytes, brokerMessageMaxBytes), options.MaxBatchedBytes, ) }) @@ -757,16 +685,21 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t } func TestValidateReplicationFactor(t *testing.T) { - adminFixture := newKafkaAdminFixture(t) - admin := adminFixture.admin - adminFixture.setMinInsyncReplicas("2") + ctrl := gomock.NewController(t) + adminClient := NewMockAdminClient(ctrl) + gomock.InOrder( + adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + Return("2", true, nil), + adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + Return("", false, nil), + ) topicConfig := &AutoCreateTopicConfig{ AutoCreate: true, ReplicationFactor: 1, RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(admin) + err := topicConfig.ValidateReplicationFactor(adminClient) require.Regexp( t, ".*`replication-factor` 1 is smaller than the `min.insync.replicas` 2 of broker.*", @@ -778,17 +711,65 @@ func TestValidateReplicationFactor(t *testing.T) { ReplicationFactor: 1, RequiredAcks: WaitForLocal, } - err = localAcksConfig.ValidateReplicationFactor(admin) + err = localAcksConfig.ValidateReplicationFactor(adminClient) require.NoError(t, err) - adminFixture.dropBrokerConfig(MinInsyncReplicasConfigName) missingBrokerConfig := &AutoCreateTopicConfig{ AutoCreate: true, ReplicationFactor: 1, RequiredAcks: WaitForAll, } - err = missingBrokerConfig.ValidateReplicationFactor(admin) + err = missingBrokerConfig.ValidateReplicationFactor(adminClient) require.NoError(t, err) + + t.Run("replication factor satisfies min insync replicas", func(t *testing.T) { + ctrl := gomock.NewController(t) + adminClient := NewMockAdminClient(ctrl) + adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + Return("2", true, nil) + + topicConfig := &AutoCreateTopicConfig{ + ReplicationFactor: 3, + RequiredAcks: WaitForAll, + } + + err := topicConfig.ValidateReplicationFactor(adminClient) + require.NoError(t, err) + }) + + t.Run("invalid min insync replicas", func(t *testing.T) { + ctrl := gomock.NewController(t) + adminClient := NewMockAdminClient(ctrl) + adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + Return("invalid", true, nil) + + topicConfig := &AutoCreateTopicConfig{ + ReplicationFactor: 3, + RequiredAcks: WaitForAll, + } + + err := topicConfig.ValidateReplicationFactor(adminClient) + require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) + }) + + t.Run("broker config lookup failure skips validation", func(t *testing.T) { + ctrl := gomock.NewController(t) + adminClient := NewMockAdminClient(ctrl) + lookupErr := errors.ErrKafkaAdminAPI.GenWithStackByArgs( + "describe-config", + MinInsyncReplicasConfigName, + ) + adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + Return("", false, lookupErr) + + topicConfig := &AutoCreateTopicConfig{ + ReplicationFactor: 1, + RequiredAcks: WaitForAll, + } + + err := topicConfig.ValidateReplicationFactor(adminClient) + require.NoError(t, err) + }) } func TestConfigurationCombinations(t *testing.T) { @@ -827,13 +808,6 @@ func TestConfigurationCombinations(t *testing.T) { mockBrokerMessageMaxBytes, mockTopicMessageMaxBytes, }, - { - "new topic claim check threshold below broker", - "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", - []any{"not-created-topic", "800"}, - mockBrokerMessageMaxBytes, - mockTopicMessageMaxBytes, - }, { "new topic user below default below broker", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", @@ -897,13 +871,6 @@ func TestConfigurationCombinations(t *testing.T) { mockBrokerMessageMaxBytes, mockTopicMessageMaxBytes, }, - { - "existing topic claim check threshold below topic", - "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", - []any{defaultMockTopicName, "800"}, - mockBrokerMessageMaxBytes, - mockTopicMessageMaxBytes, - }, { "existing topic user below default below topic", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", @@ -955,39 +922,53 @@ func TestConfigurationCombinations(t *testing.T) { for _, a := range combinations { t.Run(a.name, func(t *testing.T) { - adminFixture := newKafkaAdminFixture(t) - adminFixture.setMessageMaxBytes(a.brokerMessageMaxBytes, a.topicMaxMessageBytes) - admin := adminFixture.admin - uri := fmt.Sprintf(a.uriTemplate, a.uriParams...) sinkURI, err := url.Parse(uri) require.Nil(t, err) - options := NewOptions() - err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) - require.Nil(t, err) - configuredMaxMessageBytes := options.MaxMessageBytes - topic, ok := a.uriParams[0].(string) require.True(t, ok) require.NotEqual(t, "", topic) - sourceMaxMessageBytes := adminFixture.brokerMessageMaxBytes() - if _, exists := adminFixture.topics[topic]; exists { - sourceMaxMessageBytes = adminFixture.topicMaxMessageBytes(topic) + ctrl := gomock.NewController(t) + adminClient := NewMockAdminClient(ctrl) + metadataCall := adminClient.EXPECT().GetTopicsMeta([]string{topic}, true) + sourceMaxMessageBytes := a.brokerMessageMaxBytes + if topic == defaultMockTopicName { + metadataCall.Return(map[string]TopicDetail{ + topic: {Name: topic, NumPartitions: defaultPartitionNum}, + }, nil) + gomock.InOrder( + metadataCall, + adminClient.EXPECT().GetTopicConfig(topic, TopicMaxMessageBytesConfigName). + Return(a.topicMaxMessageBytes, true, nil), + ) + sourceMaxMessageBytes = a.topicMaxMessageBytes + } else { + metadataCall.Return(map[string]TopicDetail{}, nil) + gomock.InOrder( + metadataCall, + adminClient.EXPECT().GetBrokerConfig(BrokerMessageMaxBytesConfigName). + Return(a.brokerMessageMaxBytes, true, nil), + ) } - changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") - err = adjustOptions(changefeedID, admin, options, topic) + options := NewOptions() + err = options.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Nil(t, err) - require.Equal(t, sourceMaxMessageBytes, options.MaxMessageBytes) + configuredMaxMessageBytes := options.MaxMessageBytes + + expectedMaxMessageBytes, err := strconv.Atoi(sourceMaxMessageBytes) + require.NoError(t, err) + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") + err = adjustOptions(changefeedID, adminClient, options, topic) + require.Nil(t, err) + require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) require.Equal( t, - min(configuredMaxMessageBytes, sourceMaxMessageBytes), + min(configuredMaxMessageBytes, expectedMaxMessageBytes), options.MaxBatchedBytes, ) - - admin.Close() }) } } @@ -1024,7 +1005,7 @@ func TestMerge(t *testing.T) { Key: aws.String("key.pem"), } c := NewOptions() - err = c.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, replicaConfig.Sink) + err = c.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, replicaConfig.Sink) require.NoError(t, err) require.Equal(t, int32(12), c.PartitionNum) require.Equal(t, int16(5), c.ReplicationFactor) @@ -1106,7 +1087,7 @@ func TestMerge(t *testing.T) { Key: aws.String("key2.pem"), } c = NewOptions() - err = c.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, replicaConfig.Sink) + err = c.Apply(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), sinkURI, replicaConfig.Sink) require.NoError(t, err) require.Equal(t, int32(12), c.PartitionNum) require.Equal(t, int16(5), c.ReplicationFactor) diff --git a/pkg/sink/kafka/sarama_admin_mock.go b/pkg/sink/kafka/sarama_admin_mock.go new file mode 100644 index 0000000000..588e08986b --- /dev/null +++ b/pkg/sink/kafka/sarama_admin_mock.go @@ -0,0 +1,175 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/sink/kafka/admin.go + +// Package kafka is a generated GoMock package. +package kafka + +import ( + reflect "reflect" + + sarama "github.com/IBM/sarama" + gomock "github.com/golang/mock/gomock" +) + +// MocksaramaClient is a mock of saramaClient interface. +type MocksaramaClient struct { + ctrl *gomock.Controller + recorder *MocksaramaClientMockRecorder +} + +// MocksaramaClientMockRecorder is the mock recorder for MocksaramaClient. +type MocksaramaClientMockRecorder struct { + mock *MocksaramaClient +} + +// NewMocksaramaClient creates a new mock instance. +func NewMocksaramaClient(ctrl *gomock.Controller) *MocksaramaClient { + mock := &MocksaramaClient{ctrl: ctrl} + mock.recorder = &MocksaramaClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocksaramaClient) EXPECT() *MocksaramaClientMockRecorder { + return m.recorder +} + +// Brokers mocks base method. +func (m *MocksaramaClient) Brokers() []*sarama.Broker { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Brokers") + ret0, _ := ret[0].([]*sarama.Broker) + return ret0 +} + +// Brokers indicates an expected call of Brokers. +func (mr *MocksaramaClientMockRecorder) Brokers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Brokers", reflect.TypeOf((*MocksaramaClient)(nil).Brokers)) +} + +// Close mocks base method. +func (m *MocksaramaClient) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MocksaramaClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaClient)(nil).Close)) +} + +// Partitions mocks base method. +func (m *MocksaramaClient) Partitions(topic string) ([]int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Partitions", topic) + ret0, _ := ret[0].([]int32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Partitions indicates an expected call of Partitions. +func (mr *MocksaramaClientMockRecorder) Partitions(topic interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Partitions", reflect.TypeOf((*MocksaramaClient)(nil).Partitions), topic) +} + +// MocksaramaClusterAdmin is a mock of saramaClusterAdmin interface. +type MocksaramaClusterAdmin struct { + ctrl *gomock.Controller + recorder *MocksaramaClusterAdminMockRecorder +} + +// MocksaramaClusterAdminMockRecorder is the mock recorder for MocksaramaClusterAdmin. +type MocksaramaClusterAdminMockRecorder struct { + mock *MocksaramaClusterAdmin +} + +// NewMocksaramaClusterAdmin creates a new mock instance. +func NewMocksaramaClusterAdmin(ctrl *gomock.Controller) *MocksaramaClusterAdmin { + mock := &MocksaramaClusterAdmin{ctrl: ctrl} + mock.recorder = &MocksaramaClusterAdminMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocksaramaClusterAdmin) EXPECT() *MocksaramaClusterAdminMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MocksaramaClusterAdmin) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MocksaramaClusterAdminMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).Close)) +} + +// CreateTopic mocks base method. +func (m *MocksaramaClusterAdmin) CreateTopic(topic string, detail *sarama.TopicDetail, validateOnly bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateTopic", topic, detail, validateOnly) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateTopic indicates an expected call of CreateTopic. +func (mr *MocksaramaClusterAdminMockRecorder) CreateTopic(topic, detail, validateOnly interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).CreateTopic), topic, detail, validateOnly) +} + +// DescribeCluster mocks base method. +func (m *MocksaramaClusterAdmin) DescribeCluster() ([]*sarama.Broker, int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeCluster") + ret0, _ := ret[0].([]*sarama.Broker) + ret1, _ := ret[1].(int32) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// DescribeCluster indicates an expected call of DescribeCluster. +func (mr *MocksaramaClusterAdminMockRecorder) DescribeCluster() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeCluster", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).DescribeCluster)) +} + +// DescribeConfig mocks base method. +func (m *MocksaramaClusterAdmin) DescribeConfig(resource sarama.ConfigResource) ([]sarama.ConfigEntry, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeConfig", resource) + ret0, _ := ret[0].([]sarama.ConfigEntry) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeConfig indicates an expected call of DescribeConfig. +func (mr *MocksaramaClusterAdminMockRecorder) DescribeConfig(resource interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeConfig", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).DescribeConfig), resource) +} + +// DescribeTopics mocks base method. +func (m *MocksaramaClusterAdmin) DescribeTopics(topics []string) ([]*sarama.TopicMetadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeTopics", topics) + ret0, _ := ret[0].([]*sarama.TopicMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeTopics indicates an expected call of DescribeTopics. +func (mr *MocksaramaClusterAdminMockRecorder) DescribeTopics(topics interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTopics", reflect.TypeOf((*MocksaramaClusterAdmin)(nil).DescribeTopics), topics) +} diff --git a/pkg/sink/kafka/sarama_admin_test.go b/pkg/sink/kafka/sarama_admin_test.go new file mode 100644 index 0000000000..90fc3530dd --- /dev/null +++ b/pkg/sink/kafka/sarama_admin_test.go @@ -0,0 +1,377 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "io" + "testing" + + "github.com/IBM/sarama" + "github.com/golang/mock/gomock" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestGetBrokerConfig(t *testing.T) { + t.Parallel() + + t.Run("found", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeCluster().Return(nil, int32(1), nil) + admin.EXPECT().DescribeConfig(sarama.ConfigResource{ + Type: sarama.BrokerResource, + Name: "1", + ConfigNames: []string{"message.max.bytes"}, + }).Return([]sarama.ConfigEntry{ + {Name: "unrelated", Value: "value"}, + {Name: "message.max.bytes", Value: "1048576"}, + }, nil) + + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + value, found, err := client.GetBrokerConfig("message.max.bytes") + + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "1048576", value) + }) + + t.Run("not found", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeCluster().Return(nil, int32(1), nil) + admin.EXPECT().DescribeConfig(gomock.Any()).Return([]sarama.ConfigEntry{}, nil) + + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + value, found, err := client.GetBrokerConfig("missing") + + require.NoError(t, err) + require.False(t, found) + require.Empty(t, value) + }) + + t.Run("admin error", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + cause := io.ErrUnexpectedEOF + admin.EXPECT().DescribeCluster().Return(nil, int32(0), cause) + + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + _, _, err := client.GetBrokerConfig("missing") + + require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.ErrorIs(t, err, cause) + }) +} + +func TestGetTopicConfig(t *testing.T) { + t.Parallel() + + t.Run("found", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeConfig(sarama.ConfigResource{ + Type: sarama.TopicResource, + Name: "test-topic", + ConfigNames: []string{"max.message.bytes"}, + }).Return([]sarama.ConfigEntry{ + {Name: "max.message.bytes", Value: "1048576"}, + }, nil) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + value, found, err := client.GetTopicConfig("test-topic", "max.message.bytes") + + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "1048576", value) + }) + + t.Run("not found", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeConfig(gomock.Any()).Return([]sarama.ConfigEntry{}, nil) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + value, found, err := client.GetTopicConfig("test-topic", "missing") + + require.NoError(t, err) + require.False(t, found) + require.Empty(t, value) + }) + + t.Run("admin error", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeConfig(gomock.Any()).Return(nil, context.DeadlineExceeded) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + _, _, err := client.GetTopicConfig("test-topic", "missing") + + require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.False(t, IsAuthorizationFailed(err)) + }) +} + +func TestGetTopicsMeta(t *testing.T) { + t.Parallel() + + t.Run("returns valid topics and ignores unknown topics", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeTopics([]string{"valid-topic", "missing-topic"}).Return([]*sarama.TopicMetadata{ + { + Name: "valid-topic", + Partitions: []*sarama.PartitionMetadata{{}, {}}, + }, + { + Name: "missing-topic", + Err: sarama.ErrUnknownTopicOrPartition, + }, + }, nil) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + topics, err := client.GetTopicsMeta([]string{"valid-topic", "missing-topic"}, false) + + require.NoError(t, err) + require.Equal(t, map[string]TopicDetail{ + "valid-topic": { + Name: "valid-topic", + NumPartitions: 2, + }, + }, topics) + }) + + t.Run("missing response", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeTopics([]string{"missing-topic"}).Return(nil, nil) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + topics, err := client.GetTopicsMeta([]string{"missing-topic"}, false) + + require.NoError(t, err) + require.Empty(t, topics) + }) + + t.Run("topic error", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeTopics([]string{"test-topic"}).Return([]*sarama.TopicMetadata{ + {Name: "test-topic", Err: sarama.ErrInvalidTopic}, + }, nil) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + + require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.ErrorIs(t, err, sarama.ErrInvalidTopic) + require.False(t, IsAuthorizationFailed(err)) + }) + + t.Run("topic authorization error", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeTopics([]string{"test-topic"}).Return([]*sarama.TopicMetadata{ + {Name: "test-topic", Err: sarama.ErrTopicAuthorizationFailed}, + }, nil) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + + require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) + require.NotErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.ErrorIs(t, err, sarama.ErrTopicAuthorizationFailed) + require.True(t, IsAuthorizationFailed(err)) + code, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrKafkaAuthorizationFailed.RFCCode(), code) + }) + + t.Run("cluster authorization error", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeTopics([]string{"test-topic"}).Return(nil, sarama.ErrClusterAuthorizationFailed) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + + require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) + require.NotErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.ErrorIs(t, err, sarama.ErrClusterAuthorizationFailed) + require.True(t, IsAuthorizationFailed(err)) + }) + + t.Run("ignored topic error", func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().DescribeTopics([]string{"test-topic"}).Return([]*sarama.TopicMetadata{ + {Name: "test-topic", Err: sarama.ErrInvalidTopic}, + }, nil) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + topics, err := client.GetTopicsMeta([]string{"test-topic"}, true) + + require.NoError(t, err) + require.Empty(t, topics) + }) +} + +func TestIsAuthorizationFailed(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + expected bool + }{ + {name: "TiCDC authorization error", err: errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs("describe-topic", "test-topic"), expected: true}, + {name: "topic authorization error", err: sarama.ErrTopicAuthorizationFailed, expected: true}, + {name: "cluster authorization error", err: sarama.ErrClusterAuthorizationFailed, expected: true}, + {name: "general error", err: sarama.ErrInvalidTopic}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.expected, IsAuthorizationFailed(test.err)) + }) + } +} + +func TestCreateTopic(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + adminErr error + expectedErr error + authorization bool + }{ + {name: "success"}, + {name: "topic already exists", adminErr: sarama.ErrTopicAlreadyExists}, + {name: "authorization error", adminErr: sarama.ErrClusterAuthorizationFailed, expectedErr: errors.ErrKafkaAuthorizationFailed, authorization: true}, + {name: "general error", adminErr: sarama.ErrInvalidReplicationFactor, expectedErr: errors.ErrKafkaAdminAPI}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().CreateTopic("test-topic", &sarama.TopicDetail{ + NumPartitions: 3, + ReplicationFactor: 2, + }, false).Return(test.adminErr) + client := &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + admin: admin, + } + + err := client.CreateTopic(&TopicDetail{ + Name: "test-topic", + NumPartitions: 3, + ReplicationFactor: 2, + }) + + if test.expectedErr == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, test.expectedErr) + require.ErrorIs(t, err, test.adminErr) + if test.authorization { + require.NotErrorIs(t, err, errors.ErrKafkaAdminAPI) + } + }) + } +} + +func TestAdminClientClose(t *testing.T) { + tests := []struct { + name string + setup func(*gomock.Controller) *saramaAdminClient + }{ + { + name: "uses admin close", + setup: func(ctrl *gomock.Controller) *saramaAdminClient { + client := NewMocksaramaClient(ctrl) + admin := NewMocksaramaClusterAdmin(ctrl) + admin.EXPECT().Close().Return(nil) + client.EXPECT().Close().Times(0) + return &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + client: client, + admin: admin, + } + }, + }, + { + name: "falls back to client when admin is nil", + setup: func(ctrl *gomock.Controller) *saramaAdminClient { + client := NewMocksaramaClient(ctrl) + client.EXPECT().Close().Return(nil) + return &saramaAdminClient{ + changefeed: common.NewChangeFeedIDWithName("test", "default"), + client: client, + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + adminClient := test.setup(ctrl) + + require.NotPanics(t, func() { adminClient.Close() }) + }) + } +} diff --git a/pkg/sink/kafka/sarama_async_producer.go b/pkg/sink/kafka/sarama_async_producer.go new file mode 100644 index 0000000000..5a9ea691e1 --- /dev/null +++ b/pkg/sink/kafka/sarama_async_producer.go @@ -0,0 +1,173 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "time" + + "github.com/IBM/sarama" + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +type saramaAsyncProducer struct { + client sarama.Client + producer sarama.AsyncProducer + changefeedID common.ChangeFeedID + + closed *atomic.Bool +} + +type messageMetadata struct { + callback func() + logInfo *codecCommon.MessageLogInfo +} + +func (p *saramaAsyncProducer) Close() { + p.closed.Store(true) + go func() { + // We need to close it asynchronously. Otherwise, we might get stuck + // with an unhealthy(i.e. Network jitter, isolation) state of Kafka. + // Safety: + // * If the kafka cluster is running well, it will be closed as soon as possible. + // Also, we cancel all table pipelines before closed, so it's safe. + // * If there is a problem with the kafka cluster, it will shut down the client first, + // which means no more data will be sent because the connection to the broker is dropped. + // Also, we cancel all table pipelines before closed, so it's safe. + // * For Kafka Sink, duplicate data is acceptable. + // * There is a risk of goroutine leakage, but it is acceptable and our main + // goal is not to get stuck with the processor tick. + + // `client` is mainly used by `asyncProducer` to fetch metadata and perform other related + // operations. When we close the `kafkaSaramaProducer`, + // there is no need for TiCDC to make sure that all buffered messages are flushed. + // Consider the situation where the broker is irresponsive. If the client were not + // closed, `asyncProducer.Close()` would waste a mount of time to try flush all messages. + // To prevent the scenario mentioned above, close the client first. + start := time.Now() + if err := p.client.Close(); err != nil { + log.Warn("kafka async producer client close failed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.Duration("duration", time.Since(start)), + zap.Error(err)) + } else { + log.Info("kafka async producer client closed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.Duration("duration", time.Since(start))) + } + + start = time.Now() + if err := p.producer.Close(); err != nil { + log.Warn("kafka async producer close failed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.Duration("duration", time.Since(start)), + zap.Error(err)) + } else { + log.Info("kafka async producer closed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.Duration("duration", time.Since(start))) + } + }() +} + +func (p *saramaAsyncProducer) AsyncRunCallback( + ctx context.Context, +) error { + defer p.closed.Store(true) + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case ack := <-p.producer.Successes(): + if ack != nil { + switch meta := ack.Metadata.(type) { + case *messageMetadata: + if meta != nil && meta.callback != nil { + meta.callback() + } + default: + log.Error("kafka producer received unknown message metadata type", + zap.Any("metadata", ack.Metadata)) + } + } + case err := <-p.producer.Errors(): + // We should not wrap a nil pointer if the pointer + // is of a subtype of `error` because Go would store the type info + // and the resulted `error` variable would not be nil, + // which will cause the pkg/error library to malfunction. + // See: https://go.dev/doc/faq#nil_error + if err == nil { + return nil + } + return p.handleProducerError(err.Err, extractLogInfo(err.Msg)) + } + } +} + +func (p *saramaAsyncProducer) handleProducerError(err error, logInfo *codecCommon.MessageLogInfo) error { + log.Error("kafka message send failed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.String("eventContext", BuildEventLogContext( + p.changefeedID.Keyspace(), p.changefeedID.Name(), logInfo)), + zap.Error(err)) + return errors.WrapError(errors.ErrKafkaSendMessage, err) +} + +// AsyncSend is the input channel for the user to write messages to that they +// wish to send. +func (p *saramaAsyncProducer) AsyncSend( + ctx context.Context, topic string, partition int32, message *codecCommon.Message, +) error { + if p.closed.Load() { + return errors.ErrKafkaSinkClosed.GenWithStackByArgs() + } + meta := &messageMetadata{ + callback: message.Callback, + logInfo: message.LogInfo, + } + msg := &sarama.ProducerMessage{ + Topic: topic, + Partition: partition, + Key: sarama.StringEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), + Metadata: meta, + } + select { + case <-ctx.Done(): + return context.Cause(ctx) + case p.producer.Input() <- msg: + } + return nil +} + +func extractLogInfo(msg *sarama.ProducerMessage) *codecCommon.MessageLogInfo { + if msg == nil { + return nil + } + meta, ok := msg.Metadata.(*messageMetadata) + if !ok || meta == nil { + return nil + } + return meta.logInfo +} diff --git a/pkg/sink/kafka/sarama_config.go b/pkg/sink/kafka/sarama_config.go new file mode 100644 index 0000000000..932de956b6 --- /dev/null +++ b/pkg/sink/kafka/sarama_config.go @@ -0,0 +1,269 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "crypto/tls" + "math/rand" + "strings" + "time" + + "github.com/IBM/sarama" + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +var ( + defaultKafkaVersion = sarama.V2_0_0_0 + maxKafkaVersion = sarama.V2_8_0_0 +) + +// newSaramaConfig return the default config and set the according version and metrics +func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { + config := sarama.NewConfig() + config.ClientID = o.ClientID + var err error + // Admin client would refresh metadata periodically, + // if metadata cannot be refreshed easily, this would indicate the network condition between the + // capture server and kafka broker is not good. + // Set the timeout to 2 minutes to ensure that the underlying client does not retry for too long. + // If retrying to obtain the metadata fails, simply return the error and let sinkManager rebuild the sink. + config.Metadata.Retry.Max = 10 + config.Metadata.Retry.Backoff = 200 * time.Millisecond + config.Metadata.Timeout = 2 * time.Minute + config.Admin.Retry.Max = 10 + config.Admin.Retry.Backoff = 200 * time.Millisecond + // This timeout control the request timeout for each admin request. + // set it as the read timeout. + config.Admin.Timeout = 10 * time.Second + + // Keep a bounded producer retry budget to tolerate transient broker-side + // connection failures such as stale connections or broken pipe errors. + // The PingCAP Sarama fork includes the partition-muting ordering fix, while + // Net.MaxOpenRequests=1 below remains an extra ordering guard. + config.Producer.Retry.Max = o.MaxRetry + config.Producer.Retry.Backoff = 100 * time.Millisecond + + // make sure sarama producer flush messages as soon as possible. + config.Producer.Flush.Bytes = 0 + config.Producer.Flush.Messages = 0 + config.Producer.Flush.Frequency = time.Duration(0) + config.Producer.Flush.MaxMessages = 0 + + config.Net.MaxOpenRequests = 1 + config.Net.DialTimeout = o.DialTimeout + config.Net.WriteTimeout = o.WriteTimeout + config.Net.ReadTimeout = o.ReadTimeout + + config.Producer.Partitioner = sarama.NewManualPartitioner + config.Producer.MaxMessageBytes = o.MaxMessageBytes + config.Producer.Return.Successes = true + config.Producer.Return.Errors = true + config.Producer.RequiredAcks = sarama.RequiredAcks(o.RequiredAcks) + compression := strings.ToLower(strings.TrimSpace(o.Compression)) + switch compression { + case "none": + config.Producer.Compression = sarama.CompressionNone + case "gzip": + config.Producer.Compression = sarama.CompressionGZIP + case "snappy": + config.Producer.Compression = sarama.CompressionSnappy + case "lz4": + config.Producer.Compression = sarama.CompressionLZ4 + case "zstd": + config.Producer.Compression = sarama.CompressionZSTD + default: + log.Warn("unsupported kafka compression algorithm", zap.String("compression", o.Compression)) + config.Producer.Compression = sarama.CompressionNone + } + + if o.EnableTLS { + // for SSL encryption with a trust CA certificate, we must populate the + // following two params of config.Net.TLS + config.Net.TLS.Enable = true + config.Net.TLS.Config = &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + } + + // for SSL encryption with self-signed CA certificate, we reassign the + // config.Net.TLS.Config using the relevant credential files. + if o.Credential != nil && o.Credential.IsTLSEnabled() { + config.Net.TLS.Config, err = o.Credential.ToTLSConfig() + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + } + + config.Net.TLS.Config.InsecureSkipVerify = o.InsecureSkipVerify + } + + err = completeSaramaSASLConfig(ctx, config, o) + if err != nil { + return nil, err + } + + err = completeSaramaKafkaVersion(config, o) + if err != nil { + return nil, err + } + return config, nil +} + +func completeSaramaKafkaVersion(config *sarama.Config, o *options) error { + detectedVersion, err := detectKafkaVersion(config, o) + if err != nil { + log.Warn("kafka version detection failed, using fallback version", + zap.Strings("brokers", o.BrokerEndpoints), + zap.String("fallbackVersion", detectedVersion.String()), + zap.Error(err)) + } + kafkaVersion, err := selectKafkaVersion(detectedVersion, o) + if err != nil { + return err + } + config.Version = kafkaVersion + return nil +} + +func completeSaramaSASLConfig(ctx context.Context, config *sarama.Config, o *options) error { + if o.sasl != nil && o.sasl.mechanism != "" { + config.Net.SASL.Enable = true + config.Net.SASL.Mechanism = sarama.SASLMechanism(o.sasl.mechanism) + switch o.sasl.mechanism { + case scram256Mechanism, scram512Mechanism, plainMechanism: + config.Net.SASL.User = o.sasl.user + config.Net.SASL.Password = o.sasl.password + if strings.EqualFold(string(o.sasl.mechanism), string(scram256Mechanism)) { + config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { + return &xdgSCRAMClient{HashGeneratorFcn: sha256HashGenerator} + } + } else if strings.EqualFold(string(o.sasl.mechanism), string(scram512Mechanism)) { + config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { + return &xdgSCRAMClient{HashGeneratorFcn: sha512HashGenerator} + } + } + case gssapiMechanism: + config.Net.SASL.GSSAPI.AuthType = int(o.sasl.gssapi.authType) + config.Net.SASL.GSSAPI.Username = o.sasl.gssapi.username + config.Net.SASL.GSSAPI.ServiceName = o.sasl.gssapi.serviceName + config.Net.SASL.GSSAPI.KerberosConfigPath = o.sasl.gssapi.kerberosConfigPath + config.Net.SASL.GSSAPI.Realm = o.sasl.gssapi.realm + config.Net.SASL.GSSAPI.DisablePAFXFAST = o.sasl.gssapi.disablePAFXFAST + switch o.sasl.gssapi.authType { + case userAuth: + config.Net.SASL.GSSAPI.Password = o.sasl.gssapi.password + case keyTabAuth: + config.Net.SASL.GSSAPI.KeyTabPath = o.sasl.gssapi.keyTabPath + } + + case oauthMechanism: + p, err := newTokenProvider(ctx, o) + if err != nil { + return err + } + config.Net.SASL.TokenProvider = p + } + } + + return nil +} + +func detectKafkaVersion(config *sarama.Config, o *options) (sarama.KafkaVersion, error) { + addrs := o.BrokerEndpoints + if len(addrs) > 1 { + // Shuffle the list of addresses to randomize the order in which + // connections are attempted. This prevents routing all connections + // to the first broker (which will usually succeed). + rand.Shuffle(len(addrs), func(i, j int) { + addrs[i], addrs[j] = addrs[j], addrs[i] + }) + } + + var ( + err error + targetVersion sarama.KafkaVersion + ) + for i := range addrs { + targetVersion, err = getKafkaVersionFromBroker(config, o.RequestVersion, addrs[i]) + if err == nil { + break + } + } + if err != nil { + targetVersion = defaultKafkaVersion + } + return targetVersion, err +} + +func selectKafkaVersion(detectedVersion sarama.KafkaVersion, o *options) (sarama.KafkaVersion, error) { + if !o.IsAssignedVersion { + return detectedVersion, nil + } + assignedVersion, err := sarama.ParseKafkaVersion(o.Version) + if err != nil { + return assignedVersion, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if !assignedVersion.IsAtLeast(maxKafkaVersion) && + assignedVersion.String() != detectedVersion.String() { + log.Warn("configured kafka version differs from detected version", + zap.String("assignedVersion", assignedVersion.String()), + zap.String("desiredVersion", detectedVersion.String())) + } + return assignedVersion, nil +} + +func getKafkaVersionFromBroker(config *sarama.Config, requestVersion int16, addr string) (sarama.KafkaVersion, error) { + KafkaVersion := defaultKafkaVersion + broker := sarama.NewBroker(addr) + err := broker.Open(config) + defer func() { + _ = broker.Close() + }() + if err != nil { + return KafkaVersion, err + } + apiResponse, err := broker.ApiVersions(&sarama.ApiVersionsRequest{Version: requestVersion}) + if err != nil { + return KafkaVersion, err + } + // ApiKey method + // 0 Produce + // 3 Metadata (default) + version := apiResponse.ApiKeys[3].MaxVersion + if version >= 10 { + KafkaVersion = sarama.V2_8_0_0 + } else if version >= 9 { + KafkaVersion = sarama.V2_4_0_0 + } else if version >= 8 { + KafkaVersion = sarama.V2_3_0_0 + } else if version >= 7 { + KafkaVersion = sarama.V2_1_0_0 + } else if version >= 6 { + KafkaVersion = sarama.V2_0_0_0 + } else if version >= 5 { + KafkaVersion = sarama.V1_0_0_0 + } else if version >= 3 { + KafkaVersion = sarama.V0_11_0_0 + } else if version >= 2 { + KafkaVersion = sarama.V0_10_1_0 + } else if version >= 1 { + KafkaVersion = sarama.V0_10_0_0 + } else if version >= 0 { + KafkaVersion = sarama.V0_8_2_0 + } + return KafkaVersion, nil +} diff --git a/pkg/sink/kafka/sarama_config_test.go b/pkg/sink/kafka/sarama_config_test.go new file mode 100644 index 0000000000..d8070361ba --- /dev/null +++ b/pkg/sink/kafka/sarama_config_test.go @@ -0,0 +1,316 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "net/url" + "testing" + + "github.com/IBM/sarama" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" + "github.com/stretchr/testify/require" +) + +func TestNewSaramaConfig(t *testing.T) { + options := NewOptions() + options.Version = "invalid" + options.IsAssignedVersion = true + ctx := context.Background() + _, err := newSaramaConfig(ctx, options) + require.Regexp(t, "invalid version.*", errors.Cause(err)) + options.Version = "2.6.0" + + options.ClientID = "test-kafka-client" + compressionCases := []struct { + algorithm string + expected sarama.CompressionCodec + }{ + {"none", sarama.CompressionNone}, + {"gzip", sarama.CompressionGZIP}, + {"snappy", sarama.CompressionSnappy}, + {"lz4", sarama.CompressionLZ4}, + {"zstd", sarama.CompressionZSTD}, + {"others", sarama.CompressionNone}, + } + for _, cc := range compressionCases { + options.Compression = cc.algorithm + cfg, err := newSaramaConfig(ctx, options) + require.NoError(t, err) + require.Equal(t, cc.expected, cfg.Producer.Compression) + } + cfg, err := newSaramaConfig(ctx, options) + require.NoError(t, err) + require.Equal(t, defaultMaxRetry, cfg.Producer.Retry.Max) + require.Equal(t, options.MaxMessageBytes, cfg.Producer.MaxMessageBytes) + + options.EnableTLS = true + options.Credential = &security.Credential{ + CAPath: "/invalid/ca/path", + CertPath: "/invalid/cert/path", + KeyPath: "/invalid/key/path", + } + _, err = newSaramaConfig(ctx, options) + require.Regexp(t, ".*no such file or directory", errors.Cause(err)) + + saslOptions := NewOptions() + saslOptions.Version = "2.6.0" + saslOptions.ClientID = "test-sasl-scram" + saslOptions.sasl = &saslConfig{ + user: "user", + password: "password", + mechanism: scram256Mechanism, + } + + cfg, err = newSaramaConfig(ctx, saslOptions) + require.NoError(t, err) + require.NotNil(t, cfg) + require.Equal(t, "user", cfg.Net.SASL.User) + require.Equal(t, "password", cfg.Net.SASL.Password) + require.Equal(t, sarama.SASLMechanism("SCRAM-SHA-256"), cfg.Net.SASL.Mechanism) +} + +func TestSelectKafkaVersion(t *testing.T) { + tests := []struct { + name string + detectedVersion sarama.KafkaVersion + assignedVersion string + expectedVersion sarama.KafkaVersion + expectedErr error + }{ + { + name: "use detected version", + detectedVersion: sarama.V2_4_0_0, + expectedVersion: sarama.V2_4_0_0, + }, + { + name: "use fallback version", + detectedVersion: defaultKafkaVersion, + expectedVersion: defaultKafkaVersion, + }, + { + name: "assigned version overrides detected version", + detectedVersion: sarama.V2_4_0_0, + assignedVersion: "2.6.0", + expectedVersion: sarama.V2_6_0_0, + }, + { + name: "assigned version overrides fallback version", + detectedVersion: defaultKafkaVersion, + assignedVersion: "2.6.0", + expectedVersion: sarama.V2_6_0_0, + }, + { + name: "reject invalid assigned version", + detectedVersion: sarama.V2_4_0_0, + assignedVersion: "invalid", + expectedErr: errors.ErrKafkaInvalidConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + options := NewOptions() + if test.assignedVersion != "" { + options.IsAssignedVersion = true + options.Version = test.assignedVersion + } + + version, err := selectKafkaVersion(test.detectedVersion, options) + if test.expectedErr != nil { + require.ErrorIs(t, err, test.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, test.expectedVersion, version) + }) + } +} + +func TestNewSaramaConfigMaxRetryFromSinkURI(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sinkURI string + expected int + }{ + { + name: "default max retry", + sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&kafka-client-id=unit-test", + expected: defaultMaxRetry, + }, + { + name: "set max retry", + sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0" + + "&kafka-client-id=unit-test&max-retry=7", + expected: 7, + }, + { + name: "zero max retry", + sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0" + + "&kafka-client-id=unit-test&max-retry=0", + expected: 0, + }, + { + name: "negative max retry", + sinkURI: "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0" + + "&kafka-client-id=unit-test&max-retry=-1", + expected: defaultMaxRetry, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + options := NewOptions() + sinkURI, err := url.Parse(test.sinkURI) + require.NoError(t, err) + err = options.Apply( + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), + sinkURI, + config.GetDefaultReplicaConfig().Sink, + ) + require.NoError(t, err) + + cfg, err := newSaramaConfig(context.Background(), options) + require.NoError(t, err) + require.Equal(t, test.expected, cfg.Producer.Retry.Max) + }) + } +} + +func TestCompleteSaramaSASLConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sasl *saslConfig + verify func(*testing.T, *sarama.Config) + }{ + { + name: "disabled", + sasl: &saslConfig{}, + verify: func(t *testing.T, config *sarama.Config) { + require.False(t, config.Net.SASL.Enable) + }, + }, + { + name: "PLAIN", + sasl: &saslConfig{user: "user", password: "password", mechanism: plainMechanism}, + verify: func(t *testing.T, config *sarama.Config) { + require.Equal(t, "user", config.Net.SASL.User) + require.Equal(t, "password", config.Net.SASL.Password) + require.Nil(t, config.Net.SASL.SCRAMClientGeneratorFunc) + }, + }, + { + name: "SCRAM-SHA-256", + sasl: &saslConfig{user: "user", password: "password", mechanism: scram256Mechanism}, + verify: func(t *testing.T, config *sarama.Config) { + require.Equal(t, "user", config.Net.SASL.User) + require.Equal(t, "password", config.Net.SASL.Password) + require.NotNil(t, config.Net.SASL.SCRAMClientGeneratorFunc) + }, + }, + { + name: "SCRAM-SHA-512", + sasl: &saslConfig{user: "user", password: "password", mechanism: scram512Mechanism}, + verify: func(t *testing.T, config *sarama.Config) { + require.Equal(t, "user", config.Net.SASL.User) + require.Equal(t, "password", config.Net.SASL.Password) + require.NotNil(t, config.Net.SASL.SCRAMClientGeneratorFunc) + }, + }, + { + name: "GSSAPI user auth", + sasl: &saslConfig{mechanism: gssapiMechanism, gssapi: gssapiConfig{ + authType: userAuth, + kerberosConfigPath: "/etc/krb5.conf", + serviceName: "kafka", + username: "user", + password: "password", + realm: "EXAMPLE.COM", + disablePAFXFAST: true, + }}, + verify: func(t *testing.T, config *sarama.Config) { + require.Equal(t, int(userAuth), config.Net.SASL.GSSAPI.AuthType) + require.Equal(t, "/etc/krb5.conf", config.Net.SASL.GSSAPI.KerberosConfigPath) + require.Equal(t, "kafka", config.Net.SASL.GSSAPI.ServiceName) + require.Equal(t, "user", config.Net.SASL.GSSAPI.Username) + require.Equal(t, "password", config.Net.SASL.GSSAPI.Password) + require.Empty(t, config.Net.SASL.GSSAPI.KeyTabPath) + require.Equal(t, "EXAMPLE.COM", config.Net.SASL.GSSAPI.Realm) + require.True(t, config.Net.SASL.GSSAPI.DisablePAFXFAST) + }, + }, + { + name: "GSSAPI keytab auth", + sasl: &saslConfig{mechanism: gssapiMechanism, gssapi: gssapiConfig{ + authType: keyTabAuth, + keyTabPath: "/tmp/user.keytab", + kerberosConfigPath: "/etc/krb5.conf", + serviceName: "kafka", + username: "user", + password: "unused", + realm: "EXAMPLE.COM", + }}, + verify: func(t *testing.T, config *sarama.Config) { + require.Equal(t, int(keyTabAuth), config.Net.SASL.GSSAPI.AuthType) + require.Equal(t, "/tmp/user.keytab", config.Net.SASL.GSSAPI.KeyTabPath) + require.Empty(t, config.Net.SASL.GSSAPI.Password) + }, + }, + { + name: "OAUTHBEARER", + sasl: &saslConfig{mechanism: oauthMechanism, oauth2: oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: "http://127.0.0.1/token", + }}, + verify: func(t *testing.T, config *sarama.Config) { + require.NotNil(t, config.Net.SASL.TokenProvider) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + config := sarama.NewConfig() + options := NewOptions() + options.sasl = test.sasl + err := completeSaramaSASLConfig(t.Context(), config, options) + require.NoError(t, err) + if test.sasl.mechanism != "" { + require.True(t, config.Net.SASL.Enable) + require.Equal(t, sarama.SASLMechanism(test.sasl.mechanism), config.Net.SASL.Mechanism) + } + test.verify(t, config) + }) + } +} + +func TestSaramaTimeout(t *testing.T) { + options := NewOptions() + saramaConfig, err := newSaramaConfig(context.Background(), options) + require.NoError(t, err) + require.Equal(t, options.DialTimeout, saramaConfig.Net.DialTimeout) + require.Equal(t, options.WriteTimeout, saramaConfig.Net.WriteTimeout) + require.Equal(t, options.ReadTimeout, saramaConfig.Net.ReadTimeout) +} diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go new file mode 100644 index 0000000000..e86cdc78a3 --- /dev/null +++ b/pkg/sink/kafka/sarama_factory.go @@ -0,0 +1,194 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "time" + + "github.com/IBM/sarama" + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/rcrowley/go-metrics" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +type saramaFactory struct { + changefeedID common.ChangeFeedID + option *options + metricRegistry metrics.Registry +} + +// NewSaramaFactory constructs a Factory with sarama implementation. +func NewSaramaFactory( + ctx context.Context, + o *options, + changefeedID common.ChangeFeedID, +) (Factory, error) { + start := time.Now() + config, err := newSaramaConfig(ctx, o) + duration := time.Since(start) + if duration > 2*time.Second { + log.Warn("kafka configuration initialization is slow", + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.Duration("duration", duration)) + } + if err != nil { + return nil, err + } + + admin, err := newAdminClient(changefeedID, o.BrokerEndpoints, config) + if err != nil { + return nil, err + } + defer func() { + admin.Close() + }() + + if err = adjustOptions(changefeedID, admin, o, o.Topic); err != nil { + return nil, err + } + log.Info("kafka sink configuration resolved", + zap.String("namespace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.String("topic", o.Topic), + zap.Int32("partitionNum", o.PartitionNum), + zap.Int("maxMessageBytes", o.MaxMessageBytes), + zap.Int("maxBatchedBytes", o.MaxBatchedBytes), + zap.String("compression", config.Producer.Compression.String()), + zap.Int16("requiredAcks", int16(o.RequiredAcks)), + zap.Int("maxRetry", o.MaxRetry), + zap.Duration("dialTimeout", o.DialTimeout), + zap.Duration("readTimeout", o.ReadTimeout), + zap.Duration("writeTimeout", o.WriteTimeout)) + + return &saramaFactory{ + changefeedID: changefeedID, + option: o, + metricRegistry: metrics.NewRegistry(), + }, nil +} + +func newAdminClient(changefeedID common.ChangeFeedID, endpoints []string, config *sarama.Config) (AdminClient, error) { + start := time.Now() + client, err := sarama.NewClient(endpoints, config) + duration := time.Since(start) + if duration > 2*time.Second { + log.Warn("kafka client initialization is slow", + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.Duration("duration", duration)) + } + if err != nil { + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + + start = time.Now() + admin, err := sarama.NewClusterAdminFromClient(client) + duration = time.Since(start) + if duration > 2*time.Second { + log.Warn("kafka admin client initialization is slow", + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.Duration("duration", duration)) + } + if err != nil { + // `sarama.NewClusterAdminFromClient` does not take ownership of the client, + // so we need to close it on failures to avoid leaking background goroutines. + _ = client.Close() + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + return &saramaAdminClient{ + client: client, + admin: admin, + changefeed: changefeedID, + }, nil +} + +func (f *saramaFactory) AdminClient(ctx context.Context) (AdminClient, error) { + config, err := newSaramaConfig(ctx, f.option) + if err != nil { + return nil, err + } + return newAdminClient(f.changefeedID, f.option.BrokerEndpoints, config) +} + +// SyncProducer returns a Sync SyncProducer, +// it should be the caller's responsibility to close the producer +func (f *saramaFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { + config, err := newSaramaConfig(ctx, f.option) + if err != nil { + return nil, err + } + config.MetricRegistry = f.metricRegistry + + client, err := sarama.NewClient(f.option.BrokerEndpoints, config) + if err != nil { + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + + p, err := sarama.NewSyncProducerFromClient(client) + if err != nil { + _ = client.Close() + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + + return &saramaSyncProducer{ + id: f.changefeedID, + client: client, + producer: p, + closed: atomic.NewBool(false), + }, nil +} + +// AsyncProducer return an Async SyncProducer, +// it should be the caller's responsibility to close the producer +func (f *saramaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { + config, err := newSaramaConfig(ctx, f.option) + if err != nil { + return nil, err + } + config.MetricRegistry = f.metricRegistry + + client, err := sarama.NewClient(f.option.BrokerEndpoints, config) + if err != nil { + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + + p, err := sarama.NewAsyncProducerFromClient(client) + if err != nil { + _ = client.Close() + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + return &saramaAsyncProducer{ + client: client, + producer: p, + changefeedID: f.changefeedID, + closed: atomic.NewBool(false), + }, nil +} + +func (f *saramaFactory) MetricsCollector( + adminClient AdminClient, +) MetricsCollector { + return &saramaMetricsCollector{ + changefeedID: f.changefeedID, + adminClient: adminClient, + brokers: make(map[int32]struct{}), + registry: f.metricRegistry, + } +} diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider.go b/pkg/sink/kafka/sarama_oauth2_token_provider.go new file mode 100644 index 0000000000..6a474b6725 --- /dev/null +++ b/pkg/sink/kafka/sarama_oauth2_token_provider.go @@ -0,0 +1,84 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "net/url" + + "github.com/IBM/sarama" + "github.com/pingcap/ticdc/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +// tokenProvider is a user-defined callback for generating +// access tokens for SASL/OAUTHBEARER auth. +type tokenProvider struct { + tokenSource oauth2.TokenSource +} + +var _ sarama.AccessTokenProvider = (*tokenProvider)(nil) + +// Token implements the sarama.AccessTokenProvider interface. +// Token returns an access token. The implementation should ensure token +// reuse so that multiple calls at connect time do not create multiple +// tokens. The implementation should also periodically refresh the token in +// order to guarantee that each call returns an unexpired token. This +// method should not block indefinitely--a timeout error should be returned +// after a short period of inactivity so that the broker connection logic +// can log debugging information and retry. +func (t *tokenProvider) Token() (*sarama.AccessToken, error) { + token, err := t.tokenSource.Token() + if err != nil { + // Errors will result in Sarama retrying the broker connection and logging + // the transient error, with a Broker connection error surfacing after retry + // attempts have been exhausted. + return nil, err + } + + return &sarama.AccessToken{Token: token.AccessToken}, nil +} + +func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvider, error) { + // grant_type is by default going to be set to 'client_credentials' by the + // client credentials library as defined by the spec, however non-compliant + // auth server implementations may want a custom type + endpointParams := url.Values{} + if o.sasl.oauth2.grantType != "" { + endpointParams.Set("grant_type", o.sasl.oauth2.grantType) + } + + // audience is an optional parameter that can be used to specify the + // intended audience of the token. + if o.sasl.oauth2.audience != "" { + endpointParams.Set("audience", o.sasl.oauth2.audience) + } + + tokenURL, err := url.Parse(o.sasl.oauth2.tokenURL) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + + cfg := clientcredentials.Config{ + ClientID: o.sasl.oauth2.clientID, + ClientSecret: o.sasl.oauth2.clientSecret, + TokenURL: tokenURL.String(), + EndpointParams: endpointParams, + Scopes: o.sasl.oauth2.scopes, + } + return &tokenProvider{ + tokenSource: cfg.TokenSource(ctx), + }, nil +} diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go new file mode 100644 index 0000000000..d7d0473c32 --- /dev/null +++ b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go @@ -0,0 +1,134 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" + "golang.org/x/oauth2" +) + +func TestNewTokenProviderRejectsInvalidTokenURL(t *testing.T) { + t.Parallel() + + options := &options{ + sasl: &saslConfig{ + oauth2: oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: "http://test.com/Segment%%2815197306101420000%29", + scopes: []string{"scope1", "scope2"}, + grantType: "client_credentials", + }, + }, + } + + _, err := newTokenProvider(t.Context(), options) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + var escapeErr url.EscapeError + require.ErrorAs(t, err, &escapeErr) + require.ErrorContains(t, err, "invalid URL escape") +} + +func TestTokenProviderRequestsToken(t *testing.T) { + t.Parallel() + + type tokenRequest struct { + method string + path string + form url.Values + err error + } + requestCh := make(chan tokenRequest, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + err := r.ParseForm() + requestCh <- tokenRequest{ + method: r.Method, + path: r.URL.Path, + form: r.PostForm, + err: err, + } + + w.Header().Set("Content-Type", "application/json") + if _, err := io.WriteString(w, `{"access_token":"access-token","token_type":"bearer"}`); err != nil { + t.Errorf("write token response: %v", err) + } + })) + t.Cleanup(server.Close) + + options := &options{ + sasl: &saslConfig{ + oauth2: oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: server.URL + "/oauth2/token", + scopes: []string{"scope1", "scope2"}, + grantType: "custom_grant", + audience: "test-audience", + }, + }, + } + + provider, err := newTokenProvider(t.Context(), options) + require.NoError(t, err) + token, err := provider.Token() + require.NoError(t, err) + require.Equal(t, "access-token", token.Token) + + request := <-requestCh + require.NoError(t, request.err) + require.Equal(t, http.MethodPost, request.method) + require.Equal(t, "/oauth2/token", request.path) + require.Equal(t, "custom_grant", request.form.Get("grant_type")) + require.Equal(t, "test-audience", request.form.Get("audience")) + require.Equal(t, "scope1 scope2", request.form.Get("scope")) +} + +func TestTokenProviderPropagatesEndpointError(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + if _, err := io.WriteString(w, `{"error":"invalid_client","error_description":"bad credentials"}`); err != nil { + t.Errorf("write token error response: %v", err) + } + })) + t.Cleanup(server.Close) + + options := &options{ + sasl: &saslConfig{ + oauth2: oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: server.URL, + }, + }, + } + + provider, err := newTokenProvider(t.Context(), options) + require.NoError(t, err) + _, err = provider.Token() + var retrieveErr *oauth2.RetrieveError + require.ErrorAs(t, err, &retrieveErr) + require.Equal(t, http.StatusUnauthorized, retrieveErr.Response.StatusCode) + require.Equal(t, "invalid_client", retrieveErr.ErrorCode) + require.Equal(t, "bad credentials", retrieveErr.ErrorDescription) +} diff --git a/pkg/sink/kafka/sarama_sync_producer.go b/pkg/sink/kafka/sarama_sync_producer.go new file mode 100644 index 0000000000..fcf1c9c258 --- /dev/null +++ b/pkg/sink/kafka/sarama_sync_producer.go @@ -0,0 +1,130 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "time" + + "github.com/IBM/sarama" + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +type saramaSyncClient interface { + Brokers() []*sarama.Broker + Close() error +} + +type saramaSyncProducerClient interface { + SendMessage(msg *sarama.ProducerMessage) (partition int32, offset int64, err error) + SendMessages(msgs []*sarama.ProducerMessage) error + Close() error +} + +type saramaSyncProducer struct { + id common.ChangeFeedID + client saramaSyncClient + producer saramaSyncProducerClient + closed *atomic.Bool +} + +func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, message *codecCommon.Message) error { + if p.closed.Load() { + return errors.ErrKafkaSinkClosed.GenWithStackByArgs() + } + + msg := &sarama.ProducerMessage{ + Topic: topic, + Key: sarama.ByteEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), + Partition: partitionNum, + } + _, _, err := p.producer.SendMessage(msg) + if err == nil { + return nil + } + log.Error("kafka message send failed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name()), + zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), + zap.Error(err)) + return errors.WrapError(errors.ErrKafkaSendMessage, err) +} + +func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, message *codecCommon.Message) error { + if p.closed.Load() { + return errors.ErrKafkaSinkClosed.GenWithStackByArgs() + } + + msgs := make([]*sarama.ProducerMessage, partitionNum) + for i := 0; i < int(partitionNum); i++ { + msgs[i] = &sarama.ProducerMessage{ + Topic: topic, + Key: sarama.ByteEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), + Partition: int32(i), + } + } + err := p.producer.SendMessages(msgs) + if err == nil { + return nil + } + log.Error("kafka message send failed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name()), + zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), + zap.Error(err)) + return errors.WrapError(errors.ErrKafkaSendMessage, err) +} + +func (p *saramaSyncProducer) Close() { + if p.closed.Load() { + log.Warn("kafka ddl producer already closed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name())) + return + } + + p.closed.Store(true) + start := time.Now() + // sarama.NewSyncProducerFromClient wraps the provided client with a nopCloserClient, + // so producer.Close() alone won't release the underlying client resources. + if p.client != nil { + if err := p.client.Close(); err != nil { + log.Warn("kafka ddl producer client close failed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name()), + zap.Duration("duration", time.Since(start)), + zap.Error(err)) + } + } + if p.producer != nil { + if err := p.producer.Close(); err != nil { + log.Error("kafka ddl producer close failed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name()), + zap.Duration("duration", time.Since(start)), + zap.Error(err)) + return + } + } + log.Info("kafka ddl producer closed", + zap.String("keyspace", p.id.Keyspace()), + zap.String("changefeed", p.id.Name()), + zap.Duration("duration", time.Since(start))) +} diff --git a/pkg/sink/kafka/sarama_sync_producer_mock.go b/pkg/sink/kafka/sarama_sync_producer_mock.go new file mode 100644 index 0000000000..78671e02f2 --- /dev/null +++ b/pkg/sink/kafka/sarama_sync_producer_mock.go @@ -0,0 +1,130 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/sink/kafka/sarama_sync_producer.go + +// Package kafka is a generated GoMock package. +package kafka + +import ( + reflect "reflect" + + sarama "github.com/IBM/sarama" + gomock "github.com/golang/mock/gomock" +) + +// MocksaramaSyncClient is a mock of saramaSyncClient interface. +type MocksaramaSyncClient struct { + ctrl *gomock.Controller + recorder *MocksaramaSyncClientMockRecorder +} + +// MocksaramaSyncClientMockRecorder is the mock recorder for MocksaramaSyncClient. +type MocksaramaSyncClientMockRecorder struct { + mock *MocksaramaSyncClient +} + +// NewMocksaramaSyncClient creates a new mock instance. +func NewMocksaramaSyncClient(ctrl *gomock.Controller) *MocksaramaSyncClient { + mock := &MocksaramaSyncClient{ctrl: ctrl} + mock.recorder = &MocksaramaSyncClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocksaramaSyncClient) EXPECT() *MocksaramaSyncClientMockRecorder { + return m.recorder +} + +// Brokers mocks base method. +func (m *MocksaramaSyncClient) Brokers() []*sarama.Broker { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Brokers") + ret0, _ := ret[0].([]*sarama.Broker) + return ret0 +} + +// Brokers indicates an expected call of Brokers. +func (mr *MocksaramaSyncClientMockRecorder) Brokers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Brokers", reflect.TypeOf((*MocksaramaSyncClient)(nil).Brokers)) +} + +// Close mocks base method. +func (m *MocksaramaSyncClient) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MocksaramaSyncClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaSyncClient)(nil).Close)) +} + +// MocksaramaSyncProducerClient is a mock of saramaSyncProducerClient interface. +type MocksaramaSyncProducerClient struct { + ctrl *gomock.Controller + recorder *MocksaramaSyncProducerClientMockRecorder +} + +// MocksaramaSyncProducerClientMockRecorder is the mock recorder for MocksaramaSyncProducerClient. +type MocksaramaSyncProducerClientMockRecorder struct { + mock *MocksaramaSyncProducerClient +} + +// NewMocksaramaSyncProducerClient creates a new mock instance. +func NewMocksaramaSyncProducerClient(ctrl *gomock.Controller) *MocksaramaSyncProducerClient { + mock := &MocksaramaSyncProducerClient{ctrl: ctrl} + mock.recorder = &MocksaramaSyncProducerClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocksaramaSyncProducerClient) EXPECT() *MocksaramaSyncProducerClientMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MocksaramaSyncProducerClient) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MocksaramaSyncProducerClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocksaramaSyncProducerClient)(nil).Close)) +} + +// SendMessage mocks base method. +func (m *MocksaramaSyncProducerClient) SendMessage(msg *sarama.ProducerMessage) (int32, int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessage", msg) + ret0, _ := ret[0].(int32) + ret1, _ := ret[1].(int64) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// SendMessage indicates an expected call of SendMessage. +func (mr *MocksaramaSyncProducerClientMockRecorder) SendMessage(msg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MocksaramaSyncProducerClient)(nil).SendMessage), msg) +} + +// SendMessages mocks base method. +func (m *MocksaramaSyncProducerClient) SendMessages(msgs []*sarama.ProducerMessage) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessages", msgs) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessages indicates an expected call of SendMessages. +func (mr *MocksaramaSyncProducerClientMockRecorder) SendMessages(msgs interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MocksaramaSyncProducerClient)(nil).SendMessages), msgs) +} diff --git a/pkg/sink/kafka/sarama_sync_producer_test.go b/pkg/sink/kafka/sarama_sync_producer_test.go new file mode 100644 index 0000000000..e00e0dafa8 --- /dev/null +++ b/pkg/sink/kafka/sarama_sync_producer_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "io" + "strings" + "testing" + + "github.com/golang/mock/gomock" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" +) + +func TestProducerRejectsSendAfterClose(t *testing.T) { + t.Parallel() + + message := &codecCommon.Message{} + syncProducer := &saramaSyncProducer{closed: atomic.NewBool(true)} + require.ErrorIs(t, syncProducer.SendMessage("topic", 1, message), errors.ErrKafkaSinkClosed) + require.ErrorIs(t, syncProducer.SendMessages("topic", 1, message), errors.ErrKafkaSinkClosed) + + asyncProducer := &saramaAsyncProducer{closed: atomic.NewBool(true)} + require.ErrorIs(t, asyncProducer.AsyncSend(context.Background(), "topic", 0, message), errors.ErrKafkaSinkClosed) +} + +func TestSyncProducerClose(t *testing.T) { + tests := []struct { + name string + clientCloseErr error + }{ + { + name: "closes client and producer", + }, + { + name: "still closes producer when client close fails", + clientCloseErr: io.ErrClosedPipe, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + client := NewMocksaramaSyncClient(ctrl) + producer := NewMocksaramaSyncProducerClient(ctrl) + gomock.InOrder( + client.EXPECT().Close().Return(test.clientCloseErr), + producer.EXPECT().Close().Return(nil), + ) + + p := &saramaSyncProducer{ + id: common.NewChangeFeedIDWithName("test", "default"), + client: client, + producer: producer, + closed: atomic.NewBool(false), + } + + p.Close() + }) + } +} + +func TestSyncProducerErrorWrappedOnce(t *testing.T) { + cause := io.ErrClosedPipe + tests := []struct { + name string + expectSend func(*MocksaramaSyncProducerClient) + send func(*saramaSyncProducer, *codecCommon.Message) error + }{ + { + name: "single message", + expectSend: func(producer *MocksaramaSyncProducerClient) { + producer.EXPECT().SendMessage(gomock.Any()).Return(int32(0), int64(0), cause) + }, + send: func(producer *saramaSyncProducer, message *codecCommon.Message) error { + return producer.SendMessage("topic", 0, message) + }, + }, + { + name: "message batch", + expectSend: func(producer *MocksaramaSyncProducerClient) { + producer.EXPECT().SendMessages(gomock.Any()).Return(cause) + }, + send: func(producer *saramaSyncProducer, message *codecCommon.Message) error { + return producer.SendMessages("topic", 1, message) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + producer := NewMocksaramaSyncProducerClient(ctrl) + test.expectSend(producer) + p := &saramaSyncProducer{ + id: common.NewChangeFeedIDWithName("test", "default"), + producer: producer, + closed: atomic.NewBool(false), + } + message := &codecCommon.Message{LogInfo: &codecCommon.MessageLogInfo{}} + + err := test.send(p, message) + + requireKafkaSendError(t, err, cause) + }) + } +} + +func TestAsyncProducerErrorWrappedOnce(t *testing.T) { + cause := io.ErrClosedPipe + producer := &saramaAsyncProducer{ + changefeedID: common.NewChangeFeedIDWithName("test", "default"), + } + + err := producer.handleProducerError(cause, &codecCommon.MessageLogInfo{}) + + requireKafkaSendError(t, err, cause) +} + +func requireKafkaSendError(t *testing.T, err, cause error) { + t.Helper() + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) + require.ErrorIs(t, err, cause) + require.Equal(t, 1, strings.Count(err.Error(), string(errors.ErrKafkaSendMessage.RFCCode()))) + require.NotContains(t, err.Error(), "keyspace=test") +} diff --git a/pkg/sink/kafka/sasl_config.go b/pkg/sink/kafka/sasl_config.go index e5f775e2d1..95389a1774 100644 --- a/pkg/sink/kafka/sasl_config.go +++ b/pkg/sink/kafka/sasl_config.go @@ -30,8 +30,8 @@ const ( scram256Mechanism saslMechanism = "SCRAM-SHA-256" // scram512Mechanism means the SASL mechanism is SCRAM-SHA-512. scram512Mechanism saslMechanism = "SCRAM-SHA-512" - // gssapiMechanismName means the SASL mechanism is GSSAPI. - gssapiMechanismName saslMechanism = "GSSAPI" + // gssapiMechanism means the SASL mechanism is GSSAPI. + gssapiMechanism saslMechanism = "GSSAPI" // oauthMechanism means the SASL mechanism is OAUTHBEARER. oauthMechanism saslMechanism = "OAUTHBEARER" ) @@ -46,7 +46,7 @@ func saslMechanismFromString(s string) (saslMechanism, error) { case "scram-sha-512": return scram512Mechanism, nil case "gssapi": - return gssapiMechanismName, nil + return gssapiMechanism, nil case "oauthbearer": return oauthMechanism, nil default: diff --git a/pkg/sink/kafka/sasl_test.go b/pkg/sink/kafka/sasl_test.go deleted file mode 100644 index 14b5e80ded..0000000000 --- a/pkg/sink/kafka/sasl_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestBuildSaslMechanismGSSAPI(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - authType gssapiAuthType - password string - keyTabPath string - }{ - {name: "user", authType: userAuth, password: "pwd"}, - {name: "keytab", authType: keyTabAuth, keyTabPath: "/tmp/a.keytab"}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - o := &options{sasl: &saslConfig{ - mechanism: gssapiMechanismName, - gssapi: gssapiConfig{ - authType: tc.authType, - kerberosConfigPath: "/etc/krb5.conf", - serviceName: "kafka", - username: "alice", - password: tc.password, - keyTabPath: tc.keyTabPath, - realm: "EXAMPLE.COM", - }, - }} - - mechanism, err := buildSaslMechanism(context.Background(), o) - require.NoError(t, err) - require.Equal(t, "GSSAPI", mechanism.Name()) - }) - } -} diff --git a/pkg/sink/kafka/selector.go b/pkg/sink/kafka/selector.go new file mode 100644 index 0000000000..4cbf67bef4 --- /dev/null +++ b/pkg/sink/kafka/selector.go @@ -0,0 +1,36 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + + "github.com/pingcap/ticdc/pkg/common" +) + +// NewFactory selects the Kafka client for one changefeed without automatic fallback. +func NewFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { + if o.Client == KafkaClientSarama { + return NewSaramaFactory(ctx, o, changefeedID) + } + return NewFranzFactory(ctx, o, changefeedID) +} + +// CleanupFactoryMetrics removes metrics owned directly by a client factory. +func CleanupFactoryMetrics(factory Factory) { + if cleaner, ok := factory.(interface{ CleanupMetrics() }); ok { + cleaner.CleanupMetrics() + } +} diff --git a/pkg/sink/kafka/selector_test.go b/pkg/sink/kafka/selector_test.go new file mode 100644 index 0000000000..7eb7598627 --- /dev/null +++ b/pkg/sink/kafka/selector_test.go @@ -0,0 +1,118 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" +) + +func TestFactorySelection(t *testing.T) { + const topic = "factory-selection" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory-selection") + for _, test := range []struct { + client string + expected Factory + }{ + {client: KafkaClientFranz, expected: &franzFactoryAdapter{}}, + {client: KafkaClientSarama, expected: &saramaFactory{}}, + } { + t.Run(test.client, func(t *testing.T) { + options := NewOptions() + options.Client = test.client + options.ClientID = "ticdc-test" + options.BrokerEndpoints = cluster.ListenAddrs() + options.Topic = topic + + factory, err := NewFactory(context.Background(), options, changefeedID) + require.NoError(t, err) + require.IsType(t, test.expected, factory) + + CleanupFactoryMetrics(factory) + }) + } +} + +func TestFactoryDoesNotFallbackAfterFranzFailure(t *testing.T) { + options := NewOptions() + options.ClientID = "ticdc-test" + options.BrokerEndpoints = []string{"127.0.0.1:9092"} + options.Topic = "no-fallback" + options.Version = "invalid" + options.IsAssignedVersion = true + + _, err := NewFactory( + context.Background(), + options, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "no-fallback"), + ) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} + +func TestFranzAndSaramaFactoriesAreIndependent(t *testing.T) { + const topic = "factory-independence" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + franzOptions := NewOptions() + franzOptions.ClientID = "ticdc-franz-test" + franzOptions.BrokerEndpoints = cluster.ListenAddrs() + franzOptions.Topic = topic + + franzFactory, err := NewFactory( + context.Background(), + franzOptions, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz"), + ) + require.NoError(t, err) + t.Cleanup(func() { CleanupFactoryMetrics(franzFactory) }) + + saramaOptions := NewOptions() + saramaOptions.Client = KafkaClientSarama + saramaOptions.ClientID = "ticdc-sarama-test" + saramaOptions.BrokerEndpoints = cluster.ListenAddrs() + saramaOptions.Topic = topic + + saramaFactory, err := NewFactory( + context.Background(), + saramaOptions, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sarama"), + ) + require.NoError(t, err) + + franzProducer, err := franzFactory.SyncProducer(context.Background()) + require.NoError(t, err) + t.Cleanup(franzProducer.Close) + + saramaProducer, err := saramaFactory.SyncProducer(context.Background()) + require.NoError(t, err) + t.Cleanup(saramaProducer.Close) + + message := &codeccommon.Message{Value: []byte("value")} + require.NoError(t, franzProducer.SendMessage(topic, 0, message)) + require.NoError(t, saramaProducer.SendMessage(topic, 0, message)) + + franzProducer.Close() + require.NoError(t, saramaProducer.SendMessage(topic, 0, message)) +} diff --git a/pkg/sink/kafka/sync_producer_mock.go b/pkg/sink/kafka/sync_producer_mock.go deleted file mode 100644 index 58429b08be..0000000000 --- a/pkg/sink/kafka/sync_producer_mock.go +++ /dev/null @@ -1,75 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: pkg/sink/kafka/sync_producer.go - -// Package kafka is a generated GoMock package. -package kafka - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - common "github.com/pingcap/ticdc/pkg/sink/codec/common" -) - -// MockSyncProducer is a mock of SyncProducer interface. -type MockSyncProducer struct { - ctrl *gomock.Controller - recorder *MockSyncProducerMockRecorder -} - -// MockSyncProducerMockRecorder is the mock recorder for MockSyncProducer. -type MockSyncProducerMockRecorder struct { - mock *MockSyncProducer -} - -// NewMockSyncProducer creates a new mock instance. -func NewMockSyncProducer(ctrl *gomock.Controller) *MockSyncProducer { - mock := &MockSyncProducer{ctrl: ctrl} - mock.recorder = &MockSyncProducerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSyncProducer) EXPECT() *MockSyncProducerMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockSyncProducer) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockSyncProducerMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSyncProducer)(nil).Close)) -} - -// SendMessage mocks base method. -func (m *MockSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessage", topic, partitionNum, message) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMessage indicates an expected call of SendMessage. -func (mr *MockSyncProducerMockRecorder) SendMessage(topic, partitionNum, message interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), topic, partitionNum, message) -} - -// SendMessages mocks base method. -func (m *MockSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessages", topic, partitionNum, message) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMessages indicates an expected call of SendMessages. -func (mr *MockSyncProducerMockRecorder) SendMessages(topic, partitionNum, message interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), topic, partitionNum, message) -} diff --git a/pkg/sink/kafka/sync_producer_test.go b/pkg/sink/kafka/sync_producer_test.go deleted file mode 100644 index 8d3abbb704..0000000000 --- a/pkg/sink/kafka/sync_producer_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "testing" - - "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/codec/common" - "github.com/stretchr/testify/require" -) - -func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { - producer := &syncProducer{} - producer.closed.Store(true) - - err := producer.SendMessage("topic", 1, &common.Message{}) - require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) - - err = producer.SendMessages("topic", 1, &common.Message{}) - require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) -} diff --git a/scripts/generate-mock.sh b/scripts/generate-mock.sh index 9ba7ce91ad..f1db9a6a6a 100755 --- a/scripts/generate-mock.sh +++ b/scripts/generate-mock.sh @@ -34,10 +34,10 @@ fi "$MOCKGEN" -source pkg/api/v2/changefeed.go -destination pkg/api/v2/mock/changefeed_mock.go -package mock "$MOCKGEN" -source pkg/api/v2/api_client.go -destination pkg/api/v2/mock/api_client_mock.go -package mock "$MOCKGEN" -source pkg/sink/codec/simple/marshaller.go -destination pkg/sink/codec/simple/mock/marshaller.go -"$MOCKGEN" -source pkg/sink/kafka/admin.go -destination pkg/sink/kafka/admin_mock.go -package kafka +"$MOCKGEN" -source pkg/sink/kafka/admin_client.go -destination pkg/sink/kafka/admin_client_mock.go -package kafka "$MOCKGEN" -source pkg/sink/kafka/factory.go -destination pkg/sink/kafka/factory_mock.go -package kafka -"$MOCKGEN" -source pkg/sink/kafka/sync_producer.go -destination pkg/sink/kafka/sync_producer_mock.go -package kafka -"$MOCKGEN" -source pkg/sink/kafka/async_producer.go -destination pkg/sink/kafka/async_producer_mock.go -package kafka +"$MOCKGEN" -source pkg/sink/kafka/admin.go -destination pkg/sink/kafka/sarama_admin_mock.go -package kafka +"$MOCKGEN" -source pkg/sink/kafka/sarama_sync_producer.go -destination pkg/sink/kafka/sarama_sync_producer_mock.go -package kafka "$MOCKGEN" -source downstreamadapter/sink/topicmanager/topic_manager.go -destination downstreamadapter/sink/topicmanager/topic_manager_mock.go -package topicmanager "$MOCKGEN" -source pkg/keyspace/keyspace_manager.go -destination pkg/keyspace/keyspace_manager_mock.go -package keyspace "$MOCKGEN" -source pkg/txnutil/gc/gc_manager.go -destination pkg/txnutil/gc/gc_manager_mock.go -package gc diff --git a/tests/integration_tests/kafka_compression/data/gzip_data.sql b/tests/integration_tests/kafka_compression/data/gzip_data.sql new file mode 100644 index 0000000000..f1c7671010 --- /dev/null +++ b/tests/integration_tests/kafka_compression/data/gzip_data.sql @@ -0,0 +1,21 @@ +use test; + +create table tp_int_gzip +( + id int auto_increment, + c_tinyint tinyint null, + c_smallint smallint null, + c_mediumint mediumint null, + c_int int null, + c_bigint bigint null, + constraint pk + primary key (id) +); + +insert into tp_int_gzip(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) +values (1, 2, 3, 4, 5); + +create table gzip_finish_mark +( + id int PRIMARY KEY +); diff --git a/tests/integration_tests/kafka_compression/data/lz4_data.sql b/tests/integration_tests/kafka_compression/data/lz4_data.sql new file mode 100644 index 0000000000..6f00b24faa --- /dev/null +++ b/tests/integration_tests/kafka_compression/data/lz4_data.sql @@ -0,0 +1,21 @@ +use test; + +create table tp_int_lz4 +( + id int auto_increment, + c_tinyint tinyint null, + c_smallint smallint null, + c_mediumint mediumint null, + c_int int null, + c_bigint bigint null, + constraint pk + primary key (id) +); + +insert into tp_int_lz4(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) +values (1, 2, 3, 4, 5); + +create table lz4_finish_mark +( + id int PRIMARY KEY +); diff --git a/tests/integration_tests/kafka_compression/data/snappy_data.sql b/tests/integration_tests/kafka_compression/data/snappy_data.sql new file mode 100644 index 0000000000..435e9f6f7f --- /dev/null +++ b/tests/integration_tests/kafka_compression/data/snappy_data.sql @@ -0,0 +1,21 @@ +use test; + +create table tp_int_snappy +( + id int auto_increment, + c_tinyint tinyint null, + c_smallint smallint null, + c_mediumint mediumint null, + c_int int null, + c_bigint bigint null, + constraint pk + primary key (id) +); + +insert into tp_int_snappy(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) +values (1, 2, 3, 4, 5); + +create table snappy_finish_mark +( + id int PRIMARY KEY +); diff --git a/tests/integration_tests/kafka_compression/data/zstd_data.sql b/tests/integration_tests/kafka_compression/data/zstd_data.sql new file mode 100644 index 0000000000..82b78dc6e4 --- /dev/null +++ b/tests/integration_tests/kafka_compression/data/zstd_data.sql @@ -0,0 +1,21 @@ +use test; + +create table tp_int_zstd +( + id int auto_increment, + c_tinyint tinyint null, + c_smallint smallint null, + c_mediumint mediumint null, + c_int int null, + c_bigint bigint null, + constraint pk + primary key (id) +); + +insert into tp_int_zstd(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) +values (1, 2, 3, 4, 5); + +create table zstd_finish_mark +( + id int PRIMARY KEY +); diff --git a/tests/integration_tests/kafka_compression/run.sh b/tests/integration_tests/kafka_compression/run.sh index de9f6cd7e0..a57cd937b1 100755 --- a/tests/integration_tests/kafka_compression/run.sh +++ b/tests/integration_tests/kafka_compression/run.sh @@ -9,33 +9,19 @@ CDC_BINARY=cdc.test SINK_TYPE=$1 function test_compression() { - local compression=$1 - # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) - TOPIC_NAME="ticdc-kafka-compression-$compression-test-$RANDOM" - SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true&kafka-version=${KAFKA_VERSION}&compression=$compression" - cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" -c $compression + TOPIC_NAME="ticdc-kafka-compression-$1-test-$RANDOM" + SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true&kafka-version=${KAFKA_VERSION}&compression=$1" + cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" -c $1 run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=canal-json&version=${KAFKA_VERSION}&enable-tidb-extension=true" + run_sql_file $CUR/data/$1_data.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} - run_sql "CREATE TABLE test.tp_int_$compression ( - id INT AUTO_INCREMENT, - c_tinyint TINYINT NULL, - c_smallint SMALLINT NULL, - c_mediumint MEDIUMINT NULL, - c_int INT NULL, - c_bigint BIGINT NULL, - PRIMARY KEY (id) - );" ${UP_TIDB_HOST} ${UP_TIDB_PORT} - run_sql "INSERT INTO test.tp_int_$compression(c_tinyint, c_smallint, c_mediumint, c_int, c_bigint) - VALUES (1, 2, 3, 4, 5);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} - run_sql "CREATE TABLE test.${compression}_finish_mark (id INT PRIMARY KEY);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} - - check_table_exists test.${compression}_finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 + check_table_exists test.$1_finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - cdc_cli_changefeed pause -c $compression - cdc_cli_changefeed remove -c $compression + cdc_cli_changefeed pause -c $1 + cdc_cli_changefeed remove -c $1 } function run() { diff --git a/tests/utils/kafka_topic/main.go b/tests/utils/kafka_topic/main.go index 5a55f33ec1..6227492cf9 100644 --- a/tests/utils/kafka_topic/main.go +++ b/tests/utils/kafka_topic/main.go @@ -14,14 +14,12 @@ package main import ( - "context" "flag" "log" "strconv" "strings" - "github.com/twmb/franz-go/pkg/kadm" - "github.com/twmb/franz-go/pkg/kgo" + "github.com/IBM/sarama" ) func main() { @@ -38,47 +36,33 @@ func main() { log.Fatal("max-message-bytes must be greater than zero") } - ctx := context.Background() value := strconv.Itoa(*maxMessageBytes) - client, err := kgo.NewClient( - kgo.SeedBrokers(strings.Split(*brokers, ",")...), - kgo.ClientID("ticdc-integration-test-kafka-topic"), - ) + config := sarama.NewConfig() + config.ClientID = "ticdc-integration-test-kafka-topic" + admin, err := sarama.NewClusterAdmin(strings.Split(*brokers, ","), config) if err != nil { log.Fatalf("create Kafka admin client: %v", err) } - defer client.Close() - admin := kadm.NewClient(client) + defer func() { + if err := admin.Close(); err != nil { + log.Printf("close Kafka admin client: %v", err) + } + }() + configEntries := map[string]*string{"max.message.bytes": &value} if *alter { - responses, err := admin.AlterTopicConfigsState(ctx, []kadm.AlterConfig{{ - Name: "max.message.bytes", - Value: &value, - }}, *topic) - if err != nil { + if err := admin.AlterConfig(sarama.TopicResource, *topic, configEntries, false); err != nil { log.Fatalf("alter Kafka topic %s: %v", *topic, err) } - response, err := responses.On(*topic, nil) - if err != nil { - log.Fatalf("find altered Kafka topic %s response: %v", *topic, err) - } - if response.Err != nil { - log.Fatalf("alter Kafka topic %s: %v", *topic, response.Err) - } return } - responses, err := admin.CreateTopics(ctx, 1, 1, map[string]*string{ - "max.message.bytes": &value, - }, *topic) - if err != nil { - log.Fatalf("create Kafka topic %s: %v", *topic, err) - } - response, err := responses.On(*topic, nil) - if err != nil { - log.Fatalf("find created Kafka topic %s response: %v", *topic, err) + detail := &sarama.TopicDetail{ + NumPartitions: 1, + ReplicationFactor: 1, + ConfigEntries: configEntries, } - if response.Err != nil { - log.Fatalf("create Kafka topic %s: %v", *topic, response.Err) + if err := admin.CreateTopic(*topic, detail, false); err != nil { + log.Fatalf("create Kafka topic %s: %v", *topic, err) } } From 9309281a073f88aafe4e18a5dd399d66d22e146b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 14 Aug 2026 19:23:54 +0800 Subject: [PATCH 39/61] add franz package and monitoring --- go.mod | 4 +- go.sum | 8 +- metrics/grafana/ticdc_new_arch.json | 960 ++++++++++++++++-- .../ticdc_new_arch_next_gen.json | 960 ++++++++++++++++-- .../ticdc_new_arch_with_keyspace_name.json | 960 ++++++++++++++++-- pkg/sink/kafka/franz/config.go | 35 +- pkg/sink/kafka/franz/config_test.go | 27 +- pkg/sink/kafka/franz/factory_test.go | 3 +- pkg/sink/kafka/franz/gssapi.go | 8 +- pkg/sink/kafka/franz/logger.go | 1 + pkg/sink/kafka/franz/metrics.go | 9 + pkg/sink/kafka/franz/metrics_hook.go | 16 + pkg/sink/kafka/franz/metrics_hook_test.go | 14 + pkg/sink/kafka/franz_adapter.go | 5 +- pkg/sink/kafka/metrics.go | 8 + pkg/sink/kafka/metrics_collector.go | 29 + pkg/sink/kafka/metrics_collector_test.go | 59 ++ pkg/sink/kafka/selector_test.go | 19 +- 18 files changed, 2822 insertions(+), 303 deletions(-) create mode 100644 pkg/sink/kafka/metrics_collector_test.go diff --git a/go.mod b/go.mod index 57e7997fd4..532fd5b676 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/jcmturner/gofork v1.7.6 github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.18.6 + github.com/klauspost/compress v1.18.7 github.com/linkedin/goavro/v2 v2.14.0 github.com/mailru/easyjson v0.9.1 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 @@ -76,7 +76,7 @@ require ( github.com/tikv/pd v1.1.0-beta.0.20260604125942-9f1c47b1e851 github.com/tikv/pd/client v0.0.0-20260604125942-9f1c47b1e851 github.com/tinylib/msgp v1.5.0 - github.com/twmb/franz-go v1.21.5 + github.com/twmb/franz-go v1.21.6 github.com/twmb/franz-go/pkg/kadm v1.18.0 github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 github.com/twmb/franz-go/pkg/kmsg v1.13.1 diff --git a/go.sum b/go.sum index e56d86c295..7b50651d2f 100644 --- a/go.sum +++ b/go.sum @@ -602,8 +602,8 @@ github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -965,8 +965,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/twmb/franz-go v1.21.5 h1:cVYI2+JTTKSvohhy8bCOleYrS7G79ZBrLVFIJsoHm8M= -github.com/twmb/franz-go v1.21.5/go.mod h1:rfoMTnVk7107fhTGxfEKIHP/e7tPe6oyij/ywzO0czk= +github.com/twmb/franz-go v1.21.6 h1:+v0dQJVIIuw9uPmPWmPrkoUHs1pPeV8MSwA4eU/Y2kY= +github.com/twmb/franz-go v1.21.6/go.mod h1:wMepkgCatAdV9vCsuwM+wr+C1fl7KV/41+uHGAjt/wc= github.com/twmb/franz-go/pkg/kadm v1.18.0 h1:WRf/LZmDdcDXwX7WMbtDU++v+b3NzYh2bCGoPMmzirw= github.com/twmb/franz-go/pkg/kadm v1.18.0/go.mod h1:XeLhGoLXLFzK8/ryv5FfpxPxGwj4oFEGpPJMB/x6KDE= github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 h1:OLL/h1pOsMuCnJEp3IW0L2W3jHATeTUMAmE6PxxbhtU= diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index a30616fa7b..5423a193a1 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -17794,15 +17794,6 @@ "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", - "refId": "B" } ], "thresholds": [], @@ -17906,15 +17897,6 @@ "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", - "refId": "B" } ], "thresholds": [], @@ -17964,7 +17946,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The request latency in seconds for all brokers. Franz values show the one-minute average and p99.", + "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean", "fieldConfig": { "defaults": { "links": [] @@ -18018,24 +18000,6 @@ "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-avg", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-p99", - "refId": "C" } ], "thresholds": [], @@ -18139,24 +18103,6 @@ "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-request-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "B" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-response-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "C" } ], "thresholds": [], @@ -18206,7 +18152,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records per producer batch for franz-go and per request for Sarama.", + "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate", "fieldConfig": { "defaults": { "links": [] @@ -18260,31 +18206,13 @@ "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-avg", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}-p99", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Batch or Request", + "title": "Kafka Records Per Request", "tooltip": { "shared": true, "sort": 0, @@ -18379,15 +18307,6 @@ "intervalFactor": 1, "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{namespace}}-{{changefeed}}-{{instance}}", - "refId": "B" } ], "thresholds": [], @@ -19247,6 +19166,109 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum average and p99 broker throttle time reported by the producer.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 67 + }, + "hiddenSeries": false, + "id": 62108, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_producer_throttle_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Kafka Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Sink - MQ Sink", @@ -28106,6 +28128,776 @@ ], "title": "Pulsar Sink", "type": "row" + }, + { + "collapsed": true, + "datasource": "${DS_TEST-CLUSTER}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 62100, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Bytes written to brokers per second.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 62101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests currently awaiting a broker response.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 62102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Inflight Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 request latency for each broker.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 62103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests and responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 62104, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "request-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "response-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 records per topic-partition batch.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 62105, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Per Batch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute ratio of uncompressed bytes to compressed bytes.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 62106, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compression Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum one-minute average and p99 broker throttle time.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 62107, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Kafka Sink V2", + "type": "row" } ], "refresh": "10s", diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index a230576074..0f1a094920 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -17794,15 +17794,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", - "refId": "B" } ], "thresholds": [], @@ -17906,15 +17897,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", - "refId": "B" } ], "thresholds": [], @@ -17964,7 +17946,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The request latency in seconds for all brokers. Franz values show the one-minute average and p99.", + "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean", "fieldConfig": { "defaults": { "links": [] @@ -18018,24 +18000,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", - "refId": "C" } ], "thresholds": [], @@ -18139,24 +18103,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "B" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "C" } ], "thresholds": [], @@ -18206,7 +18152,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records per producer batch for franz-go and per request for Sarama.", + "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate", "fieldConfig": { "defaults": { "links": [] @@ -18260,31 +18206,13 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Batch or Request", + "title": "Kafka Records Per Request", "tooltip": { "shared": true, "sort": 0, @@ -18379,15 +18307,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}", - "refId": "B" } ], "thresholds": [], @@ -19247,6 +19166,109 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum average and p99 broker throttle time reported by the producer.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 67 + }, + "hiddenSeries": false, + "id": 62108, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_producer_throttle_time{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Kafka Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Sink - MQ Sink", @@ -28106,6 +28128,776 @@ ], "title": "Pulsar Sink", "type": "row" + }, + { + "collapsed": true, + "datasource": "${DS_TEST-CLUSTER}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 62100, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Bytes written to brokers per second.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 62101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests currently awaiting a broker response.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 62102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Inflight Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 request latency for each broker.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 62103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests and responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 62104, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 records per topic-partition batch.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 62105, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Per Batch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute ratio of uncompressed bytes to compressed bytes.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 62106, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compression Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum one-minute average and p99 broker throttle time.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 62107, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Kafka Sink V2", + "type": "row" } ], "refresh": "10s", diff --git a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json index 6cab1ef756..f899b05d21 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json +++ b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json @@ -6319,15 +6319,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", - "refId": "B" } ], "thresholds": [], @@ -6431,15 +6422,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", - "refId": "B" } ], "thresholds": [], @@ -6489,7 +6471,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The request latency in seconds for all brokers. Franz values show the one-minute average and p99.", + "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean", "fieldConfig": { "defaults": { "links": [] @@ -6543,24 +6525,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", - "refId": "C" } ], "thresholds": [], @@ -6664,24 +6628,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "B" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "C" } ], "thresholds": [], @@ -6731,7 +6677,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Records per producer batch for franz-go and per request for Sarama.", + "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate", "fieldConfig": { "defaults": { "links": [] @@ -6785,31 +6731,13 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Kafka Records Per Batch or Request", + "title": "Kafka Records Per Request", "tooltip": { "shared": true, "sort": 0, @@ -6904,15 +6832,6 @@ "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "franz-{{keyspace_name}}-{{changefeed}}-{{instance}}", - "refId": "B" } ], "thresholds": [], @@ -7772,6 +7691,109 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum average and p99 broker throttle time reported by the producer.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 67 + }, + "hiddenSeries": false, + "id": 62108, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_producer_throttle_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Kafka Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Sink - MQ Sink", @@ -11513,6 +11535,776 @@ ], "title": "DDL", "type": "row" + }, + { + "collapsed": true, + "datasource": "${DS_TEST-CLUSTER}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 62100, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Bytes written to brokers per second.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 62101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests currently awaiting a broker response.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 62102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Inflight Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 request latency for each broker.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 62103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests and responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 62104, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 records per topic-partition batch.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 62105, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Per Batch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute ratio of uncompressed bytes to compressed bytes.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 62106, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compression Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum one-minute average and p99 broker throttle time.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 62107, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Kafka Sink V2", + "type": "row" } ], "refresh": "10s", diff --git a/pkg/sink/kafka/franz/config.go b/pkg/sink/kafka/franz/config.go index d1f2f1803a..5b6a753783 100644 --- a/pkg/sink/kafka/franz/config.go +++ b/pkg/sink/kafka/franz/config.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/twmb/franz-go/pkg/kgo" - "github.com/twmb/franz-go/pkg/kversion" "github.com/twmb/franz-go/pkg/sasl" "github.com/twmb/franz-go/pkg/sasl/oauth" "github.com/twmb/franz-go/pkg/sasl/plain" @@ -36,21 +35,31 @@ import ( ) const ( + // defaultMaxBufferedBytes bounds the producer's per-client byte buffer under normal configurations. defaultMaxBufferedBytes = 64 << 20 + // defaultBrokerWriteBytes matches Kafka's default socket.request.max.bytes limit. defaultBrokerWriteBytes = 100 << 20 - minProducerBatchBytes = 512 - maxProducerBatchBytes = 1 << 30 - - NoResponse = int16(0) + // minProducerBatchBytes and maxProducerBatchBytes are franz-go's accepted batch-size bounds. + minProducerBatchBytes = 512 + maxProducerBatchBytes = 1 << 30 + + // NoResponse requests no broker acknowledgement. A send completes after the + // request is written. Broker-side failures are not reported, so messages can be lost. + NoResponse = int16(0) + // WaitForLocal requests acknowledgement from the partition leader. A send completes + // after the leader writes the message locally. An acknowledged message can be lost + // if the leader fails before follower replication. WaitForLocal = int16(1) - WaitForAll = int16(-1) + // WaitForAll requests acknowledgement from all in-sync replicas. A send completes + // after the replication requirement is met. It is the default and provides the + // strongest durability, at the cost of higher latency or failed sends when too few + // replicas are in sync. + WaitForAll = int16(-1) ) type Config struct { BrokerEndpoints []string ClientID string - Version string - AssignedVersion bool MaxMessageBytes int MaxRetry int Compression string @@ -111,14 +120,6 @@ func newClientOptions( opts = append(opts, kgo.WithHooks(hook)) } - if cfg.AssignedVersion { - versions := kversion.FromString(cfg.Version) - if versions == nil { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("invalid kafka version %s", cfg.Version) - } - opts = append(opts, kgo.MaxVersions(versions)) - } - if cfg.TLSConfig != nil { opts = append(opts, kgo.DialTLSConfig(cfg.TLSConfig)) } @@ -201,7 +202,9 @@ func producerOptions(cfg Config) ([]kgo.Opt, error) { return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), kgo.RequiredAcks(requiredAcks(cfg.RequiredAcks)), + // Retried requests may create duplicates because broker-side producer ID deduplication is disabled. kgo.DisableIdempotentWrite(), + // More than one in-flight request can reorder records when an earlier request is retried. kgo.MaxProduceRequestsInflightPerBroker(1), kgo.RecordRetries(cfg.MaxRetry), kgo.UnknownTopicRetries(cfg.MaxRetry), diff --git a/pkg/sink/kafka/franz/config_test.go b/pkg/sink/kafka/franz/config_test.go index 2b010c586f..bc2270dc6a 100644 --- a/pkg/sink/kafka/franz/config_test.go +++ b/pkg/sink/kafka/franz/config_test.go @@ -92,6 +92,20 @@ func TestProducerOptionsBoundBufferAndBatch(t *testing.T) { require.Equal(t, int32(defaultBrokerWriteBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) } +func TestProducerOptionsUseSingleNonIdempotentRequest(t *testing.T) { + config := testConfig([]string{"127.0.0.1:9092"}) + + producerOpts, err := producerOptions(config) + require.NoError(t, err) + + client, err := kgo.NewClient(producerOpts...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, true, client.OptValue(kgo.DisableIdempotentWrite)) + require.Equal(t, 1, client.OptValue(kgo.MaxProduceRequestsInflightPerBroker)) +} + func TestProducerLimitsScaleWithConfiguredMessage(t *testing.T) { maxMessageBytes := defaultBrokerWriteBytes + 1 config := testConfig([]string{"127.0.0.1:9092"}) @@ -159,19 +173,6 @@ func TestCompressionOptions(t *testing.T) { } } -func TestInvalidAssignedVersionUsesInvalidConfigError(t *testing.T) { - cfg := testConfig([]string{"127.0.0.1:9092"}) - cfg.Version = "invalid" - cfg.AssignedVersion = true - - _, err := NewAdmin( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "invalid-version"), - cfg, - ) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) -} - func TestBuildGSSAPIMechanism(t *testing.T) { for _, cfg := range []GSSAPIConfig{ {AuthType: userAuth, Password: "pwd"}, diff --git a/pkg/sink/kafka/franz/factory_test.go b/pkg/sink/kafka/franz/factory_test.go index 477cc37b54..b5ff03aceb 100644 --- a/pkg/sink/kafka/franz/factory_test.go +++ b/pkg/sink/kafka/franz/factory_test.go @@ -49,8 +49,7 @@ func TestFactoryCreatesAllClients(t *testing.T) { func TestFactoryCleansMetricsAfterProducerConstructionFailure(t *testing.T) { changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory-error") config := testConfig([]string{"127.0.0.1:9092"}) - config.Version = "invalid" - config.AssignedVersion = true + config.MaxMessageBytes = maxProducerBatchBytes + 1 factory := NewFactory(config, changefeedID) _, err := factory.SyncProducer(context.Background()) diff --git a/pkg/sink/kafka/franz/gssapi.go b/pkg/sink/kafka/franz/gssapi.go index df2b6ef93d..2a061b48d6 100644 --- a/pkg/sink/kafka/franz/gssapi.go +++ b/pkg/sink/kafka/franz/gssapi.go @@ -33,10 +33,12 @@ import ( ) const ( - tokIDKrbAPReq = 256 + // tokIDKrbAPReq and gssAPIGeneric identify a Kerberos AP-REQ inside a GSS-API initial context token. + tokIDKrbAPReq = 0x0100 gssAPIGeneric = 0x60 - userAuth = 1 - keyTabAuth = 2 + // Authentication type values are part of sink URI compatibility and must remain stable. + userAuth = 1 + keyTabAuth = 2 ) type gssapiMechanism struct { diff --git a/pkg/sink/kafka/franz/logger.go b/pkg/sink/kafka/franz/logger.go index 8c8c8d773f..3d5076ec07 100644 --- a/pkg/sink/kafka/franz/logger.go +++ b/pkg/sink/kafka/franz/logger.go @@ -26,6 +26,7 @@ import ( "go.uber.org/zap/zapcore" ) +// logValueLimit bounds individual string fields emitted by the franz-go logger. const logValueLimit = 1024 type logger struct { diff --git a/pkg/sink/kafka/franz/metrics.go b/pkg/sink/kafka/franz/metrics.go index db18c28e93..97116df7a5 100644 --- a/pkg/sink/kafka/franz/metrics.go +++ b/pkg/sink/kafka/franz/metrics.go @@ -52,6 +52,14 @@ var ( Buckets: prometheus.DefBuckets, }, []string{"namespace", "changefeed", "broker"}) + throttleTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_throttle_time_seconds", + Help: "Kafka broker throttle time reported to the producer in seconds.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), + }, []string{"namespace", "changefeed", "broker"}) + recordsPerBatch = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "ticdc", Subsystem: "sink", @@ -82,6 +90,7 @@ func InitMetrics(registry *prometheus.Registry) { requestsTotal, responsesTotal, requestDuration, + throttleTime, recordsPerBatch, uncompressedBytesTotal, compressedBytesTotal, diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/franz/metrics_hook.go index db959a7a10..2e41bb7267 100644 --- a/pkg/sink/kafka/franz/metrics_hook.go +++ b/pkg/sink/kafka/franz/metrics_hook.go @@ -46,9 +46,11 @@ type brokerMetrics struct { responsesReadError prometheus.Counter requestsInFlight prometheus.Gauge requestDuration prometheus.Observer + throttleTime prometheus.Observer } const ( + // Result values are fixed to keep the broker-level metric label cardinality bounded. metricResultSuccess = "success" metricResultWriteError = "write_error" metricResultReadError = "read_error" @@ -67,6 +69,18 @@ func newMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { } } +func (h *metricsHook) OnBrokerThrottle( + meta kgo.BrokerMetadata, + throttleInterval time.Duration, + _ bool, +) { + if meta.NodeID < 0 { + return + } + + h.broker(meta.NodeID).throttleTime.Observe(throttleInterval.Seconds()) +} + func (h *metricsHook) broker(nodeID int32) *brokerMetrics { if cached, ok := h.brokers.Load(nodeID); ok { return cached.(*brokerMetrics) @@ -85,6 +99,7 @@ func (h *metricsHook) broker(nodeID int32) *brokerMetrics { h.keyspace, h.changefeed, brokerID, metricResultReadError), requestsInFlight: requestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID), requestDuration: requestDuration.WithLabelValues(h.keyspace, h.changefeed, brokerID), + throttleTime: throttleTime.WithLabelValues(h.keyspace, h.changefeed, brokerID), } actual, _ := h.brokers.LoadOrStore(nodeID, metrics) @@ -104,6 +119,7 @@ func CleanupMetrics(changefeedID common.ChangeFeedID) { responsesTotal.DeletePartialMatch(labels) requestsInFlight.DeletePartialMatch(labels) requestDuration.DeletePartialMatch(labels) + throttleTime.DeletePartialMatch(labels) recordsPerBatch.DeletePartialMatch(labels) uncompressedBytesTotal.DeletePartialMatch(labels) compressedBytesTotal.DeletePartialMatch(labels) diff --git a/pkg/sink/kafka/franz/metrics_hook_test.go b/pkg/sink/kafka/franz/metrics_hook_test.go index 377d28e477..a5cb9532f4 100644 --- a/pkg/sink/kafka/franz/metrics_hook_test.go +++ b/pkg/sink/kafka/franz/metrics_hook_test.go @@ -43,6 +43,7 @@ func TestInitMetrics(t *testing.T) { CompressedBytes: 1, }, ) + hook.OnBrokerThrottle(kgo.BrokerMetadata{NodeID: 1}, time.Millisecond, true) registry := prometheus.NewRegistry() InitMetrics(registry) @@ -58,6 +59,7 @@ func TestInitMetrics(t *testing.T) { require.Contains(t, names, "ticdc_sink_kafka_franz_producer_records_per_batch") require.Contains(t, names, "ticdc_sink_kafka_franz_producer_uncompressed_bytes_total") require.Contains(t, names, "ticdc_sink_kafka_franz_producer_compressed_bytes_total") + require.Contains(t, names, "ticdc_sink_kafka_franz_producer_throttle_time_seconds") } func TestMetricsHookRecordsRawValues(t *testing.T) { @@ -118,4 +120,16 @@ func TestMetricsHookRecordsRawValues(t *testing.T) { hook.OnBrokerWrite(kgo.BrokerMetadata{NodeID: -1}, 0, 1, 0, 0, nil) hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: -1}, 0, kgo.BrokerE2E{}) + + hook.OnBrokerThrottle(meta, 10*time.Millisecond, true) + hook.OnBrokerThrottle(meta, 50*time.Millisecond, false) + hook.OnBrokerThrottle(kgo.BrokerMetadata{NodeID: 2}, 30*time.Millisecond, true) + hook.OnBrokerThrottle(kgo.BrokerMetadata{NodeID: -1}, time.Second, true) + + throttleMetric, ok := metrics.throttleTime.(prometheus.Metric) + require.True(t, ok) + throttleHistogram := &dto.Metric{} + require.NoError(t, throttleMetric.Write(throttleHistogram)) + require.Equal(t, uint64(2), throttleHistogram.GetHistogram().GetSampleCount()) + require.InDelta(t, 0.06, throttleHistogram.GetHistogram().GetSampleSum(), 0.000001) } diff --git a/pkg/sink/kafka/franz_adapter.go b/pkg/sink/kafka/franz_adapter.go index 6ba17e20c6..c27cd64d56 100644 --- a/pkg/sink/kafka/franz_adapter.go +++ b/pkg/sink/kafka/franz_adapter.go @@ -30,8 +30,7 @@ type franzFactoryAdapter struct { inner *franz.Factory } -// NewFranzFactory constructs the additive franz-go implementation while the -// existing Sarama factory remains unchanged. +// NewFranzFactory constructs a franz-go Kafka client factory. func NewFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { config, err := newFranzConfig(o) if err != nil { @@ -159,8 +158,6 @@ func newFranzConfig(o *options) (franz.Config, error) { config := franz.Config{ BrokerEndpoints: append([]string(nil), o.BrokerEndpoints...), ClientID: o.ClientID, - Version: o.Version, - AssignedVersion: o.IsAssignedVersion, MaxMessageBytes: o.MaxMessageBytes, MaxRetry: o.MaxRetry, Compression: o.Compression, diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index ba3ec0a553..bd2ed80445 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -70,6 +70,13 @@ var ( Name: "kafka_producer_records_per_request", Help: "The number of records per request for all topics.", }, []string{"namespace", "changefeed", "type"}) + throttleTimeGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_producer_throttle_time", + Help: "Kafka broker throttle time in milliseconds, aggregated as the maximum across brokers.", + }, []string{"namespace", "changefeed", "type"}) // Meter mark by 1 once a response received. responseRateGauge = prometheus.NewGaugeVec( @@ -86,6 +93,7 @@ func InitMetrics(registry *prometheus.Registry) { franz.InitMetrics(registry) registry.MustRegister(compressionRatioGauge) registry.MustRegister(recordsPerRequestGauge) + registry.MustRegister(throttleTimeGauge) registry.MustRegister(OutgoingByteRateGauge) registry.MustRegister(RequestRateGauge) registry.MustRegister(RequestLatencyGauge) diff --git a/pkg/sink/kafka/metrics_collector.go b/pkg/sink/kafka/metrics_collector.go index 3e54c7d82f..8609386ffa 100644 --- a/pkg/sink/kafka/metrics_collector.go +++ b/pkg/sink/kafka/metrics_collector.go @@ -50,6 +50,7 @@ const ( requestLatencyInMsMetricNamePrefix = "request-latency-in-ms-for-broker-" requestsInFlightMetricNamePrefix = "requests-in-flight-for-broker-" responseRateMetricNamePrefix = "response-rate-for-broker-" + throttleTimeMetricNamePrefix = "throttle-time-in-ms-for-broker-" p99 = "p99" avg = "avg" @@ -124,6 +125,8 @@ func (m *saramaMetricsCollector) collectProducerMetrics() { func (m *saramaMetricsCollector) collectBrokerMetrics() { keyspace := m.changefeedID.Keyspace() changefeedID := m.changefeedID.Name() + var maxThrottleAvg, maxThrottleP99 float64 + for id := range m.brokers { brokerID := strconv.Itoa(int(id)) outgoingByteRateMetric := m.registry.Get( @@ -168,7 +171,17 @@ func (m *saramaMetricsCollector) collectBrokerMetrics() { WithLabelValues(keyspace, changefeedID, brokerID). Set(meter.Snapshot().Rate1()) } + + throttleTimeMetric := m.registry.Get(getBrokerMetricName( + throttleTimeMetricNamePrefix, brokerID)) + if histogram, ok := throttleTimeMetric.(metrics.Histogram); ok { + snapshot := histogram.Snapshot() + maxThrottleAvg = max(maxThrottleAvg, snapshot.Mean()) + maxThrottleP99 = max(maxThrottleP99, snapshot.Percentile(0.99)) + } } + + setThrottleTime(m.changefeedID, maxThrottleAvg, maxThrottleP99) } func getBrokerMetricName(prefix, brokerID string) string { @@ -185,6 +198,8 @@ func (m *saramaMetricsCollector) cleanupProducerMetrics() { DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), avg) recordsPerRequestGauge. DeleteLabelValues(m.changefeedID.Keyspace(), m.changefeedID.Name(), p99) + + cleanupThrottleTime(m.changefeedID) } func (m *saramaMetricsCollector) cleanupBrokerMetrics() { @@ -212,3 +227,17 @@ func (m *saramaMetricsCollector) cleanupMetrics() { m.cleanupProducerMetrics() m.cleanupBrokerMetrics() } + +func setThrottleTime(changefeedID common.ChangeFeedID, average, percentile99 float64) { + throttleTimeGauge. + WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), avg). + Set(average) + throttleTimeGauge. + WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), p99). + Set(percentile99) +} + +func cleanupThrottleTime(changefeedID common.ChangeFeedID) { + throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), avg) + throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), p99) +} diff --git a/pkg/sink/kafka/metrics_collector_test.go b/pkg/sink/kafka/metrics_collector_test.go new file mode 100644 index 0000000000..7f8cdc04f3 --- /dev/null +++ b/pkg/sink/kafka/metrics_collector_test.go @@ -0,0 +1,59 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rcrowley/go-metrics" + "github.com/stretchr/testify/require" +) + +func TestCollectBrokerThrottleTime(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "throttle-time") + cleanupThrottleTime(changefeedID) + + registry := metrics.NewRegistry() + firstBroker := metrics.NewHistogram(metrics.NewUniformSample(10)) + firstBroker.Update(10) + firstBroker.Update(50) + require.NoError(t, registry.Register( + getBrokerMetricName(throttleTimeMetricNamePrefix, "1"), firstBroker)) + + secondBroker := metrics.NewHistogram(metrics.NewUniformSample(10)) + secondBroker.Update(40) + require.NoError(t, registry.Register( + getBrokerMetricName(throttleTimeMetricNamePrefix, "2"), secondBroker)) + + collector := saramaMetricsCollector{ + changefeedID: changefeedID, + brokers: map[int32]struct{}{1: {}, 2: {}}, + registry: registry, + } + collector.collectBrokerMetrics() + + require.Equal(t, float64(40), testutil.ToFloat64(throttleTimeGauge.WithLabelValues( + changefeedID.Keyspace(), changefeedID.Name(), avg))) + require.Equal(t, float64(50), testutil.ToFloat64(throttleTimeGauge.WithLabelValues( + changefeedID.Keyspace(), changefeedID.Name(), p99))) + + collector.cleanupMetrics() + require.False(t, throttleTimeGauge.DeleteLabelValues( + changefeedID.Keyspace(), changefeedID.Name(), avg)) + require.False(t, throttleTimeGauge.DeleteLabelValues( + changefeedID.Keyspace(), changefeedID.Name(), p99)) +} diff --git a/pkg/sink/kafka/selector_test.go b/pkg/sink/kafka/selector_test.go index 7eb7598627..8552ba647b 100644 --- a/pkg/sink/kafka/selector_test.go +++ b/pkg/sink/kafka/selector_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kfake" @@ -54,20 +53,26 @@ func TestFactorySelection(t *testing.T) { } } -func TestFactoryDoesNotFallbackAfterFranzFailure(t *testing.T) { +func TestFranzIgnoresConfiguredKafkaVersion(t *testing.T) { + const topic = "version-negotiation" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + options := NewOptions() options.ClientID = "ticdc-test" - options.BrokerEndpoints = []string{"127.0.0.1:9092"} - options.Topic = "no-fallback" + options.BrokerEndpoints = cluster.ListenAddrs() + options.Topic = topic options.Version = "invalid" options.IsAssignedVersion = true - _, err := NewFactory( + factory, err := NewFactory( context.Background(), options, - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "no-fallback"), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "version-negotiation"), ) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.NoError(t, err) + require.IsType(t, &franzFactoryAdapter{}, factory) + CleanupFactoryMetrics(factory) } func TestFranzAndSaramaFactoriesAreIndependent(t *testing.T) { From 1119feeb451a6322d8c1a87e84995cb7d1471aaa Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Mon, 17 Aug 2026 14:31:18 +0800 Subject: [PATCH 40/61] add more metrics --- metrics/grafana/ticdc_new_arch.json | 27 ++++++++++++------- .../ticdc_new_arch_next_gen.json | 27 ++++++++++++------- .../ticdc_new_arch_with_keyspace_name.json | 27 ++++++++++++------- pkg/sink/kafka/franz/admin.go | 3 +++ pkg/sink/kafka/franz/admin_test.go | 13 ++++++--- 5 files changed, 64 insertions(+), 33 deletions(-) diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 5423a193a1..06004735d8 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -28222,7 +28222,8 @@ }, "yaxes": [ { - "format": "bytes", + "decimals": 1, + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -28325,7 +28326,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 0, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -28400,7 +28402,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "expr": "1000 * sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28409,7 +28411,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))", + "expr": "1000 * histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28437,7 +28439,8 @@ }, "yaxes": [ { - "format": "s", + "decimals": 1, + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -28549,7 +28552,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -28661,7 +28665,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -28762,6 +28767,7 @@ }, "yaxes": [ { + "decimals": 1, "format": "percent", "label": null, "logBase": 1, @@ -28837,7 +28843,7 @@ "targets": [ { "exemplar": true, - "expr": "max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)) by (namespace,changefeed, instance)", + "expr": "1000 * max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)) by (namespace,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28846,7 +28852,7 @@ }, { "exemplar": true, - "expr": "max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))) by (namespace,changefeed, instance)", + "expr": "1000 * max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))) by (namespace,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28874,7 +28880,8 @@ }, "yaxes": [ { - "format": "s", + "decimals": 1, + "format": "ms", "label": null, "logBase": 1, "max": null, diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index 0f1a094920..0559b5332f 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -28222,7 +28222,8 @@ }, "yaxes": [ { - "format": "bytes", + "decimals": 1, + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -28325,7 +28326,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 0, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -28400,7 +28402,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "expr": "1000 * sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28409,7 +28411,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "expr": "1000 * histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28437,7 +28439,8 @@ }, "yaxes": [ { - "format": "s", + "decimals": 1, + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -28549,7 +28552,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -28661,7 +28665,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -28762,6 +28767,7 @@ }, "yaxes": [ { + "decimals": 1, "format": "percent", "label": null, "logBase": 1, @@ -28837,7 +28843,7 @@ "targets": [ { "exemplar": true, - "expr": "max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", + "expr": "1000 * max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28846,7 +28852,7 @@ }, { "exemplar": true, - "expr": "max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", + "expr": "1000 * max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -28874,7 +28880,8 @@ }, "yaxes": [ { - "format": "s", + "decimals": 1, + "format": "ms", "label": null, "logBase": 1, "max": null, diff --git a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json index f899b05d21..eb2e8a2d75 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json +++ b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json @@ -11629,7 +11629,8 @@ }, "yaxes": [ { - "format": "bytes", + "decimals": 1, + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -11732,7 +11733,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 0, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -11807,7 +11809,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "expr": "1000 * sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -11816,7 +11818,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "expr": "1000 * histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -11844,7 +11846,8 @@ }, "yaxes": [ { - "format": "s", + "decimals": 1, + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -11956,7 +11959,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -12068,7 +12072,8 @@ }, "yaxes": [ { - "format": "none", + "decimals": 1, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -12169,6 +12174,7 @@ }, "yaxes": [ { + "decimals": 1, "format": "percent", "label": null, "logBase": 1, @@ -12244,7 +12250,7 @@ "targets": [ { "exemplar": true, - "expr": "max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", + "expr": "1000 * max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12253,7 +12259,7 @@ }, { "exemplar": true, - "expr": "max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", + "expr": "1000 * max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12281,7 +12287,8 @@ }, "yaxes": [ { - "format": "s", + "decimals": 1, + "format": "ms", "label": null, "logBase": 1, "max": null, diff --git a/pkg/sink/kafka/franz/admin.go b/pkg/sink/kafka/franz/admin.go index 297f695dc5..bdf54da706 100644 --- a/pkg/sink/kafka/franz/admin.go +++ b/pkg/sink/kafka/franz/admin.go @@ -51,6 +51,9 @@ func NewAdmin( if err != nil { return nil, err } + // MetadataMinAge is the minimum interval between metadata requests. + // It must stay below the visibility retry interval to avoid retrying a cached topic-not-found result. + opts = append(opts, kgo.MetadataMinAge(100*time.Millisecond)) client, err := kgo.NewClient(opts...) if err != nil { diff --git a/pkg/sink/kafka/franz/admin_test.go b/pkg/sink/kafka/franz/admin_test.go index d80180fc0b..54450a2930 100644 --- a/pkg/sink/kafka/franz/admin_test.go +++ b/pkg/sink/kafka/franz/admin_test.go @@ -16,6 +16,7 @@ package franz import ( "context" "testing" + "time" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" @@ -178,6 +179,10 @@ func TestAdminOperations(t *testing.T) { require.Equal(t, map[string]int32{existingTopic: 3}, partitions) const topic = "test-topic" + topics, err := admin.GetTopicsMeta([]string{topic}, true) + require.NoError(t, err) + require.Empty(t, topics) + err = admin.CreateTopic(&TopicDetail{ Name: topic, NumPartitions: 3, @@ -185,9 +190,11 @@ func TestAdminOperations(t *testing.T) { }) require.NoError(t, err) - topics, err := admin.GetTopicsMeta([]string{topic}, false) - require.NoError(t, err) - require.Equal(t, int32(3), topics[topic].NumPartitions) + require.Eventually(t, func() bool { + topics, err = admin.GetTopicsMeta([]string{topic}, false) + return err == nil && topics[topic].NumPartitions == 3 + }, time.Second, 20*time.Millisecond) + require.NoError(t, admin.CreateTopic(&TopicDetail{Name: topic, NumPartitions: 3, ReplicationFactor: 1})) } From f8e352c11b9f8cebe640e9215c586bb5e7705612 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 1 Sep 2026 11:37:13 +0800 Subject: [PATCH 41/61] add kafka sasl oauth ca --- api/v2/model.go | 3 + api/v2/model_test.go | 18 +++ cmd/cdc/cli/cli_changefeed_create_test.go | 6 + pkg/config/replica_config_test.go | 19 +++ pkg/config/sink.go | 1 + pkg/sink/kafka/options.go | 9 ++ pkg/sink/kafka/options_test.go | 41 ++++++ .../kafka/sarama_oauth2_token_provider.go | 45 ++++++ .../sarama_oauth2_token_provider_test.go | 131 ++++++++++++++++++ pkg/sink/kafka/sasl_config.go | 1 + 10 files changed, 274 insertions(+) diff --git a/api/v2/model.go b/api/v2/model.go index 0c99b46b18..e8ebc38d7d 100644 --- a/api/v2/model.go +++ b/api/v2/model.go @@ -490,6 +490,7 @@ func (c *ReplicaConfig) toInternalReplicaConfigWithOriginConfig( SASLOAuthClientID: c.Sink.KafkaConfig.SASLOAuthClientID, SASLOAuthClientSecret: c.Sink.KafkaConfig.SASLOAuthClientSecret, SASLOAuthTokenURL: c.Sink.KafkaConfig.SASLOAuthTokenURL, + SASLOAuthCA: c.Sink.KafkaConfig.SASLOAuthCA, SASLOAuthScopes: c.Sink.KafkaConfig.SASLOAuthScopes, SASLOAuthGrantType: c.Sink.KafkaConfig.SASLOAuthGrantType, SASLOAuthAudience: c.Sink.KafkaConfig.SASLOAuthAudience, @@ -832,6 +833,7 @@ func ToAPIReplicaConfig(c *config.ReplicaConfig) *ReplicaConfig { SASLOAuthClientID: cloned.Sink.KafkaConfig.SASLOAuthClientID, SASLOAuthClientSecret: cloned.Sink.KafkaConfig.SASLOAuthClientSecret, SASLOAuthTokenURL: cloned.Sink.KafkaConfig.SASLOAuthTokenURL, + SASLOAuthCA: cloned.Sink.KafkaConfig.SASLOAuthCA, SASLOAuthScopes: cloned.Sink.KafkaConfig.SASLOAuthScopes, SASLOAuthGrantType: cloned.Sink.KafkaConfig.SASLOAuthGrantType, SASLOAuthAudience: cloned.Sink.KafkaConfig.SASLOAuthAudience, @@ -1581,6 +1583,7 @@ type KafkaConfig struct { SASLOAuthClientID *string `json:"sasl_oauth_client_id,omitempty" toml:"sasl-oauth-client-id,omitempty"` SASLOAuthClientSecret *string `json:"sasl_oauth_client_secret,omitempty" toml:"sasl-oauth-client-secret,omitempty"` SASLOAuthTokenURL *string `json:"sasl_oauth_token_url,omitempty" toml:"sasl-oauth-token-url,omitempty"` + SASLOAuthCA *string `json:"sasl_oauth_ca,omitempty" toml:"sasl-oauth-ca,omitempty"` SASLOAuthScopes []string `json:"sasl_oauth_scopes,omitempty" toml:"sasl-oauth-scopes,omitempty"` SASLOAuthGrantType *string `json:"sasl_oauth_grant_type,omitempty" toml:"sasl-oauth-grant-type,omitempty"` SASLOAuthAudience *string `json:"sasl_oauth_audience,omitempty" toml:"sasl-oauth-audience,omitempty"` diff --git a/api/v2/model_test.go b/api/v2/model_test.go index 98779fc4eb..db9d58ff0a 100644 --- a/api/v2/model_test.go +++ b/api/v2/model_test.go @@ -326,3 +326,21 @@ func TestReplicaConfigCodecConfigConversion(t *testing.T) { require.NotNil(t, apiCfgBack.Sink.KafkaConfig.CodecConfig) require.True(t, util.GetOrZero(apiCfgBack.Sink.KafkaConfig.CodecConfig.AvroIncludeBeforeValue)) } + +func TestReplicaConfigKafkaOAuthCAConversion(t *testing.T) { + t.Parallel() + + apiCfg := &ReplicaConfig{ + Sink: &SinkConfig{ + KafkaConfig: &KafkaConfig{ + SASLOAuthCA: util.AddressOf("/etc/ssl/oauth-ca.pem"), + }, + }, + } + + internalCfg := apiCfg.ToInternalReplicaConfig() + require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(internalCfg.Sink.KafkaConfig.SASLOAuthCA)) + + apiCfgBack := ToAPIReplicaConfig(internalCfg) + require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(apiCfgBack.Sink.KafkaConfig.SASLOAuthCA)) +} diff --git a/cmd/cdc/cli/cli_changefeed_create_test.go b/cmd/cdc/cli/cli_changefeed_create_test.go index e69c9372ec..77d487e999 100644 --- a/cmd/cdc/cli/cli_changefeed_create_test.go +++ b/cmd/cdc/cli/cli_changefeed_create_test.go @@ -77,6 +77,9 @@ func TestTomlFileToApiModel(t *testing.T) { [sink.mysql-config] async-ddl-timeout = "45m" + + [sink.kafka-config] + sasl-oauth-ca = "/etc/ssl/oauth-ca.pem" ` err := os.WriteFile(path, []byte(content), 0o644) require.Nil(t, err) @@ -87,7 +90,10 @@ func TestTomlFileToApiModel(t *testing.T) { err = o.strictDecodeConfig("cdc", cfg) require.Nil(t, err) apiModel := v2.ToAPIReplicaConfig(cfg) + require.Equal(t, "/etc/ssl/oauth-ca.pem", *cfg.Sink.KafkaConfig.SASLOAuthCA) + require.Equal(t, "/etc/ssl/oauth-ca.pem", *apiModel.Sink.KafkaConfig.SASLOAuthCA) cfg2 := apiModel.ToInternalReplicaConfig() + require.Equal(t, "/etc/ssl/oauth-ca.pem", *cfg2.Sink.KafkaConfig.SASLOAuthCA) cfgBuf, err := json.MarshalIndent(cfg, "", " ") require.NoError(t, err) cfg2Buf, err := json.MarshalIndent(cfg2, "", " ") diff --git a/pkg/config/replica_config_test.go b/pkg/config/replica_config_test.go index 3a5aba8b58..ee968ee93c 100644 --- a/pkg/config/replica_config_test.go +++ b/pkg/config/replica_config_test.go @@ -194,6 +194,25 @@ func TestReplicaConfig_EnableSplittableCheck_DefaultValue(t *testing.T) { require.False(t, util.GetOrZero(config.Scheduler.EnableSplittableCheck)) } +func TestReplicaConfigClonePreservesKafkaOAuthCA(t *testing.T) { + t.Parallel() + + cfg := GetDefaultReplicaConfig() + cfg.Sink.KafkaConfig = &KafkaConfig{ + SASLOAuthCA: util.AddressOf("/etc/ssl/oauth-ca.pem"), + } + + cloned := cfg.Clone() + require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(cloned.Sink.KafkaConfig.SASLOAuthCA)) + require.NotSame(t, cfg.Sink.KafkaConfig.SASLOAuthCA, cloned.Sink.KafkaConfig.SASLOAuthCA) + encoded, err := cfg.Marshal() + require.NoError(t, err) + require.Contains(t, encoded, `"sasl-oauth-ca":"/etc/ssl/oauth-ca.pem"`) + + *cloned.Sink.KafkaConfig.SASLOAuthCA = "/etc/ssl/other-ca.pem" + require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(cfg.Sink.KafkaConfig.SASLOAuthCA)) +} + func TestReplicaConfigPerformanceMode(t *testing.T) { sinkURI, err := url.Parse("mysql://localhost:3306/test") require.NoError(t, err) diff --git a/pkg/config/sink.go b/pkg/config/sink.go index 5e83ca67d9..12e93a531a 100644 --- a/pkg/config/sink.go +++ b/pkg/config/sink.go @@ -489,6 +489,7 @@ type KafkaConfig struct { SASLOAuthClientID *string `toml:"sasl-oauth-client-id" json:"sasl-oauth-client-id,omitempty"` SASLOAuthClientSecret *string `toml:"sasl-oauth-client-secret" json:"sasl-oauth-client-secret,omitempty"` SASLOAuthTokenURL *string `toml:"sasl-oauth-token-url" json:"sasl-oauth-token-url,omitempty"` + SASLOAuthCA *string `toml:"sasl-oauth-ca" json:"sasl-oauth-ca,omitempty"` SASLOAuthScopes []string `toml:"sasl-oauth-scopes" json:"sasl-oauth-scopes,omitempty"` SASLOAuthGrantType *string `toml:"sasl-oauth-grant-type" json:"sasl-oauth-grant-type,omitempty"` SASLOAuthAudience *string `toml:"sasl-oauth-audience" json:"sasl-oauth-audience,omitempty"` diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 7f62babdc2..07c44901ac 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -515,6 +515,15 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf o.sasl.oauth2.tokenURL = tokenURL } + if sinkConfig.KafkaConfig.SASLOAuthCA != nil { + caPath := *sinkConfig.KafkaConfig.SASLOAuthCA + if caPath == "" { + return errors.ErrKafkaInvalidConfig.GenWithStack( + "OAuth2 CA path cannot be empty") + } + o.sasl.oauth2.caPath = caPath + } + if o.sasl.oauth2.clientID != "" || o.sasl.oauth2.clientSecret != "" || o.sasl.oauth2.tokenURL != "" { diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 1fe52bab86..32bb58436f 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -259,6 +259,7 @@ func TestApplySASL(t *testing.T) { SASLOAuthClientID: aws.String("client_id"), SASLOAuthClientSecret: aws.String("Y2xpZW50X3NlY3JldA=="), SASLOAuthTokenURL: aws.String("127.0.0.1:9093/token"), + SASLOAuthCA: aws.String("/etc/ssl/oauth-ca.pem"), }, expected: saslConfig{ mechanism: oauthMechanism, @@ -266,6 +267,7 @@ func TestApplySASL(t *testing.T) { clientID: "client_id", clientSecret: "client_secret", tokenURL: "127.0.0.1:9093/token", + caPath: "/etc/ssl/oauth-ca.pem", grantType: "client_credentials", }, }, @@ -317,6 +319,17 @@ func TestApplySASL(t *testing.T) { }, expectErr: "OAuth2 is only supported with SASL mechanism type OAUTHBEARER", }, + { + name: "invalid OAUTHBEARER SASL: empty CA path", + uri: baseURI + "?sasl-mechanism=OAUTHBEARER", + kafkaConfig: &config.KafkaConfig{ + SASLOAuthClientID: aws.String("client_id"), + SASLOAuthClientSecret: aws.String("Y2xpZW50X3NlY3JldA=="), + SASLOAuthTokenURL: aws.String("127.0.0.1:9093/token"), + SASLOAuthCA: aws.String(""), + }, + expectErr: "OAuth2 CA path cannot be empty", + }, } for _, test := range tests { @@ -345,6 +358,34 @@ func TestApplySASL(t *testing.T) { } } +func TestOAuthCAIsIndependentFromBrokerTLS(t *testing.T) { + t.Parallel() + + sinkURI, err := url.Parse("kafka://127.0.0.1:9092/abc?sasl-mechanism=OAUTHBEARER") + require.NoError(t, err) + replicaConfig := config.GetDefaultReplicaConfig() + replicaConfig.Sink.KafkaConfig = &config.KafkaConfig{ + SASLOAuthClientID: aws.String("client_id"), + SASLOAuthClientSecret: aws.String("Y2xpZW50X3NlY3JldA=="), + SASLOAuthTokenURL: aws.String("https://oauth.example.com/token"), + SASLOAuthCA: aws.String("/etc/ssl/oauth-ca.pem"), + CA: aws.String("/etc/ssl/broker-ca.pem"), + Cert: aws.String("/etc/ssl/broker-cert.pem"), + Key: aws.String("/etc/ssl/broker-key.pem"), + } + + options := NewOptions() + require.NoError(t, options.Apply( + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), + sinkURI, + replicaConfig.Sink, + )) + require.Equal(t, "/etc/ssl/oauth-ca.pem", options.sasl.oauth2.caPath) + require.Equal(t, "/etc/ssl/broker-ca.pem", options.Credential.CAPath) + require.Equal(t, "/etc/ssl/broker-cert.pem", options.Credential.CertPath) + require.Equal(t, "/etc/ssl/broker-key.pem", options.Credential.KeyPath) +} + func TestApplyTLS(t *testing.T) { t.Parallel() diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider.go b/pkg/sink/kafka/sarama_oauth2_token_provider.go index 6a474b6725..96aabd080b 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider.go @@ -15,7 +15,11 @@ package kafka import ( "context" + "crypto/tls" + "crypto/x509" + "net/http" "net/url" + "os" "github.com/IBM/sarama" "github.com/pingcap/ticdc/pkg/errors" @@ -71,6 +75,13 @@ func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvid return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } + if o.sasl.oauth2.caPath != "" { + ctx, err = contextWithOAuthCA(ctx, o.sasl.oauth2.caPath) + if err != nil { + return nil, err + } + } + cfg := clientcredentials.Config{ ClientID: o.sasl.oauth2.clientID, ClientSecret: o.sasl.oauth2.clientSecret, @@ -82,3 +93,37 @@ func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvid tokenSource: cfg.TokenSource(ctx), }, nil } + +func contextWithOAuthCA(ctx context.Context, caPath string) (context.Context, error) { + caPEM, err := os.ReadFile(caPath) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if !rootCAs.AppendCertsFromPEM(caPEM) { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "OAuth2 CA file %q does not contain a valid certificate", caPath) + } + + defaultTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "cannot configure OAuth2 CA file %q with HTTP transport type %T", + caPath, http.DefaultTransport) + } + transport := defaultTransport.Clone() + tlsConfig := transport.TLSClientConfig + if tlsConfig == nil { + tlsConfig = &tls.Config{} + } else { + tlsConfig = tlsConfig.Clone() + } + tlsConfig.RootCAs = rootCAs + transport.TLSClientConfig = tlsConfig + httpClient := &http.Client{Transport: transport} + return context.WithValue(ctx, oauth2.HTTPClient, httpClient), nil +} diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go index d7d0473c32..0dcbaa42fa 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go @@ -14,11 +14,20 @@ package kafka import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/pem" "io" + "math/big" "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "testing" + "time" "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" @@ -132,3 +141,125 @@ func TestTokenProviderPropagatesEndpointError(t *testing.T) { require.Equal(t, "invalid_client", retrieveErr.ErrorCode) require.Equal(t, "bad credentials", retrieveErr.ErrorDescription) } + +func TestTokenProviderUsesOAuthCA(t *testing.T) { + t.Parallel() + + server := newTLSTokenServer(t) + caPath := writeServerCA(t, server) + options := newOAuthOptions(server.URL, caPath) + + provider, err := newTokenProvider(t.Context(), options) + require.NoError(t, err) + token, err := provider.Token() + require.NoError(t, err) + require.Equal(t, "access-token", token.Token) +} + +func TestTokenProviderRejectsInvalidOAuthCA(t *testing.T) { + t.Parallel() + + t.Run("missing file", func(t *testing.T) { + t.Parallel() + + caPath := filepath.Join(t.TempDir(), "missing-ca.pem") + _, err := newTokenProvider(t.Context(), newOAuthOptions("https://example.com/token", caPath)) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.ErrorContains(t, err, caPath) + require.ErrorContains(t, err, "no such file") + }) + + t.Run("invalid PEM", func(t *testing.T) { + t.Parallel() + + caPath := filepath.Join(t.TempDir(), "invalid-ca.pem") + require.NoError(t, os.WriteFile(caPath, []byte("not a certificate"), 0o600)) + _, err := newTokenProvider(t.Context(), newOAuthOptions("https://example.com/token", caPath)) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.ErrorContains(t, err, caPath) + require.ErrorContains(t, err, "does not contain a valid certificate") + }) +} + +func TestTokenProviderRejectsMismatchedOAuthCA(t *testing.T) { + t.Parallel() + + tokenServer := newTLSTokenServer(t) + options := newOAuthOptions(tokenServer.URL, writeUnrelatedCA(t)) + + provider, err := newTokenProvider(t.Context(), options) + require.NoError(t, err) + _, err = provider.Token() + require.ErrorContains(t, err, "certificate signed by unknown authority") +} + +func writeUnrelatedCA(t *testing.T) string { + t.Helper() + + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + certificate, err := x509.CreateCertificate(rand.Reader, template, template, publicKey, privateKey) + require.NoError(t, err) + + caPath := filepath.Join(t.TempDir(), "unrelated-ca.pem") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate}) + require.NotNil(t, caPEM) + require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) + return caPath +} + +func TestTokenProviderWithoutOAuthCAKeepsContextHTTPClient(t *testing.T) { + t.Parallel() + + server := newTLSTokenServer(t) + ctx := context.WithValue(t.Context(), oauth2.HTTPClient, server.Client()) + provider, err := newTokenProvider(ctx, newOAuthOptions(server.URL, "")) + require.NoError(t, err) + token, err := provider.Token() + require.NoError(t, err) + require.Equal(t, "access-token", token.Token) +} + +func newTLSTokenServer(t *testing.T) *httptest.Server { + t.Helper() + + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if _, err := io.WriteString(w, `{"access_token":"access-token","token_type":"bearer"}`); err != nil { + t.Errorf("write token response: %v", err) + } + })) + t.Cleanup(server.Close) + return server +} + +func writeServerCA(t *testing.T, server *httptest.Server) string { + t.Helper() + + caPath := filepath.Join(t.TempDir(), "ca.pem") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + require.NotNil(t, caPEM) + require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) + return caPath +} + +func newOAuthOptions(tokenURL, caPath string) *options { + return &options{ + sasl: &saslConfig{ + oauth2: oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: tokenURL, + caPath: caPath, + }, + }, + } +} diff --git a/pkg/sink/kafka/sasl_config.go b/pkg/sink/kafka/sasl_config.go index 95389a1774..75ff6156e2 100644 --- a/pkg/sink/kafka/sasl_config.go +++ b/pkg/sink/kafka/sasl_config.go @@ -68,6 +68,7 @@ type oauth2Config struct { clientID string clientSecret string tokenURL string + caPath string scopes []string grantType string audience string From cbe8a31aa881a402a3a28c5cbd02bfd936dbcbad Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 1 Sep 2026 12:12:58 +0800 Subject: [PATCH 42/61] kafka: simplify OAuth CA handling and tests --- api/v2/model_test.go | 23 +++--------- .../kafka/sarama_oauth2_token_provider.go | 13 +++---- .../sarama_oauth2_token_provider_test.go | 35 ++++--------------- 3 files changed, 15 insertions(+), 56 deletions(-) diff --git a/api/v2/model_test.go b/api/v2/model_test.go index db9d58ff0a..c3fb71e03a 100644 --- a/api/v2/model_test.go +++ b/api/v2/model_test.go @@ -90,6 +90,9 @@ func TestReplicaConfigConversion(t *testing.T) { DebeziumConfig: &DebeziumConfig{ IncludeStartTs: util.AddressOf(true), }, + KafkaConfig: &KafkaConfig{ + SASLOAuthCA: util.AddressOf("/etc/ssl/oauth-ca.pem"), + }, }, Mounter: &MounterConfig{ WorkerNum: util.AddressOf(16), @@ -124,6 +127,7 @@ func TestReplicaConfigConversion(t *testing.T) { require.Equal(t, int64(1024), util.GetOrZero(internalCfg.Sink.CloudStorageConfig.SpoolDiskQuota)) require.Equal(t, "/tmp/ticdc-spool", util.GetOrZero(internalCfg.Sink.CloudStorageConfig.SpoolBaseDir)) require.True(t, util.GetOrZero(internalCfg.Sink.Debezium.IncludeStartTs)) + require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(internalCfg.Sink.KafkaConfig.SASLOAuthCA)) require.Equal(t, internalCfg.Mounter.WorkerNum, *apiCfg.Mounter.WorkerNum) require.True(t, util.GetOrZero(internalCfg.Scheduler.EnableTableAcrossNodes)) require.Equal(t, 1000, util.GetOrZero(internalCfg.Scheduler.RegionThreshold)) @@ -169,6 +173,7 @@ func TestReplicaConfigConversion(t *testing.T) { require.Equal(t, "/tmp/ticdc-spool", *apiCfgBack.Sink.CloudStorageConfig.SpoolBaseDir) require.True(t, util.GetOrZero(apiCfgBack.Sink.DebeziumConfig.IncludeStartTs)) require.True(t, util.GetOrZero(apiCfgBack.Sink.DebeziumConfig.OutputOldValue)) + require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(apiCfgBack.Sink.KafkaConfig.SASLOAuthCA)) require.Equal(t, 16, *apiCfgBack.Mounter.WorkerNum) require.True(t, *apiCfgBack.Scheduler.EnableTableAcrossNodes) require.Equal(t, "correctness", *apiCfgBack.Integrity.IntegrityCheckLevel) @@ -326,21 +331,3 @@ func TestReplicaConfigCodecConfigConversion(t *testing.T) { require.NotNil(t, apiCfgBack.Sink.KafkaConfig.CodecConfig) require.True(t, util.GetOrZero(apiCfgBack.Sink.KafkaConfig.CodecConfig.AvroIncludeBeforeValue)) } - -func TestReplicaConfigKafkaOAuthCAConversion(t *testing.T) { - t.Parallel() - - apiCfg := &ReplicaConfig{ - Sink: &SinkConfig{ - KafkaConfig: &KafkaConfig{ - SASLOAuthCA: util.AddressOf("/etc/ssl/oauth-ca.pem"), - }, - }, - } - - internalCfg := apiCfg.ToInternalReplicaConfig() - require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(internalCfg.Sink.KafkaConfig.SASLOAuthCA)) - - apiCfgBack := ToAPIReplicaConfig(internalCfg) - require.Equal(t, "/etc/ssl/oauth-ca.pem", util.GetOrZero(apiCfgBack.Sink.KafkaConfig.SASLOAuthCA)) -} diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider.go b/pkg/sink/kafka/sarama_oauth2_token_provider.go index 96aabd080b..6adbc56659 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider.go @@ -116,14 +116,9 @@ func contextWithOAuthCA(ctx context.Context, caPath string) (context.Context, er caPath, http.DefaultTransport) } transport := defaultTransport.Clone() - tlsConfig := transport.TLSClientConfig - if tlsConfig == nil { - tlsConfig = &tls.Config{} - } else { - tlsConfig = tlsConfig.Clone() + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} } - tlsConfig.RootCAs = rootCAs - transport.TLSClientConfig = tlsConfig - httpClient := &http.Client{Transport: transport} - return context.WithValue(ctx, oauth2.HTTPClient, httpClient), nil + transport.TLSClientConfig.RootCAs = rootCAs + return context.WithValue(ctx, oauth2.HTTPClient, &http.Client{Transport: transport}), nil } diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go index 0dcbaa42fa..fc2dffa6a2 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go @@ -15,21 +15,17 @@ package kafka import ( "context" - "crypto/ed25519" - "crypto/rand" - "crypto/x509" "encoding/pem" "io" - "math/big" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "testing" - "time" "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" "golang.org/x/oauth2" ) @@ -185,7 +181,11 @@ func TestTokenProviderRejectsMismatchedOAuthCA(t *testing.T) { t.Parallel() tokenServer := newTLSTokenServer(t) - options := newOAuthOptions(tokenServer.URL, writeUnrelatedCA(t)) + unrelatedCA, err := security.NewCA() + require.NoError(t, err) + caPath := filepath.Join(t.TempDir(), "unrelated-ca.pem") + require.NoError(t, os.WriteFile(caPath, unrelatedCA.CAPEM, 0o600)) + options := newOAuthOptions(tokenServer.URL, caPath) provider, err := newTokenProvider(t.Context(), options) require.NoError(t, err) @@ -193,29 +193,6 @@ func TestTokenProviderRejectsMismatchedOAuthCA(t *testing.T) { require.ErrorContains(t, err, "certificate signed by unknown authority") } -func writeUnrelatedCA(t *testing.T) string { - t.Helper() - - publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) - require.NoError(t, err) - template := &x509.Certificate{ - SerialNumber: big.NewInt(1), - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - IsCA: true, - BasicConstraintsValid: true, - KeyUsage: x509.KeyUsageCertSign, - } - certificate, err := x509.CreateCertificate(rand.Reader, template, template, publicKey, privateKey) - require.NoError(t, err) - - caPath := filepath.Join(t.TempDir(), "unrelated-ca.pem") - caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate}) - require.NotNil(t, caPEM) - require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) - return caPath -} - func TestTokenProviderWithoutOAuthCAKeepsContextHTTPClient(t *testing.T) { t.Parallel() From 31d41374fb587dee32719108d07f72a933417f3a Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 1 Sep 2026 15:50:58 +0800 Subject: [PATCH 43/61] kafka: fix franz security authentication --- go.mod | 3 +- go.sum | 11 + pkg/sink/kafka/franz/config.go | 6 + pkg/sink/kafka/franz/config_test.go | 19 ++ pkg/sink/kafka/franz/gssapi.go | 221 ++---------------- pkg/sink/kafka/franz/gssapi_test.go | 178 +------------- pkg/sink/kafka/franz_adapter.go | 7 + .../kafka/sarama_oauth2_token_provider.go | 7 +- .../sarama_oauth2_token_provider_test.go | 16 ++ 9 files changed, 95 insertions(+), 373 deletions(-) diff --git a/go.mod b/go.mod index de3a3a5d33..8744adfd19 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,6 @@ require ( github.com/imdario/mergo v0.3.16 github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc github.com/jarcoal/httpmock v1.2.0 - github.com/jcmturner/gofork v1.7.6 github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/json-iterator/go v1.1.12 github.com/klauspost/compress v1.19.0 @@ -80,6 +79,7 @@ require ( github.com/twmb/franz-go/pkg/kadm v1.18.0 github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 github.com/twmb/franz-go/pkg/kmsg v1.13.1 + github.com/twmb/franz-go/pkg/sasl/kerberos v1.1.0 github.com/uber-go/atomic v1.4.0 github.com/xdg/scram v1.0.5 github.com/zeebo/assert v1.3.0 @@ -248,6 +248,7 @@ require ( github.com/influxdata/tdigest v0.0.1 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jedib0t/go-pretty/v6 v6.2.2 // indirect github.com/jellydator/ttlcache/v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 8448cf1816..54946c2407 100644 --- a/go.sum +++ b/go.sum @@ -549,6 +549,7 @@ github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVET github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= @@ -962,14 +963,18 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/twmb/franz-go v1.7.0/go.mod h1:PMze0jNfNghhih2XHbkmTFykbMF5sJqmNJB31DOOzro= github.com/twmb/franz-go v1.21.6 h1:+v0dQJVIIuw9uPmPWmPrkoUHs1pPeV8MSwA4eU/Y2kY= github.com/twmb/franz-go v1.21.6/go.mod h1:wMepkgCatAdV9vCsuwM+wr+C1fl7KV/41+uHGAjt/wc= github.com/twmb/franz-go/pkg/kadm v1.18.0 h1:WRf/LZmDdcDXwX7WMbtDU++v+b3NzYh2bCGoPMmzirw= github.com/twmb/franz-go/pkg/kadm v1.18.0/go.mod h1:XeLhGoLXLFzK8/ryv5FfpxPxGwj4oFEGpPJMB/x6KDE= github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 h1:OLL/h1pOsMuCnJEp3IW0L2W3jHATeTUMAmE6PxxbhtU= github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7/go.mod h1:9j4VxU2ng6tHgD4lIkNJ5OJ3D6vgPhhIp3tBa7dJgLA= +github.com/twmb/franz-go/pkg/kmsg v1.2.0/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY= github.com/twmb/franz-go/pkg/kmsg v1.13.1 h1:fG5kItwysTk5UXqVwb64EpQEy3TydF3vYYK21nUQ+bI= github.com/twmb/franz-go/pkg/kmsg v1.13.1/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY= +github.com/twmb/franz-go/pkg/sasl/kerberos v1.1.0 h1:alKdbddkPw3rDh+AwmUEwh6HNYgTvDSFIe/GWYRR9RM= +github.com/twmb/franz-go/pkg/sasl/kerberos v1.1.0/go.mod h1:k8BoBjyUbFj34f0rRbn+Ky12sZFAPbmShrg0karAIMo= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= @@ -1119,6 +1124,8 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= @@ -1174,8 +1181,11 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -1236,6 +1246,7 @@ golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/sink/kafka/franz/config.go b/pkg/sink/kafka/franz/config.go index 5b6a753783..8ee65eb35a 100644 --- a/pkg/sink/kafka/franz/config.go +++ b/pkg/sink/kafka/franz/config.go @@ -17,6 +17,7 @@ package franz import ( "context" "crypto/tls" + "net/http" "net/url" "strings" "time" @@ -97,6 +98,7 @@ type OAuth2Config struct { Scopes []string GrantType string Audience string + HTTPClient *http.Client } func (c Config) requestTimeout() time.Duration { return max(c.ReadTimeout, c.WriteTimeout) } @@ -163,6 +165,10 @@ func buildSASLMechanism(ctx context.Context, cfg SASLConfig) (sasl.Mechanism, er } func newOAuthTokenSource(ctx context.Context, cfg OAuth2Config) (oauth2.TokenSource, error) { + if cfg.HTTPClient != nil { + ctx = context.WithValue(ctx, oauth2.HTTPClient, cfg.HTTPClient) + } + endpointParams := url.Values{} if cfg.GrantType != "" { endpointParams.Set("grant_type", cfg.GrantType) diff --git a/pkg/sink/kafka/franz/config_test.go b/pkg/sink/kafka/franz/config_test.go index bc2270dc6a..87456b84dc 100644 --- a/pkg/sink/kafka/franz/config_test.go +++ b/pkg/sink/kafka/franz/config_test.go @@ -244,6 +244,25 @@ func TestOAuthTokenSource(t *testing.T) { require.Equal(t, "scope-a scope-b", form.Get("scope")) } +func TestOAuthTokenSourceUsesHTTPClient(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := io.WriteString(w, `{"access_token":"token","token_type":"bearer"}`) + require.NoError(t, err) + })) + defer server.Close() + + source, err := newOAuthTokenSource(context.Background(), OAuth2Config{ + TokenURL: server.URL, + HTTPClient: server.Client(), + }) + require.NoError(t, err) + + token, err := source.Token() + require.NoError(t, err) + require.Equal(t, "token", token.AccessToken) +} + func TestOAuthTokenSourceRejectsInvalidURL(t *testing.T) { _, err := newOAuthTokenSource(context.Background(), OAuth2Config{TokenURL: "http://example.com/%%"}) require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) diff --git a/pkg/sink/kafka/franz/gssapi.go b/pkg/sink/kafka/franz/gssapi.go index 2a061b48d6..d88584c085 100644 --- a/pkg/sink/kafka/franz/gssapi.go +++ b/pkg/sink/kafka/franz/gssapi.go @@ -8,6 +8,7 @@ // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. @@ -15,262 +16,90 @@ package franz import ( "context" - "encoding/binary" - "net" - "github.com/jcmturner/gofork/encoding/asn1" - "github.com/jcmturner/gokrb5/v8/asn1tools" "github.com/jcmturner/gokrb5/v8/client" "github.com/jcmturner/gokrb5/v8/config" - "github.com/jcmturner/gokrb5/v8/gssapi" - "github.com/jcmturner/gokrb5/v8/iana/chksumtype" - "github.com/jcmturner/gokrb5/v8/iana/keyusage" "github.com/jcmturner/gokrb5/v8/keytab" - "github.com/jcmturner/gokrb5/v8/messages" - "github.com/jcmturner/gokrb5/v8/types" "github.com/pingcap/ticdc/pkg/errors" "github.com/twmb/franz-go/pkg/sasl" + "github.com/twmb/franz-go/pkg/sasl/kerberos" ) const ( - // tokIDKrbAPReq and gssAPIGeneric identify a Kerberos AP-REQ inside a GSS-API initial context token. - tokIDKrbAPReq = 0x0100 - gssAPIGeneric = 0x60 // Authentication type values are part of sink URI compatibility and must remain stable. userAuth = 1 keyTabAuth = 2 ) -type gssapiMechanism struct { - config GSSAPIConfig - newClient func(GSSAPIConfig) (kerberosClient, error) - newToken func(string, types.PrincipalName, messages.Ticket, types.EncryptionKey) ([]byte, error) -} - -type kerberosClient interface { - Login() error - Destroy() - GetServiceTicket(string) (messages.Ticket, types.EncryptionKey, error) - Domain() string - CName() types.PrincipalName -} - -type gokrb5Client struct { - *client.Client -} - -func (m *gssapiMechanism) Name() string { - return "GSSAPI" -} - -func (m *gssapiMechanism) Authenticate( - _ context.Context, - host string, -) (sasl.Session, []byte, error) { - client, err := m.newClient(m.config) - if err != nil { - return nil, nil, err - } - - if err = client.Login(); err != nil { - client.Destroy() - return nil, nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - serverHost, err := brokerHost(host) - if err != nil { - client.Destroy() - return nil, nil, err - } - - spn := m.config.ServiceName + "/" + serverHost - ticket, encKey, err := client.GetServiceTicket(spn) - if err != nil { - client.Destroy() - return nil, nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - token, err := m.newToken(client.Domain(), client.CName(), ticket, encKey) - if err != nil { - client.Destroy() - return nil, nil, err - } - - firstMessage, err := appendGSSAPIHeader(token) - if err != nil { - client.Destroy() - return nil, nil, err - } - - return &gssapiSession{client: client, encKey: encKey}, firstMessage, nil -} - -func brokerHost(address string) (string, error) { - host, _, err := net.SplitHostPort(address) - if err != nil { - return "", errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - - return host, nil -} - -type gssapiSession struct { - client kerberosClient - encKey types.EncryptionKey -} - -func (s *gssapiSession) Challenge(challenge []byte) (bool, []byte, error) { - defer s.client.Destroy() - - wrapTokenReq := gssapi.WrapToken{} - if err := wrapTokenReq.Unmarshal(challenge, true); err != nil { - return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) +func buildGSSAPIMechanism(g GSSAPIConfig) (sasl.Mechanism, error) { + if err := validateGSSAPIConfig(g); err != nil { + return nil, err } - isValid, err := wrapTokenReq.Verify(s.encKey, keyusage.GSSAPI_ACCEPTOR_SEAL) - if !isValid { + return kerberos.Kerberos(func(context.Context) (kerberos.Auth, error) { + krbClient, err := newKerberosClient(g) if err != nil { - return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) + return kerberos.Auth{}, err } - - return false, nil, errors.ErrNewKafkaSink.GenWithStackByArgs() - } - - wrapTokenResp, err := gssapi.NewInitiatorWrapToken(wrapTokenReq.Payload, s.encKey) - if err != nil { - return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - msg, err := wrapTokenResp.Marshal() - if err != nil { - return false, nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - return true, msg, nil + return kerberos.Auth{Client: krbClient, Service: g.ServiceName}, nil + }), nil } -func buildGSSAPIMechanism(g GSSAPIConfig) (sasl.Mechanism, error) { +func validateGSSAPIConfig(g GSSAPIConfig) error { if g.ServiceName == "" { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") } if g.KerberosConfigPath == "" { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") } if g.Username == "" { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") } if g.Realm == "" { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") } switch g.AuthType { case userAuth: if g.Password == "" { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") } case keyTabAuth: if g.KeyTabPath == "" { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") } default: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "unsupported sasl-gssapi-auth-type %d", g.AuthType) } - - return &gssapiMechanism{ - config: g, - newClient: newKerberosClient, - newToken: newKrb5Token, - }, nil + return nil } -func newKerberosClient(g GSSAPIConfig) (kerberosClient, error) { +func newKerberosClient(g GSSAPIConfig) (*client.Client, error) { cfg, err := config.Load(g.KerberosConfigPath) if err != nil { return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - var krbClient *client.Client switch g.AuthType { + case userAuth: + return client.NewWithPassword( + g.Username, g.Realm, g.Password, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)), nil case keyTabAuth: kt, err := keytab.Load(g.KeyTabPath) if err != nil { return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - krbClient = client.NewWithKeytab( - g.Username, g.Realm, kt, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)) - case userAuth: - krbClient = client.NewWithPassword( - g.Username, g.Realm, g.Password, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)) + return client.NewWithKeytab( + g.Username, g.Realm, kt, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)), nil default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "unsupported sasl-gssapi-auth-type %d", g.AuthType) } - - return &gokrb5Client{Client: krbClient}, nil -} - -func (c *gokrb5Client) Domain() string { return c.Credentials.Domain() } - -func (c *gokrb5Client) CName() types.PrincipalName { return c.Credentials.CName() } - -func newKrb5Token( - domain string, - cname types.PrincipalName, - ticket messages.Ticket, - sessionKey types.EncryptionKey, -) ([]byte, error) { - authenticator, err := types.NewAuthenticator(domain, cname) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - authenticator.Cksum = types.Checksum{ - CksumType: chksumtype.GSSAPI, - Checksum: newAuthenticatorChecksum(), - } - - apReq, err := messages.NewAPReq(ticket, sessionKey, authenticator) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - body, err := apReq.Marshal() - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - prefix := make([]byte, 2, 2+len(body)) - binary.BigEndian.PutUint16(prefix, tokIDKrbAPReq) - - return append(prefix, body...), nil -} - -func newAuthenticatorChecksum() []byte { - sum := make([]byte, 24) - binary.LittleEndian.PutUint32(sum[:4], 16) - - flags := uint32(gssapi.ContextFlagInteg | gssapi.ContextFlagConf) - binary.LittleEndian.PutUint32(sum[20:24], flags) - - return sum -} - -func appendGSSAPIHeader(payload []byte) ([]byte, error) { - oidBytes, err := asn1.Marshal(gssapi.OIDKRB5.OID()) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - tkoLengthBytes := asn1tools.MarshalLengthBytes(len(oidBytes) + len(payload)) - header := append([]byte{gssAPIGeneric}, tkoLengthBytes...) - header = append(header, oidBytes...) - - return append(header, payload...), nil } diff --git a/pkg/sink/kafka/franz/gssapi_test.go b/pkg/sink/kafka/franz/gssapi_test.go index 1995cf219f..982da3327d 100644 --- a/pkg/sink/kafka/franz/gssapi_test.go +++ b/pkg/sink/kafka/franz/gssapi_test.go @@ -16,63 +16,12 @@ package franz import ( "context" - "encoding/binary" "testing" - "github.com/jcmturner/gokrb5/v8/gssapi" - "github.com/jcmturner/gokrb5/v8/iana/etypeID" - "github.com/jcmturner/gokrb5/v8/iana/keyusage" - "github.com/jcmturner/gokrb5/v8/iana/nametype" - "github.com/jcmturner/gokrb5/v8/messages" - "github.com/jcmturner/gokrb5/v8/types" "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" ) -type fakeKerberosClient struct { - loginError error - ticketError error - spn string - destroyed bool -} - -func (c *fakeKerberosClient) Login() error { return c.loginError } - -func (c *fakeKerberosClient) Destroy() { c.destroyed = true } - -func (c *fakeKerberosClient) GetServiceTicket(spn string) ( - messages.Ticket, - types.EncryptionKey, - error, -) { - c.spn = spn - - return messages.Ticket{}, testEncryptionKey(), c.ticketError -} - -func (c *fakeKerberosClient) Domain() string { return "EXAMPLE.COM" } - -func (c *fakeKerberosClient) CName() types.PrincipalName { - return types.NewPrincipalName(nametype.KRB_NT_PRINCIPAL, "alice") -} - -func TestBrokerHost(t *testing.T) { - for _, test := range []struct { - address string - expected string - }{ - {address: "broker.example.com:9092", expected: "broker.example.com"}, - {address: "[2001:db8::1]:9092", expected: "2001:db8::1"}, - } { - host, err := brokerHost(test.address) - require.NoError(t, err) - require.Equal(t, test.expected, host) - } - - _, err := brokerHost("2001:db8::1:9092") - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) -} - func TestGSSAPIConfigValidation(t *testing.T) { valid := GSSAPIConfig{ AuthType: userAuth, @@ -100,134 +49,17 @@ func TestGSSAPIConfigValidation(t *testing.T) { } } -func TestGSSAPIEncodingHelpers(t *testing.T) { - checksum := newAuthenticatorChecksum() - require.Len(t, checksum, 24) - require.Equal(t, uint32(16), binary.LittleEndian.Uint32(checksum[:4])) - require.Equal(t, uint32(gssapi.ContextFlagInteg|gssapi.ContextFlagConf), binary.LittleEndian.Uint32(checksum[20:24])) - - payload := []byte{1, 2, 3} - message, err := appendGSSAPIHeader(payload) - require.NoError(t, err) - require.Equal(t, byte(gssAPIGeneric), message[0]) - require.Equal(t, payload, message[len(message)-len(payload):]) -} - -func TestNewKerberosClientRejectsMissingConfig(t *testing.T) { - _, err := newKerberosClient(GSSAPIConfig{ +func TestGSSAPIRejectsMissingKerberosConfig(t *testing.T) { + mechanism, err := buildGSSAPIMechanism(GSSAPIConfig{ AuthType: userAuth, KerberosConfigPath: "/path/that/does/not/exist", + ServiceName: "kafka", Username: "alice", Password: "secret", Realm: "EXAMPLE.COM", }) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) -} - -func TestGSSAPIAuthenticate(t *testing.T) { - client := &fakeKerberosClient{} - mechanism := &gssapiMechanism{ - config: GSSAPIConfig{ServiceName: "kafka"}, - newClient: func(GSSAPIConfig) (kerberosClient, error) { - return client, nil - }, - newToken: func( - domain string, - cname types.PrincipalName, - _ messages.Ticket, - _ types.EncryptionKey, - ) ([]byte, error) { - require.Equal(t, "EXAMPLE.COM", domain) - require.Equal(t, "alice", cname.PrincipalNameString()) - - return []byte{1, 2, 3}, nil - }, - } - - session, message, err := mechanism.Authenticate(context.Background(), "[2001:db8::1]:9092") require.NoError(t, err) - require.NotNil(t, session) - require.Equal(t, "kafka/2001:db8::1", client.spn) - require.Equal(t, []byte{1, 2, 3}, message[len(message)-3:]) - require.False(t, client.destroyed) - - done, _, err := session.Challenge([]byte("invalid")) - require.False(t, done) - require.ErrorIs(t, err, errors.ErrNewKafkaSink) - require.True(t, client.destroyed) -} -func TestGSSAPIAuthenticateDestroysClientAfterFailure(t *testing.T) { - for _, test := range []struct { - name string - host string - loginError error - ticketError error - tokenError error - }{ - {name: "login", host: "broker:9092", loginError: context.Canceled}, - {name: "broker address", host: "invalid"}, - {name: "service ticket", host: "broker:9092", ticketError: context.DeadlineExceeded}, - {name: "AP request", host: "broker:9092", tokenError: context.Canceled}, - } { - t.Run(test.name, func(t *testing.T) { - client := &fakeKerberosClient{ - loginError: test.loginError, - ticketError: test.ticketError, - } - mechanism := &gssapiMechanism{ - config: GSSAPIConfig{ServiceName: "kafka"}, - newClient: func(GSSAPIConfig) (kerberosClient, error) { - return client, nil - }, - newToken: func( - string, - types.PrincipalName, - messages.Ticket, - types.EncryptionKey, - ) ([]byte, error) { - return nil, test.tokenError - }, - } - - _, _, err := mechanism.Authenticate(context.Background(), test.host) - require.Error(t, err) - require.True(t, client.destroyed) - }) - } -} - -func TestGSSAPIChallenge(t *testing.T) { - key := testEncryptionKey() - request := gssapi.WrapToken{ - Flags: 1, - EC: 12, - Payload: []byte{1, 2, 3, 4}, - } - require.NoError(t, request.SetCheckSum(key, keyusage.GSSAPI_ACCEPTOR_SEAL)) - - challenge, err := request.Marshal() - require.NoError(t, err) - - client := &fakeKerberosClient{} - session := &gssapiSession{client: client, encKey: key} - done, response, err := session.Challenge(challenge) - require.NoError(t, err) - require.True(t, done) - require.True(t, client.destroyed) - - initiatorToken := gssapi.WrapToken{} - require.NoError(t, initiatorToken.Unmarshal(response, false)) - require.Equal(t, request.Payload, initiatorToken.Payload) - - valid, err := initiatorToken.Verify(key, keyusage.GSSAPI_INITIATOR_SEAL) - require.NoError(t, err) - require.True(t, valid) -} - -func testEncryptionKey() types.EncryptionKey { - return types.EncryptionKey{ - KeyType: etypeID.AES128_CTS_HMAC_SHA1_96, - KeyValue: []byte("0123456789abcdef"), - } + _, _, err = mechanism.Authenticate(context.Background(), "broker:9092") + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } diff --git a/pkg/sink/kafka/franz_adapter.go b/pkg/sink/kafka/franz_adapter.go index c27cd64d56..2f3a7a2e90 100644 --- a/pkg/sink/kafka/franz_adapter.go +++ b/pkg/sink/kafka/franz_adapter.go @@ -209,6 +209,13 @@ func newFranzConfig(o *options) (franz.Config, error) { Audience: o.sasl.oauth2.audience, }, } + if o.sasl.oauth2.caPath != "" { + httpClient, err := oauthHTTPClient(o.sasl.oauth2.caPath) + if err != nil { + return franz.Config{}, err + } + config.SASL.OAuth2.HTTPClient = httpClient + } } return config, nil diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider.go b/pkg/sink/kafka/sarama_oauth2_token_provider.go index 6adbc56659..3b5d6fed2e 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider.go @@ -76,10 +76,11 @@ func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvid } if o.sasl.oauth2.caPath != "" { - ctx, err = contextWithOAuthCA(ctx, o.sasl.oauth2.caPath) + httpClient, err := oauthHTTPClient(o.sasl.oauth2.caPath) if err != nil { return nil, err } + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) } cfg := clientcredentials.Config{ @@ -94,7 +95,7 @@ func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvid }, nil } -func contextWithOAuthCA(ctx context.Context, caPath string) (context.Context, error) { +func oauthHTTPClient(caPath string) (*http.Client, error) { caPEM, err := os.ReadFile(caPath) if err != nil { return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) @@ -120,5 +121,5 @@ func contextWithOAuthCA(ctx context.Context, caPath string) (context.Context, er transport.TLSClientConfig = &tls.Config{} } transport.TLSClientConfig.RootCAs = rootCAs - return context.WithValue(ctx, oauth2.HTTPClient, &http.Client{Transport: transport}), nil + return &http.Client{Transport: transport}, nil } diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go index fc2dffa6a2..a831747f4e 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go @@ -152,6 +152,22 @@ func TestTokenProviderUsesOAuthCA(t *testing.T) { require.Equal(t, "access-token", token.Token) } +func TestFranzConfigUsesOAuthCA(t *testing.T) { + t.Parallel() + + server := newTLSTokenServer(t) + caPath := writeServerCA(t, server) + options := newOAuthOptions(server.URL, caPath) + options.sasl.mechanism = oauthMechanism + config, err := newFranzConfig(options) + require.NoError(t, err) + require.NotNil(t, config.SASL.OAuth2.HTTPClient) + + response, err := config.SASL.OAuth2.HTTPClient.Get(server.URL) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) +} + func TestTokenProviderRejectsInvalidOAuthCA(t *testing.T) { t.Parallel() From 5cbe3b34c3e06909904d58045bef2a714d3f0b62 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 2 Sep 2026 14:38:51 +0800 Subject: [PATCH 44/61] kafka: classify franz-go admin errors --- .../topicmanager/kafka_topic_manager_test.go | 47 ++++++++++++------- pkg/sink/kafka/admin.go | 9 +++- pkg/sink/kafka/franz_adapter.go | 16 +++++++ pkg/sink/kafka/sarama_admin_test.go | 4 +- pkg/sink/kafka/selector_test.go | 39 +++++++++++++++ 5 files changed, 94 insertions(+), 21 deletions(-) diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index 9ba464da7c..8a261d8a97 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kerr" ) const kafkaTopicManagerTestTopic = "mock_topic" @@ -192,23 +193,35 @@ func TestCreateTopicValidatesReplicationFactor(t *testing.T) { func TestWaitUntilTopicVisibleUnretryableError(t *testing.T) { t.Parallel() - ctrl := gomock.NewController(t) - adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"invalid-topic"}, false).Return( - nil, - errors.WrapError(errors.ErrKafkaAdminAPI, sarama.ErrInvalidTopic, "describe-topic", "invalid-topic"), - ).Times(1) - manager := newKafkaTopicManager( - "invalid-topic", - common.NewChangefeedID4Test("test", "test"), - adminClient, - &kafka.AutoCreateTopicConfig{PartitionNum: 2}, - ) - - err := manager.waitUntilTopicVisible(context.Background(), "invalid-topic") - - require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) - require.ErrorIs(t, err, sarama.ErrInvalidTopic) + for _, test := range []struct { + name string + cause error + }{ + {name: "sarama", cause: sarama.ErrInvalidTopic}, + {name: "franz-go", cause: kerr.InvalidTopicException}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta([]string{"invalid-topic"}, false).Return( + nil, + errors.WrapError(errors.ErrKafkaAdminAPI, test.cause, "describe-topic", "invalid-topic"), + ).Times(1) + manager := newKafkaTopicManager( + "invalid-topic", + common.NewChangefeedID4Test("test", "test"), + adminClient, + &kafka.AutoCreateTopicConfig{PartitionNum: 2}, + ) + + err := manager.waitUntilTopicVisible(context.Background(), "invalid-topic") + + require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.ErrorIs(t, err, test.cause) + }) + } } func TestGetTopicManagerStartsBackgroundRefreshAfterTopicReady(t *testing.T) { diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index 8c1a54f2ec..651dd8330f 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -156,10 +156,10 @@ func IsAuthorizationFailed(err error) bool { errors.Is(err, sarama.ErrClusterAuthorizationFailed) } -// IsUnretryableKafkaError reports whether err is not retryable. +// IsUnretryableSaramaError reports whether a Sarama error is not retryable. // See Apache Kafka protocol error definitions: // https://kafka.apache.org/38/generated/protocol_errors.html -func IsUnretryableKafkaError(err error) bool { +func IsUnretryableSaramaError(err error) bool { if IsAuthorizationFailed(err) || errors.Is(err, errors.ErrKafkaInvalidConfig) || errors.Is(err, sarama.ErrInvalidTopic) || @@ -176,6 +176,11 @@ func IsUnretryableKafkaError(err error) bool { return errors.As(err, &configErr) } +// IsUnretryableKafkaError reports whether a Kafka error is not retryable. +func IsUnretryableKafkaError(err error) bool { + return IsUnretryableFranzError(err) || IsUnretryableSaramaError(err) +} + func (a *saramaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { result := make(map[string]int32, len(topics)) for _, topic := range topics { diff --git a/pkg/sink/kafka/franz_adapter.go b/pkg/sink/kafka/franz_adapter.go index 2f3a7a2e90..38cd35fdca 100644 --- a/pkg/sink/kafka/franz_adapter.go +++ b/pkg/sink/kafka/franz_adapter.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/kafka/franz" + "github.com/twmb/franz-go/pkg/kerr" "go.uber.org/zap" ) @@ -105,6 +106,21 @@ func (franzMetricsCollector) Run(ctx context.Context) { <-ctx.Done() } type franzAdminAdapter struct{ inner *franz.Admin } +// IsUnretryableFranzError reports whether a franz-go error is not retryable. +func IsUnretryableFranzError(err error) bool { + return errors.Is(err, errors.ErrKafkaAuthorizationFailed) || + errors.Is(err, errors.ErrKafkaInvalidConfig) || + errors.Is(err, kerr.TopicAuthorizationFailed) || + errors.Is(err, kerr.ClusterAuthorizationFailed) || + errors.Is(err, kerr.InvalidTopicException) || + errors.Is(err, kerr.InvalidConfig) || + errors.Is(err, kerr.SaslAuthenticationFailed) || + errors.Is(err, kerr.UnsupportedSaslMechanism) || + errors.Is(err, kerr.IllegalSaslState) || + errors.Is(err, kerr.UnsupportedVersion) || + errors.Is(err, kerr.InvalidRequest) +} + func (a *franzAdminAdapter) GetAllBrokers() []Broker { inner := a.inner.GetAllBrokers() brokers := make([]Broker, 0, len(inner)) diff --git a/pkg/sink/kafka/sarama_admin_test.go b/pkg/sink/kafka/sarama_admin_test.go index d0ce07fc1d..eba888f002 100644 --- a/pkg/sink/kafka/sarama_admin_test.go +++ b/pkg/sink/kafka/sarama_admin_test.go @@ -312,7 +312,7 @@ func TestIsAuthorizationFailed(t *testing.T) { } } -func TestIsUnretryableKafkaError(t *testing.T) { +func TestIsUnretryableSaramaError(t *testing.T) { t.Parallel() tests := []struct { @@ -354,7 +354,7 @@ func TestIsUnretryableKafkaError(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - require.Equal(t, test.unretryable, IsUnretryableKafkaError(test.err)) + require.Equal(t, test.unretryable, IsUnretryableSaramaError(test.err)) }) } } diff --git a/pkg/sink/kafka/selector_test.go b/pkg/sink/kafka/selector_test.go index 8552ba647b..774cf0c829 100644 --- a/pkg/sink/kafka/selector_test.go +++ b/pkg/sink/kafka/selector_test.go @@ -16,14 +16,53 @@ package kafka import ( "context" + "io" "testing" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kfake" ) +func TestIsUnretryableFranzError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + unretryable bool + }{ + {name: "unknown topic", err: kerr.UnknownTopicOrPartition}, + {name: "leader unavailable", err: kerr.LeaderNotAvailable}, + {name: "request timeout", err: kerr.RequestTimedOut}, + {name: "network exception", err: kerr.NetworkException}, + {name: "controller changed", err: kerr.NotController}, + {name: "EOF", err: io.EOF}, + {name: "invalid topic", err: kerr.InvalidTopicException, unretryable: true}, + {name: "invalid config", err: kerr.InvalidConfig, unretryable: true}, + {name: "SASL authentication failure", err: kerr.SaslAuthenticationFailed, unretryable: true}, + {name: "unsupported SASL mechanism", err: kerr.UnsupportedSaslMechanism, unretryable: true}, + {name: "illegal SASL state", err: kerr.IllegalSaslState, unretryable: true}, + {name: "unsupported version", err: kerr.UnsupportedVersion, unretryable: true}, + {name: "invalid request", err: kerr.InvalidRequest, unretryable: true}, + { + name: "wrapped invalid topic", + err: errors.WrapError(errors.ErrKafkaAdminAPI, kerr.InvalidTopicException, "describe-topic", "test-topic"), + unretryable: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, test.unretryable, IsUnretryableFranzError(test.err)) + }) + } +} + func TestFactorySelection(t *testing.T) { const topic = "factory-selection" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) From 3c74dd91c64f1bde4140675ea2067820163fdf68 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 2 Sep 2026 14:56:59 +0800 Subject: [PATCH 45/61] metrics: split franz-go request and response rates --- metrics/grafana/ticdc_new_arch.json | 117 ++++++++++++++++-- .../ticdc_new_arch_next_gen.json | 117 ++++++++++++++++-- .../ticdc_new_arch_with_keyspace_name.json | 117 ++++++++++++++++-- 3 files changed, 318 insertions(+), 33 deletions(-) diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 1d85d04a63..b02af79518 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -28683,7 +28683,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Requests and responses per second, grouped by broker and result.", + "description": "Requests per second, grouped by broker and result.", "fieldConfig": { "defaults": { "links": [] @@ -28735,17 +28735,8 @@ "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "request-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "response-{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "B" } ], "thresholds": [], @@ -29117,6 +29108,110 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 62109, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Kafka Sink V2", diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index 9f08cddf9e..1f13b87a4e 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -28683,7 +28683,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Requests and responses per second, grouped by broker and result.", + "description": "Requests per second, grouped by broker and result.", "fieldConfig": { "defaults": { "links": [] @@ -28735,17 +28735,8 @@ "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "B" } ], "thresholds": [], @@ -29117,6 +29108,110 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 62109, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Kafka Sink V2", diff --git a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json index 1b12e08209..2511a575b5 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json +++ b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json @@ -11875,7 +11875,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Requests and responses per second, grouped by broker and result.", + "description": "Requests per second, grouped by broker and result.", "fieldConfig": { "defaults": { "links": [] @@ -11927,17 +11927,8 @@ "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "request-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "response-{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", - "refId": "B" } ], "thresholds": [], @@ -12309,6 +12300,110 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 62109, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Kafka Sink V2", From 526da7041b666c733254ec920de57944396c6eb1 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 2 Sep 2026 17:20:45 +0800 Subject: [PATCH 46/61] kafka: simplify franz-go client implementation Merge the franz-go implementation into the Kafka package, remove adapter and wrapper layers, and pass contexts explicitly through the admin client interface. --- downstreamadapter/sink/kafka/sink.go | 4 +- .../sink/topicmanager/kafka_topic_manager.go | 16 +- .../topicmanager/kafka_topic_manager_test.go | 32 +-- pkg/sink/kafka/admin.go | 26 +- pkg/sink/kafka/admin_client.go | 14 +- pkg/sink/kafka/admin_client_mock.go | 49 ++-- pkg/sink/kafka/factory.go | 2 + pkg/sink/kafka/factory_mock.go | 12 + pkg/sink/kafka/franz/factory.go | 52 ---- pkg/sink/kafka/franz/factory_test.go | 62 ----- pkg/sink/kafka/franz/logutil.go | 65 ----- pkg/sink/kafka/franz/logutil_test.go | 75 ------ pkg/sink/kafka/franz_adapter.go | 238 ------------------ .../kafka/{franz/admin.go => franz_admin.go} | 52 ++-- .../admin_test.go => franz_admin_test.go} | 60 +++-- ...nc_producer.go => franz_async_producer.go} | 37 +-- ...r_test.go => franz_async_producer_test.go} | 22 +- .../{franz/config.go => franz_config.go} | 193 +++++++------- .../config_test.go => franz_config_test.go} | 134 +++++----- pkg/sink/kafka/franz_factory.go | 93 +++++++ .../{franz/gssapi.go => franz_gssapi.go} | 44 ++-- .../gssapi_test.go => franz_gssapi_test.go} | 46 ++-- .../{franz/logger.go => franz_logger.go} | 61 ++--- .../logger_test.go => franz_logger_test.go} | 26 +- .../{franz/metrics.go => franz_metrics.go} | 16 +- .../metrics_hook.go => franz_metrics_hook.go} | 8 +- ...ook_test.go => franz_metrics_hook_test.go} | 10 +- ...ync_producer.go => franz_sync_producer.go} | 58 ++--- ...er_test.go => franz_sync_producer_test.go} | 19 +- pkg/sink/kafka/metrics.go | 13 +- pkg/sink/kafka/metrics_collector.go | 2 +- pkg/sink/kafka/options.go | 30 ++- pkg/sink/kafka/options_test.go | 38 +-- pkg/sink/kafka/sarama_admin_test.go | 28 +-- pkg/sink/kafka/sarama_factory.go | 4 +- .../sarama_oauth2_token_provider_test.go | 16 -- pkg/sink/kafka/selector.go | 9 +- pkg/sink/kafka/selector_test.go | 60 +---- 38 files changed, 586 insertions(+), 1140 deletions(-) delete mode 100644 pkg/sink/kafka/franz/factory.go delete mode 100644 pkg/sink/kafka/franz/factory_test.go delete mode 100644 pkg/sink/kafka/franz/logutil.go delete mode 100644 pkg/sink/kafka/franz/logutil_test.go delete mode 100644 pkg/sink/kafka/franz_adapter.go rename pkg/sink/kafka/{franz/admin.go => franz_admin.go} (85%) rename pkg/sink/kafka/{franz/admin_test.go => franz_admin_test.go} (80%) rename pkg/sink/kafka/{franz/async_producer.go => franz_async_producer.go} (79%) rename pkg/sink/kafka/{franz/async_producer_test.go => franz_async_producer_test.go} (94%) rename pkg/sink/kafka/{franz/config.go => franz_config.go} (53%) rename pkg/sink/kafka/{franz/config_test.go => franz_config_test.go} (65%) create mode 100644 pkg/sink/kafka/franz_factory.go rename pkg/sink/kafka/{franz/gssapi.go => franz_gssapi.go} (72%) rename pkg/sink/kafka/{franz/gssapi_test.go => franz_gssapi_test.go} (54%) rename pkg/sink/kafka/{franz/logger.go => franz_logger.go} (60%) rename pkg/sink/kafka/{franz/logger_test.go => franz_logger_test.go} (76%) rename pkg/sink/kafka/{franz/metrics.go => franz_metrics.go} (92%) rename pkg/sink/kafka/{franz/metrics_hook.go => franz_metrics_hook.go} (96%) rename pkg/sink/kafka/{franz/metrics_hook_test.go => franz_metrics_hook_test.go} (96%) rename pkg/sink/kafka/{franz/sync_producer.go => franz_sync_producer.go} (67%) rename pkg/sink/kafka/{franz/sync_producer_test.go => franz_sync_producer_test.go} (92%) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index eaf028c18a..6e00857de2 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -170,7 +170,7 @@ func newWithComponents( } comp.close() statistics.Close() - kafka.CleanupFactoryMetrics(comp.factory) + comp.factory.CleanupMetrics() }() asyncProducer, err = comp.factory.AsyncProducer(ctx) @@ -576,7 +576,7 @@ func (s *sink) Close() { s.dmlProducer.Close() s.comp.close() s.statistics.Close() - kafka.CleanupFactoryMetrics(s.comp.factory) + s.comp.factory.CleanupMetrics() } func (s *sink) BatchCount() int { diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index 7dbfe31ac5..7f53eeb1b0 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -125,7 +125,7 @@ func (m *kafkaTopicManager) backgroundRefreshMeta(ctx context.Context) { case <-ticker.C: // We ignore the error here, because the error may be caused by the // network problem, and we can try to get the metadata next time. - topicPartitionNums, _ := m.fetchAllTopicsPartitionsNum() + topicPartitionNums, _ := m.fetchAllTopicsPartitionsNum(ctx) for topic, partitionNum := range topicPartitionNums { m.tryUpdatePartitionsAndLogging(topic, partitionNum) } @@ -157,7 +157,7 @@ func (m *kafkaTopicManager) tryUpdatePartitionsAndLogging(topic string, partitio // The error returned by this method could be a transient error that is fixable by the underlying logic. // When handling this error, please be cautious. // If you simply throw the error to the caller, it may impact the robustness of your program. -func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, error) { +func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum(ctx context.Context) (map[string]int32, error) { var topics []string m.topics.Range(func(key, _ any) bool { topics = append(topics, key.(string)) @@ -165,7 +165,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err }) start := time.Now() - numPartitions, err := m.admin.GetTopicsPartitionsNum(topics) + numPartitions, err := m.admin.GetTopicsPartitionsNum(ctx, topics) if err != nil { log.Warn( "kafka topic metadata refresh failed", @@ -202,7 +202,7 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( err := retry.Do(ctx, func() error { // ignoreTopicError is set to false since we just create the topic, // make sure the topic is visible. - meta, err := m.admin.GetTopicsMeta(topics, false) + meta, err := m.admin.GetTopicsMeta(ctx, topics, false) if err != nil { return err } @@ -232,19 +232,19 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( // createTopic creates a topic with the given name // and returns the number of partitions. func (m *kafkaTopicManager) createTopic( - _ context.Context, + ctx context.Context, topicName string, ) (int32, error) { if !m.cfg.AutoCreate { return 0, errors.ErrKafkaInvalidConfig.GenWithStack("`auto-create-topic` is false, and %s not found", topicName) } - if err := m.cfg.ValidateReplicationFactor(m.admin); err != nil { + if err := m.cfg.ValidateReplicationFactor(ctx, m.admin); err != nil { return 0, err } start := time.Now() - err := m.admin.CreateTopic(&kafka.TopicDetail{ + err := m.admin.CreateTopic(ctx, &kafka.TopicDetail{ Name: topicName, NumPartitions: m.cfg.PartitionNum, ReplicationFactor: m.cfg.ReplicationFactor, @@ -274,7 +274,7 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( ctx context.Context, topicName string, ) (int32, error) { // If the topic is not in the cache, try to get its metadata. - topicDetails, err := m.admin.GetTopicsMeta([]string{topicName}, false) + topicDetails, err := m.admin.GetTopicsMeta(ctx, []string{topicName}, false) if err == nil { if numPartition, ok := m.tryStoreTopicMeta(topicName, topicDetails); ok { return numPartition, nil diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index 8a261d8a97..a992d2d6a9 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -39,7 +39,7 @@ func TestCreateTopic(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{kafkaTopicManagerTestTopic}, false).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{kafkaTopicManagerTestTopic}, false).Return( map[string]kafka.TopicDetail{ kafkaTopicManagerTestTopic: {Name: kafkaTopicManagerTestTopic, NumPartitions: 2}, }, nil) @@ -64,8 +64,8 @@ func TestCreateTopic(t *testing.T) { var createdTopic *kafka.TopicDetail postCreateDescribeCount := 0 var manager *kafkaTopicManager - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).DoAndReturn( - func([]string, bool) (map[string]kafka.TopicDetail, error) { + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).DoAndReturn( + func(context.Context, []string, bool) (map[string]kafka.TopicDetail, error) { if createdTopic == nil { return nil, errors.WrapError(errors.ErrKafkaAdminAPI, sarama.ErrUnknownTopicOrPartition, "describe-topic", "new-topic") } @@ -79,8 +79,8 @@ func TestCreateTopic(t *testing.T) { createdTopic.Name: {Name: createdTopic.Name, NumPartitions: createdTopic.NumPartitions}, }, nil }).Times(3) - adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail *kafka.TopicDetail) error { + adminClient.EXPECT().CreateTopic(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, detail *kafka.TopicDetail) error { copy := *detail createdTopic = © return nil @@ -117,7 +117,7 @@ func TestCreateTopic(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) manager := newKafkaTopicManager( "new-topic", changefeedID, @@ -140,10 +140,10 @@ func TestCreateTopic(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) var createdTopic *kafka.TopicDetail - adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail *kafka.TopicDetail) error { + adminClient.EXPECT().CreateTopic(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, detail *kafka.TopicDetail) error { copy := *detail createdTopic = © return errors.ErrKafkaAdminAPI.GenWithStackByArgs("create-topic", detail.Name) @@ -171,8 +171,8 @@ func TestCreateTopicValidatesReplicationFactor(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) - adminClient.EXPECT().GetBrokerConfig(kafka.MinInsyncReplicasConfigName).Return("2", true, nil) + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), kafka.MinInsyncReplicasConfigName).Return("2", true, nil) manager := newKafkaTopicManager( "new-topic", common.NewChangefeedID4Test("test", "test"), @@ -205,7 +205,7 @@ func TestWaitUntilTopicVisibleUnretryableError(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"invalid-topic"}, false).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"invalid-topic"}, false).Return( nil, errors.WrapError(errors.ErrKafkaAdminAPI, test.cause, "describe-topic", "invalid-topic"), ).Times(1) @@ -229,7 +229,7 @@ func TestGetTopicManagerStartsBackgroundRefreshAfterTopicReady(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"existing-topic"}, false).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"existing-topic"}, false).Return( map[string]kafka.TopicDetail{ "existing-topic": {Name: "existing-topic", NumPartitions: 2}, }, nil) @@ -252,7 +252,7 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"default-topic"}, false).Return( nil, errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs("describe-topic", "default-topic")) manager := newKafkaTopicManager( "default-topic", @@ -279,8 +279,8 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) - adminClient.EXPECT().CreateTopic(&kafka.TopicDetail{ + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"default-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().CreateTopic(gomock.Any(), &kafka.TopicDetail{ Name: "default-topic", NumPartitions: 2, ReplicationFactor: 1, diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index 651dd8330f..c9d87098b2 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "strconv" "strings" @@ -21,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kerr" "go.uber.org/zap" ) @@ -47,7 +49,7 @@ type saramaClusterAdmin interface { Close() error } -func (a *saramaAdminClient) GetAllBrokers() []Broker { +func (a *saramaAdminClient) GetAllBrokers(_ context.Context) []Broker { brokers := a.client.Brokers() result := make([]Broker, 0, len(brokers)) for _, broker := range brokers { @@ -58,7 +60,7 @@ func (a *saramaAdminClient) GetAllBrokers() []Broker { return result } -func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, bool, error) { +func (a *saramaAdminClient) GetBrokerConfig(_ context.Context, configName string) (string, bool, error) { _, controller, err := a.admin.DescribeCluster() if err != nil { if IsAuthorizationFailed(err) { @@ -90,7 +92,7 @@ func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, bool, er return "", false, nil } -func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) (string, bool, error) { +func (a *saramaAdminClient) GetTopicConfig(_ context.Context, topicName string, configName string) (string, bool, error) { configEntries, err := a.admin.DescribeConfig(sarama.ConfigResource{ Type: sarama.TopicResource, Name: topicName, @@ -114,7 +116,7 @@ func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) return "", false, nil } -func (a *saramaAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { +func (a *saramaAdminClient) GetTopicsMeta(_ context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { result := make(map[string]TopicDetail, len(topics)) metaList, err := a.admin.DescribeTopics(topics) @@ -178,10 +180,20 @@ func IsUnretryableSaramaError(err error) bool { // IsUnretryableKafkaError reports whether a Kafka error is not retryable. func IsUnretryableKafkaError(err error) bool { - return IsUnretryableFranzError(err) || IsUnretryableSaramaError(err) + if errors.Is(err, errors.ErrKafkaAuthorizationFailed) || + errors.Is(err, errors.ErrKafkaInvalidConfig) { + return true + } + + var kafkaErr *kerr.Error + if errors.As(err, &kafkaErr) { + return !kafkaErr.Retriable + } + + return IsUnretryableSaramaError(err) } -func (a *saramaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { +func (a *saramaAdminClient) GetTopicsPartitionsNum(_ context.Context, topics []string) (map[string]int32, error) { result := make(map[string]int32, len(topics)) for _, topic := range topics { partition, err := a.client.Partitions(topic) @@ -197,7 +209,7 @@ func (a *saramaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string] return result, nil } -func (a *saramaAdminClient) CreateTopic(detail *TopicDetail) error { +func (a *saramaAdminClient) CreateTopic(_ context.Context, detail *TopicDetail) error { request := &sarama.TopicDetail{ NumPartitions: detail.NumPartitions, ReplicationFactor: detail.ReplicationFactor, diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index cea348c2a2..bb7680e73c 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -13,6 +13,8 @@ package kafka +import "context" + // TopicDetail represent a topic's detail information. type TopicDetail struct { Name string @@ -29,23 +31,23 @@ type Broker struct { // which supports managing and inspecting topics, brokers, configurations and ACLs. type AdminClient interface { // GetAllBrokers return all brokers among the cluster - GetAllBrokers() []Broker + GetAllBrokers(ctx context.Context) []Broker // GetBrokerConfig returns the broker-level configuration and whether it exists. - GetBrokerConfig(configName string) (value string, found bool, err error) + GetBrokerConfig(ctx context.Context, configName string) (value string, found bool, err error) // GetTopicConfig returns the topic-level configuration and whether it exists. - GetTopicConfig(topicName string, configName string) (value string, found bool, err error) + GetTopicConfig(ctx context.Context, topicName string, configName string) (value string, found bool, err error) // GetTopicsMeta return all target topics' metadata // if `ignoreTopicError` is true, ignore the topic error and return the metadata of valid topics - GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) + GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) // GetTopicsPartitionsNum return the number of partitions of each topic. - GetTopicsPartitionsNum(topics []string) (map[string]int32, error) + GetTopicsPartitionsNum(ctx context.Context, topics []string) (map[string]int32, error) // CreateTopic creates a new topic. - CreateTopic(detail *TopicDetail) error + CreateTopic(ctx context.Context, detail *TopicDetail) error // Close shuts down the admin client. Close() diff --git a/pkg/sink/kafka/admin_client_mock.go b/pkg/sink/kafka/admin_client_mock.go index 2d2778ea3b..50e80191c9 100644 --- a/pkg/sink/kafka/admin_client_mock.go +++ b/pkg/sink/kafka/admin_client_mock.go @@ -5,6 +5,7 @@ package kafka import ( + context "context" reflect "reflect" gomock "github.com/golang/mock/gomock" @@ -46,37 +47,37 @@ func (mr *MockAdminClientMockRecorder) Close() *gomock.Call { } // CreateTopic mocks base method. -func (m *MockAdminClient) CreateTopic(detail *TopicDetail) error { +func (m *MockAdminClient) CreateTopic(ctx context.Context, detail *TopicDetail) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTopic", detail) + ret := m.ctrl.Call(m, "CreateTopic", ctx, detail) ret0, _ := ret[0].(error) return ret0 } // CreateTopic indicates an expected call of CreateTopic. -func (mr *MockAdminClientMockRecorder) CreateTopic(detail interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) CreateTopic(ctx, detail interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockAdminClient)(nil).CreateTopic), detail) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockAdminClient)(nil).CreateTopic), ctx, detail) } // GetAllBrokers mocks base method. -func (m *MockAdminClient) GetAllBrokers() []Broker { +func (m *MockAdminClient) GetAllBrokers(ctx context.Context) []Broker { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllBrokers") + ret := m.ctrl.Call(m, "GetAllBrokers", ctx) ret0, _ := ret[0].([]Broker) return ret0 } // GetAllBrokers indicates an expected call of GetAllBrokers. -func (mr *MockAdminClientMockRecorder) GetAllBrokers() *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetAllBrokers(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrokers", reflect.TypeOf((*MockAdminClient)(nil).GetAllBrokers)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrokers", reflect.TypeOf((*MockAdminClient)(nil).GetAllBrokers), ctx) } // GetBrokerConfig mocks base method. -func (m *MockAdminClient) GetBrokerConfig(configName string) (string, bool, error) { +func (m *MockAdminClient) GetBrokerConfig(ctx context.Context, configName string) (string, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBrokerConfig", configName) + ret := m.ctrl.Call(m, "GetBrokerConfig", ctx, configName) ret0, _ := ret[0].(string) ret1, _ := ret[1].(bool) ret2, _ := ret[2].(error) @@ -84,15 +85,15 @@ func (m *MockAdminClient) GetBrokerConfig(configName string) (string, bool, erro } // GetBrokerConfig indicates an expected call of GetBrokerConfig. -func (mr *MockAdminClientMockRecorder) GetBrokerConfig(configName interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetBrokerConfig(ctx, configName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockAdminClient)(nil).GetBrokerConfig), configName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockAdminClient)(nil).GetBrokerConfig), ctx, configName) } // GetTopicConfig mocks base method. -func (m *MockAdminClient) GetTopicConfig(topicName, configName string) (string, bool, error) { +func (m *MockAdminClient) GetTopicConfig(ctx context.Context, topicName, configName string) (string, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicConfig", topicName, configName) + ret := m.ctrl.Call(m, "GetTopicConfig", ctx, topicName, configName) ret0, _ := ret[0].(string) ret1, _ := ret[1].(bool) ret2, _ := ret[2].(error) @@ -100,37 +101,37 @@ func (m *MockAdminClient) GetTopicConfig(topicName, configName string) (string, } // GetTopicConfig indicates an expected call of GetTopicConfig. -func (mr *MockAdminClientMockRecorder) GetTopicConfig(topicName, configName interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetTopicConfig(ctx, topicName, configName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockAdminClient)(nil).GetTopicConfig), topicName, configName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockAdminClient)(nil).GetTopicConfig), ctx, topicName, configName) } // GetTopicsMeta mocks base method. -func (m *MockAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { +func (m *MockAdminClient) GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsMeta", topics, ignoreTopicError) + ret := m.ctrl.Call(m, "GetTopicsMeta", ctx, topics, ignoreTopicError) ret0, _ := ret[0].(map[string]TopicDetail) ret1, _ := ret[1].(error) return ret0, ret1 } // GetTopicsMeta indicates an expected call of GetTopicsMeta. -func (mr *MockAdminClientMockRecorder) GetTopicsMeta(topics, ignoreTopicError interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetTopicsMeta(ctx, topics, ignoreTopicError interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsMeta), topics, ignoreTopicError) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsMeta), ctx, topics, ignoreTopicError) } // GetTopicsPartitionsNum mocks base method. -func (m *MockAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { +func (m *MockAdminClient) GetTopicsPartitionsNum(ctx context.Context, topics []string) (map[string]int32, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", topics) + ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", ctx, topics) ret0, _ := ret[0].(map[string]int32) ret1, _ := ret[1].(error) return ret0, ret1 } // GetTopicsPartitionsNum indicates an expected call of GetTopicsPartitionsNum. -func (mr *MockAdminClientMockRecorder) GetTopicsPartitionsNum(topics interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetTopicsPartitionsNum(ctx, topics interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsPartitionsNum), topics) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsPartitionsNum), ctx, topics) } diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index c19089de4c..83d5ec8cb7 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -29,6 +29,8 @@ type Factory interface { AsyncProducer(ctx context.Context) (AsyncProducer, error) // MetricsCollector returns the kafka metrics collector MetricsCollector(adminClient AdminClient) MetricsCollector + // CleanupMetrics removes metrics owned directly by the factory. + CleanupMetrics() } // SyncProducer is the kafka sync producer diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index ecc8fe131c..baf1ca289e 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -65,6 +65,18 @@ func (mr *MockFactoryMockRecorder) AsyncProducer(ctx interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), ctx) } +// CleanupMetrics mocks base method. +func (m *MockFactory) CleanupMetrics() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "CleanupMetrics") +} + +// CleanupMetrics indicates an expected call of CleanupMetrics. +func (mr *MockFactoryMockRecorder) CleanupMetrics() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanupMetrics", reflect.TypeOf((*MockFactory)(nil).CleanupMetrics)) +} + // MetricsCollector mocks base method. func (m *MockFactory) MetricsCollector(adminClient AdminClient) MetricsCollector { m.ctrl.T.Helper() diff --git a/pkg/sink/kafka/franz/factory.go b/pkg/sink/kafka/franz/factory.go deleted file mode 100644 index 9e77eda44d..0000000000 --- a/pkg/sink/kafka/franz/factory.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package franz - -import ( - "context" - - "github.com/pingcap/ticdc/pkg/common" -) - -type Factory struct { - changefeedID common.ChangeFeedID - config Config -} - -func NewFactory(config Config, changefeedID common.ChangeFeedID) *Factory { - return &Factory{changefeedID: changefeedID, config: config} -} - -func (f *Factory) Admin(ctx context.Context) (*Admin, error) { - return NewAdmin(ctx, f.changefeedID, f.config) -} - -func (f *Factory) SyncProducer(ctx context.Context) (*SyncProducer, error) { - producer, err := NewSyncProducer(ctx, f.changefeedID, f.config, newMetricsHook(f.changefeedID)) - if err != nil { - CleanupMetrics(f.changefeedID) - } - return producer, err -} - -func (f *Factory) AsyncProducer(ctx context.Context) (*AsyncProducer, error) { - producer, err := NewAsyncProducer(ctx, f.changefeedID, f.config, newMetricsHook(f.changefeedID)) - if err != nil { - CleanupMetrics(f.changefeedID) - } - return producer, err -} - -func (f *Factory) CleanupMetrics() { CleanupMetrics(f.changefeedID) } diff --git a/pkg/sink/kafka/franz/factory_test.go b/pkg/sink/kafka/franz/factory_test.go deleted file mode 100644 index b5ff03aceb..0000000000 --- a/pkg/sink/kafka/franz/factory_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package franz - -import ( - "context" - "testing" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kfake" -) - -func TestFactoryCreatesAllClients(t *testing.T) { - cluster := kfake.MustCluster(kfake.NumBrokers(1)) - defer cluster.Close() - - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory") - factory := NewFactory(testConfig(cluster.ListenAddrs()), changefeedID) - - admin, err := factory.Admin(context.Background()) - require.NoError(t, err) - admin.Close() - - syncProducer, err := factory.SyncProducer(context.Background()) - require.NoError(t, err) - syncProducer.Close() - - asyncProducer, err := factory.AsyncProducer(context.Background()) - require.NoError(t, err) - asyncProducer.Close() - - factory.CleanupMetrics() -} - -func TestFactoryCleansMetricsAfterProducerConstructionFailure(t *testing.T) { - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory-error") - config := testConfig([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = maxProducerBatchBytes + 1 - factory := NewFactory(config, changefeedID) - - _, err := factory.SyncProducer(context.Background()) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) - require.False(t, recordsPerBatch.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name())) - - _, err = factory.AsyncProducer(context.Background()) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) - require.False(t, recordsPerBatch.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name())) -} diff --git a/pkg/sink/kafka/franz/logutil.go b/pkg/sink/kafka/franz/logutil.go deleted file mode 100644 index c7aa49abcd..0000000000 --- a/pkg/sink/kafka/franz/logutil.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package franz - -import ( - "encoding/json" - "strconv" - "strings" - - "github.com/pingcap/ticdc/pkg/sink/codec/common" -) - -func buildEventLogContext(keyspace, changefeed string, info *common.MessageLogInfo) string { - var text strings.Builder - text.WriteString("keyspace=" + keyspace + ", changefeed=" + changefeed + ", eventType=" + eventType(info)) - if info == nil { - return text.String() - } - if rows, err := json.Marshal(info.Rows); len(info.Rows) > 0 && err == nil { - text.WriteString(", dmlInfo=" + string(rows)) - } - if info.DDL != nil { - if info.DDL.Query != "" { - text.WriteString(", ddlQuery=" + strconv.Quote(info.DDL.Query)) - } - if info.DDL.StartTs != 0 { - text.WriteString(", ddlStartTs=" + strconv.FormatUint(info.DDL.StartTs, 10)) - } - if info.DDL.CommitTs != 0 { - text.WriteString(", ddlCommitTs=" + strconv.FormatUint(info.DDL.CommitTs, 10)) - } - } - if info.Checkpoint != nil && info.Checkpoint.CommitTs != 0 { - text.WriteString(", checkpointTs=" + strconv.FormatUint(info.Checkpoint.CommitTs, 10)) - } - return text.String() -} - -func eventType(info *common.MessageLogInfo) string { - if info == nil { - return "unknown" - } - if info.DDL != nil { - return "ddl" - } - if info.Checkpoint != nil { - return "checkpoint" - } - if len(info.Rows) > 0 { - return "dml" - } - return "unknown" -} diff --git a/pkg/sink/kafka/franz/logutil_test.go b/pkg/sink/kafka/franz/logutil_test.go deleted file mode 100644 index cdf831f435..0000000000 --- a/pkg/sink/kafka/franz/logutil_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package franz - -import ( - "testing" - - "github.com/pingcap/ticdc/pkg/sink/codec/common" - "github.com/stretchr/testify/require" -) - -func TestEventType(t *testing.T) { - require.Equal(t, "unknown", eventType(nil)) - require.Equal(t, "ddl", eventType(&common.MessageLogInfo{DDL: &common.DDLLogInfo{}})) - require.Equal(t, "checkpoint", eventType(&common.MessageLogInfo{ - Checkpoint: &common.CheckpointLogInfo{}, - })) - require.Equal(t, "dml", eventType(&common.MessageLogInfo{Rows: []common.RowLogInfo{{}}})) - require.Equal(t, "unknown", eventType(&common.MessageLogInfo{})) -} - -func TestBuildEventLogContext(t *testing.T) { - info := &common.MessageLogInfo{ - Rows: []common.RowLogInfo{ - { - Type: "insert", - Database: "database", - Table: "table", - CommitTs: 1, - }, - }, - } - - context := buildEventLogContext("keyspace", "changefeed", info) - require.Contains(t, context, "keyspace=keyspace") - require.Contains(t, context, "changefeed=changefeed") - require.Contains(t, context, "eventType=dml") - require.Contains(t, context, `dmlInfo=[{"Type":"insert"`) - require.Contains(t, context, `"Database":"database"`) - require.Contains(t, context, `"Table":"table"`) - require.Contains(t, context, `"CommitTs":1`) -} - -func TestBuildEventLogContextForBlockEvents(t *testing.T) { - ddlContext := buildEventLogContext("keyspace", "changefeed", &common.MessageLogInfo{ - DDL: &common.DDLLogInfo{ - Query: "CREATE TABLE t(id INT PRIMARY KEY)", - StartTs: 1, - CommitTs: 2, - }, - }) - - require.Contains(t, ddlContext, "eventType=ddl") - require.Contains(t, ddlContext, `ddlQuery="CREATE TABLE t(id INT PRIMARY KEY)"`) - require.Contains(t, ddlContext, "ddlStartTs=1") - require.Contains(t, ddlContext, "ddlCommitTs=2") - - checkpointContext := buildEventLogContext("keyspace", "changefeed", &common.MessageLogInfo{ - Checkpoint: &common.CheckpointLogInfo{CommitTs: 3}, - }) - require.Contains(t, checkpointContext, "eventType=checkpoint") - require.Contains(t, checkpointContext, "checkpointTs=3") -} diff --git a/pkg/sink/kafka/franz_adapter.go b/pkg/sink/kafka/franz_adapter.go deleted file mode 100644 index 38cd35fdca..0000000000 --- a/pkg/sink/kafka/franz_adapter.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "crypto/tls" - "strings" - - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/kafka/franz" - "github.com/twmb/franz-go/pkg/kerr" - "go.uber.org/zap" -) - -type franzFactoryAdapter struct { - inner *franz.Factory -} - -// NewFranzFactory constructs a franz-go Kafka client factory. -func NewFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { - config, err := newFranzConfig(o) - if err != nil { - return nil, err - } - - innerAdmin, err := franz.NewAdmin(ctx, changefeedID, config) - if err != nil { - return nil, err - } - - admin := &franzAdminAdapter{inner: innerAdmin} - defer admin.Close() - - if err := adjustOptions(changefeedID, admin, o, o.Topic); err != nil { - return nil, err - } - - config, err = newFranzConfig(o) - if err != nil { - return nil, err - } - - compression := strings.ToLower(strings.TrimSpace(o.Compression)) - if compression == "" { - compression = "none" - } - - log.Info("kafka sink configuration resolved", - zap.String("namespace", changefeedID.Keyspace()), - zap.String("changefeed", changefeedID.Name()), - zap.String("client", KafkaClientFranz), - zap.String("topic", o.Topic), - zap.Int32("partitionNum", o.PartitionNum), - zap.Int("maxMessageBytes", o.MaxMessageBytes), - zap.Int("maxBatchedBytes", o.MaxBatchedBytes), - zap.String("compression", compression), - zap.Int16("requiredAcks", int16(o.RequiredAcks)), - zap.Int("maxRetry", o.MaxRetry), - zap.Duration("dialTimeout", o.DialTimeout), - zap.Duration("readTimeout", o.ReadTimeout), - zap.Duration("writeTimeout", o.WriteTimeout)) - - return &franzFactoryAdapter{inner: franz.NewFactory(config, changefeedID)}, nil -} - -func (f *franzFactoryAdapter) AdminClient(ctx context.Context) (AdminClient, error) { - admin, err := f.inner.Admin(ctx) - if err != nil { - return nil, err - } - return &franzAdminAdapter{inner: admin}, nil -} - -func (f *franzFactoryAdapter) SyncProducer(ctx context.Context) (SyncProducer, error) { - return f.inner.SyncProducer(ctx) -} - -func (f *franzFactoryAdapter) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - return f.inner.AsyncProducer(ctx) -} - -func (f *franzFactoryAdapter) MetricsCollector(AdminClient) MetricsCollector { - return franzMetricsCollector{} -} - -func (f *franzFactoryAdapter) CleanupMetrics() { f.inner.CleanupMetrics() } - -type franzMetricsCollector struct{} - -func (franzMetricsCollector) Run(ctx context.Context) { <-ctx.Done() } - -type franzAdminAdapter struct{ inner *franz.Admin } - -// IsUnretryableFranzError reports whether a franz-go error is not retryable. -func IsUnretryableFranzError(err error) bool { - return errors.Is(err, errors.ErrKafkaAuthorizationFailed) || - errors.Is(err, errors.ErrKafkaInvalidConfig) || - errors.Is(err, kerr.TopicAuthorizationFailed) || - errors.Is(err, kerr.ClusterAuthorizationFailed) || - errors.Is(err, kerr.InvalidTopicException) || - errors.Is(err, kerr.InvalidConfig) || - errors.Is(err, kerr.SaslAuthenticationFailed) || - errors.Is(err, kerr.UnsupportedSaslMechanism) || - errors.Is(err, kerr.IllegalSaslState) || - errors.Is(err, kerr.UnsupportedVersion) || - errors.Is(err, kerr.InvalidRequest) -} - -func (a *franzAdminAdapter) GetAllBrokers() []Broker { - inner := a.inner.GetAllBrokers() - brokers := make([]Broker, 0, len(inner)) - for _, broker := range inner { - brokers = append(brokers, Broker{ID: broker.ID}) - } - return brokers -} - -func (a *franzAdminAdapter) GetBrokerConfig(name string) (string, bool, error) { - return a.inner.GetBrokerConfig(name) -} - -func (a *franzAdminAdapter) GetTopicConfig(topic, name string) (string, bool, error) { - return a.inner.GetTopicConfig(topic, name) -} - -func (a *franzAdminAdapter) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { - inner, err := a.inner.GetTopicsMeta(topics, ignoreTopicError) - if err != nil { - return nil, err - } - - details := make(map[string]TopicDetail, len(inner)) - for topic, detail := range inner { - details[topic] = TopicDetail{ - Name: detail.Name, - NumPartitions: detail.NumPartitions, - ReplicationFactor: detail.ReplicationFactor, - } - } - - return details, nil -} - -func (a *franzAdminAdapter) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - return a.inner.GetTopicsPartitionsNum(topics) -} - -func (a *franzAdminAdapter) CreateTopic(detail *TopicDetail) error { - return a.inner.CreateTopic(&franz.TopicDetail{ - Name: detail.Name, - NumPartitions: detail.NumPartitions, - ReplicationFactor: detail.ReplicationFactor, - }) -} - -func (a *franzAdminAdapter) Close() { a.inner.Close() } - -func newFranzConfig(o *options) (franz.Config, error) { - config := franz.Config{ - BrokerEndpoints: append([]string(nil), o.BrokerEndpoints...), - ClientID: o.ClientID, - MaxMessageBytes: o.MaxMessageBytes, - MaxRetry: o.MaxRetry, - Compression: o.Compression, - RequiredAcks: int16(o.RequiredAcks), - DialTimeout: o.DialTimeout, - ReadTimeout: o.ReadTimeout, - WriteTimeout: o.WriteTimeout, - } - - if o.EnableTLS { - config.TLSConfig = &tls.Config{ - MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", "http/1.1"}, - } - - if o.Credential != nil && o.Credential.IsTLSEnabled() { - tlsConfig, err := o.Credential.ToTLSConfig() - if err != nil { - return franz.Config{}, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - - config.TLSConfig = tlsConfig - } - - config.TLSConfig.InsecureSkipVerify = o.InsecureSkipVerify - } - - if o.sasl != nil && o.sasl.mechanism != "" { - config.SASL = &franz.SASLConfig{ - Mechanism: string(o.sasl.mechanism), - User: o.sasl.user, - Password: o.sasl.password, - GSSAPI: franz.GSSAPIConfig{ - AuthType: int(o.sasl.gssapi.authType), - KeyTabPath: o.sasl.gssapi.keyTabPath, - KerberosConfigPath: o.sasl.gssapi.kerberosConfigPath, - ServiceName: o.sasl.gssapi.serviceName, - Username: o.sasl.gssapi.username, - Password: o.sasl.gssapi.password, - Realm: o.sasl.gssapi.realm, - DisablePAFXFAST: o.sasl.gssapi.disablePAFXFAST, - }, - OAuth2: franz.OAuth2Config{ - ClientID: o.sasl.oauth2.clientID, - ClientSecret: o.sasl.oauth2.clientSecret, - TokenURL: o.sasl.oauth2.tokenURL, - Scopes: append([]string(nil), o.sasl.oauth2.scopes...), - GrantType: o.sasl.oauth2.grantType, - Audience: o.sasl.oauth2.audience, - }, - } - if o.sasl.oauth2.caPath != "" { - httpClient, err := oauthHTTPClient(o.sasl.oauth2.caPath) - if err != nil { - return franz.Config{}, err - } - config.SASL.OAuth2.HTTPClient = httpClient - } - } - - return config, nil -} diff --git a/pkg/sink/kafka/franz/admin.go b/pkg/sink/kafka/franz_admin.go similarity index 85% rename from pkg/sink/kafka/franz/admin.go rename to pkg/sink/kafka/franz_admin.go index bdf54da706..d648ace2d4 100644 --- a/pkg/sink/kafka/franz/admin.go +++ b/pkg/sink/kafka/franz_admin.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -26,28 +26,19 @@ import ( "github.com/twmb/franz-go/pkg/kgo" ) -type Broker struct{ ID int32 } - -type TopicDetail struct { - Name string - NumPartitions int32 - ReplicationFactor int16 -} - -type Admin struct { +type admin struct { changefeed common.ChangeFeedID - client *kgo.Client admin *kadm.Client timeout time.Duration } -func NewAdmin( +func newAdmin( ctx context.Context, changefeedID common.ChangeFeedID, - cfg Config, -) (*Admin, error) { - opts, err := newClientOptions(ctx, changefeedID, "admin", cfg, nil) + o *options, +) (*admin, error) { + opts, err := newClientOptions(ctx, changefeedID, "admin", o, nil) if err != nil { return nil, err } @@ -60,16 +51,15 @@ func NewAdmin( return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } - return &Admin{ + return &admin{ changefeed: changefeedID, - client: client, admin: kadm.NewClient(client), - timeout: cfg.requestTimeout(), + timeout: requestTimeout(o), }, nil } -func (a *Admin) GetAllBrokers() []Broker { - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) +func (a *admin) GetAllBrokers(ctx context.Context) []Broker { + ctx, cancel := context.WithTimeout(ctx, a.timeout) defer cancel() meta, err := a.admin.BrokerMetadata(ctx) @@ -85,8 +75,8 @@ func (a *Admin) GetAllBrokers() []Broker { return brokers } -func (a *Admin) GetBrokerConfig(configName string) (string, bool, error) { - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) +func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, bool, error) { + ctx, cancel := context.WithTimeout(ctx, a.timeout) defer cancel() meta, err := a.admin.BrokerMetadata(ctx) @@ -138,8 +128,8 @@ func (a *Admin) GetBrokerConfig(configName string) (string, bool, error) { return "", false, nil } -func (a *Admin) GetTopicConfig(topicName string, configName string) (string, bool, error) { - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) +func (a *admin) GetTopicConfig(ctx context.Context, topicName string, configName string) (string, bool, error) { + ctx, cancel := context.WithTimeout(ctx, a.timeout) defer cancel() configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) @@ -177,12 +167,12 @@ func (a *Admin) GetTopicConfig(topicName string, configName string) (string, boo return "", false, nil } -func (a *Admin) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { +func (a *admin) GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { if len(topics) == 0 { return make(map[string]TopicDetail), nil } - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) + ctx, cancel := context.WithTimeout(ctx, a.timeout) defer cancel() meta, err := a.admin.Metadata(ctx, topics...) @@ -238,8 +228,8 @@ func isAuthorizationFailed(err error) bool { errors.Is(err, kerr.ClusterAuthorizationFailed) } -func (a *Admin) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { - details, err := a.GetTopicsMeta(topics, false) +func (a *admin) GetTopicsPartitionsNum(ctx context.Context, topics []string) (map[string]int32, error) { + details, err := a.GetTopicsMeta(ctx, topics, false) if err != nil { return nil, err } @@ -252,8 +242,8 @@ func (a *Admin) GetTopicsPartitionsNum(topics []string) (map[string]int32, error return partitions, nil } -func (a *Admin) CreateTopic(detail *TopicDetail) error { - ctx, cancel := context.WithTimeout(a.client.Context(), a.timeout) +func (a *admin) CreateTopic(ctx context.Context, detail *TopicDetail) error { + ctx, cancel := context.WithTimeout(ctx, a.timeout) defer cancel() responses, err := a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) @@ -289,6 +279,6 @@ func (a *Admin) CreateTopic(detail *TopicDetail) error { return errors.WrapError(errors.ErrKafkaAdminAPI, resp.Err, "create-topic", detail.Name) } -func (a *Admin) Close() { +func (a *admin) Close() { a.admin.Close() } diff --git a/pkg/sink/kafka/franz/admin_test.go b/pkg/sink/kafka/franz_admin_test.go similarity index 80% rename from pkg/sink/kafka/franz/admin_test.go rename to pkg/sink/kafka/franz_admin_test.go index 54450a2930..4f3a512308 100644 --- a/pkg/sink/kafka/franz/admin_test.go +++ b/pkg/sink/kafka/franz_admin_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -27,7 +27,7 @@ import ( "github.com/twmb/franz-go/pkg/kmsg" ) -func TestTopicDetailsFromMetadata(t *testing.T) { +func TestFranzTopicDetailsFromMetadata(t *testing.T) { t.Parallel() const topic = "topic" @@ -116,7 +116,7 @@ func TestTopicDetailsFromMetadata(t *testing.T) { } } -func TestIsAuthorizationFailed(t *testing.T) { +func TestFranzIsAuthorizationFailed(t *testing.T) { t.Parallel() tests := []struct { @@ -141,49 +141,66 @@ func TestIsAuthorizationFailed(t *testing.T) { } } +func TestAdminHonorsCallContext(t *testing.T) { + admin, err := newAdmin( + t.Context(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "context"), + testOptions([]string{"127.0.0.1:1"}), + ) + require.NoError(t, err) + t.Cleanup(admin.Close) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err = admin.GetTopicsMeta(ctx, []string{"topic"}, false) + require.ErrorIs(t, err, context.Canceled) +} + func TestAdminOperations(t *testing.T) { const existingTopic = "existing-topic" + ctx := t.Context() cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, existingTopic)) defer cluster.Close() - admin, err := NewAdmin( - context.Background(), + admin, err := newAdmin( + ctx, common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), - testConfig(cluster.ListenAddrs()), + testOptions(cluster.ListenAddrs()), ) require.NoError(t, err) defer admin.Close() - require.Len(t, admin.GetAllBrokers(), 1) + require.Len(t, admin.GetAllBrokers(ctx), 1) - value, found, err := admin.GetBrokerConfig("message.max.bytes") + value, found, err := admin.GetBrokerConfig(ctx, "message.max.bytes") require.NoError(t, err) require.True(t, found) require.Equal(t, "1048588", value) - _, found, err = admin.GetBrokerConfig("missing") + _, found, err = admin.GetBrokerConfig(ctx, "missing") require.NoError(t, err) require.False(t, found) - value, found, err = admin.GetTopicConfig(existingTopic, "max.message.bytes") + value, found, err = admin.GetTopicConfig(ctx, existingTopic, "max.message.bytes") require.NoError(t, err) require.True(t, found) require.Equal(t, "1048588", value) - _, found, err = admin.GetTopicConfig(existingTopic, "missing") + _, found, err = admin.GetTopicConfig(ctx, existingTopic, "missing") require.NoError(t, err) require.False(t, found) - partitions, err := admin.GetTopicsPartitionsNum([]string{existingTopic}) + partitions, err := admin.GetTopicsPartitionsNum(ctx, []string{existingTopic}) require.NoError(t, err) require.Equal(t, map[string]int32{existingTopic: 3}, partitions) const topic = "test-topic" - topics, err := admin.GetTopicsMeta([]string{topic}, true) + topics, err := admin.GetTopicsMeta(ctx, []string{topic}, true) require.NoError(t, err) require.Empty(t, topics) - err = admin.CreateTopic(&TopicDetail{ + err = admin.CreateTopic(ctx, &TopicDetail{ Name: topic, NumPartitions: 3, ReplicationFactor: 1, @@ -191,21 +208,22 @@ func TestAdminOperations(t *testing.T) { require.NoError(t, err) require.Eventually(t, func() bool { - topics, err = admin.GetTopicsMeta([]string{topic}, false) + topics, err = admin.GetTopicsMeta(ctx, []string{topic}, false) return err == nil && topics[topic].NumPartitions == 3 }, time.Second, 20*time.Millisecond) - require.NoError(t, admin.CreateTopic(&TopicDetail{Name: topic, NumPartitions: 3, ReplicationFactor: 1})) + require.NoError(t, admin.CreateTopic(ctx, &TopicDetail{Name: topic, NumPartitions: 3, ReplicationFactor: 1})) } func TestCreateTopicErrors(t *testing.T) { + ctx := t.Context() cluster := kfake.MustCluster(kfake.NumBrokers(1)) defer cluster.Close() - admin, err := NewAdmin( - context.Background(), + admin, err := newAdmin( + ctx, common.NewChangefeedID4Test(common.DefaultKeyspaceName, "create-errors"), - testConfig(cluster.ListenAddrs()), + testOptions(cluster.ListenAddrs()), ) require.NoError(t, err) defer admin.Close() @@ -215,7 +233,7 @@ func TestCreateTopicErrors(t *testing.T) { cluster.ControlKey(int16(kmsg.CreateTopics), func(req kmsg.Request) (kmsg.Response, error, bool) { return req.ResponseKind(), nil, true }) - require.ErrorIs(t, admin.CreateTopic(detail), errors.ErrKafkaAdminAPI) + require.ErrorIs(t, admin.CreateTopic(ctx, detail), errors.ErrKafkaAdminAPI) for _, test := range []struct { name string @@ -248,7 +266,7 @@ func TestCreateTopicErrors(t *testing.T) { return response, nil, true }) - require.ErrorIs(t, admin.CreateTopic(detail), test.expected) + require.ErrorIs(t, admin.CreateTopic(ctx, detail), test.expected) }) } } diff --git a/pkg/sink/kafka/franz/async_producer.go b/pkg/sink/kafka/franz_async_producer.go similarity index 79% rename from pkg/sink/kafka/franz/async_producer.go rename to pkg/sink/kafka/franz_async_producer.go index 89f2787ece..d730f11427 100644 --- a/pkg/sink/kafka/franz/async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -26,7 +26,7 @@ import ( "go.uber.org/zap" ) -type AsyncProducer struct { +type asyncProducer struct { client *kgo.Client changefeedID common.ChangeFeedID @@ -35,37 +35,24 @@ type AsyncProducer struct { errCh chan error } -func NewAsyncProducer( +func newAsyncProducer( ctx context.Context, changefeedID common.ChangeFeedID, - cfg Config, - hook *metricsHook, -) (*AsyncProducer, error) { - opts, err := newClientOptions(ctx, changefeedID, "async-producer", cfg, hook) + o *options, +) (*asyncProducer, error) { + client, err := newProducerClient(ctx, changefeedID, "async-producer", o) if err != nil { return nil, err } - producerOpts, err := producerOptions(cfg) - if err != nil { - return nil, err - } - - opts = append(opts, producerOpts...) - - client, err := kgo.NewClient(opts...) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - return &AsyncProducer{ + return &asyncProducer{ client: client, changefeedID: changefeedID, errCh: make(chan error, 1), }, nil } -func (p *AsyncProducer) Close() { +func (p *asyncProducer) Close() { if !p.closeStarted.CompareAndSwap(false, true) { return } @@ -80,7 +67,7 @@ func (p *AsyncProducer) Close() { zap.Duration("duration", time.Since(start))) } -func (p *AsyncProducer) AsyncSend( +func (p *asyncProducer) AsyncSend( ctx context.Context, topic string, partition int32, @@ -121,14 +108,14 @@ func (p *AsyncProducer) AsyncSend( return nil } -func (p *AsyncProducer) enqueueAsyncSendError( +func (p *asyncProducer) enqueueAsyncSendError( logInfo *codeccommon.MessageLogInfo, err error, ) { log.Error("kafka message send failed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), - zap.String("eventContext", buildEventLogContext( + zap.String("eventContext", BuildEventLogContext( p.changefeedID.Keyspace(), p.changefeedID.Name(), logInfo)), zap.Error(err)) @@ -139,7 +126,7 @@ func (p *AsyncProducer) enqueueAsyncSendError( } } -func (p *AsyncProducer) AsyncRunCallback(ctx context.Context) error { +func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { defer p.closed.Store(true) for { select { diff --git a/pkg/sink/kafka/franz/async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go similarity index 94% rename from pkg/sink/kafka/franz/async_producer_test.go rename to pkg/sink/kafka/franz_async_producer_test.go index 9a928e5bc9..f7cdd3a44a 100644 --- a/pkg/sink/kafka/franz/async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -30,7 +30,7 @@ import ( ) func TestAsyncSendClosedProducer(t *testing.T) { - producer := &AsyncProducer{} + producer := &asyncProducer{} producer.closed.Store(true) err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) @@ -41,12 +41,12 @@ func TestAsyncSendClosedProducer(t *testing.T) { func TestAsyncSendCanceledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - producer := &AsyncProducer{} + producer := &asyncProducer{} require.ErrorIs(t, producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{}), context.Canceled) } func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { - producer := &AsyncProducer{ + producer := &asyncProducer{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), errCh: make(chan error, 1), } @@ -63,7 +63,7 @@ func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { require.NoError(t, err) var callbackCalled atomic.Bool - producer := &AsyncProducer{ + producer := &asyncProducer{ client: client, changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), errCh: make(chan error, 1), @@ -87,11 +87,10 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) defer cluster.Close() - producer, err := NewAsyncProducer( + producer, err := newAsyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), - testConfig(cluster.ListenAddrs()), - nil, + testOptions(cluster.ListenAddrs()), ) require.NoError(t, err) defer producer.Close() @@ -126,11 +125,10 @@ func TestAsyncProducerReportsProduceFailure(t *testing.T) { return produceResponseWithError(req, 0, kerr.InvalidTopicException.Code) }) - producer, err := NewAsyncProducer( + producer, err := newAsyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), - testConfig(cluster.ListenAddrs()), - nil, + testOptions(cluster.ListenAddrs()), ) require.NoError(t, err) defer producer.Close() @@ -159,7 +157,7 @@ func TestBufferBackpressureCanBeCanceled(t *testing.T) { ) require.NoError(t, err) - producer := &AsyncProducer{ + producer := &asyncProducer{ client: client, changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "backpressure"), errCh: make(chan error, 1), diff --git a/pkg/sink/kafka/franz/config.go b/pkg/sink/kafka/franz_config.go similarity index 53% rename from pkg/sink/kafka/franz/config.go rename to pkg/sink/kafka/franz_config.go index 8ee65eb35a..16104615a2 100644 --- a/pkg/sink/kafka/franz/config.go +++ b/pkg/sink/kafka/franz_config.go @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" "crypto/tls" - "net/http" "net/url" "strings" "time" @@ -43,91 +42,47 @@ const ( // minProducerBatchBytes and maxProducerBatchBytes are franz-go's accepted batch-size bounds. minProducerBatchBytes = 512 maxProducerBatchBytes = 1 << 30 - - // NoResponse requests no broker acknowledgement. A send completes after the - // request is written. Broker-side failures are not reported, so messages can be lost. - NoResponse = int16(0) - // WaitForLocal requests acknowledgement from the partition leader. A send completes - // after the leader writes the message locally. An acknowledged message can be lost - // if the leader fails before follower replication. - WaitForLocal = int16(1) - // WaitForAll requests acknowledgement from all in-sync replicas. A send completes - // after the replication requirement is met. It is the default and provides the - // strongest durability, at the cost of higher latency or failed sends when too few - // replicas are in sync. - WaitForAll = int16(-1) ) -type Config struct { - BrokerEndpoints []string - ClientID string - MaxMessageBytes int - MaxRetry int - Compression string - RequiredAcks int16 - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - TLSConfig *tls.Config - SASL *SASLConfig -} - -type SASLConfig struct { - Mechanism string - User string - Password string - GSSAPI GSSAPIConfig - OAuth2 OAuth2Config -} - -type GSSAPIConfig struct { - AuthType int - KeyTabPath string - KerberosConfigPath string - ServiceName string - Username string - Password string - Realm string - DisablePAFXFAST bool -} - -type OAuth2Config struct { - ClientID string - ClientSecret string - TokenURL string - Scopes []string - GrantType string - Audience string - HTTPClient *http.Client -} - -func (c Config) requestTimeout() time.Duration { return max(c.ReadTimeout, c.WriteTimeout) } +func requestTimeout(o *options) time.Duration { return max(o.ReadTimeout, o.WriteTimeout) } func newClientOptions( ctx context.Context, changefeedID common.ChangeFeedID, role string, - cfg Config, + o *options, hook *metricsHook, ) ([]kgo.Opt, error) { opts := []kgo.Opt{ kgo.WithContext(ctx), - kgo.SeedBrokers(cfg.BrokerEndpoints...), - kgo.ClientID(cfg.ClientID), - kgo.DialTimeout(cfg.DialTimeout), - kgo.RequestTimeoutOverhead(cfg.requestTimeout()), - kgo.WithLogger(newLogger(changefeedID, role)), + kgo.SeedBrokers(o.BrokerEndpoints...), + kgo.ClientID(o.ClientID), + kgo.DialTimeout(o.DialTimeout), + kgo.RequestTimeoutOverhead(requestTimeout(o)), + kgo.WithLogger(newClientLogger(changefeedID, role)), } if hook != nil { opts = append(opts, kgo.WithHooks(hook)) } - if cfg.TLSConfig != nil { - opts = append(opts, kgo.DialTLSConfig(cfg.TLSConfig)) + if o.EnableTLS { + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + } + if o.Credential != nil && o.Credential.IsTLSEnabled() { + var err error + tlsConfig, err = o.Credential.ToTLSConfig() + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + } + tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify + opts = append(opts, kgo.DialTLSConfig(tlsConfig)) } - if cfg.SASL != nil && cfg.SASL.Mechanism != "" { - mechanism, err := buildSASLMechanism(ctx, *cfg.SASL) + if o.sasl != nil && o.sasl.mechanism != "" { + mechanism, err := buildSASLMechanism(ctx, o.sasl) if err != nil { return nil, err } @@ -137,16 +92,16 @@ func newClientOptions( return opts, nil } -func buildSASLMechanism(ctx context.Context, cfg SASLConfig) (sasl.Mechanism, error) { - switch strings.ToUpper(cfg.Mechanism) { - case "PLAIN": - return plain.Auth{User: cfg.User, Pass: cfg.Password}.AsMechanism(), nil - case "SCRAM-SHA-256": - return scram.Auth{User: cfg.User, Pass: cfg.Password}.AsSha256Mechanism(), nil - case "SCRAM-SHA-512": - return scram.Auth{User: cfg.User, Pass: cfg.Password}.AsSha512Mechanism(), nil - case "OAUTHBEARER": - tokenSource, err := newOAuthTokenSource(ctx, cfg.OAuth2) +func buildSASLMechanism(ctx context.Context, cfg *saslConfig) (sasl.Mechanism, error) { + switch cfg.mechanism { + case plainMechanism: + return plain.Auth{User: cfg.user, Pass: cfg.password}.AsMechanism(), nil + case scram256Mechanism: + return scram.Auth{User: cfg.user, Pass: cfg.password}.AsSha256Mechanism(), nil + case scram512Mechanism: + return scram.Auth{User: cfg.user, Pass: cfg.password}.AsSha512Mechanism(), nil + case oauthMechanism: + tokenSource, err := newOAuthTokenSource(ctx, cfg.oauth2) if err != nil { return nil, err } @@ -157,73 +112,101 @@ func buildSASLMechanism(ctx context.Context, cfg SASLConfig) (sasl.Mechanism, er } return oauth.Auth{Token: token.AccessToken}, nil }), nil - case "GSSAPI": - return buildGSSAPIMechanism(cfg.GSSAPI) + case gssapiMechanism: + return buildGSSAPIMechanism(cfg.gssapi) default: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", cfg.Mechanism) + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "unsupported sasl mechanism %s", cfg.mechanism) } } -func newOAuthTokenSource(ctx context.Context, cfg OAuth2Config) (oauth2.TokenSource, error) { - if cfg.HTTPClient != nil { - ctx = context.WithValue(ctx, oauth2.HTTPClient, cfg.HTTPClient) +func newOAuthTokenSource(ctx context.Context, cfg oauth2Config) (oauth2.TokenSource, error) { + if cfg.caPath != "" { + httpClient, err := oauthHTTPClient(cfg.caPath) + if err != nil { + return nil, err + } + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) } endpointParams := url.Values{} - if cfg.GrantType != "" { - endpointParams.Set("grant_type", cfg.GrantType) + if cfg.grantType != "" { + endpointParams.Set("grant_type", cfg.grantType) } - if cfg.Audience != "" { - endpointParams.Set("audience", cfg.Audience) + if cfg.audience != "" { + endpointParams.Set("audience", cfg.audience) } - tokenURL, err := url.Parse(cfg.TokenURL) + tokenURL, err := url.Parse(cfg.tokenURL) if err != nil { return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } config := &clientcredentials.Config{ - ClientID: cfg.ClientID, - ClientSecret: cfg.ClientSecret, + ClientID: cfg.clientID, + ClientSecret: cfg.clientSecret, TokenURL: tokenURL.String(), EndpointParams: endpointParams, - Scopes: cfg.Scopes, + Scopes: cfg.scopes, } return config.TokenSource(ctx), nil } -func producerOptions(cfg Config) ([]kgo.Opt, error) { - if cfg.MaxMessageBytes > maxProducerBatchBytes { +func newProducerClient( + ctx context.Context, + changefeedID common.ChangeFeedID, + role string, + o *options, +) (*kgo.Client, error) { + opts, err := newClientOptions(ctx, changefeedID, role, o, newMetricsHook(changefeedID)) + if err != nil { + return nil, err + } + + producerOpts, err := producerOptions(o) + if err != nil { + return nil, err + } + + client, err := kgo.NewClient(append(opts, producerOpts...)...) + if err != nil { + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + return client, nil +} + +func producerOptions(o *options) ([]kgo.Opt, error) { + if o.MaxMessageBytes > maxProducerBatchBytes { return nil, errors.ErrKafkaInvalidConfig.GenWithStack( "max-message-bytes %d exceeds franz-go limit %d", - cfg.MaxMessageBytes, + o.MaxMessageBytes, maxProducerBatchBytes, ) } // Use 64 MiB as the default budget, but never make it smaller than the configured message limit. // Keep franz-go's 10,000-record default as a second bound. - maxBufferedBytes := max(defaultMaxBufferedBytes, cfg.MaxMessageBytes) - maxBatchBytes := max(minProducerBatchBytes, cfg.MaxMessageBytes) + maxBufferedBytes := max(defaultMaxBufferedBytes, o.MaxMessageBytes) + maxBatchBytes := max(minProducerBatchBytes, o.MaxMessageBytes) maxBrokerWriteBytes := max(defaultBrokerWriteBytes, maxBatchBytes) return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), - kgo.RequiredAcks(requiredAcks(cfg.RequiredAcks)), + kgo.RequiredAcks(requiredAcks(o.RequiredAcks)), // Retried requests may create duplicates because broker-side producer ID deduplication is disabled. kgo.DisableIdempotentWrite(), // More than one in-flight request can reorder records when an earlier request is retried. kgo.MaxProduceRequestsInflightPerBroker(1), - kgo.RecordRetries(cfg.MaxRetry), - kgo.UnknownTopicRetries(cfg.MaxRetry), + kgo.RecordRetries(o.MaxRetry), + kgo.UnknownTopicRetries(o.MaxRetry), kgo.MaxBufferedBytes(maxBufferedBytes), kgo.ProducerBatchMaxBytes(int32(maxBatchBytes)), kgo.BrokerMaxWriteBytes(int32(maxBrokerWriteBytes)), - kgo.ProduceRequestTimeout(cfg.requestTimeout()), + kgo.ProduceRequestTimeout(requestTimeout(o)), kgo.ProducerLinger(0), - compressionOption(cfg.Compression), + compressionOption(o.Compression), }, nil } -func requiredAcks(required int16) kgo.Acks { +func requiredAcks(required RequiredAcks) kgo.Acks { switch required { case WaitForAll: return kgo.AllISRAcks() @@ -232,7 +215,7 @@ func requiredAcks(required int16) kgo.Acks { case NoResponse: return kgo.NoAck() default: - log.Warn("unsupported required acks", zap.Int16("requiredAcks", required)) + log.Warn("unsupported required acks", zap.Int16("requiredAcks", int16(required))) return kgo.AllISRAcks() } } diff --git a/pkg/sink/kafka/franz/config_test.go b/pkg/sink/kafka/franz_config_test.go similarity index 65% rename from pkg/sink/kafka/franz/config_test.go rename to pkg/sink/kafka/franz_config_test.go index 87456b84dc..2f284fd52b 100644 --- a/pkg/sink/kafka/franz/config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -29,55 +29,54 @@ import ( "github.com/twmb/franz-go/pkg/kgo" ) -func testConfig(brokers []string) Config { - return Config{ - BrokerEndpoints: brokers, - MaxMessageBytes: 1 << 20, - MaxRetry: 1, - RequiredAcks: WaitForAll, - DialTimeout: time.Second, - ReadTimeout: time.Second, - WriteTimeout: time.Second, - } +func testOptions(brokers []string) *options { + o := NewOptions() + o.BrokerEndpoints = brokers + o.MaxMessageBytes = 1 << 20 + o.MaxRetry = 1 + o.DialTimeout = time.Second + o.ReadTimeout = time.Second + o.WriteTimeout = time.Second + return o } -func TestRequiredAcks(t *testing.T) { +func TestFranzRequiredAcks(t *testing.T) { for _, test := range []struct { - required int16 + required RequiredAcks expected kgo.Acks }{ {required: WaitForAll, expected: kgo.AllISRAcks()}, {required: WaitForLocal, expected: kgo.LeaderAck()}, {required: NoResponse, expected: kgo.NoAck()}, - {required: 2, expected: kgo.AllISRAcks()}, + {required: RequiredAcks(2), expected: kgo.AllISRAcks()}, } { require.Equal(t, test.expected, requiredAcks(test.required)) } } -func TestRequestTimeoutUsesLargerTimeout(t *testing.T) { - cfg := Config{ReadTimeout: time.Second, WriteTimeout: 2 * time.Second} - require.Equal(t, 2*time.Second, cfg.requestTimeout()) +func TestFranzRequestTimeoutUsesLargerTimeout(t *testing.T) { + o := &options{ReadTimeout: time.Second, WriteTimeout: 2 * time.Second} + require.Equal(t, 2*time.Second, requestTimeout(o)) - cfg.ReadTimeout = 3 * time.Second - require.Equal(t, 3*time.Second, cfg.requestTimeout()) + o.ReadTimeout = 3 * time.Second + require.Equal(t, 3*time.Second, requestTimeout(o)) } func TestProducerOptionsBoundBufferAndBatch(t *testing.T) { const batchBytes = 1048588 - cfg := testConfig([]string{"127.0.0.1:9092"}) - cfg.MaxMessageBytes = batchBytes + o := testOptions([]string{"127.0.0.1:9092"}) + o.MaxMessageBytes = batchBytes opts, err := newClientOptions( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "config"), "test", - cfg, + o, nil, ) require.NoError(t, err) - producerOpts, err := producerOptions(cfg) + producerOpts, err := producerOptions(o) require.NoError(t, err) client, err := kgo.NewClient(append(opts, producerOpts...)...) @@ -93,7 +92,7 @@ func TestProducerOptionsBoundBufferAndBatch(t *testing.T) { } func TestProducerOptionsUseSingleNonIdempotentRequest(t *testing.T) { - config := testConfig([]string{"127.0.0.1:9092"}) + config := testOptions([]string{"127.0.0.1:9092"}) producerOpts, err := producerOptions(config) require.NoError(t, err) @@ -108,7 +107,7 @@ func TestProducerOptionsUseSingleNonIdempotentRequest(t *testing.T) { func TestProducerLimitsScaleWithConfiguredMessage(t *testing.T) { maxMessageBytes := defaultBrokerWriteBytes + 1 - config := testConfig([]string{"127.0.0.1:9092"}) + config := testOptions([]string{"127.0.0.1:9092"}) config.MaxMessageBytes = maxMessageBytes producerOpts, err := producerOptions(config) @@ -123,7 +122,7 @@ func TestProducerLimitsScaleWithConfiguredMessage(t *testing.T) { } func TestProducerOptionsClampSmallBatch(t *testing.T) { - config := testConfig([]string{"127.0.0.1:9092"}) + config := testOptions([]string{"127.0.0.1:9092"}) config.MaxMessageBytes = minProducerBatchBytes - 1 producerOpts, err := producerOptions(config) @@ -137,7 +136,7 @@ func TestProducerOptionsClampSmallBatch(t *testing.T) { } func TestProducerOptionsRejectOversizedBatch(t *testing.T) { - config := testConfig([]string{"127.0.0.1:9092"}) + config := testOptions([]string{"127.0.0.1:9092"}) config.MaxMessageBytes = maxProducerBatchBytes + 1 _, err := producerOptions(config) @@ -158,7 +157,7 @@ func TestCompressionOptions(t *testing.T) { {name: "unknown falls back to none", compression: "unknown", expected: kgo.NoCompression()}, } { t.Run(test.name, func(t *testing.T) { - cfg := testConfig([]string{"127.0.0.1:9092"}) + cfg := testOptions([]string{"127.0.0.1:9092"}) cfg.Compression = test.compression producerOpts, err := producerOptions(cfg) @@ -173,41 +172,41 @@ func TestCompressionOptions(t *testing.T) { } } -func TestBuildGSSAPIMechanism(t *testing.T) { - for _, cfg := range []GSSAPIConfig{ - {AuthType: userAuth, Password: "pwd"}, - {AuthType: keyTabAuth, KeyTabPath: "/tmp/a.keytab"}, +func TestBuildFranzGSSAPIMechanism(t *testing.T) { + for _, cfg := range []gssapiConfig{ + {authType: userAuth, password: "pwd"}, + {authType: keyTabAuth, keyTabPath: "/tmp/a.keytab"}, } { - cfg.KerberosConfigPath = "/etc/krb5.conf" - cfg.ServiceName = "kafka" - cfg.Username = "alice" - cfg.Realm = "EXAMPLE.COM" - - mechanism, err := buildSASLMechanism(context.Background(), SASLConfig{ - Mechanism: "GSSAPI", - GSSAPI: cfg, + cfg.kerberosConfigPath = "/etc/krb5.conf" + cfg.serviceName = "kafka" + cfg.username = "alice" + cfg.realm = "EXAMPLE.COM" + + mechanism, err := buildSASLMechanism(context.Background(), &saslConfig{ + mechanism: gssapiMechanism, + gssapi: cfg, }) require.NoError(t, err) require.Equal(t, "GSSAPI", mechanism.Name()) } } -func TestBuildSASLMechanisms(t *testing.T) { - for _, mechanism := range []string{"PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"} { - actual, err := buildSASLMechanism(context.Background(), SASLConfig{ - Mechanism: mechanism, - User: "alice", - Password: "secret", +func TestBuildFranzSASLMechanisms(t *testing.T) { + for _, mechanism := range []saslMechanism{plainMechanism, scram256Mechanism, scram512Mechanism} { + actual, err := buildSASLMechanism(context.Background(), &saslConfig{ + mechanism: mechanism, + user: "alice", + password: "secret", }) require.NoError(t, err) - require.Equal(t, mechanism, actual.Name()) + require.Equal(t, string(mechanism), actual.Name()) } - _, err := buildSASLMechanism(context.Background(), SASLConfig{Mechanism: "unknown"}) + _, err := buildSASLMechanism(context.Background(), &saslConfig{mechanism: "unknown"}) require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } -func TestOAuthTokenSource(t *testing.T) { +func TestFranzOAuthTokenSource(t *testing.T) { request := make(chan url.Values, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { @@ -224,13 +223,13 @@ func TestOAuthTokenSource(t *testing.T) { })) defer server.Close() - source, err := newOAuthTokenSource(context.Background(), OAuth2Config{ - ClientID: "client", - ClientSecret: "secret", - TokenURL: server.URL, - Scopes: []string{"scope-a", "scope-b"}, - GrantType: "custom", - Audience: "audience", + source, err := newOAuthTokenSource(context.Background(), oauth2Config{ + clientID: "client", + clientSecret: "secret", + tokenURL: server.URL, + scopes: []string{"scope-a", "scope-b"}, + grantType: "custom", + audience: "audience", }) require.NoError(t, err) @@ -244,26 +243,7 @@ func TestOAuthTokenSource(t *testing.T) { require.Equal(t, "scope-a scope-b", form.Get("scope")) } -func TestOAuthTokenSourceUsesHTTPClient(t *testing.T) { - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, err := io.WriteString(w, `{"access_token":"token","token_type":"bearer"}`) - require.NoError(t, err) - })) - defer server.Close() - - source, err := newOAuthTokenSource(context.Background(), OAuth2Config{ - TokenURL: server.URL, - HTTPClient: server.Client(), - }) - require.NoError(t, err) - - token, err := source.Token() - require.NoError(t, err) - require.Equal(t, "token", token.AccessToken) -} - -func TestOAuthTokenSourceRejectsInvalidURL(t *testing.T) { - _, err := newOAuthTokenSource(context.Background(), OAuth2Config{TokenURL: "http://example.com/%%"}) +func TestFranzOAuthTokenSourceRejectsInvalidURL(t *testing.T) { + _, err := newOAuthTokenSource(context.Background(), oauth2Config{tokenURL: "http://example.com/%%"}) require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go new file mode 100644 index 0000000000..a56e03f15d --- /dev/null +++ b/pkg/sink/kafka/franz_factory.go @@ -0,0 +1,93 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strings" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "go.uber.org/zap" +) + +type franzFactory struct { + options *options + changefeedID common.ChangeFeedID +} + +func newFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { + admin, err := newAdmin(ctx, changefeedID, o) + if err != nil { + return nil, err + } + defer admin.Close() + + if err := adjustOptions(ctx, changefeedID, admin, o, o.Topic); err != nil { + return nil, err + } + + compression := strings.ToLower(strings.TrimSpace(o.Compression)) + if compression == "" { + compression = "none" + } + + log.Info("kafka sink configuration resolved", + zap.String("namespace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.String("client", KafkaClientFranz), + zap.String("topic", o.Topic), + zap.Int32("partitionNum", o.PartitionNum), + zap.Int("maxMessageBytes", o.MaxMessageBytes), + zap.Int("maxBatchedBytes", o.MaxBatchedBytes), + zap.String("compression", compression), + zap.Int16("requiredAcks", int16(o.RequiredAcks)), + zap.Int("maxRetry", o.MaxRetry), + zap.Duration("dialTimeout", o.DialTimeout), + zap.Duration("readTimeout", o.ReadTimeout), + zap.Duration("writeTimeout", o.WriteTimeout)) + + return &franzFactory{options: o, changefeedID: changefeedID}, nil +} + +func (f *franzFactory) AdminClient(ctx context.Context) (AdminClient, error) { + return newAdmin(ctx, f.changefeedID, f.options) +} + +func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { + producer, err := newSyncProducer(ctx, f.changefeedID, f.options) + if err != nil { + cleanupMetrics(f.changefeedID) + } + return producer, err +} + +func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { + producer, err := newAsyncProducer(ctx, f.changefeedID, f.options) + if err != nil { + cleanupMetrics(f.changefeedID) + } + return producer, err +} + +func (f *franzFactory) MetricsCollector(AdminClient) MetricsCollector { + return noopMetricsCollector{} +} + +func (f *franzFactory) CleanupMetrics() { cleanupMetrics(f.changefeedID) } + +type noopMetricsCollector struct{} + +func (noopMetricsCollector) Run(ctx context.Context) { <-ctx.Done() } diff --git a/pkg/sink/kafka/franz/gssapi.go b/pkg/sink/kafka/franz_gssapi.go similarity index 72% rename from pkg/sink/kafka/franz/gssapi.go rename to pkg/sink/kafka/franz_gssapi.go index d88584c085..3f77d26777 100644 --- a/pkg/sink/kafka/franz/gssapi.go +++ b/pkg/sink/kafka/franz_gssapi.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -25,13 +25,7 @@ import ( "github.com/twmb/franz-go/pkg/sasl/kerberos" ) -const ( - // Authentication type values are part of sink URI compatibility and must remain stable. - userAuth = 1 - keyTabAuth = 2 -) - -func buildGSSAPIMechanism(g GSSAPIConfig) (sasl.Mechanism, error) { +func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { if err := validateGSSAPIConfig(g); err != nil { return nil, err } @@ -41,65 +35,65 @@ func buildGSSAPIMechanism(g GSSAPIConfig) (sasl.Mechanism, error) { if err != nil { return kerberos.Auth{}, err } - return kerberos.Auth{Client: krbClient, Service: g.ServiceName}, nil + return kerberos.Auth{Client: krbClient, Service: g.serviceName}, nil }), nil } -func validateGSSAPIConfig(g GSSAPIConfig) error { - if g.ServiceName == "" { +func validateGSSAPIConfig(g gssapiConfig) error { + if g.serviceName == "" { return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") } - if g.KerberosConfigPath == "" { + if g.kerberosConfigPath == "" { return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") } - if g.Username == "" { + if g.username == "" { return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") } - if g.Realm == "" { + if g.realm == "" { return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") } - switch g.AuthType { + switch g.authType { case userAuth: - if g.Password == "" { + if g.password == "" { return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") } case keyTabAuth: - if g.KeyTabPath == "" { + if g.keyTabPath == "" { return errors.ErrKafkaInvalidConfig.GenWithStack( "sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") } default: return errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.AuthType) + "unsupported sasl-gssapi-auth-type %d", g.authType) } return nil } -func newKerberosClient(g GSSAPIConfig) (*client.Client, error) { - cfg, err := config.Load(g.KerberosConfigPath) +func newKerberosClient(g gssapiConfig) (*client.Client, error) { + cfg, err := config.Load(g.kerberosConfigPath) if err != nil { return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - switch g.AuthType { + switch g.authType { case userAuth: return client.NewWithPassword( - g.Username, g.Realm, g.Password, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)), nil + g.username, g.realm, g.password, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil case keyTabAuth: - kt, err := keytab.Load(g.KeyTabPath) + kt, err := keytab.Load(g.keyTabPath) if err != nil { return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } return client.NewWithKeytab( - g.Username, g.Realm, kt, cfg, client.DisablePAFXFAST(g.DisablePAFXFAST)), nil + g.username, g.realm, kt, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil default: return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.AuthType) + "unsupported sasl-gssapi-auth-type %d", g.authType) } } diff --git a/pkg/sink/kafka/franz/gssapi_test.go b/pkg/sink/kafka/franz_gssapi_test.go similarity index 54% rename from pkg/sink/kafka/franz/gssapi_test.go rename to pkg/sink/kafka/franz_gssapi_test.go index 982da3327d..fbd1975438 100644 --- a/pkg/sink/kafka/franz/gssapi_test.go +++ b/pkg/sink/kafka/franz_gssapi_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -23,23 +23,23 @@ import ( ) func TestGSSAPIConfigValidation(t *testing.T) { - valid := GSSAPIConfig{ - AuthType: userAuth, - KerberosConfigPath: "/etc/krb5.conf", - ServiceName: "kafka", - Username: "alice", - Password: "secret", - Realm: "EXAMPLE.COM", + valid := gssapiConfig{ + authType: userAuth, + kerberosConfigPath: "/etc/krb5.conf", + serviceName: "kafka", + username: "alice", + password: "secret", + realm: "EXAMPLE.COM", } - for _, mutate := range []func(*GSSAPIConfig){ - func(cfg *GSSAPIConfig) { cfg.ServiceName = "" }, - func(cfg *GSSAPIConfig) { cfg.KerberosConfigPath = "" }, - func(cfg *GSSAPIConfig) { cfg.Username = "" }, - func(cfg *GSSAPIConfig) { cfg.Realm = "" }, - func(cfg *GSSAPIConfig) { cfg.Password = "" }, - func(cfg *GSSAPIConfig) { cfg.AuthType = 0 }, - func(cfg *GSSAPIConfig) { cfg.AuthType, cfg.KeyTabPath = keyTabAuth, "" }, + for _, mutate := range []func(*gssapiConfig){ + func(cfg *gssapiConfig) { cfg.serviceName = "" }, + func(cfg *gssapiConfig) { cfg.kerberosConfigPath = "" }, + func(cfg *gssapiConfig) { cfg.username = "" }, + func(cfg *gssapiConfig) { cfg.realm = "" }, + func(cfg *gssapiConfig) { cfg.password = "" }, + func(cfg *gssapiConfig) { cfg.authType = 0 }, + func(cfg *gssapiConfig) { cfg.authType, cfg.keyTabPath = keyTabAuth, "" }, } { cfg := valid mutate(&cfg) @@ -50,13 +50,13 @@ func TestGSSAPIConfigValidation(t *testing.T) { } func TestGSSAPIRejectsMissingKerberosConfig(t *testing.T) { - mechanism, err := buildGSSAPIMechanism(GSSAPIConfig{ - AuthType: userAuth, - KerberosConfigPath: "/path/that/does/not/exist", - ServiceName: "kafka", - Username: "alice", - Password: "secret", - Realm: "EXAMPLE.COM", + mechanism, err := buildGSSAPIMechanism(gssapiConfig{ + authType: userAuth, + kerberosConfigPath: "/path/that/does/not/exist", + serviceName: "kafka", + username: "alice", + password: "secret", + realm: "EXAMPLE.COM", }) require.NoError(t, err) diff --git a/pkg/sink/kafka/franz/logger.go b/pkg/sink/kafka/franz_logger.go similarity index 60% rename from pkg/sink/kafka/franz/logger.go rename to pkg/sink/kafka/franz_logger.go index 3d5076ec07..45f12890b3 100644 --- a/pkg/sink/kafka/franz/logger.go +++ b/pkg/sink/kafka/franz_logger.go @@ -11,12 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "fmt" "strings" - "sync" "time" "github.com/pingcap/log" @@ -29,36 +28,32 @@ import ( // logValueLimit bounds individual string fields emitted by the franz-go logger. const logValueLimit = 1024 -type logger struct { - changefeedID common.ChangeFeedID - role string - now func() time.Time - mu sync.Mutex - windowStart time.Time - counts map[string]uint64 +type clientLogger struct { + logger *zap.Logger } -func newLogger(changefeedID common.ChangeFeedID, role string) kgo.Logger { - return &logger{changefeedID: changefeedID, role: role, now: time.Now, counts: make(map[string]uint64)} +func newClientLogger(changefeedID common.ChangeFeedID, role string) kgo.Logger { + logger := log.L().With( + zap.String("component", "kafka-client"), + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.String("role", role), + ).WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewSamplerWithOptions(core, time.Minute, 5, 100) + })) + + return &clientLogger{logger: logger} } -func (l *logger) Level() kgo.LogLevel { +func (l *clientLogger) Level() kgo.LogLevel { if log.GetLevel() <= zapcore.DebugLevel { return kgo.LogLevelInfo } return kgo.LogLevelWarn } -func (l *logger) Log(level kgo.LogLevel, msg string, keyvals ...any) { - if !l.shouldLog(level, msg) { - return - } - fields := []zap.Field{ - zap.String("component", "kafka-client"), - zap.String("keyspace", l.changefeedID.Keyspace()), - zap.String("changefeed", l.changefeedID.Name()), - zap.String("role", l.role), - } +func (l *clientLogger) Log(level kgo.LogLevel, msg string, keyvals ...any) { + fields := make([]zap.Field, 0, (len(keyvals)+1)/2) for i := 0; i < len(keyvals); i += 2 { key := fmt.Sprint(keyvals[i]) @@ -77,30 +72,14 @@ func (l *logger) Log(level kgo.LogLevel, msg string, keyvals ...any) { switch level { case kgo.LogLevelError: - log.Error(msg, fields...) + l.logger.Error(msg, fields...) case kgo.LogLevelWarn: - log.Warn(msg, fields...) + l.logger.Warn(msg, fields...) default: - log.Debug(msg, fields...) + l.logger.Debug(msg, fields...) } } -func (l *logger) shouldLog(level kgo.LogLevel, msg string) bool { - now := l.now() - key := fmt.Sprintf("%d:%s", level, msg) - - l.mu.Lock() - defer l.mu.Unlock() - - if l.windowStart.IsZero() || now.Sub(l.windowStart) >= time.Minute { - l.windowStart, l.counts = now, make(map[string]uint64) - } - - l.counts[key]++ - - return l.counts[key] <= 5 || l.counts[key]%100 == 0 -} - func isSensitiveLogKey(key string) bool { key = strings.ToLower(key) if key == "key" || key == "value" { diff --git a/pkg/sink/kafka/franz/logger_test.go b/pkg/sink/kafka/franz_logger_test.go similarity index 76% rename from pkg/sink/kafka/franz/logger_test.go rename to pkg/sink/kafka/franz_logger_test.go index 2c0fea9164..31b97c06a2 100644 --- a/pkg/sink/kafka/franz/logger_test.go +++ b/pkg/sink/kafka/franz_logger_test.go @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "strings" - "sync" "testing" - "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" @@ -33,7 +31,7 @@ func TestLoggerLevelAndFiltering(t *testing.T) { oldLevel := log.GetLevel() defer log.SetLevel(oldLevel) - clientLogger := newLogger(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "logger"), "producer").(*logger) + clientLogger := newClientLogger(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "logger"), "producer").(*clientLogger) log.SetLevel(zapcore.InfoLevel) require.Equal(t, kgo.LogLevelWarn, clientLogger.Level()) @@ -50,28 +48,12 @@ func TestLoggerLevelAndFiltering(t *testing.T) { }) } -func TestLoggerSamplingIsConcurrent(t *testing.T) { - clientLogger := newLogger(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "logger"), "producer").(*logger) - now := time.Now() - clientLogger.now = func() time.Time { return now } - - var wg sync.WaitGroup - for range 20 { - wg.Go(func() { - clientLogger.shouldLog(kgo.LogLevelWarn, "repeat") - }) - } - - wg.Wait() - require.Equal(t, uint64(20), clientLogger.counts["2:repeat"]) -} - func TestLoggerPreservesContextAndRedactsValues(t *testing.T) { core, logs := observer.New(zapcore.DebugLevel) restore := log.ReplaceGlobals(zap.New(core), nil) defer restore() - clientLogger := newLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") + clientLogger := newClientLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") clientLogger.Log( kgo.LogLevelWarn, "connection failed", @@ -98,7 +80,7 @@ func TestLoggerSamplesRepeatedMessages(t *testing.T) { restore := log.ReplaceGlobals(zap.New(core), nil) defer restore() - clientLogger := newLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") + clientLogger := newClientLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") for range 105 { clientLogger.Log(kgo.LogLevelWarn, "repeated") } diff --git a/pkg/sink/kafka/franz/metrics.go b/pkg/sink/kafka/franz_metrics.go similarity index 92% rename from pkg/sink/kafka/franz/metrics.go rename to pkg/sink/kafka/franz_metrics.go index 97116df7a5..5489960389 100644 --- a/pkg/sink/kafka/franz/metrics.go +++ b/pkg/sink/kafka/franz_metrics.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import "github.com/prometheus/client_golang/prometheus" @@ -82,17 +82,3 @@ var ( Help: "Total record bytes after compression in successfully written franz-go batches.", }, []string{"namespace", "changefeed"}) ) - -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister( - requestsInFlight, - outgoingBytesTotal, - requestsTotal, - responsesTotal, - requestDuration, - throttleTime, - recordsPerBatch, - uncompressedBytesTotal, - compressedBytesTotal, - ) -} diff --git a/pkg/sink/kafka/franz/metrics_hook.go b/pkg/sink/kafka/franz_metrics_hook.go similarity index 96% rename from pkg/sink/kafka/franz/metrics_hook.go rename to pkg/sink/kafka/franz_metrics_hook.go index 2e41bb7267..0ee3d8414f 100644 --- a/pkg/sink/kafka/franz/metrics_hook.go +++ b/pkg/sink/kafka/franz_metrics_hook.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "strconv" @@ -23,7 +23,7 @@ import ( "github.com/twmb/franz-go/pkg/kgo" ) -// metricsHook adapts franz-go client callbacks to TiCDC's Kafka sink metrics. +// metricsHook adapts client callbacks to TiCDC's Kafka sink metrics. // franz-go calls these hook methods while writing requests, receiving responses, // and flushing produce batches. The hook does not poll Kafka; it only records // raw callback values for Prometheus. @@ -107,8 +107,8 @@ func (h *metricsHook) broker(nodeID int32) *brokerMetrics { return actual.(*brokerMetrics) } -// CleanupMetrics removes producer series after all clients are closed. -func CleanupMetrics(changefeedID common.ChangeFeedID) { +// cleanupMetrics removes producer series after all clients are closed. +func cleanupMetrics(changefeedID common.ChangeFeedID) { labels := prometheus.Labels{ "namespace": changefeedID.Keyspace(), "changefeed": changefeedID.Name(), diff --git a/pkg/sink/kafka/franz/metrics_hook_test.go b/pkg/sink/kafka/franz_metrics_hook_test.go similarity index 96% rename from pkg/sink/kafka/franz/metrics_hook_test.go rename to pkg/sink/kafka/franz_metrics_hook_test.go index a5cb9532f4..7033a4ffff 100644 --- a/pkg/sink/kafka/franz/metrics_hook_test.go +++ b/pkg/sink/kafka/franz_metrics_hook_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -29,8 +29,8 @@ import ( func TestInitMetrics(t *testing.T) { changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "metrics-registration") - CleanupMetrics(changefeedID) - t.Cleanup(func() { CleanupMetrics(changefeedID) }) + cleanupMetrics(changefeedID) + t.Cleanup(func() { cleanupMetrics(changefeedID) }) hook := newMetricsHook(changefeedID) hook.OnProduceBatchWritten( @@ -64,8 +64,8 @@ func TestInitMetrics(t *testing.T) { func TestMetricsHookRecordsRawValues(t *testing.T) { changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "metrics-hook") - CleanupMetrics(changefeedID) - t.Cleanup(func() { CleanupMetrics(changefeedID) }) + cleanupMetrics(changefeedID) + t.Cleanup(func() { cleanupMetrics(changefeedID) }) hook := newMetricsHook(changefeedID) meta := kgo.BrokerMetadata{NodeID: 1} diff --git a/pkg/sink/kafka/franz/sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go similarity index 67% rename from pkg/sink/kafka/franz/sync_producer.go rename to pkg/sink/kafka/franz_sync_producer.go index 59c39c37ba..c88927927b 100644 --- a/pkg/sink/kafka/franz/sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -26,7 +26,7 @@ import ( "go.uber.org/zap" ) -type SyncProducer struct { +type syncProducer struct { id common.ChangeFeedID client *kgo.Client @@ -34,66 +34,38 @@ type SyncProducer struct { timeout time.Duration } -func NewSyncProducer( +func newSyncProducer( ctx context.Context, changefeedID common.ChangeFeedID, - cfg Config, - hook *metricsHook, -) (*SyncProducer, error) { - opts, err := newClientOptions(ctx, changefeedID, "sync-producer", cfg, hook) + o *options, +) (*syncProducer, error) { + client, err := newProducerClient(ctx, changefeedID, "sync-producer", o) if err != nil { return nil, err } - producerOpts, err := producerOptions(cfg) - if err != nil { - return nil, err - } - - opts = append(opts, producerOpts...) - - client, err := kgo.NewClient(opts...) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - - return &SyncProducer{ + return &syncProducer{ id: changefeedID, client: client, - timeout: cfg.requestTimeout(), + timeout: requestTimeout(o), }, nil } -func (p *SyncProducer) SendMessage(topic string, partitionNum int32, message *codeccommon.Message) error { +func (p *syncProducer) SendMessage(topic string, partitionNum int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } - ctx, cancel := context.WithTimeout(p.client.Context(), p.timeout) - defer cancel() - record := &kgo.Record{ Topic: topic, Partition: partitionNum, Key: message.Key, Value: message.Value, } - err := p.client.ProduceSync(ctx, record).FirstErr() - if err == nil { - return nil - } - - log.Error("kafka message send failed", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name()), - zap.String("eventContext", buildEventLogContext( - p.id.Keyspace(), p.id.Name(), message.LogInfo)), - zap.Error(err)) - - return errors.WrapError(errors.ErrKafkaSendMessage, err) + return p.sendRecords(message, record) } -func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *codeccommon.Message) error { +func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -108,6 +80,10 @@ func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *c }) } + return p.sendRecords(message, records...) +} + +func (p *syncProducer) sendRecords(message *codeccommon.Message, records ...*kgo.Record) error { ctx, cancel := context.WithTimeout(p.client.Context(), p.timeout) defer cancel() @@ -119,14 +95,14 @@ func (p *SyncProducer) SendMessages(topic string, partitionNum int32, message *c log.Error("kafka message send failed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), - zap.String("eventContext", buildEventLogContext( + zap.String("eventContext", BuildEventLogContext( p.id.Keyspace(), p.id.Name(), message.LogInfo)), zap.Error(err)) return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *SyncProducer) Close() { +func (p *syncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { log.Warn("kafka ddl producer already closed", zap.String("keyspace", p.id.Keyspace()), diff --git a/pkg/sink/kafka/franz/sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go similarity index 92% rename from pkg/sink/kafka/franz/sync_producer_test.go rename to pkg/sink/kafka/franz_sync_producer_test.go index 90e387f988..ce62f951e6 100644 --- a/pkg/sink/kafka/franz/sync_producer_test.go +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package franz +package kafka import ( "context" @@ -27,7 +27,7 @@ import ( ) func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { - producer := &SyncProducer{} + producer := &syncProducer{} producer.closed.Store(true) err := producer.SendMessage("topic", 1, &codeccommon.Message{}) @@ -42,11 +42,10 @@ func TestSyncProducerSendsToRequestedPartitions(t *testing.T) { cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) defer cluster.Close() - producer, err := NewSyncProducer( + producer, err := newSyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), - testConfig(cluster.ListenAddrs()), - nil, + testOptions(cluster.ListenAddrs()), ) require.NoError(t, err) defer producer.Close() @@ -64,11 +63,10 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { return produceResponseWithError(req, 1, kerr.InvalidTopicException.Code) }) - producer, err := NewSyncProducer( + producer, err := newSyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), - testConfig(cluster.ListenAddrs()), - nil, + testOptions(cluster.ListenAddrs()), ) require.NoError(t, err) defer producer.Close() @@ -79,11 +77,10 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { } func TestSyncProducerCloseIsIdempotent(t *testing.T) { - client, err := NewSyncProducer( + client, err := newSyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), - testConfig([]string{"127.0.0.1:1"}), - nil, + testOptions([]string{"127.0.0.1:1"}), ) require.NoError(t, err) diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index b4f0776a41..f75f700cc0 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -16,7 +16,6 @@ package kafka import ( "github.com/pingcap/ticdc/pkg/sink/codec" "github.com/pingcap/ticdc/pkg/sink/kafka/claimcheck" - "github.com/pingcap/ticdc/pkg/sink/kafka/franz" "github.com/prometheus/client_golang/prometheus" ) @@ -90,7 +89,17 @@ var ( // InitMetrics registers all metrics in this file. func InitMetrics(registry *prometheus.Registry) { - franz.InitMetrics(registry) + registry.MustRegister( + requestsInFlight, + outgoingBytesTotal, + requestsTotal, + responsesTotal, + requestDuration, + throttleTime, + recordsPerBatch, + uncompressedBytesTotal, + compressedBytesTotal, + ) registry.MustRegister(compressionRatioGauge) registry.MustRegister(recordsPerRequestGauge) registry.MustRegister(throttleTimeGauge) diff --git a/pkg/sink/kafka/metrics_collector.go b/pkg/sink/kafka/metrics_collector.go index 4c7ea5c43f..60b48ac628 100644 --- a/pkg/sink/kafka/metrics_collector.go +++ b/pkg/sink/kafka/metrics_collector.go @@ -93,7 +93,7 @@ func (m *saramaMetricsCollector) Run(ctx context.Context) { } func (m *saramaMetricsCollector) updateBrokers(ctx context.Context) { - brokers := m.adminClient.GetAllBrokers() + brokers := m.adminClient.GetAllBrokers(ctx) for _, b := range brokers { m.brokers[b.ID] = struct{}{} } diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 89b279cf2f..d05bdfd116 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "encoding/base64" "fmt" "net/http" @@ -587,12 +588,12 @@ func (o *options) DeriveTopicConfig() *AutoCreateTopicConfig { // ValidateReplicationFactor checks whether a topic created with this config // can satisfy the configured acknowledgment requirement. -func (c *AutoCreateTopicConfig) ValidateReplicationFactor(admin AdminClient) error { +func (c *AutoCreateTopicConfig) ValidateReplicationFactor(ctx context.Context, admin AdminClient) error { if c.RequiredAcks != WaitForAll { return nil } - raw, found, err := admin.GetBrokerConfig(MinInsyncReplicasConfigName) + raw, found, err := admin.GetBrokerConfig(ctx, MinInsyncReplicasConfigName) if err != nil { log.Warn("kafka broker configuration lookup failed, skipping replication factor validation", zap.String("configName", MinInsyncReplicasConfigName), @@ -650,6 +651,7 @@ func NewKafkaClientID(captureAddr string, // It overwrites MaxMessageBytes with the final producer message limit derived // from the topic or broker configuration. func adjustOptions( + ctx context.Context, changefeedID common.ChangeFeedID, admin AdminClient, options *options, @@ -657,7 +659,7 @@ func adjustOptions( ) error { // The topic may not exist yet and will be created later by the topic manager, // so ignore per-topic metadata errors here. - topics, err := admin.GetTopicsMeta([]string{topic}, true) + topics, err := admin.GetTopicsMeta(ctx, []string{topic}, true) if err != nil { return err } @@ -666,9 +668,9 @@ func adjustOptions( // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. if exists { - err = adjustExistingTopicOption(changefeedID, admin, options, info) + err = adjustExistingTopicOption(ctx, changefeedID, admin, options, info) } else { - adjustNewTopicOptions(admin, changefeedID, options) + adjustNewTopicOptions(ctx, admin, changefeedID, options) } if err != nil { return err @@ -679,12 +681,13 @@ func adjustOptions( } func adjustExistingTopicOption( + ctx context.Context, changefeedID common.ChangeFeedID, admin AdminClient, options *options, info TopicDetail, ) error { - maxMessageBytes, found, err := getTopicMaxMessageBytes(admin, info.Name) + maxMessageBytes, found, err := getTopicMaxMessageBytes(ctx, admin, info.Name) if err != nil || !found { log.Warn("kafka topic `max.message.bytes` unavailable, using configured value", zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), @@ -700,13 +703,14 @@ func adjustExistingTopicOption( } func adjustNewTopicOptions( + ctx context.Context, admin AdminClient, changefeedID common.ChangeFeedID, options *options, ) { // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - messageMaxBytes, found, err := getBrokerMaxMessageBytes(admin) + messageMaxBytes, found, err := getBrokerMaxMessageBytes(ctx, admin) if err != nil || !found { log.Warn("kafka broker `message.max.bytes` unavailable, using configured value", zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), @@ -722,11 +726,12 @@ func adjustNewTopicOptions( } func getTopicMaxMessageBytes( + ctx context.Context, admin AdminClient, topic string, ) (int, bool, error) { raw, found, err := getTopicConfig( - admin, topic, + ctx, admin, topic, TopicMaxMessageBytesConfigName, BrokerMessageMaxBytesConfigName, ) @@ -743,8 +748,8 @@ func getTopicMaxMessageBytes( return maxMessageBytes, true, nil } -func getBrokerMaxMessageBytes(admin AdminClient) (int, bool, error) { - raw, found, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) +func getBrokerMaxMessageBytes(ctx context.Context, admin AdminClient) (int, bool, error) { + raw, found, err := admin.GetBrokerConfig(ctx, BrokerMessageMaxBytesConfigName) if err != nil { return 0, false, err } @@ -763,15 +768,16 @@ func getBrokerMaxMessageBytes(admin AdminClient) (int, bool, error) { // we will try to get it from the broker's configuration. // NOTICE: The configuration names of topic and broker may be different for the same configuration. func getTopicConfig( + ctx context.Context, admin AdminClient, topicName string, topicConfigName string, brokerConfigName string, ) (string, bool, error) { - c, found, err := admin.GetTopicConfig(topicName, topicConfigName) + c, found, err := admin.GetTopicConfig(ctx, topicName, topicConfigName) if err == nil && found { return c, true, nil } - return admin.GetBrokerConfig(brokerConfigName) + return admin.GetBrokerConfig(ctx, brokerConfigName) } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index ab656b6b75..4adb6ec886 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -690,13 +690,13 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) gomock.InOrder( - adminClient.EXPECT().GetTopicsMeta([]string{topicName}, true).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{topicName}, true).Return( map[string]TopicDetail{ topicName: {Name: topicName, NumPartitions: 3}, }, nil), - adminClient.EXPECT().GetTopicConfig(topicName, TopicMaxMessageBytesConfigName). + adminClient.EXPECT().GetTopicConfig(gomock.Any(), topicName, TopicMaxMessageBytesConfigName). Return("", false, nil), - adminClient.EXPECT().GetBrokerConfig(BrokerMessageMaxBytesConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), BrokerMessageMaxBytesConfigName). Return(mockBrokerMessageMaxBytes, true, nil), ) sinkURI, err := url.Parse(fmt.Sprintf( @@ -711,7 +711,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t require.Equal(t, test.configuredMaxMessageBytes, options.MaxMessageBytes) require.Equal(t, test.configuredMaxMessageBytes, options.MaxBatchedBytes) - err = adjustOptions(changefeedID, adminClient, options, topicName) + err = adjustOptions(t.Context(), changefeedID, adminClient, options, topicName) require.NoError(t, err) require.NotEqual(t, test.configuredMaxMessageBytes, options.MaxMessageBytes) @@ -729,9 +729,9 @@ func TestValidateReplicationFactor(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) gomock.InOrder( - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("2", true, nil), - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("", false, nil), ) @@ -740,7 +740,7 @@ func TestValidateReplicationFactor(t *testing.T) { ReplicationFactor: 1, RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.Regexp( t, ".*`replication-factor` 1 is smaller than the `min.insync.replicas` 2 of broker.*", @@ -752,7 +752,7 @@ func TestValidateReplicationFactor(t *testing.T) { ReplicationFactor: 1, RequiredAcks: WaitForLocal, } - err = localAcksConfig.ValidateReplicationFactor(adminClient) + err = localAcksConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) missingBrokerConfig := &AutoCreateTopicConfig{ @@ -760,13 +760,13 @@ func TestValidateReplicationFactor(t *testing.T) { ReplicationFactor: 1, RequiredAcks: WaitForAll, } - err = missingBrokerConfig.ValidateReplicationFactor(adminClient) + err = missingBrokerConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) t.Run("replication factor satisfies min insync replicas", func(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("2", true, nil) topicConfig := &AutoCreateTopicConfig{ @@ -774,14 +774,14 @@ func TestValidateReplicationFactor(t *testing.T) { RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) }) t.Run("invalid min insync replicas", func(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("invalid", true, nil) topicConfig := &AutoCreateTopicConfig{ @@ -789,7 +789,7 @@ func TestValidateReplicationFactor(t *testing.T) { RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) }) @@ -800,7 +800,7 @@ func TestValidateReplicationFactor(t *testing.T) { "describe-config", MinInsyncReplicasConfigName, ) - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("", false, lookupErr) topicConfig := &AutoCreateTopicConfig{ @@ -808,7 +808,7 @@ func TestValidateReplicationFactor(t *testing.T) { RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) }) } @@ -973,7 +973,7 @@ func TestConfigurationCombinations(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) - metadataCall := adminClient.EXPECT().GetTopicsMeta([]string{topic}, true) + metadataCall := adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{topic}, true) sourceMaxMessageBytes := a.brokerMessageMaxBytes if topic == defaultMockTopicName { metadataCall.Return(map[string]TopicDetail{ @@ -981,7 +981,7 @@ func TestConfigurationCombinations(t *testing.T) { }, nil) gomock.InOrder( metadataCall, - adminClient.EXPECT().GetTopicConfig(topic, TopicMaxMessageBytesConfigName). + adminClient.EXPECT().GetTopicConfig(gomock.Any(), topic, TopicMaxMessageBytesConfigName). Return(a.topicMaxMessageBytes, true, nil), ) sourceMaxMessageBytes = a.topicMaxMessageBytes @@ -989,7 +989,7 @@ func TestConfigurationCombinations(t *testing.T) { metadataCall.Return(map[string]TopicDetail{}, nil) gomock.InOrder( metadataCall, - adminClient.EXPECT().GetBrokerConfig(BrokerMessageMaxBytesConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), BrokerMessageMaxBytesConfigName). Return(a.brokerMessageMaxBytes, true, nil), ) } @@ -1002,7 +1002,7 @@ func TestConfigurationCombinations(t *testing.T) { expectedMaxMessageBytes, err := strconv.Atoi(sourceMaxMessageBytes) require.NoError(t, err) changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") - err = adjustOptions(changefeedID, adminClient, options, topic) + err = adjustOptions(t.Context(), changefeedID, adminClient, options, topic) require.Nil(t, err) require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) require.Equal( diff --git a/pkg/sink/kafka/sarama_admin_test.go b/pkg/sink/kafka/sarama_admin_test.go index eba888f002..9afa80c407 100644 --- a/pkg/sink/kafka/sarama_admin_test.go +++ b/pkg/sink/kafka/sarama_admin_test.go @@ -45,7 +45,7 @@ func TestGetBrokerConfig(t *testing.T) { changefeed: common.NewChangeFeedIDWithName("test", "default"), admin: admin, } - value, found, err := client.GetBrokerConfig("message.max.bytes") + value, found, err := client.GetBrokerConfig(t.Context(), "message.max.bytes") require.NoError(t, err) require.True(t, found) @@ -62,7 +62,7 @@ func TestGetBrokerConfig(t *testing.T) { changefeed: common.NewChangeFeedIDWithName("test", "default"), admin: admin, } - value, found, err := client.GetBrokerConfig("missing") + value, found, err := client.GetBrokerConfig(t.Context(), "missing") require.NoError(t, err) require.False(t, found) @@ -79,7 +79,7 @@ func TestGetBrokerConfig(t *testing.T) { changefeed: common.NewChangeFeedIDWithName("test", "default"), admin: admin, } - _, _, err := client.GetBrokerConfig("missing") + _, _, err := client.GetBrokerConfig(t.Context(), "missing") require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) require.ErrorIs(t, err, cause) @@ -104,7 +104,7 @@ func TestGetTopicConfig(t *testing.T) { admin: admin, } - value, found, err := client.GetTopicConfig("test-topic", "max.message.bytes") + value, found, err := client.GetTopicConfig(t.Context(), "test-topic", "max.message.bytes") require.NoError(t, err) require.True(t, found) @@ -120,7 +120,7 @@ func TestGetTopicConfig(t *testing.T) { admin: admin, } - value, found, err := client.GetTopicConfig("test-topic", "missing") + value, found, err := client.GetTopicConfig(t.Context(), "test-topic", "missing") require.NoError(t, err) require.False(t, found) @@ -136,7 +136,7 @@ func TestGetTopicConfig(t *testing.T) { admin: admin, } - _, _, err := client.GetTopicConfig("test-topic", "missing") + _, _, err := client.GetTopicConfig(t.Context(), "test-topic", "missing") require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) require.ErrorIs(t, err, context.DeadlineExceeded) @@ -165,7 +165,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"valid-topic", "missing-topic"}, false) + topics, err := client.GetTopicsMeta(t.Context(), []string{"valid-topic", "missing-topic"}, false) require.Nil(t, topics) require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) @@ -190,7 +190,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"valid-topic", "missing-topic"}, true) + topics, err := client.GetTopicsMeta(t.Context(), []string{"valid-topic", "missing-topic"}, true) require.NoError(t, err) require.Equal(t, map[string]TopicDetail{ @@ -210,7 +210,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"missing-topic"}, false) + topics, err := client.GetTopicsMeta(t.Context(), []string{"missing-topic"}, false) require.NoError(t, err) require.Empty(t, topics) @@ -227,7 +227,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + _, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, false) require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) require.ErrorIs(t, err, sarama.ErrInvalidTopic) @@ -245,7 +245,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + _, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, false) require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) require.NotErrorIs(t, err, errors.ErrKafkaAdminAPI) @@ -265,7 +265,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + _, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, false) require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) require.NotErrorIs(t, err, errors.ErrKafkaAdminAPI) @@ -284,7 +284,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"test-topic"}, true) + topics, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, true) require.NoError(t, err) require.Empty(t, topics) @@ -387,7 +387,7 @@ func TestCreateTopic(t *testing.T) { admin: admin, } - err := client.CreateTopic(&TopicDetail{ + err := client.CreateTopic(t.Context(), &TopicDetail{ Name: "test-topic", NumPartitions: 3, ReplicationFactor: 2, diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index e86cdc78a3..87d5faf2cc 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -32,6 +32,8 @@ type saramaFactory struct { metricRegistry metrics.Registry } +func (*saramaFactory) CleanupMetrics() {} + // NewSaramaFactory constructs a Factory with sarama implementation. func NewSaramaFactory( ctx context.Context, @@ -59,7 +61,7 @@ func NewSaramaFactory( admin.Close() }() - if err = adjustOptions(changefeedID, admin, o, o.Topic); err != nil { + if err = adjustOptions(ctx, changefeedID, admin, o, o.Topic); err != nil { return nil, err } log.Info("kafka sink configuration resolved", diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go index a831747f4e..fc2dffa6a2 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go @@ -152,22 +152,6 @@ func TestTokenProviderUsesOAuthCA(t *testing.T) { require.Equal(t, "access-token", token.Token) } -func TestFranzConfigUsesOAuthCA(t *testing.T) { - t.Parallel() - - server := newTLSTokenServer(t) - caPath := writeServerCA(t, server) - options := newOAuthOptions(server.URL, caPath) - options.sasl.mechanism = oauthMechanism - config, err := newFranzConfig(options) - require.NoError(t, err) - require.NotNil(t, config.SASL.OAuth2.HTTPClient) - - response, err := config.SASL.OAuth2.HTTPClient.Get(server.URL) - require.NoError(t, err) - require.NoError(t, response.Body.Close()) -} - func TestTokenProviderRejectsInvalidOAuthCA(t *testing.T) { t.Parallel() diff --git a/pkg/sink/kafka/selector.go b/pkg/sink/kafka/selector.go index 4cbf67bef4..a26c0664d3 100644 --- a/pkg/sink/kafka/selector.go +++ b/pkg/sink/kafka/selector.go @@ -25,12 +25,5 @@ func NewFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedI if o.Client == KafkaClientSarama { return NewSaramaFactory(ctx, o, changefeedID) } - return NewFranzFactory(ctx, o, changefeedID) -} - -// CleanupFactoryMetrics removes metrics owned directly by a client factory. -func CleanupFactoryMetrics(factory Factory) { - if cleaner, ok := factory.(interface{ CleanupMetrics() }); ok { - cleaner.CleanupMetrics() - } + return newFactory(ctx, o, changefeedID) } diff --git a/pkg/sink/kafka/selector_test.go b/pkg/sink/kafka/selector_test.go index 774cf0c829..298e07b63f 100644 --- a/pkg/sink/kafka/selector_test.go +++ b/pkg/sink/kafka/selector_test.go @@ -21,13 +21,12 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kfake" ) -func TestIsUnretryableFranzError(t *testing.T) { +func TestIsUnretryableClientError(t *testing.T) { t.Parallel() tests := []struct { @@ -58,7 +57,7 @@ func TestIsUnretryableFranzError(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - require.Equal(t, test.unretryable, IsUnretryableFranzError(test.err)) + require.Equal(t, test.unretryable, IsUnretryableKafkaError(test.err)) }) } } @@ -73,7 +72,7 @@ func TestFactorySelection(t *testing.T) { client string expected Factory }{ - {client: KafkaClientFranz, expected: &franzFactoryAdapter{}}, + {client: KafkaClientFranz, expected: &franzFactory{}}, {client: KafkaClientSarama, expected: &saramaFactory{}}, } { t.Run(test.client, func(t *testing.T) { @@ -87,7 +86,7 @@ func TestFactorySelection(t *testing.T) { require.NoError(t, err) require.IsType(t, test.expected, factory) - CleanupFactoryMetrics(factory) + factory.CleanupMetrics() }) } } @@ -110,53 +109,6 @@ func TestFranzIgnoresConfiguredKafkaVersion(t *testing.T) { common.NewChangefeedID4Test(common.DefaultKeyspaceName, "version-negotiation"), ) require.NoError(t, err) - require.IsType(t, &franzFactoryAdapter{}, factory) - CleanupFactoryMetrics(factory) -} - -func TestFranzAndSaramaFactoriesAreIndependent(t *testing.T) { - const topic = "factory-independence" - cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) - defer cluster.Close() - - franzOptions := NewOptions() - franzOptions.ClientID = "ticdc-franz-test" - franzOptions.BrokerEndpoints = cluster.ListenAddrs() - franzOptions.Topic = topic - - franzFactory, err := NewFactory( - context.Background(), - franzOptions, - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "franz"), - ) - require.NoError(t, err) - t.Cleanup(func() { CleanupFactoryMetrics(franzFactory) }) - - saramaOptions := NewOptions() - saramaOptions.Client = KafkaClientSarama - saramaOptions.ClientID = "ticdc-sarama-test" - saramaOptions.BrokerEndpoints = cluster.ListenAddrs() - saramaOptions.Topic = topic - - saramaFactory, err := NewFactory( - context.Background(), - saramaOptions, - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sarama"), - ) - require.NoError(t, err) - - franzProducer, err := franzFactory.SyncProducer(context.Background()) - require.NoError(t, err) - t.Cleanup(franzProducer.Close) - - saramaProducer, err := saramaFactory.SyncProducer(context.Background()) - require.NoError(t, err) - t.Cleanup(saramaProducer.Close) - - message := &codeccommon.Message{Value: []byte("value")} - require.NoError(t, franzProducer.SendMessage(topic, 0, message)) - require.NoError(t, saramaProducer.SendMessage(topic, 0, message)) - - franzProducer.Close() - require.NoError(t, saramaProducer.SendMessage(topic, 0, message)) + require.IsType(t, &franzFactory{}, factory) + factory.CleanupMetrics() } From 8bfc31c6e631b19875332f332f729c2f3c4da114 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 2 Sep 2026 17:43:19 +0800 Subject: [PATCH 47/61] kafka: pass context to sync producer --- downstreamadapter/sink/kafka/sink.go | 8 +++--- downstreamadapter/sink/kafka/sink_test.go | 20 +++++++-------- pkg/sink/kafka/factory.go | 4 +-- pkg/sink/kafka/factory_mock.go | 16 ++++++------ pkg/sink/kafka/franz_sync_producer.go | 18 +++++++++----- pkg/sink/kafka/franz_sync_producer_test.go | 27 +++++++++++++++++---- pkg/sink/kafka/sarama_sync_producer.go | 9 +++++-- pkg/sink/kafka/sarama_sync_producer_test.go | 8 +++--- 8 files changed, 69 insertions(+), 41 deletions(-) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 6e00857de2..b4471019e2 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -466,11 +466,11 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { ddlType := e.GetDDLType().String() if s.partitionRule == helper.PartitionAll { err = s.statistics.RecordDDLExecution(func() (string, error) { - return ddlType, s.ddlProducer.SendMessages(topic, partitionNum, message) + return ddlType, s.ddlProducer.SendMessages(s.ctx, topic, partitionNum, message) }) } else { err = s.statistics.RecordDDLExecution(func() (string, error) { - return ddlType, s.ddlProducer.SendMessage(topic, 0, message) + return ddlType, s.ddlProducer.SendMessage(s.ctx, topic, 0, message) }) } if err != nil { @@ -536,7 +536,7 @@ func (s *sink) sendCheckpoint(ctx context.Context) error { if err != nil { return err } - err = s.ddlProducer.SendMessages(topic, partitionNum, msg) + err = s.ddlProducer.SendMessages(ctx, topic, partitionNum, msg) if err != nil { return err } @@ -547,7 +547,7 @@ func (s *sink) sendCheckpoint(ctx context.Context) error { if err != nil { return err } - err = s.ddlProducer.SendMessages(topic, partitionNum, msg) + err = s.ddlProducer.SendMessages(ctx, topic, partitionNum, msg) if err != nil { return err } diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 13f163569a..5fb3689b3b 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -194,7 +194,7 @@ func TestKafkaSinkBasicFunctionality(t *testing.T) { } return nil }).Times(2) - syncProducer.EXPECT().SendMessages(gomock.Any(), int32(1), gomock.Any()).Return(nil) + syncProducer.EXPECT().SendMessages(gomock.Any(), gomock.Any(), int32(1), gomock.Any()).Return(nil) defer cancel() go kafkaSink.Run(ctx) @@ -429,8 +429,8 @@ func TestKafkaSinkDDL(t *testing.T) { kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(4), nil) - syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(4), gomock.Any()). - DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { + syncProducer.EXPECT().SendMessages(gomock.Any(), kafkaSinkTestTopic, int32(4), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ int32, message *codecCommon.Message) error { require.NotEmpty(t, message.Key) require.NotEmpty(t, message.Value) return nil @@ -443,8 +443,8 @@ func TestKafkaSinkDDL(t *testing.T) { kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( t, t.Context(), config.ProtocolCanalJSON, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(4), nil) - syncProducer.EXPECT().SendMessage(kafkaSinkTestTopic, int32(0), gomock.Any()). - DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { + syncProducer.EXPECT().SendMessage(gomock.Any(), kafkaSinkTestTopic, int32(0), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ int32, message *codecCommon.Message) error { require.NotEmpty(t, message.Value) return nil }) @@ -466,7 +466,7 @@ func TestKafkaSinkDDL(t *testing.T) { t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) cause := errors.ErrKafkaSendMessage.GenWithStackByArgs() topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(2), nil) - syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(2), gomock.Any()).Return(cause) + syncProducer.EXPECT().SendMessages(gomock.Any(), kafkaSinkTestTopic, int32(2), gomock.Any()).Return(cause) require.Equal(t, cause, kafkaSink.WriteBlockEvent(ddlEvent)) require.False(t, kafkaSink.IsNormal()) @@ -494,8 +494,8 @@ func TestKafkaSinkCheckpoint(t *testing.T) { kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(3), nil) - syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(3), gomock.Any()). - DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { + syncProducer.EXPECT().SendMessages(gomock.Any(), kafkaSinkTestTopic, int32(3), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ int32, message *codecCommon.Message) error { require.NotEmpty(t, message.Key) return nil }) @@ -521,7 +521,7 @@ func TestKafkaSinkCheckpoint(t *testing.T) { partitionCounts := map[string]int32{"topic-a": 2, "topic-b": 3, kafkaSinkTestTopic: 4} for topic, partitionCount := range partitionCounts { topicManager.EXPECT().GetPartitionNum(gomock.Any(), topic).Return(partitionCount, nil) - syncProducer.EXPECT().SendMessages(topic, partitionCount, gomock.Any()).Return(nil) + syncProducer.EXPECT().SendMessages(gomock.Any(), topic, partitionCount, gomock.Any()).Return(nil) } kafkaSink.checkpointChan <- 100 close(kafkaSink.checkpointChan) @@ -553,7 +553,7 @@ func TestKafkaSinkCheckpoint(t *testing.T) { // return the error and stop, so exactly one GetPartitionNum and one // SendMessages call are expected regardless of the topic order. topicManager.EXPECT().GetPartitionNum(gomock.Any(), gomock.Any()).Return(int32(2), nil) - syncProducer.EXPECT().SendMessages(gomock.Any(), int32(2), gomock.Any()).Return(cause) + syncProducer.EXPECT().SendMessages(gomock.Any(), gomock.Any(), int32(2), gomock.Any()).Return(cause) kafkaSink.checkpointChan <- 100 require.Equal(t, cause, kafkaSink.sendCheckpoint(t.Context())) diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index 83d5ec8cb7..aeb16bc7a7 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -38,13 +38,13 @@ type SyncProducer interface { // SendMessage produces a given message, and returns only when it either has // succeeded or failed to produce. It will return the partition and the offset // of the produced message, or an error if the message failed to produce. - SendMessage(topic string, partitionNum int32, message *common.Message) error + SendMessage(ctx context.Context, topic string, partitionNum int32, message *common.Message) error // SendMessages produces a given set of messages, and returns only when all // messages in the set have either succeeded or failed. Note that messages // can succeed and fail individually; if some succeed and some fail, // SendMessages will return an error. - SendMessages(topic string, partitionNum int32, message *common.Message) error + SendMessages(ctx context.Context, topic string, partitionNum int32, message *common.Message) error // Close shuts down the producer; you must call this function before a producer // object passes out of scope, as it may otherwise leak memory. diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index baf1ca289e..e1d892fb00 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -142,31 +142,31 @@ func (mr *MockSyncProducerMockRecorder) Close() *gomock.Call { } // SendMessage mocks base method. -func (m *MockSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { +func (m *MockSyncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *common.Message) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessage", topic, partitionNum, message) + ret := m.ctrl.Call(m, "SendMessage", ctx, topic, partitionNum, message) ret0, _ := ret[0].(error) return ret0 } // SendMessage indicates an expected call of SendMessage. -func (mr *MockSyncProducerMockRecorder) SendMessage(topic, partitionNum, message interface{}) *gomock.Call { +func (mr *MockSyncProducerMockRecorder) SendMessage(ctx, topic, partitionNum, message interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), topic, partitionNum, message) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), ctx, topic, partitionNum, message) } // SendMessages mocks base method. -func (m *MockSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { +func (m *MockSyncProducer) SendMessages(ctx context.Context, topic string, partitionNum int32, message *common.Message) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessages", topic, partitionNum, message) + ret := m.ctrl.Call(m, "SendMessages", ctx, topic, partitionNum, message) ret0, _ := ret[0].(error) return ret0 } // SendMessages indicates an expected call of SendMessages. -func (mr *MockSyncProducerMockRecorder) SendMessages(topic, partitionNum, message interface{}) *gomock.Call { +func (mr *MockSyncProducerMockRecorder) SendMessages(ctx, topic, partitionNum, message interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), topic, partitionNum, message) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), ctx, topic, partitionNum, message) } // MockAsyncProducer is a mock of AsyncProducer interface. diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go index c88927927b..5b84511441 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -51,7 +51,9 @@ func newSyncProducer( }, nil } -func (p *syncProducer) SendMessage(topic string, partitionNum int32, message *codeccommon.Message) error { +func (p *syncProducer) SendMessage( + ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message, +) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -62,10 +64,12 @@ func (p *syncProducer) SendMessage(topic string, partitionNum int32, message *co Key: message.Key, Value: message.Value, } - return p.sendRecords(message, record) + return p.sendRecords(ctx, message, record) } -func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *codeccommon.Message) error { +func (p *syncProducer) SendMessages( + ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message, +) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -80,11 +84,13 @@ func (p *syncProducer) SendMessages(topic string, partitionNum int32, message *c }) } - return p.sendRecords(message, records...) + return p.sendRecords(ctx, message, records...) } -func (p *syncProducer) sendRecords(message *codeccommon.Message, records ...*kgo.Record) error { - ctx, cancel := context.WithTimeout(p.client.Context(), p.timeout) +func (p *syncProducer) sendRecords( + ctx context.Context, message *codeccommon.Message, records ...*kgo.Record, +) error { + ctx, cancel := context.WithTimeout(ctx, p.timeout) defer cancel() err := p.client.ProduceSync(ctx, records...).FirstErr() diff --git a/pkg/sink/kafka/franz_sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go index ce62f951e6..c4701c6f84 100644 --- a/pkg/sink/kafka/franz_sync_producer_test.go +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -30,10 +30,10 @@ func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { producer := &syncProducer{} producer.closed.Store(true) - err := producer.SendMessage("topic", 1, &codeccommon.Message{}) + err := producer.SendMessage(t.Context(), "topic", 1, &codeccommon.Message{}) require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) - err = producer.SendMessages("topic", 1, &codeccommon.Message{}) + err = producer.SendMessages(t.Context(), "topic", 1, &codeccommon.Message{}) require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) } @@ -50,8 +50,8 @@ func TestSyncProducerSendsToRequestedPartitions(t *testing.T) { require.NoError(t, err) defer producer.Close() - require.NoError(t, producer.SendMessage(topic, 2, &codeccommon.Message{Key: []byte("key"), Value: []byte("value")})) - require.NoError(t, producer.SendMessages(topic, 3, &codeccommon.Message{Value: []byte("all")})) + require.NoError(t, producer.SendMessage(t.Context(), topic, 2, &codeccommon.Message{Key: []byte("key"), Value: []byte("value")})) + require.NoError(t, producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("all")})) } func TestSyncProducerReturnsPartialFailure(t *testing.T) { @@ -71,11 +71,28 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { require.NoError(t, err) defer producer.Close() - err = producer.SendMessages(topic, 3, &codeccommon.Message{Value: []byte("value")}) + err = producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("value")}) require.ErrorIs(t, err, errors.ErrKafkaSendMessage) require.ErrorIs(t, err, kerr.InvalidTopicException) } +func TestSyncProducerUsesSendContext(t *testing.T) { + producer, err := newSyncProducer( + t.Context(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), + testOptions([]string{"127.0.0.1:1"}), + ) + require.NoError(t, err) + defer producer.Close() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + err = producer.SendMessage(ctx, "topic", 0, &codeccommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) + require.ErrorIs(t, err, context.Canceled) +} + func TestSyncProducerCloseIsIdempotent(t *testing.T) { client, err := newSyncProducer( context.Background(), diff --git a/pkg/sink/kafka/sarama_sync_producer.go b/pkg/sink/kafka/sarama_sync_producer.go index fcf1c9c258..dd482dcab8 100644 --- a/pkg/sink/kafka/sarama_sync_producer.go +++ b/pkg/sink/kafka/sarama_sync_producer.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "time" "github.com/IBM/sarama" @@ -43,7 +44,9 @@ type saramaSyncProducer struct { closed *atomic.Bool } -func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, message *codecCommon.Message) error { +func (p *saramaSyncProducer) SendMessage( + _ context.Context, topic string, partitionNum int32, message *codecCommon.Message, +) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -66,7 +69,9 @@ func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, messa return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, message *codecCommon.Message) error { +func (p *saramaSyncProducer) SendMessages( + _ context.Context, topic string, partitionNum int32, message *codecCommon.Message, +) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } diff --git a/pkg/sink/kafka/sarama_sync_producer_test.go b/pkg/sink/kafka/sarama_sync_producer_test.go index e00e0dafa8..92e93c8a8c 100644 --- a/pkg/sink/kafka/sarama_sync_producer_test.go +++ b/pkg/sink/kafka/sarama_sync_producer_test.go @@ -32,8 +32,8 @@ func TestProducerRejectsSendAfterClose(t *testing.T) { message := &codecCommon.Message{} syncProducer := &saramaSyncProducer{closed: atomic.NewBool(true)} - require.ErrorIs(t, syncProducer.SendMessage("topic", 1, message), errors.ErrKafkaSinkClosed) - require.ErrorIs(t, syncProducer.SendMessages("topic", 1, message), errors.ErrKafkaSinkClosed) + require.ErrorIs(t, syncProducer.SendMessage(t.Context(), "topic", 1, message), errors.ErrKafkaSinkClosed) + require.ErrorIs(t, syncProducer.SendMessages(t.Context(), "topic", 1, message), errors.ErrKafkaSinkClosed) asyncProducer := &saramaAsyncProducer{closed: atomic.NewBool(true)} require.ErrorIs(t, asyncProducer.AsyncSend(context.Background(), "topic", 0, message), errors.ErrKafkaSinkClosed) @@ -88,7 +88,7 @@ func TestSyncProducerErrorWrappedOnce(t *testing.T) { producer.EXPECT().SendMessage(gomock.Any()).Return(int32(0), int64(0), cause) }, send: func(producer *saramaSyncProducer, message *codecCommon.Message) error { - return producer.SendMessage("topic", 0, message) + return producer.SendMessage(t.Context(), "topic", 0, message) }, }, { @@ -97,7 +97,7 @@ func TestSyncProducerErrorWrappedOnce(t *testing.T) { producer.EXPECT().SendMessages(gomock.Any()).Return(cause) }, send: func(producer *saramaSyncProducer, message *codecCommon.Message) error { - return producer.SendMessages("topic", 1, message) + return producer.SendMessages(t.Context(), "topic", 1, message) }, }, } From ce78340b24ed0a4b4eb0ed146e3ba136079ce56d Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 2 Sep 2026 18:31:25 +0800 Subject: [PATCH 48/61] kafka: simplify franz-go factory configuration --- pkg/sink/kafka/factory.go | 17 ++- pkg/sink/kafka/franz_admin.go | 35 ++---- pkg/sink/kafka/franz_admin_test.go | 50 ++++++++- pkg/sink/kafka/franz_async_producer.go | 21 +--- pkg/sink/kafka/franz_async_producer_test.go | 8 +- pkg/sink/kafka/franz_config.go | 76 +++++-------- pkg/sink/kafka/franz_config_test.go | 76 +++++++++---- pkg/sink/kafka/franz_factory.go | 33 ++++-- pkg/sink/kafka/franz_gssapi.go | 30 ++---- pkg/sink/kafka/franz_logger.go | 3 - pkg/sink/kafka/franz_metrics_hook.go | 48 ++------- pkg/sink/kafka/franz_sync_producer.go | 23 ++-- pkg/sink/kafka/franz_sync_producer_test.go | 21 +++- pkg/sink/kafka/options_test.go | 31 ++++++ pkg/sink/kafka/sarama_factory.go | 12 +-- pkg/sink/kafka/selector.go | 29 ----- pkg/sink/kafka/selector_test.go | 114 -------------------- 17 files changed, 264 insertions(+), 363 deletions(-) delete mode 100644 pkg/sink/kafka/selector.go delete mode 100644 pkg/sink/kafka/selector_test.go diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index aeb16bc7a7..dc394d4b7b 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -16,9 +16,18 @@ package kafka import ( "context" - "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/common" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" ) +// NewFactory selects the Kafka client for one changefeed without automatic fallback. +func NewFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { + if o.Client == KafkaClientSarama { + return newSaramaFactory(ctx, o, changefeedID) + } + return newFranzFactory(ctx, o, changefeedID) +} + // Factory is used to produce all kafka components. type Factory interface { // AdminClient return a kafka cluster admin client @@ -38,13 +47,13 @@ type SyncProducer interface { // SendMessage produces a given message, and returns only when it either has // succeeded or failed to produce. It will return the partition and the offset // of the produced message, or an error if the message failed to produce. - SendMessage(ctx context.Context, topic string, partitionNum int32, message *common.Message) error + SendMessage(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error // SendMessages produces a given set of messages, and returns only when all // messages in the set have either succeeded or failed. Note that messages // can succeed and fail individually; if some succeed and some fail, // SendMessages will return an error. - SendMessages(ctx context.Context, topic string, partitionNum int32, message *common.Message) error + SendMessages(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error // Close shuts down the producer; you must call this function before a producer // object passes out of scope, as it may otherwise leak memory. @@ -63,7 +72,7 @@ type AsyncProducer interface { // AsyncSend is the input channel for the user to write messages to that they // wish to send. - AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error + AsyncSend(ctx context.Context, topic string, partition int32, message *codecCommon.Message) error // AsyncRunCallback process the messages that has sent to kafka, // and run tha attached callback. the caller should call this diff --git a/pkg/sink/kafka/franz_admin.go b/pkg/sink/kafka/franz_admin.go index d648ace2d4..ae6305afe2 100644 --- a/pkg/sink/kafka/franz_admin.go +++ b/pkg/sink/kafka/franz_admin.go @@ -33,15 +33,13 @@ type admin struct { timeout time.Duration } -func newAdmin( - ctx context.Context, - changefeedID common.ChangeFeedID, - o *options, -) (*admin, error) { - opts, err := newClientOptions(ctx, changefeedID, "admin", o, nil) - if err != nil { - return nil, err - } +func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt, timeout time.Duration) (*admin, error) { + opts := make([]kgo.Opt, 0, len(clientOpts)+3) + opts = append(opts, clientOpts...) + opts = append(opts, + kgo.WithContext(ctx), + kgo.WithLogger(newClientLogger(changefeedID, "admin")), + ) // MetadataMinAge is the minimum interval between metadata requests. // It must stay below the visibility retry interval to avoid retrying a cached topic-not-found result. opts = append(opts, kgo.MetadataMinAge(100*time.Millisecond)) @@ -54,7 +52,7 @@ func newAdmin( return &admin{ changefeed: changefeedID, admin: kadm.NewClient(client), - timeout: requestTimeout(o), + timeout: timeout, }, nil } @@ -71,7 +69,6 @@ func (a *admin) GetAllBrokers(ctx context.Context) []Broker { for id := range meta.Brokers { brokers = append(brokers, Broker{ID: int32(id)}) } - return brokers } @@ -84,7 +81,6 @@ func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, if isAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-cluster", "cluster") } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-cluster", "cluster") } @@ -97,7 +93,6 @@ func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, if isAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) } @@ -107,7 +102,6 @@ func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, if isAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) } @@ -115,7 +109,6 @@ func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, if isAuthorizationFailed(resource.Err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", configName) } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", configName) } @@ -124,7 +117,6 @@ func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, return entry.MaybeValue(), true, nil } } - return "", false, nil } @@ -137,7 +129,6 @@ func (a *admin) GetTopicConfig(ctx context.Context, topicName string, configName if isAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) } @@ -146,7 +137,6 @@ func (a *admin) GetTopicConfig(ctx context.Context, topicName string, configName if isAuthorizationFailed(err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) } @@ -154,7 +144,6 @@ func (a *admin) GetTopicConfig(ctx context.Context, topicName string, configName if isAuthorizationFailed(resource.Err) { return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", topicName) } - return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", topicName) } @@ -163,7 +152,6 @@ func (a *admin) GetTopicConfig(ctx context.Context, topicName string, configName return entry.MaybeValue(), true, nil } } - return "", false, nil } @@ -181,7 +169,6 @@ func (a *admin) GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicE if isAuthorizationFailed(err) { return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-topics", resource) } - return nil, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-topics", resource) } @@ -196,7 +183,6 @@ func topicDetailsFromMetadata(meta kadm.Metadata, topics []string, ignoreTopicEr if ignoreTopicError { continue } - return nil, errors.WrapError(errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", topic) } @@ -215,10 +201,8 @@ func topicDetailsFromMetadata(meta kadm.Metadata, topics []string, ignoreTopicEr if isAuthorizationFailed(detail.Err) { return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, detail.Err, "describe-topic", topic) } - return nil, errors.WrapError(errors.ErrKafkaAdminAPI, detail.Err, "describe-topic", topic) } - return result, nil } @@ -238,7 +222,6 @@ func (a *admin) GetTopicsPartitionsNum(ctx context.Context, topics []string) (ma for topic, detail := range details { partitions[topic] = detail.NumPartitions } - return partitions, nil } @@ -251,7 +234,6 @@ func (a *admin) CreateTopic(ctx context.Context, detail *TopicDetail) error { if isAuthorizationFailed(err) { return errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "create-topic", detail.Name) } - return errors.WrapError(errors.ErrKafkaAdminAPI, err, "create-topic", detail.Name) } @@ -275,7 +257,6 @@ func (a *admin) CreateTopic(ctx context.Context, detail *TopicDetail) error { if isAuthorizationFailed(resp.Err) { return errors.WrapError(errors.ErrKafkaAuthorizationFailed, resp.Err, "create-topic", detail.Name) } - return errors.WrapError(errors.ErrKafkaAdminAPI, resp.Err, "create-topic", detail.Name) } diff --git a/pkg/sink/kafka/franz_admin_test.go b/pkg/sink/kafka/franz_admin_test.go index 4f3a512308..eb4ba08294 100644 --- a/pkg/sink/kafka/franz_admin_test.go +++ b/pkg/sink/kafka/franz_admin_test.go @@ -15,6 +15,7 @@ package kafka import ( "context" + "io" "testing" "time" @@ -27,6 +28,42 @@ import ( "github.com/twmb/franz-go/pkg/kmsg" ) +func TestIsUnretryableClientError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + unretryable bool + }{ + {name: "unknown topic", err: kerr.UnknownTopicOrPartition}, + {name: "leader unavailable", err: kerr.LeaderNotAvailable}, + {name: "request timeout", err: kerr.RequestTimedOut}, + {name: "network exception", err: kerr.NetworkException}, + {name: "controller changed", err: kerr.NotController}, + {name: "EOF", err: io.EOF}, + {name: "invalid topic", err: kerr.InvalidTopicException, unretryable: true}, + {name: "invalid config", err: kerr.InvalidConfig, unretryable: true}, + {name: "SASL authentication failure", err: kerr.SaslAuthenticationFailed, unretryable: true}, + {name: "unsupported SASL mechanism", err: kerr.UnsupportedSaslMechanism, unretryable: true}, + {name: "illegal SASL state", err: kerr.IllegalSaslState, unretryable: true}, + {name: "unsupported version", err: kerr.UnsupportedVersion, unretryable: true}, + {name: "invalid request", err: kerr.InvalidRequest, unretryable: true}, + { + name: "wrapped invalid topic", + err: errors.WrapError(errors.ErrKafkaAdminAPI, kerr.InvalidTopicException, "describe-topic", "test-topic"), + unretryable: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, test.unretryable, IsUnretryableKafkaError(test.err)) + }) + } +} + func TestFranzTopicDetailsFromMetadata(t *testing.T) { t.Parallel() @@ -142,10 +179,12 @@ func TestFranzIsAuthorizationFailed(t *testing.T) { } func TestAdminHonorsCallContext(t *testing.T) { + o := testOptions([]string{"127.0.0.1:1"}) admin, err := newAdmin( t.Context(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "context"), - testOptions([]string{"127.0.0.1:1"}), + testClientOptions(t, o), + requestTimeout(o), ) require.NoError(t, err) t.Cleanup(admin.Close) @@ -162,11 +201,13 @@ func TestAdminOperations(t *testing.T) { ctx := t.Context() cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, existingTopic)) defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) admin, err := newAdmin( ctx, common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), - testOptions(cluster.ListenAddrs()), + testClientOptions(t, o), + requestTimeout(o), ) require.NoError(t, err) defer admin.Close() @@ -219,11 +260,13 @@ func TestCreateTopicErrors(t *testing.T) { ctx := t.Context() cluster := kfake.MustCluster(kfake.NumBrokers(1)) defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) admin, err := newAdmin( ctx, common.NewChangefeedID4Test(common.DefaultKeyspaceName, "create-errors"), - testOptions(cluster.ListenAddrs()), + testClientOptions(t, o), + requestTimeout(o), ) require.NoError(t, err) defer admin.Close() @@ -262,7 +305,6 @@ func TestCreateTopicErrors(t *testing.T) { topic := kmsg.NewCreateTopicsResponseTopic() topic.Topic, topic.ErrorCode = detail.Name, test.code response.Topics = append(response.Topics, topic) - return response, nil, true }) diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index d730f11427..96821d4609 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -35,12 +35,8 @@ type asyncProducer struct { errCh chan error } -func newAsyncProducer( - ctx context.Context, - changefeedID common.ChangeFeedID, - o *options, -) (*asyncProducer, error) { - client, err := newProducerClient(ctx, changefeedID, "async-producer", o) +func newAsyncProducer(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt, producerOpts []kgo.Opt) (*asyncProducer, error) { + client, err := newProducerClient(ctx, changefeedID, "async-producer", clientOpts, producerOpts) if err != nil { return nil, err } @@ -67,12 +63,7 @@ func (p *asyncProducer) Close() { zap.Duration("duration", time.Since(start))) } -func (p *asyncProducer) AsyncSend( - ctx context.Context, - topic string, - partition int32, - message *codeccommon.Message, -) error { +func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -104,14 +95,10 @@ func (p *asyncProducer) AsyncSend( } p.client.Produce(ctx, record, promise) - return nil } -func (p *asyncProducer) enqueueAsyncSendError( - logInfo *codeccommon.MessageLogInfo, - err error, -) { +func (p *asyncProducer) enqueueAsyncSendError(logInfo *codeccommon.MessageLogInfo, err error) { log.Error("kafka message send failed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index f7cdd3a44a..9044edd1db 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -86,11 +86,13 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { const topic = "async-topic" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) producer, err := newAsyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), - testOptions(cluster.ListenAddrs()), + testClientOptions(t, o), + testProducerOptions(t, o), ) require.NoError(t, err) defer producer.Close() @@ -124,11 +126,13 @@ func TestAsyncProducerReportsProduceFailure(t *testing.T) { cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { return produceResponseWithError(req, 0, kerr.InvalidTopicException.Code) }) + o := testOptions(cluster.ListenAddrs()) producer, err := newAsyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), - testOptions(cluster.ListenAddrs()), + testClientOptions(t, o), + testProducerOptions(t, o), ) require.NoError(t, err) defer producer.Close() diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go index 16104615a2..44e6dca9aa 100644 --- a/pkg/sink/kafka/franz_config.go +++ b/pkg/sink/kafka/franz_config.go @@ -17,6 +17,7 @@ package kafka import ( "context" "crypto/tls" + "net/http" "net/url" "strings" "time" @@ -46,23 +47,12 @@ const ( func requestTimeout(o *options) time.Duration { return max(o.ReadTimeout, o.WriteTimeout) } -func newClientOptions( - ctx context.Context, - changefeedID common.ChangeFeedID, - role string, - o *options, - hook *metricsHook, -) ([]kgo.Opt, error) { +func clientOptions(o *options) ([]kgo.Opt, error) { opts := []kgo.Opt{ - kgo.WithContext(ctx), kgo.SeedBrokers(o.BrokerEndpoints...), kgo.ClientID(o.ClientID), kgo.DialTimeout(o.DialTimeout), kgo.RequestTimeoutOverhead(requestTimeout(o)), - kgo.WithLogger(newClientLogger(changefeedID, role)), - } - if hook != nil { - opts = append(opts, kgo.WithHooks(hook)) } if o.EnableTLS { @@ -82,17 +72,16 @@ func newClientOptions( } if o.sasl != nil && o.sasl.mechanism != "" { - mechanism, err := buildSASLMechanism(ctx, o.sasl) + mechanism, err := buildSASLMechanism(o.sasl) if err != nil { return nil, err } opts = append(opts, kgo.SASL(mechanism)) } - return opts, nil } -func buildSASLMechanism(ctx context.Context, cfg *saslConfig) (sasl.Mechanism, error) { +func buildSASLMechanism(cfg *saslConfig) (sasl.Mechanism, error) { switch cfg.mechanism { case plainMechanism: return plain.Auth{User: cfg.user, Pass: cfg.password}.AsMechanism(), nil @@ -101,17 +90,7 @@ func buildSASLMechanism(ctx context.Context, cfg *saslConfig) (sasl.Mechanism, e case scram512Mechanism: return scram.Auth{User: cfg.user, Pass: cfg.password}.AsSha512Mechanism(), nil case oauthMechanism: - tokenSource, err := newOAuthTokenSource(ctx, cfg.oauth2) - if err != nil { - return nil, err - } - return oauth.Oauth(func(context.Context) (oauth.Auth, error) { - token, err := tokenSource.Token() - if err != nil { - return oauth.Auth{}, errors.WrapError(errors.ErrNewKafkaSink, err) - } - return oauth.Auth{Token: token.AccessToken}, nil - }), nil + return buildOAuthMechanism(cfg.oauth2) case gssapiMechanism: return buildGSSAPIMechanism(cfg.gssapi) default: @@ -120,13 +99,14 @@ func buildSASLMechanism(ctx context.Context, cfg *saslConfig) (sasl.Mechanism, e } } -func newOAuthTokenSource(ctx context.Context, cfg oauth2Config) (oauth2.TokenSource, error) { +func buildOAuthMechanism(cfg oauth2Config) (sasl.Mechanism, error) { + var httpClient *http.Client if cfg.caPath != "" { - httpClient, err := oauthHTTPClient(cfg.caPath) + var err error + httpClient, err = oauthHTTPClient(cfg.caPath) if err != nil { return nil, err } - ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) } endpointParams := url.Values{} @@ -147,26 +127,29 @@ func newOAuthTokenSource(ctx context.Context, cfg oauth2Config) (oauth2.TokenSou EndpointParams: endpointParams, Scopes: cfg.scopes, } - return config.TokenSource(ctx), nil + return oauth.Oauth(func(ctx context.Context) (oauth.Auth, error) { + if httpClient != nil { + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) + } + token, err := config.TokenSource(ctx).Token() + if err != nil { + return oauth.Auth{}, errors.WrapError(errors.ErrNewKafkaSink, err) + } + return oauth.Auth{Token: token.AccessToken}, nil + }), nil } -func newProducerClient( - ctx context.Context, - changefeedID common.ChangeFeedID, - role string, - o *options, -) (*kgo.Client, error) { - opts, err := newClientOptions(ctx, changefeedID, role, o, newMetricsHook(changefeedID)) - if err != nil { - return nil, err - } - - producerOpts, err := producerOptions(o) - if err != nil { - return nil, err - } +func newProducerClient(ctx context.Context, changefeedID common.ChangeFeedID, role string, clientOpts []kgo.Opt, producerOpts []kgo.Opt) (*kgo.Client, error) { + opts := make([]kgo.Opt, 0, len(clientOpts)+len(producerOpts)+3) + opts = append(opts, clientOpts...) + opts = append(opts, + kgo.WithContext(ctx), + kgo.WithLogger(newClientLogger(changefeedID, role)), + kgo.WithHooks(newMetricsHook(changefeedID)), + ) + opts = append(opts, producerOpts...) - client, err := kgo.NewClient(append(opts, producerOpts...)...) + client, err := kgo.NewClient(opts...) if err != nil { return nil, errors.WrapError(errors.ErrNewKafkaSink, err) } @@ -187,7 +170,6 @@ func producerOptions(o *options) ([]kgo.Opt, error) { maxBufferedBytes := max(defaultMaxBufferedBytes, o.MaxMessageBytes) maxBatchBytes := max(minProducerBatchBytes, o.MaxMessageBytes) maxBrokerWriteBytes := max(defaultBrokerWriteBytes, maxBatchBytes) - return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), kgo.RequiredAcks(requiredAcks(o.RequiredAcks)), diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index 2f284fd52b..8fc332b80b 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" "github.com/twmb/franz-go/pkg/kgo" ) @@ -40,6 +41,20 @@ func testOptions(brokers []string) *options { return o } +func testClientOptions(t *testing.T, o *options) []kgo.Opt { + t.Helper() + opts, err := clientOptions(o) + require.NoError(t, err) + return opts +} + +func testProducerOptions(t *testing.T, o *options) []kgo.Opt { + t.Helper() + opts, err := producerOptions(o) + require.NoError(t, err) + return opts +} + func TestFranzRequiredAcks(t *testing.T) { for _, test := range []struct { required RequiredAcks @@ -62,18 +77,34 @@ func TestFranzRequestTimeoutUsesLargerTimeout(t *testing.T) { require.Equal(t, 3*time.Second, requestTimeout(o)) } +func TestFranzIgnoresConfiguredKafkaVersion(t *testing.T) { + const topic = "version-negotiation" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + o := NewOptions() + o.ClientID = "ticdc-test" + o.BrokerEndpoints = cluster.ListenAddrs() + o.Topic = topic + o.Version = "invalid" + o.IsAssignedVersion = true + + factory, err := NewFactory( + context.Background(), + o, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "version-negotiation"), + ) + require.NoError(t, err) + require.IsType(t, &franzFactory{}, factory) + factory.CleanupMetrics() +} + func TestProducerOptionsBoundBufferAndBatch(t *testing.T) { const batchBytes = 1048588 o := testOptions([]string{"127.0.0.1:9092"}) o.MaxMessageBytes = batchBytes - opts, err := newClientOptions( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "config"), - "test", - o, - nil, - ) + opts, err := clientOptions(o) require.NoError(t, err) producerOpts, err := producerOptions(o) @@ -182,7 +213,7 @@ func TestBuildFranzGSSAPIMechanism(t *testing.T) { cfg.username = "alice" cfg.realm = "EXAMPLE.COM" - mechanism, err := buildSASLMechanism(context.Background(), &saslConfig{ + mechanism, err := buildSASLMechanism(&saslConfig{ mechanism: gssapiMechanism, gssapi: cfg, }) @@ -193,7 +224,7 @@ func TestBuildFranzGSSAPIMechanism(t *testing.T) { func TestBuildFranzSASLMechanisms(t *testing.T) { for _, mechanism := range []saslMechanism{plainMechanism, scram256Mechanism, scram512Mechanism} { - actual, err := buildSASLMechanism(context.Background(), &saslConfig{ + actual, err := buildSASLMechanism(&saslConfig{ mechanism: mechanism, user: "alice", password: "secret", @@ -202,7 +233,7 @@ func TestBuildFranzSASLMechanisms(t *testing.T) { require.Equal(t, string(mechanism), actual.Name()) } - _, err := buildSASLMechanism(context.Background(), &saslConfig{mechanism: "unknown"}) + _, err := buildSASLMechanism(&saslConfig{mechanism: "unknown"}) require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } @@ -223,19 +254,21 @@ func TestFranzOAuthTokenSource(t *testing.T) { })) defer server.Close() - source, err := newOAuthTokenSource(context.Background(), oauth2Config{ - clientID: "client", - clientSecret: "secret", - tokenURL: server.URL, - scopes: []string{"scope-a", "scope-b"}, - grantType: "custom", - audience: "audience", + mechanism, err := buildSASLMechanism(&saslConfig{ + mechanism: oauthMechanism, + oauth2: oauth2Config{ + clientID: "client", + clientSecret: "secret", + tokenURL: server.URL, + scopes: []string{"scope-a", "scope-b"}, + grantType: "custom", + audience: "audience", + }, }) require.NoError(t, err) - token, err := source.Token() + _, _, err = mechanism.Authenticate(context.Background(), "") require.NoError(t, err) - require.Equal(t, "token", token.AccessToken) form := <-request require.Equal(t, "custom", form.Get("grant_type")) @@ -244,6 +277,9 @@ func TestFranzOAuthTokenSource(t *testing.T) { } func TestFranzOAuthTokenSourceRejectsInvalidURL(t *testing.T) { - _, err := newOAuthTokenSource(context.Background(), oauth2Config{tokenURL: "http://example.com/%%"}) + _, err := buildSASLMechanism(&saslConfig{ + mechanism: oauthMechanism, + oauth2: oauth2Config{tokenURL: "http://example.com/%%"}, + }) require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index a56e03f15d..059d333d8b 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -17,19 +17,28 @@ package kafka import ( "context" "strings" + "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" + "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/zap" ) type franzFactory struct { - options *options changefeedID common.ChangeFeedID + clientOpts []kgo.Opt + producerOpts []kgo.Opt + timeout time.Duration } -func newFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { - admin, err := newAdmin(ctx, changefeedID, o) +func newFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { + clientOpts, err := clientOptions(o) + if err != nil { + return nil, err + } + timeout := requestTimeout(o) + admin, err := newAdmin(ctx, changefeedID, clientOpts, timeout) if err != nil { return nil, err } @@ -38,6 +47,10 @@ func newFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedI if err := adjustOptions(ctx, changefeedID, admin, o, o.Topic); err != nil { return nil, err } + producerOpts, err := producerOptions(o) + if err != nil { + return nil, err + } compression := strings.ToLower(strings.TrimSpace(o.Compression)) if compression == "" { @@ -58,16 +71,20 @@ func newFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedI zap.Duration("dialTimeout", o.DialTimeout), zap.Duration("readTimeout", o.ReadTimeout), zap.Duration("writeTimeout", o.WriteTimeout)) - - return &franzFactory{options: o, changefeedID: changefeedID}, nil + return &franzFactory{ + changefeedID: changefeedID, + clientOpts: clientOpts, + producerOpts: producerOpts, + timeout: timeout, + }, nil } func (f *franzFactory) AdminClient(ctx context.Context) (AdminClient, error) { - return newAdmin(ctx, f.changefeedID, f.options) + return newAdmin(ctx, f.changefeedID, f.clientOpts, f.timeout) } func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := newSyncProducer(ctx, f.changefeedID, f.options) + producer, err := newSyncProducer(ctx, f.changefeedID, f.clientOpts, f.producerOpts, f.timeout) if err != nil { cleanupMetrics(f.changefeedID) } @@ -75,7 +92,7 @@ func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { } func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := newAsyncProducer(ctx, f.changefeedID, f.options) + producer, err := newAsyncProducer(ctx, f.changefeedID, f.clientOpts, f.producerOpts) if err != nil { cleanupMetrics(f.changefeedID) } diff --git a/pkg/sink/kafka/franz_gssapi.go b/pkg/sink/kafka/franz_gssapi.go index 3f77d26777..0961265be8 100644 --- a/pkg/sink/kafka/franz_gssapi.go +++ b/pkg/sink/kafka/franz_gssapi.go @@ -41,36 +41,29 @@ func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { func validateGSSAPIConfig(g gssapiConfig) error { if g.serviceName == "" { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") } if g.kerberosConfigPath == "" { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") } if g.username == "" { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") } if g.realm == "" { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") } switch g.authType { case userAuth: if g.password == "" { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") } case keyTabAuth: if g.keyTabPath == "" { - return errors.ErrKafkaInvalidConfig.GenWithStack( - "sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") } default: - return errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.authType) + return errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl-gssapi-auth-type %d", g.authType) } return nil } @@ -83,17 +76,14 @@ func newKerberosClient(g gssapiConfig) (*client.Client, error) { switch g.authType { case userAuth: - return client.NewWithPassword( - g.username, g.realm, g.password, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil + return client.NewWithPassword(g.username, g.realm, g.password, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil case keyTabAuth: kt, err := keytab.Load(g.keyTabPath) if err != nil { return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) } - return client.NewWithKeytab( - g.username, g.realm, kt, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil + return client.NewWithKeytab(g.username, g.realm, kt, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil default: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl-gssapi-auth-type %d", g.authType) + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl-gssapi-auth-type %d", g.authType) } } diff --git a/pkg/sink/kafka/franz_logger.go b/pkg/sink/kafka/franz_logger.go index 45f12890b3..322534550a 100644 --- a/pkg/sink/kafka/franz_logger.go +++ b/pkg/sink/kafka/franz_logger.go @@ -41,7 +41,6 @@ func newClientLogger(changefeedID common.ChangeFeedID, role string) kgo.Logger { ).WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { return zapcore.NewSamplerWithOptions(core, time.Minute, 5, 100) })) - return &clientLogger{logger: logger} } @@ -54,7 +53,6 @@ func (l *clientLogger) Level() kgo.LogLevel { func (l *clientLogger) Log(level kgo.LogLevel, msg string, keyvals ...any) { fields := make([]zap.Field, 0, (len(keyvals)+1)/2) - for i := 0; i < len(keyvals); i += 2 { key := fmt.Sprint(keyvals[i]) value := any("") @@ -99,6 +97,5 @@ func isSensitiveLogKey(key string) bool { return true } } - return false } diff --git a/pkg/sink/kafka/franz_metrics_hook.go b/pkg/sink/kafka/franz_metrics_hook.go index 0ee3d8414f..daf19ddc12 100644 --- a/pkg/sink/kafka/franz_metrics_hook.go +++ b/pkg/sink/kafka/franz_metrics_hook.go @@ -59,7 +59,6 @@ const ( func newMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { keyspace := changefeedID.Keyspace() changefeed := changefeedID.Name() - return &metricsHook{ keyspace: keyspace, changefeed: changefeed, @@ -69,11 +68,7 @@ func newMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { } } -func (h *metricsHook) OnBrokerThrottle( - meta kgo.BrokerMetadata, - throttleInterval time.Duration, - _ bool, -) { +func (h *metricsHook) OnBrokerThrottle(meta kgo.BrokerMetadata, throttleInterval time.Duration, _ bool) { if meta.NodeID < 0 { return } @@ -89,21 +84,16 @@ func (h *metricsHook) broker(nodeID int32) *brokerMetrics { brokerID := strconv.Itoa(int(nodeID)) metrics := &brokerMetrics{ outgoingBytesTotal: outgoingBytesTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID), - requestsSuccess: requestsTotal.WithLabelValues( - h.keyspace, h.changefeed, brokerID, metricResultSuccess), - requestsWriteError: requestsTotal.WithLabelValues( - h.keyspace, h.changefeed, brokerID, metricResultWriteError), - responsesSuccess: responsesTotal.WithLabelValues( - h.keyspace, h.changefeed, brokerID, metricResultSuccess), - responsesReadError: responsesTotal.WithLabelValues( - h.keyspace, h.changefeed, brokerID, metricResultReadError), - requestsInFlight: requestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID), - requestDuration: requestDuration.WithLabelValues(h.keyspace, h.changefeed, brokerID), - throttleTime: throttleTime.WithLabelValues(h.keyspace, h.changefeed, brokerID), + requestsSuccess: requestsTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultSuccess), + requestsWriteError: requestsTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultWriteError), + responsesSuccess: responsesTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultSuccess), + responsesReadError: responsesTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultReadError), + requestsInFlight: requestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID), + requestDuration: requestDuration.WithLabelValues(h.keyspace, h.changefeed, brokerID), + throttleTime: throttleTime.WithLabelValues(h.keyspace, h.changefeed, brokerID), } actual, _ := h.brokers.LoadOrStore(nodeID, metrics) - return actual.(*brokerMetrics) } @@ -125,14 +115,7 @@ func cleanupMetrics(changefeedID common.ChangeFeedID) { compressedBytesTotal.DeletePartialMatch(labels) } -func (h *metricsHook) OnBrokerWrite( - meta kgo.BrokerMetadata, - _ int16, - bytesWritten int, - _ time.Duration, - _ time.Duration, - err error, -) { +func (h *metricsHook) OnBrokerWrite(meta kgo.BrokerMetadata, _ int16, bytesWritten int, _ time.Duration, _ time.Duration, err error) { if meta.NodeID < 0 { return } @@ -151,11 +134,7 @@ func (h *metricsHook) OnBrokerWrite( } } -func (h *metricsHook) OnBrokerE2E( - meta kgo.BrokerMetadata, - _ int16, - e2e kgo.BrokerE2E, -) { +func (h *metricsHook) OnBrokerE2E(meta kgo.BrokerMetadata, _ int16, e2e kgo.BrokerE2E) { if meta.NodeID < 0 { return } @@ -178,12 +157,7 @@ func (h *metricsHook) OnBrokerE2E( } } -func (h *metricsHook) OnProduceBatchWritten( - _ kgo.BrokerMetadata, - _ string, - _ int32, - m kgo.ProduceBatchMetrics, -) { +func (h *metricsHook) OnProduceBatchWritten(_ kgo.BrokerMetadata, _ string, _ int32, m kgo.ProduceBatchMetrics) { if m.NumRecords > 0 { h.recordsPerBatch.Observe(float64(m.NumRecords)) } diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go index 5b84511441..9b8a4d1c0a 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -34,12 +34,8 @@ type syncProducer struct { timeout time.Duration } -func newSyncProducer( - ctx context.Context, - changefeedID common.ChangeFeedID, - o *options, -) (*syncProducer, error) { - client, err := newProducerClient(ctx, changefeedID, "sync-producer", o) +func newSyncProducer(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt, producerOpts []kgo.Opt, timeout time.Duration) (*syncProducer, error) { + client, err := newProducerClient(ctx, changefeedID, "sync-producer", clientOpts, producerOpts) if err != nil { return nil, err } @@ -47,13 +43,11 @@ func newSyncProducer( return &syncProducer{ id: changefeedID, client: client, - timeout: requestTimeout(o), + timeout: timeout, }, nil } -func (p *syncProducer) SendMessage( - ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message, -) error { +func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -67,9 +61,7 @@ func (p *syncProducer) SendMessage( return p.sendRecords(ctx, message, record) } -func (p *syncProducer) SendMessages( - ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message, -) error { +func (p *syncProducer) SendMessages(ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -87,9 +79,7 @@ func (p *syncProducer) SendMessages( return p.sendRecords(ctx, message, records...) } -func (p *syncProducer) sendRecords( - ctx context.Context, message *codeccommon.Message, records ...*kgo.Record, -) error { +func (p *syncProducer) sendRecords(ctx context.Context, message *codeccommon.Message, records ...*kgo.Record) error { ctx, cancel := context.WithTimeout(ctx, p.timeout) defer cancel() @@ -104,7 +94,6 @@ func (p *syncProducer) sendRecords( zap.String("eventContext", BuildEventLogContext( p.id.Keyspace(), p.id.Name(), message.LogInfo)), zap.Error(err)) - return errors.WrapError(errors.ErrKafkaSendMessage, err) } diff --git a/pkg/sink/kafka/franz_sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go index c4701c6f84..ea0a0afa84 100644 --- a/pkg/sink/kafka/franz_sync_producer_test.go +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -41,11 +41,14 @@ func TestSyncProducerSendsToRequestedPartitions(t *testing.T) { const topic = "sync-topic" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) producer, err := newSyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), - testOptions(cluster.ListenAddrs()), + testClientOptions(t, o), + testProducerOptions(t, o), + requestTimeout(o), ) require.NoError(t, err) defer producer.Close() @@ -62,11 +65,14 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { return produceResponseWithError(req, 1, kerr.InvalidTopicException.Code) }) + o := testOptions(cluster.ListenAddrs()) producer, err := newSyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), - testOptions(cluster.ListenAddrs()), + testClientOptions(t, o), + testProducerOptions(t, o), + requestTimeout(o), ) require.NoError(t, err) defer producer.Close() @@ -77,10 +83,13 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { } func TestSyncProducerUsesSendContext(t *testing.T) { + o := testOptions([]string{"127.0.0.1:1"}) producer, err := newSyncProducer( t.Context(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), - testOptions([]string{"127.0.0.1:1"}), + testClientOptions(t, o), + testProducerOptions(t, o), + requestTimeout(o), ) require.NoError(t, err) defer producer.Close() @@ -94,10 +103,13 @@ func TestSyncProducerUsesSendContext(t *testing.T) { } func TestSyncProducerCloseIsIdempotent(t *testing.T) { + o := testOptions([]string{"127.0.0.1:1"}) client, err := newSyncProducer( context.Background(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), - testOptions([]string{"127.0.0.1:1"}), + testClientOptions(t, o), + testProducerOptions(t, o), + requestTimeout(o), ) require.NoError(t, err) @@ -125,6 +137,5 @@ func produceResponseWithError(req kmsg.Request, failedPartition int32, errorCode response.Topics = append(response.Topics, responseTopic) } - return response, nil, true } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 4adb6ec886..4674340737 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "fmt" "net/url" "strconv" @@ -28,6 +29,7 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" ) const ( @@ -81,6 +83,35 @@ func TestKafkaClientSelection(t *testing.T) { } } +func TestFactorySelection(t *testing.T) { + const topic = "factory-selection" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory-selection") + for _, test := range []struct { + client string + expected Factory + }{ + {client: KafkaClientFranz, expected: &franzFactory{}}, + {client: KafkaClientSarama, expected: &saramaFactory{}}, + } { + t.Run(test.client, func(t *testing.T) { + o := NewOptions() + o.Client = test.client + o.ClientID = "ticdc-test" + o.BrokerEndpoints = cluster.ListenAddrs() + o.Topic = topic + + factory, err := NewFactory(context.Background(), o, changefeedID) + require.NoError(t, err) + require.IsType(t, test.expected, factory) + + factory.CleanupMetrics() + }) + } +} + func TestCompleteOptions(t *testing.T) { options := NewOptions() diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index 87d5faf2cc..ecbab26407 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -34,12 +34,8 @@ type saramaFactory struct { func (*saramaFactory) CleanupMetrics() {} -// NewSaramaFactory constructs a Factory with sarama implementation. -func NewSaramaFactory( - ctx context.Context, - o *options, - changefeedID common.ChangeFeedID, -) (Factory, error) { +// newSaramaFactory constructs a Factory with sarama implementation. +func newSaramaFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { start := time.Now() config, err := newSaramaConfig(ctx, o) duration := time.Since(start) @@ -184,9 +180,7 @@ func (f *saramaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error }, nil } -func (f *saramaFactory) MetricsCollector( - adminClient AdminClient, -) MetricsCollector { +func (f *saramaFactory) MetricsCollector(adminClient AdminClient) MetricsCollector { return &saramaMetricsCollector{ changefeedID: f.changefeedID, adminClient: adminClient, diff --git a/pkg/sink/kafka/selector.go b/pkg/sink/kafka/selector.go deleted file mode 100644 index a26c0664d3..0000000000 --- a/pkg/sink/kafka/selector.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - - "github.com/pingcap/ticdc/pkg/common" -) - -// NewFactory selects the Kafka client for one changefeed without automatic fallback. -func NewFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { - if o.Client == KafkaClientSarama { - return NewSaramaFactory(ctx, o, changefeedID) - } - return newFactory(ctx, o, changefeedID) -} diff --git a/pkg/sink/kafka/selector_test.go b/pkg/sink/kafka/selector_test.go deleted file mode 100644 index 298e07b63f..0000000000 --- a/pkg/sink/kafka/selector_test.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "io" - "testing" - - "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/errors" - "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kerr" - "github.com/twmb/franz-go/pkg/kfake" -) - -func TestIsUnretryableClientError(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - err error - unretryable bool - }{ - {name: "unknown topic", err: kerr.UnknownTopicOrPartition}, - {name: "leader unavailable", err: kerr.LeaderNotAvailable}, - {name: "request timeout", err: kerr.RequestTimedOut}, - {name: "network exception", err: kerr.NetworkException}, - {name: "controller changed", err: kerr.NotController}, - {name: "EOF", err: io.EOF}, - {name: "invalid topic", err: kerr.InvalidTopicException, unretryable: true}, - {name: "invalid config", err: kerr.InvalidConfig, unretryable: true}, - {name: "SASL authentication failure", err: kerr.SaslAuthenticationFailed, unretryable: true}, - {name: "unsupported SASL mechanism", err: kerr.UnsupportedSaslMechanism, unretryable: true}, - {name: "illegal SASL state", err: kerr.IllegalSaslState, unretryable: true}, - {name: "unsupported version", err: kerr.UnsupportedVersion, unretryable: true}, - {name: "invalid request", err: kerr.InvalidRequest, unretryable: true}, - { - name: "wrapped invalid topic", - err: errors.WrapError(errors.ErrKafkaAdminAPI, kerr.InvalidTopicException, "describe-topic", "test-topic"), - unretryable: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - require.Equal(t, test.unretryable, IsUnretryableKafkaError(test.err)) - }) - } -} - -func TestFactorySelection(t *testing.T) { - const topic = "factory-selection" - cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) - defer cluster.Close() - - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory-selection") - for _, test := range []struct { - client string - expected Factory - }{ - {client: KafkaClientFranz, expected: &franzFactory{}}, - {client: KafkaClientSarama, expected: &saramaFactory{}}, - } { - t.Run(test.client, func(t *testing.T) { - options := NewOptions() - options.Client = test.client - options.ClientID = "ticdc-test" - options.BrokerEndpoints = cluster.ListenAddrs() - options.Topic = topic - - factory, err := NewFactory(context.Background(), options, changefeedID) - require.NoError(t, err) - require.IsType(t, test.expected, factory) - - factory.CleanupMetrics() - }) - } -} - -func TestFranzIgnoresConfiguredKafkaVersion(t *testing.T) { - const topic = "version-negotiation" - cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) - defer cluster.Close() - - options := NewOptions() - options.ClientID = "ticdc-test" - options.BrokerEndpoints = cluster.ListenAddrs() - options.Topic = topic - options.Version = "invalid" - options.IsAssignedVersion = true - - factory, err := NewFactory( - context.Background(), - options, - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "version-negotiation"), - ) - require.NoError(t, err) - require.IsType(t, &franzFactory{}, factory) - factory.CleanupMetrics() -} From 122d36fb409463def62a3721bc7ae96eb550d8a5 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 2 Sep 2026 18:51:19 +0800 Subject: [PATCH 49/61] kafka: inline franz producer construction --- pkg/sink/kafka/franz_async_producer.go | 13 ------ pkg/sink/kafka/franz_async_producer_test.go | 22 ++++----- pkg/sink/kafka/franz_factory.go | 12 ++--- pkg/sink/kafka/franz_sync_producer.go | 13 ------ pkg/sink/kafka/franz_sync_producer_test.go | 52 ++++++++++----------- 5 files changed, 40 insertions(+), 72 deletions(-) diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index 96821d4609..17ad372d96 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -35,19 +35,6 @@ type asyncProducer struct { errCh chan error } -func newAsyncProducer(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt, producerOpts []kgo.Opt) (*asyncProducer, error) { - client, err := newProducerClient(ctx, changefeedID, "async-producer", clientOpts, producerOpts) - if err != nil { - return nil, err - } - - return &asyncProducer{ - client: client, - changefeedID: changefeedID, - errCh: make(chan error, 1), - }, nil -} - func (p *asyncProducer) Close() { if !p.closeStarted.CompareAndSwap(false, true) { return diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index 9044edd1db..efdfb03507 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -88,12 +88,11 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { defer cluster.Close() o := testOptions(cluster.ListenAddrs()) - producer, err := newAsyncProducer( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), - testClientOptions(t, o), - testProducerOptions(t, o), - ) + producer, err := (&franzFactory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), + clientOpts: testClientOptions(t, o), + producerOpts: testProducerOptions(t, o), + }).AsyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() @@ -128,12 +127,11 @@ func TestAsyncProducerReportsProduceFailure(t *testing.T) { }) o := testOptions(cluster.ListenAddrs()) - producer, err := newAsyncProducer( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), - testClientOptions(t, o), - testProducerOptions(t, o), - ) + producer, err := (&franzFactory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), + clientOpts: testClientOptions(t, o), + producerOpts: testProducerOptions(t, o), + }).AsyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 059d333d8b..c92aeffe7f 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -84,19 +84,19 @@ func (f *franzFactory) AdminClient(ctx context.Context) (AdminClient, error) { } func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - producer, err := newSyncProducer(ctx, f.changefeedID, f.clientOpts, f.producerOpts, f.timeout) + client, err := newProducerClient(ctx, f.changefeedID, "sync-producer", f.clientOpts, f.producerOpts) if err != nil { - cleanupMetrics(f.changefeedID) + return nil, err } - return producer, err + return &syncProducer{id: f.changefeedID, client: client, timeout: f.timeout}, nil } func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - producer, err := newAsyncProducer(ctx, f.changefeedID, f.clientOpts, f.producerOpts) + client, err := newProducerClient(ctx, f.changefeedID, "async-producer", f.clientOpts, f.producerOpts) if err != nil { - cleanupMetrics(f.changefeedID) + return nil, err } - return producer, err + return &asyncProducer{client: client, changefeedID: f.changefeedID, errCh: make(chan error, 1)}, nil } func (f *franzFactory) MetricsCollector(AdminClient) MetricsCollector { diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go index 9b8a4d1c0a..c4fb3798e3 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -34,19 +34,6 @@ type syncProducer struct { timeout time.Duration } -func newSyncProducer(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt, producerOpts []kgo.Opt, timeout time.Duration) (*syncProducer, error) { - client, err := newProducerClient(ctx, changefeedID, "sync-producer", clientOpts, producerOpts) - if err != nil { - return nil, err - } - - return &syncProducer{ - id: changefeedID, - client: client, - timeout: timeout, - }, nil -} - func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() diff --git a/pkg/sink/kafka/franz_sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go index ea0a0afa84..92677d2a71 100644 --- a/pkg/sink/kafka/franz_sync_producer_test.go +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -43,13 +43,12 @@ func TestSyncProducerSendsToRequestedPartitions(t *testing.T) { defer cluster.Close() o := testOptions(cluster.ListenAddrs()) - producer, err := newSyncProducer( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), - testClientOptions(t, o), - testProducerOptions(t, o), - requestTimeout(o), - ) + producer, err := (&franzFactory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), + clientOpts: testClientOptions(t, o), + producerOpts: testProducerOptions(t, o), + timeout: requestTimeout(o), + }).SyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() @@ -67,13 +66,12 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { }) o := testOptions(cluster.ListenAddrs()) - producer, err := newSyncProducer( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), - testClientOptions(t, o), - testProducerOptions(t, o), - requestTimeout(o), - ) + producer, err := (&franzFactory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), + clientOpts: testClientOptions(t, o), + producerOpts: testProducerOptions(t, o), + timeout: requestTimeout(o), + }).SyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() @@ -84,13 +82,12 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { func TestSyncProducerUsesSendContext(t *testing.T) { o := testOptions([]string{"127.0.0.1:1"}) - producer, err := newSyncProducer( - t.Context(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), - testClientOptions(t, o), - testProducerOptions(t, o), - requestTimeout(o), - ) + producer, err := (&franzFactory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), + clientOpts: testClientOptions(t, o), + producerOpts: testProducerOptions(t, o), + timeout: requestTimeout(o), + }).SyncProducer(t.Context()) require.NoError(t, err) defer producer.Close() @@ -104,13 +101,12 @@ func TestSyncProducerUsesSendContext(t *testing.T) { func TestSyncProducerCloseIsIdempotent(t *testing.T) { o := testOptions([]string{"127.0.0.1:1"}) - client, err := newSyncProducer( - context.Background(), - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), - testClientOptions(t, o), - testProducerOptions(t, o), - requestTimeout(o), - ) + client, err := (&franzFactory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), + clientOpts: testClientOptions(t, o), + producerOpts: testProducerOptions(t, o), + timeout: requestTimeout(o), + }).SyncProducer(context.Background()) require.NoError(t, err) client.Close() From a645be1b84237eb66e9f974039fd093f23ae14e9 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 2 Sep 2026 20:05:04 +0800 Subject: [PATCH 50/61] fix --- downstreamadapter/sink/kafka/sink_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 5fb3689b3b..01630f545d 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -234,6 +234,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { gomock.InOrder( adminClient.EXPECT().Close(), topicManager.EXPECT().Close(), + factory.EXPECT().CleanupMetrics(), ) kafkaSink, err := newWithComponents( @@ -262,6 +263,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { asyncProducer.EXPECT().Close(), adminClient.EXPECT().Close(), topicManager.EXPECT().Close(), + factory.EXPECT().CleanupMetrics(), ) kafkaSink, err := newWithComponents( @@ -293,6 +295,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { asyncProducer.EXPECT().Close().Do(func() { closeCount.Add(1) }), adminClient.EXPECT().Close().Do(func() { closeCount.Add(1) }), topicManager.EXPECT().Close().Do(func() { closeCount.Add(1) }), + factory.EXPECT().CleanupMetrics(), ) kafkaSink, err := newWithComponents( @@ -609,6 +612,7 @@ func newKafkaSinkForTest( factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(syncProducer, nil) factory.EXPECT().MetricsCollector(nil).Return(noopMetricsCollector{}) + factory.EXPECT().CleanupMetrics().AnyTimes() kafkaSink, err := newWithComponents(ctx, changefeedID, common.DefaultKeyspaceID, protocol, components{ encoderGroup: encoderGroup, From 1c8dc273057ca17ed1a8baab386d00fc31647acb Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 12:16:27 +0800 Subject: [PATCH 51/61] kafka: refine franz producer limits and timeouts --- docs/franz-go/kafka-producer-size-limits.md | 249 ++++++++++++++++++ .../sink/topicmanager/kafka_topic_manager.go | 5 +- pkg/sink/kafka/admin.go | 4 +- pkg/sink/kafka/franz_admin.go | 9 +- pkg/sink/kafka/franz_admin_test.go | 4 +- pkg/sink/kafka/franz_async_producer.go | 28 +- pkg/sink/kafka/franz_async_producer_test.go | 14 +- pkg/sink/kafka/franz_config.go | 85 +++--- pkg/sink/kafka/franz_config_test.go | 67 ++--- pkg/sink/kafka/franz_factory.go | 16 +- pkg/sink/kafka/franz_logger.go | 4 +- pkg/sink/kafka/franz_logger_test.go | 4 +- pkg/sink/kafka/franz_sync_producer.go | 28 +- pkg/sink/kafka/franz_sync_producer_test.go | 12 +- pkg/sink/kafka/metrics_collector_test.go | 30 +-- pkg/sink/kafka/options.go | 43 +-- pkg/sink/kafka/sarama_sync_producer.go | 8 +- 17 files changed, 380 insertions(+), 230 deletions(-) create mode 100644 docs/franz-go/kafka-producer-size-limits.md diff --git a/docs/franz-go/kafka-producer-size-limits.md b/docs/franz-go/kafka-producer-size-limits.md new file mode 100644 index 0000000000..55ca8ce26e --- /dev/null +++ b/docs/franz-go/kafka-producer-size-limits.md @@ -0,0 +1,249 @@ +# Kafka Producer 大小限制 + +Last updated: 2026-09-03 +Status: Produce request 和 producer buffer 已确定；消息与 record batch 的边界仍在讨论 + +## 背景 + +Kafka producer 处理四种不同对象：单条消息、record batch、Produce request 和内存 buffer。它们的限制作用在不同阶段。复用一个值会产生两个问题：Kafka Topic 配置可能意外放大进程内存，单条消息也可能通过 encoder 后被 producer 拒绝。 + +本文先说明每个参数的作用范围，再用具体场景说明参数之间的关系。本文使用 MiB，因为代码以二进制移位表示容量：`1 MiB = 1 << 20 bytes`。MB 表示十进制容量：`1 MB = 1,000,000 bytes`。 + +## 参数及作用范围 + +- `max-message-bytes` + - 配置位置：Sink URI 或 SinkConfig。 + - 作用对象：encoder 生成的 `common.Message`。 + - 当前取值：默认 10 MiB。 + - 配置方式：用户可独立配置。 + +- Topic `max.message.bytes` + - 配置位置：Kafka Topic。 + - 作用对象：broker 接受的 record batch。 + - 当前取值：TiCDC 读取 Kafka 配置。 + - 配置方式：由 Kafka 管理员独立配置。 + +- Broker `message.max.bytes` + - 配置位置：Kafka Broker。 + - 作用对象：Topic 未覆盖时的 record batch。 + - 当前取值：TiCDC 读取 Kafka 配置。 + - 配置方式：由 Kafka 管理员独立配置。 + +- `ProducerBatchMaxBytes` + - 配置位置：franz-go client。 + - 作用对象：未压缩的完整 record batch。 + - 当前取值：Kafka batch 上限与 100 MiB 的较小值。 + - 配置方式：TiCDC 内部计算，不对用户开放。 + +- `BrokerMaxWriteBytes` + - 配置位置：franz-go client。 + - 作用对象：发给一个 broker 的完整 Produce request。 + - 当前取值：franz-go 默认 100 MiB。 + - 配置方式：不对用户开放。 + +- `MaxBufferedBytes` + - 配置位置：franz-go client。 + - 作用对象：一个 client 中尚未完成的 record payload 总量。 + - 当前取值：固定 64 MiB。 + - 配置方式：不对用户开放。 + +- `MaxBufferedRecords` + - 配置位置：franz-go client。 + - 作用对象：一个 client 中尚未完成的 record 数量。 + - 当前取值：franz-go 默认 10,000。 + - 配置方式：不对用户开放。 + +- `MaxProduceRequestsInflightPerBroker` + - 配置位置：franz-go client。 + - 作用对象：每个 broker 同时在途的 Produce request 数量。 + - 当前取值：固定 1。 + - 配置方式：不对用户开放。 + +`max-message-bytes` 是当前唯一由 TiCDC 用户直接配置的大小参数。Produce request 和 producer buffer 使用固定值。Kafka record batch 的服务端上限由 Kafka 管理员控制。 + +## 数据经过哪些限制 + +```text +row events + ↓ encoder +common.Message / Kafka record + ↓ 按 topic-partition 组 batch +record batch + ↓ 按目标 broker 组 request +Produce request +``` + +record 被 franz-go 接收后会计入 producer buffer，直到发送成功、发送失败或被取消。等待 metadata、等待组 batch、等待发送、等待 broker 响应和等待重试的 record 都占用 buffer。 + +## 已确定方案 + +### Produce request 固定为 100 MiB + +TiCDC 不再设置 `BrokerMaxWriteBytes`，沿用 franz-go 的 100 MiB 默认值。该值对应 Kafka `socket.request.max.bytes` 的常见默认值。 + +Kafka Topic 的 `max.message.bytes` 不再放大 Produce request。franz-go 会把发往同一 broker 的多个 record batches 放入一个 request；达到 100 MiB 后创建下一个 request。 + +TiCDC 将 `ProducerBatchMaxBytes` 设置为 Kafka batch 上限与 100 MiB 的较小值。franz-go 随后按实际 client ID、Topic 名称和协议字段扣除 request envelope。 + +TiCDC 代码继续使用 `maxMessageBytes` 命名，与 Kafka `max.message.bytes` 和原 Sarama 配置保持一致。`ProducerBatchMaxBytes` 只作为 franz-go API 名称出现。 + +`MaxProduceRequestsInflightPerBroker(1)` 保持不变。一个 broker 同时最多有一个在途 Produce request。多个 broker 可以各有一个在途 request。 + +### Producer buffer 固定为 64 MiB + +每个 franz-go client 设置： + +```go +kgo.MaxBufferedBytes(64 << 20) +``` + +不增加 `max-buffered-bytes` Sink URI 参数。64 MiB 是每个 client 的 payload 上限，实际 RSS 还包括 record 对象、batch 编码、压缩空间、request 和网络 buffer。 + +`MaxBufferedBytes` 与默认的 `MaxBufferedRecords=10000` 同时生效。任一上限先达到，`Produce` 就会等待已有 record 完成或 context 被取消。 + +64 MiB 高于 TiCDC 默认的 10 MiB `max-message-bytes`，同时限制 10,000 条 record 可能占用的 payload 内存。该值会限制单条 record：当 key、value 和 headers 的总长度超过 64 MiB 时，franz-go 直接返回 `kerr.MessageTooLarge`。 + +`MaxProduceRequestsInflightPerBroker(1)` 无法替代 buffer 限制。它只限制在途 request 数量；等待 metadata、等待重试及发往其他 broker 的 records 仍可留在 buffer 中。 + +## 配置场景 + +### 场景一：普通消息，Kafka batch 上限约 1 MiB + +配置示例： + +```text +Sink URI 不设置 max-message-bytes，使用默认值 10485760（10 MiB） +Kafka Topic max.message.bytes = 1048576（1 MiB） +Produce request = 100 MiB +Producer buffer = 64 MiB +``` + +结果： + +- encoder 的普通消息不能超过 Kafka record batch 实际可容纳的范围。 +- 一个 Produce request 可以携带多个约 1 MiB 的 record batches。 +- 如果平均 payload 为 1 MiB，buffer 大约容纳 64 条 record，随后开始反压。 + +这里的 `max-message-bytes`、Produce request 和 producer buffer 使用各自的值。Kafka record batch 上限仍会约束 encoder 的最终有效上限，具体 framing 预留量待确认。 + +### 场景二：普通消息按 1 MiB 聚合，允许 10 MiB 的单行大消息 + +配置示例： + +```text +Sink URI: kafka://broker/topic?protocol=open-protocol&max-message-bytes=1048576 +Kafka Topic max.message.bytes = 10485760 +Produce request = 100 MiB +Producer buffer = 64 MiB +``` + +`max-message-bytes=1048576` 表示 1 MiB，Kafka 的 `10485760` 表示 10 MiB。 + +预期结果： + +- Open Protocol 普通多行消息在约 1 MiB 时结束聚合。 +- 单行大消息可以继续使用 Kafka record batch 提供的空间。 +- franz-go 的 `ProducerBatchMaxBytes` 应接近 10 MiB，并为 batch framing 留出空间。 +- 100 MiB request 和 64 MiB buffer 都能容纳这类消息。 + +这个场景体现 `max-message-bytes` 与 Kafka `max.message.bytes` 的独立用途：前者控制普通消息聚合，后者控制单条大消息和 record batch 的最终上限。 + +### 场景三：Kafka 允许 80 MiB batch，单条 payload 为 70 MiB + +配置示例： + +```text +Kafka Topic max.message.bytes = 83886080（80 MiB） +单条 record payload = 73400320（70 MiB） +Produce request = 100 MiB +Producer buffer = 64 MiB +``` + +结果：franz-go 会因为单条 payload 超过 `MaxBufferedBytes` 而拒绝该 record。Kafka Topic 和 Produce request 虽然具备足够空间，固定的 64 MiB buffer 仍形成了单条 record 上限。 + +当前方案因此不支持超过 64 MiB 的单条 payload。后续需要决定 encoder 是否提前按该限制报错，保证错误发生在 producer 之前。 + +### 场景四：Kafka batch 上限超过 100 MiB + +配置示例： + +```text +Kafka Topic max.message.bytes = 134217728（128 MiB） +Produce request = 100 MiB +ProducerBatchMaxBytes = 100 MiB +``` + +Kafka 允许管理员将 `max.message.bytes` 调到 100 MiB 以上。broker 的 `socket.request.max.bytes` 也必须相应增大，才能接收这种 batch。TiCDC 当前固定使用 100 MiB Produce request，因此 franz-go 的 batch 配置会限制为 100 MiB；franz-go 还会扣除 request envelope。 + +### 场景五：Kafka 暂时不可用 + +假设平均 payload 为 1 MiB： + +- buffer 接受约 64 条 record 后达到 64 MiB。 +- 后续 `Produce` 阻塞，反压逐步传回上游。 +- Kafka 恢复后，已缓存 records 继续发送。 +- context 取消时，阻塞中的调用退出。 + +假设平均 payload 为 4 KiB，10,000 条 record 约占 39 MiB。此时 `MaxBufferedRecords=10000` 先达到，record 数量限制触发反压。 + +### 场景六：Topic 分区分布在三个 broker + +franz-go 最多可以同时存在三个在途 Produce requests，每个 broker 一个。所有等待中和在途的 records 共用同一个 64 MiB client buffer。 + +这个场景说明 `MaxProduceRequestsInflightPerBroker` 与 `MaxBufferedBytes` 可以分别设置。前者控制每个 broker 的请求并发，后者控制整个 client 的 payload 总量。 + +### 场景七：一个进程运行多个 Changefeed + +Kafka sink 会为一个 Changefeed 创建 async 和 sync 两个 producer clients。每个 client 的上限都是 64 MiB，因此一个 Changefeed 的 producer payload 理论上限为 128 MiB。sync producer 主要发送低频 DDL，通常不会长期占满。 + +例如，一个进程运行 10 个 Changefeed，producer payload 的理论上限为： + +```text +10 × 2 × 64 MiB = 1280 MiB +``` + +该数字仍未包含 encoder、event pipeline、batch、压缩和网络相关内存。 + +## 参数之间的关系 + +- `max-message-bytes` 与 Produce request 独立配置。修改 `max-message-bytes` 不会改变 100 MiB request。 +- Kafka `max.message.bytes` 与 Produce request 独立配置。一个完整 batch 加 request envelope 必须小于 100 MiB。 +- `ProducerBatchMaxBytes` 受 Kafka `max.message.bytes` 约束，不能超过 Kafka 接受的 batch。 +- `ProducerBatchMaxBytes` 受 Produce request 约束，必须能装入一个 100 MiB request。 +- 单条 record 受 `MaxBufferedBytes` 约束，payload 必须小于等于 64 MiB。 +- `MaxBufferedBytes` 与 Produce request 独立配置，两者限制不同对象，数值无需相等。 +- `MaxBufferedBytes` 与 `MaxBufferedRecords` 独立且同时生效，任一上限先达到就触发反压。 +- `MaxBufferedBytes` 与每 broker 在途请求数独立。buffer 统计整个 client，在途请求数按 broker 统计。 + +## 待讨论项 + +- encoder 应为 franz-go record batch framing 预留多少空间。 +- 单条 payload 超过固定 64 MiB 时，encoder 如何提前执行 claim-check、handle-key-only 或报错。 +- Admin API 无法读取 Topic/Broker 配置时，应该使用哪个 fallback 上限。 +- 动态 Topic 具有不同 `max.message.bytes` 时，是否需要 per-topic batch limit。 + +## 已落地的代码改动 + +`pkg/sink/kafka/franz_config.go`： + +- 删除显式 `BrokerMaxWriteBytes`，使用 franz-go 默认的 100 MiB。 +- 固定设置 `MaxBufferedBytes(64 << 20)`。 +- 删除本地的 512 bytes 和 1 GiB batch 范围常量，不再静默提升 batch 上限。 +- 将 Kafka batch 上限限制在 100 MiB 以内，再传给 franz-go。 +- 保留 `MaxBufferedRecords=10000` 默认值。 +- 保留 `MaxProduceRequestsInflightPerBroker(1)`。 +- 代码注释说明 100 MiB request 和 64 MiB buffer 的作用范围及取值原因。 + +本次没有增加 Sink URI 参数。encoder 的消息边界将在上述待讨论项确认后实现。 + +## 验证 + +当前单元测试应确认： + +- `BrokerMaxWriteBytes` 为 franz-go 默认的 100 MiB。 +- `MaxBufferedBytes` 固定为 64 MiB。 +- 修改 `max-message-bytes` 不会改变 request 或 buffer 上限。 +- Kafka batch 上限超过 100 MiB 时，`ProducerBatchMaxBytes` 限制为 100 MiB。 +- `MaxBufferedRecords` 仍为 10,000。 + +后续边界测试应覆盖 64 MiB 单条 payload、100 MiB request envelope、Kafka batch 上限超过 request 上限，以及 buffer 满后的阻塞和取消行为。 diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index 7f53eeb1b0..29d3d0d42c 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -231,10 +231,7 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( // createTopic creates a topic with the given name // and returns the number of partitions. -func (m *kafkaTopicManager) createTopic( - ctx context.Context, - topicName string, -) (int32, error) { +func (m *kafkaTopicManager) createTopic(ctx context.Context, topicName string) (int32, error) { if !m.cfg.AutoCreate { return 0, errors.ErrKafkaInvalidConfig.GenWithStack("`auto-create-topic` is false, and %s not found", topicName) } diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index c9d87098b2..df47e5c70a 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -53,9 +53,7 @@ func (a *saramaAdminClient) GetAllBrokers(_ context.Context) []Broker { brokers := a.client.Brokers() result := make([]Broker, 0, len(brokers)) for _, broker := range brokers { - result = append(result, Broker{ - ID: broker.ID(), - }) + result = append(result, Broker{ID: broker.ID()}) } return result } diff --git a/pkg/sink/kafka/franz_admin.go b/pkg/sink/kafka/franz_admin.go index ae6305afe2..5ff644f108 100644 --- a/pkg/sink/kafka/franz_admin.go +++ b/pkg/sink/kafka/franz_admin.go @@ -36,10 +36,7 @@ type admin struct { func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt, timeout time.Duration) (*admin, error) { opts := make([]kgo.Opt, 0, len(clientOpts)+3) opts = append(opts, clientOpts...) - opts = append(opts, - kgo.WithContext(ctx), - kgo.WithLogger(newClientLogger(changefeedID, "admin")), - ) + opts = append(opts, kgo.WithContext(ctx), kgo.WithLogger(newClientLogger(changefeedID, "admin"))) // MetadataMinAge is the minimum interval between metadata requests. // It must stay below the visibility retry interval to avoid retrying a cached topic-not-found result. opts = append(opts, kgo.MetadataMinAge(100*time.Millisecond)) @@ -260,6 +257,4 @@ func (a *admin) CreateTopic(ctx context.Context, detail *TopicDetail) error { return errors.WrapError(errors.ErrKafkaAdminAPI, resp.Err, "create-topic", detail.Name) } -func (a *admin) Close() { - a.admin.Close() -} +func (a *admin) Close() { a.admin.Close() } diff --git a/pkg/sink/kafka/franz_admin_test.go b/pkg/sink/kafka/franz_admin_test.go index eb4ba08294..9b44454c8f 100644 --- a/pkg/sink/kafka/franz_admin_test.go +++ b/pkg/sink/kafka/franz_admin_test.go @@ -172,9 +172,7 @@ func TestFranzIsAuthorizationFailed(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - require.Equal(t, test.expected, isAuthorizationFailed(test.err)) - }) + t.Run(test.name, func(t *testing.T) { require.Equal(t, test.expected, isAuthorizationFailed(test.err)) }) } } diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index 17ad372d96..68e19428bf 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -72,7 +72,18 @@ func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition i logInfo := message.LogInfo promise := func(_ *kgo.Record, err error) { if err != nil { - p.enqueueAsyncSendError(logInfo, err) + log.Error("kafka message send failed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.String("eventContext", BuildEventLogContext( + p.changefeedID.Keyspace(), p.changefeedID.Name(), logInfo)), + zap.Error(err)) + + select { + case p.errCh <- errors.WrapError(errors.ErrKafkaSendMessage, err): + // Keep the first error until the dispatcher can recover from multiple errors. + default: + } return } @@ -85,21 +96,6 @@ func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition i return nil } -func (p *asyncProducer) enqueueAsyncSendError(logInfo *codeccommon.MessageLogInfo, err error) { - log.Error("kafka message send failed", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.String("eventContext", BuildEventLogContext( - p.changefeedID.Keyspace(), p.changefeedID.Name(), logInfo)), - zap.Error(err)) - - select { - case p.errCh <- errors.WrapError(errors.ErrKafkaSendMessage, err): - // Keep the first error until the dispatcher can recover from multiple errors. - default: - } -} - func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { defer p.closed.Store(true) for { diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index efdfb03507..535d98faa2 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -91,7 +91,7 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { producer, err := (&franzFactory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), clientOpts: testClientOptions(t, o), - producerOpts: testProducerOptions(t, o), + producerOpts: producerOptions(o), }).AsyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() @@ -130,7 +130,7 @@ func TestAsyncProducerReportsProduceFailure(t *testing.T) { producer, err := (&franzFactory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), clientOpts: testClientOptions(t, o), - producerOpts: testProducerOptions(t, o), + producerOpts: producerOptions(o), }).AsyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() @@ -152,11 +152,7 @@ func TestAsyncProducerReportsProduceFailure(t *testing.T) { } func TestBufferBackpressureCanBeCanceled(t *testing.T) { - client, err := kgo.NewClient( - kgo.SeedBrokers("127.0.0.1:1"), - kgo.MaxBufferedBytes(10), - kgo.RecordRetries(100), - ) + client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1"), kgo.MaxBufferedBytes(10), kgo.RecordRetries(100)) require.NoError(t, err) producer := &asyncProducer{ @@ -171,9 +167,7 @@ func TestBufferBackpressureCanBeCanceled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { - done <- producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{Value: make([]byte, 10)}) - }() + go func() { done <- producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{Value: make([]byte, 10)}) }() select { case <-done: diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go index 44e6dca9aa..0e3a307cac 100644 --- a/pkg/sink/kafka/franz_config.go +++ b/pkg/sink/kafka/franz_config.go @@ -35,18 +35,19 @@ import ( "golang.org/x/oauth2/clientcredentials" ) -const ( - // defaultMaxBufferedBytes bounds the producer's per-client byte buffer under normal configurations. - defaultMaxBufferedBytes = 64 << 20 - // defaultBrokerWriteBytes matches Kafka's default socket.request.max.bytes limit. - defaultBrokerWriteBytes = 100 << 20 - // minProducerBatchBytes and maxProducerBatchBytes are franz-go's accepted batch-size bounds. - minProducerBatchBytes = 512 - maxProducerBatchBytes = 1 << 30 -) +// producerMaxBufferedBytes bounds each producer client's buffered payload. +// Produce blocks when the buffer is full and resumes when records complete +// or its context is canceled. A larger single record fails with MessageTooLarge. +const producerMaxBufferedBytes = 64 << 20 + +// Kafka defaults socket.request.max.bytes to 100 MiB. Keeping franz-go at the +// same limit prevents a Topic's batch configuration from increasing request memory. +const franzDefaultMaxRequestBytes = 100 << 20 func requestTimeout(o *options) time.Duration { return max(o.ReadTimeout, o.WriteTimeout) } +// Admin and producer clients share connection options. Producer delivery and +// resource limits stay separate so they cannot affect admin operations. func clientOptions(o *options) ([]kgo.Opt, error) { opts := []kgo.Opt{ kgo.SeedBrokers(o.BrokerEndpoints...), @@ -81,6 +82,31 @@ func clientOptions(o *options) ([]kgo.Opt, error) { return opts, nil } +func producerOptions(o *options) []kgo.Opt { + maxMessageBytes := int32(min(o.MaxMessageBytes, franzDefaultMaxRequestBytes)) + return []kgo.Opt{ + kgo.RecordPartitioner(kgo.ManualPartitioner()), + kgo.RequiredAcks(requiredAcks(o.RequiredAcks)), + // Retried requests may create duplicates because broker-side producer ID deduplication is disabled. + kgo.DisableIdempotentWrite(), + // More than one in-flight request can reorder records when an earlier request is retried. + kgo.MaxProduceRequestsInflightPerBroker(1), + kgo.RecordRetries(o.MaxRetry), + kgo.UnknownTopicRetries(o.MaxRetry), + // Limit each client to 64 MiB of buffered payload. The in-flight limit + // applies per broker and does not bound records queued for other brokers, + // metadata, or retries, so the producer needs a separate byte limit. + // 64 MiB leaves room above TiCDC's default 10 MiB message limit while + // bounding the memory hidden by the 10,000-record default. + kgo.MaxBufferedBytes(producerMaxBufferedBytes), + // ProducerBatchMaxBytes is franz-go's name for Kafka's max.message.bytes limit. + kgo.ProducerBatchMaxBytes(maxMessageBytes), + kgo.ProduceRequestTimeout(requestTimeout(o)), + kgo.ProducerLinger(0), + compressionOption(o.Compression), + } +} + func buildSASLMechanism(cfg *saslConfig) (sasl.Mechanism, error) { switch cfg.mechanism { case plainMechanism: @@ -94,8 +120,7 @@ func buildSASLMechanism(cfg *saslConfig) (sasl.Mechanism, error) { case gssapiMechanism: return buildGSSAPIMechanism(cfg.gssapi) default: - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "unsupported sasl mechanism %s", cfg.mechanism) + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", cfg.mechanism) } } @@ -142,11 +167,7 @@ func buildOAuthMechanism(cfg oauth2Config) (sasl.Mechanism, error) { func newProducerClient(ctx context.Context, changefeedID common.ChangeFeedID, role string, clientOpts []kgo.Opt, producerOpts []kgo.Opt) (*kgo.Client, error) { opts := make([]kgo.Opt, 0, len(clientOpts)+len(producerOpts)+3) opts = append(opts, clientOpts...) - opts = append(opts, - kgo.WithContext(ctx), - kgo.WithLogger(newClientLogger(changefeedID, role)), - kgo.WithHooks(newMetricsHook(changefeedID)), - ) + opts = append(opts, kgo.WithContext(ctx), kgo.WithLogger(newClientLogger(changefeedID, role)), kgo.WithHooks(newMetricsHook(changefeedID))) opts = append(opts, producerOpts...) client, err := kgo.NewClient(opts...) @@ -156,38 +177,6 @@ func newProducerClient(ctx context.Context, changefeedID common.ChangeFeedID, ro return client, nil } -func producerOptions(o *options) ([]kgo.Opt, error) { - if o.MaxMessageBytes > maxProducerBatchBytes { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "max-message-bytes %d exceeds franz-go limit %d", - o.MaxMessageBytes, - maxProducerBatchBytes, - ) - } - - // Use 64 MiB as the default budget, but never make it smaller than the configured message limit. - // Keep franz-go's 10,000-record default as a second bound. - maxBufferedBytes := max(defaultMaxBufferedBytes, o.MaxMessageBytes) - maxBatchBytes := max(minProducerBatchBytes, o.MaxMessageBytes) - maxBrokerWriteBytes := max(defaultBrokerWriteBytes, maxBatchBytes) - return []kgo.Opt{ - kgo.RecordPartitioner(kgo.ManualPartitioner()), - kgo.RequiredAcks(requiredAcks(o.RequiredAcks)), - // Retried requests may create duplicates because broker-side producer ID deduplication is disabled. - kgo.DisableIdempotentWrite(), - // More than one in-flight request can reorder records when an earlier request is retried. - kgo.MaxProduceRequestsInflightPerBroker(1), - kgo.RecordRetries(o.MaxRetry), - kgo.UnknownTopicRetries(o.MaxRetry), - kgo.MaxBufferedBytes(maxBufferedBytes), - kgo.ProducerBatchMaxBytes(int32(maxBatchBytes)), - kgo.BrokerMaxWriteBytes(int32(maxBrokerWriteBytes)), - kgo.ProduceRequestTimeout(requestTimeout(o)), - kgo.ProducerLinger(0), - compressionOption(o.Compression), - }, nil -} - func requiredAcks(required RequiredAcks) kgo.Acks { switch required { case WaitForAll: diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index 8fc332b80b..be94c0385f 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -48,13 +48,6 @@ func testClientOptions(t *testing.T, o *options) []kgo.Opt { return opts } -func testProducerOptions(t *testing.T, o *options) []kgo.Opt { - t.Helper() - opts, err := producerOptions(o) - require.NoError(t, err) - return opts -} - func TestFranzRequiredAcks(t *testing.T) { for _, test := range []struct { required RequiredAcks @@ -99,34 +92,32 @@ func TestFranzIgnoresConfiguredKafkaVersion(t *testing.T) { factory.CleanupMetrics() } -func TestProducerOptionsBoundBufferAndBatch(t *testing.T) { - const batchBytes = 1048588 +func TestProducerOptionsConfigureMessageAndBufferLimits(t *testing.T) { + const maxMessageBytes = 1048588 o := testOptions([]string{"127.0.0.1:9092"}) - o.MaxMessageBytes = batchBytes + o.MaxMessageBytes = maxMessageBytes opts, err := clientOptions(o) require.NoError(t, err) - producerOpts, err := producerOptions(o) - require.NoError(t, err) + producerOpts := producerOptions(o) client, err := kgo.NewClient(append(opts, producerOpts...)...) require.NoError(t, err) defer client.Close() - require.Equal(t, int32(batchBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) - require.Equal(t, int64(defaultMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) + require.Equal(t, int32(maxMessageBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) require.Equal(t, int64(10000), client.OptValue(kgo.MaxBufferedRecords)) require.Equal(t, int64(1), client.OptValue(kgo.RecordRetries)) require.Equal(t, int64(1), client.OptValue(kgo.UnknownTopicRetries)) - require.Equal(t, int32(defaultBrokerWriteBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) + require.Equal(t, int32(100<<20), client.OptValue(kgo.BrokerMaxWriteBytes)) } func TestProducerOptionsUseSingleNonIdempotentRequest(t *testing.T) { config := testOptions([]string{"127.0.0.1:9092"}) - producerOpts, err := producerOptions(config) - require.NoError(t, err) + producerOpts := producerOptions(config) client, err := kgo.NewClient(producerOpts...) require.NoError(t, err) @@ -136,42 +127,43 @@ func TestProducerOptionsUseSingleNonIdempotentRequest(t *testing.T) { require.Equal(t, 1, client.OptValue(kgo.MaxProduceRequestsInflightPerBroker)) } -func TestProducerLimitsScaleWithConfiguredMessage(t *testing.T) { - maxMessageBytes := defaultBrokerWriteBytes + 1 +func TestProducerLimitsDoNotScaleWithConfiguredMessage(t *testing.T) { + maxMessageBytes := 32 << 20 config := testOptions([]string{"127.0.0.1:9092"}) config.MaxMessageBytes = maxMessageBytes - producerOpts, err := producerOptions(config) - require.NoError(t, err) + producerOpts := producerOptions(config) client, err := kgo.NewClient(producerOpts...) require.NoError(t, err) defer client.Close() - require.Equal(t, int64(maxMessageBytes), client.OptValue(kgo.MaxBufferedBytes)) - require.Equal(t, int32(maxMessageBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) + require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) + require.Equal(t, int32(100<<20), client.OptValue(kgo.BrokerMaxWriteBytes)) } -func TestProducerOptionsClampSmallBatch(t *testing.T) { +func TestProducerOptionsDoNotClampSmallBatch(t *testing.T) { config := testOptions([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = minProducerBatchBytes - 1 - - producerOpts, err := producerOptions(config) - require.NoError(t, err) + config.MaxMessageBytes = 511 - client, err := kgo.NewClient(producerOpts...) - require.NoError(t, err) - defer client.Close() + producerOpts := producerOptions(config) - require.Equal(t, int32(minProducerBatchBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) + _, err := kgo.NewClient(producerOpts...) + require.Error(t, err) } -func TestProducerOptionsRejectOversizedBatch(t *testing.T) { +func TestProducerOptionsCapMessageBytesAtRequestLimit(t *testing.T) { config := testOptions([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = maxProducerBatchBytes + 1 + config.MaxMessageBytes = franzDefaultMaxRequestBytes + 1 - _, err := producerOptions(config) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + producerOpts := producerOptions(config) + + client, err := kgo.NewClient(producerOpts...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, int32(franzDefaultMaxRequestBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, int32(franzDefaultMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) } func TestCompressionOptions(t *testing.T) { @@ -191,8 +183,7 @@ func TestCompressionOptions(t *testing.T) { cfg := testOptions([]string{"127.0.0.1:9092"}) cfg.Compression = test.compression - producerOpts, err := producerOptions(cfg) - require.NoError(t, err) + producerOpts := producerOptions(cfg) client, err := kgo.NewClient(producerOpts...) require.NoError(t, err) diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index c92aeffe7f..7586359be1 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -47,10 +47,7 @@ func newFranzFactory(ctx context.Context, o *options, changefeedID common.Change if err := adjustOptions(ctx, changefeedID, admin, o, o.Topic); err != nil { return nil, err } - producerOpts, err := producerOptions(o) - if err != nil { - return nil, err - } + producerOpts := producerOptions(o) compression := strings.ToLower(strings.TrimSpace(o.Compression)) if compression == "" { @@ -88,7 +85,10 @@ func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { if err != nil { return nil, err } - return &syncProducer{id: f.changefeedID, client: client, timeout: f.timeout}, nil + return &syncProducer{ + id: f.changefeedID, + client: client, + }, nil } func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { @@ -96,7 +96,11 @@ func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) if err != nil { return nil, err } - return &asyncProducer{client: client, changefeedID: f.changefeedID, errCh: make(chan error, 1)}, nil + return &asyncProducer{ + client: client, + changefeedID: f.changefeedID, + errCh: make(chan error, 1), + }, nil } func (f *franzFactory) MetricsCollector(AdminClient) MetricsCollector { diff --git a/pkg/sink/kafka/franz_logger.go b/pkg/sink/kafka/franz_logger.go index 322534550a..54e5854683 100644 --- a/pkg/sink/kafka/franz_logger.go +++ b/pkg/sink/kafka/franz_logger.go @@ -28,9 +28,7 @@ import ( // logValueLimit bounds individual string fields emitted by the franz-go logger. const logValueLimit = 1024 -type clientLogger struct { - logger *zap.Logger -} +type clientLogger struct{ logger *zap.Logger } func newClientLogger(changefeedID common.ChangeFeedID, role string) kgo.Logger { logger := log.L().With( diff --git a/pkg/sink/kafka/franz_logger_test.go b/pkg/sink/kafka/franz_logger_test.go index 31b97c06a2..2507911ae9 100644 --- a/pkg/sink/kafka/franz_logger_test.go +++ b/pkg/sink/kafka/franz_logger_test.go @@ -43,9 +43,7 @@ func TestLoggerLevelAndFiltering(t *testing.T) { require.True(t, isSensitiveLogKey(key)) } - require.NotPanics(t, func() { - clientLogger.Log(kgo.LogLevelWarn, "odd key value", "key-only") - }) + require.NotPanics(t, func() { clientLogger.Log(kgo.LogLevelWarn, "odd key value", "key-only") }) } func TestLoggerPreservesContextAndRedactsValues(t *testing.T) { diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go index c4fb3798e3..b6349f6d9a 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -21,7 +21,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" - codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/zap" ) @@ -29,12 +29,11 @@ import ( type syncProducer struct { id common.ChangeFeedID - client *kgo.Client - closed atomic.Bool - timeout time.Duration + client *kgo.Client + closed atomic.Bool } -func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message) error { +func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -48,7 +47,7 @@ func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionN return p.sendRecords(ctx, message, record) } -func (p *syncProducer) SendMessages(ctx context.Context, topic string, partitionNum int32, message *codeccommon.Message) error { +func (p *syncProducer) SendMessages(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -66,20 +65,15 @@ func (p *syncProducer) SendMessages(ctx context.Context, topic string, partition return p.sendRecords(ctx, message, records...) } -func (p *syncProducer) sendRecords(ctx context.Context, message *codeccommon.Message, records ...*kgo.Record) error { - ctx, cancel := context.WithTimeout(ctx, p.timeout) - defer cancel() - +func (p *syncProducer) sendRecords(ctx context.Context, message *codecCommon.Message, records ...*kgo.Record) error { err := p.client.ProduceSync(ctx, records...).FirstErr() if err == nil { return nil } log.Error("kafka message send failed", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name()), - zap.String("eventContext", BuildEventLogContext( - p.id.Keyspace(), p.id.Name(), message.LogInfo)), + zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), + zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), zap.Error(err)) return errors.WrapError(errors.ErrKafkaSendMessage, err) } @@ -87,8 +81,7 @@ func (p *syncProducer) sendRecords(ctx context.Context, message *codeccommon.Mes func (p *syncProducer) Close() { if !p.closed.CompareAndSwap(false, true) { log.Warn("kafka ddl producer already closed", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name())) + zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name())) return } @@ -96,7 +89,6 @@ func (p *syncProducer) Close() { p.client.Close() log.Info("kafka ddl producer closed", - zap.String("keyspace", p.id.Keyspace()), - zap.String("changefeed", p.id.Name()), + zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start))) } diff --git a/pkg/sink/kafka/franz_sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go index 92677d2a71..a1905d07e0 100644 --- a/pkg/sink/kafka/franz_sync_producer_test.go +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -46,8 +46,7 @@ func TestSyncProducerSendsToRequestedPartitions(t *testing.T) { producer, err := (&franzFactory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), clientOpts: testClientOptions(t, o), - producerOpts: testProducerOptions(t, o), - timeout: requestTimeout(o), + producerOpts: producerOptions(o), }).SyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() @@ -69,8 +68,7 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { producer, err := (&franzFactory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), clientOpts: testClientOptions(t, o), - producerOpts: testProducerOptions(t, o), - timeout: requestTimeout(o), + producerOpts: producerOptions(o), }).SyncProducer(context.Background()) require.NoError(t, err) defer producer.Close() @@ -85,8 +83,7 @@ func TestSyncProducerUsesSendContext(t *testing.T) { producer, err := (&franzFactory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), clientOpts: testClientOptions(t, o), - producerOpts: testProducerOptions(t, o), - timeout: requestTimeout(o), + producerOpts: producerOptions(o), }).SyncProducer(t.Context()) require.NoError(t, err) defer producer.Close() @@ -104,8 +101,7 @@ func TestSyncProducerCloseIsIdempotent(t *testing.T) { client, err := (&franzFactory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), clientOpts: testClientOptions(t, o), - producerOpts: testProducerOptions(t, o), - timeout: requestTimeout(o), + producerOpts: producerOptions(o), }).SyncProducer(context.Background()) require.NoError(t, err) diff --git a/pkg/sink/kafka/metrics_collector_test.go b/pkg/sink/kafka/metrics_collector_test.go index d11bec7f4c..3f2237e331 100644 --- a/pkg/sink/kafka/metrics_collector_test.go +++ b/pkg/sink/kafka/metrics_collector_test.go @@ -30,13 +30,11 @@ func TestCollectBrokerThrottleTime(t *testing.T) { firstBroker := metrics.NewHistogram(metrics.NewUniformSample(10)) firstBroker.Update(10) firstBroker.Update(50) - require.NoError(t, registry.Register( - getBrokerMetricName(throttleTimeMetricNamePrefix, "1"), firstBroker)) + require.NoError(t, registry.Register(getBrokerMetricName(throttleTimeMetricNamePrefix, "1"), firstBroker)) secondBroker := metrics.NewHistogram(metrics.NewUniformSample(10)) secondBroker.Update(40) - require.NoError(t, registry.Register( - getBrokerMetricName(throttleTimeMetricNamePrefix, "2"), secondBroker)) + require.NoError(t, registry.Register(getBrokerMetricName(throttleTimeMetricNamePrefix, "2"), secondBroker)) collector := saramaMetricsCollector{ changefeedID: changefeedID, @@ -45,22 +43,14 @@ func TestCollectBrokerThrottleTime(t *testing.T) { } collector.collectBrokerMetrics() - require.Equal(t, 0.03, testutil.ToFloat64(throttleTimeGauge.WithLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "1", avg))) - require.Equal(t, 0.05, testutil.ToFloat64(throttleTimeGauge.WithLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "1", p99))) - require.Equal(t, 0.04, testutil.ToFloat64(throttleTimeGauge.WithLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "2", avg))) - require.Equal(t, 0.04, testutil.ToFloat64(throttleTimeGauge.WithLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "2", p99))) + require.Equal(t, 0.03, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", avg))) + require.Equal(t, 0.05, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", p99))) + require.Equal(t, 0.04, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", avg))) + require.Equal(t, 0.04, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", p99))) collector.cleanupMetrics() - require.False(t, throttleTimeGauge.DeleteLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "1", avg)) - require.False(t, throttleTimeGauge.DeleteLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "1", p99)) - require.False(t, throttleTimeGauge.DeleteLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "2", avg)) - require.False(t, throttleTimeGauge.DeleteLabelValues( - changefeedID.Keyspace(), changefeedID.Name(), "2", p99)) + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", avg)) + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", p99)) + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", avg)) + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", p99)) } diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index d05bdfd116..2f98dfee48 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -650,13 +650,7 @@ func NewKafkaClientID(captureAddr string, // adjustOptions adjusts options with Kafka runtime metadata. // It overwrites MaxMessageBytes with the final producer message limit derived // from the topic or broker configuration. -func adjustOptions( - ctx context.Context, - changefeedID common.ChangeFeedID, - admin AdminClient, - options *options, - topic string, -) error { +func adjustOptions(ctx context.Context, changefeedID common.ChangeFeedID, admin AdminClient, options *options, topic string) error { // The topic may not exist yet and will be created later by the topic manager, // so ignore per-topic metadata errors here. topics, err := admin.GetTopicsMeta(ctx, []string{topic}, true) @@ -680,13 +674,7 @@ func adjustOptions( return nil } -func adjustExistingTopicOption( - ctx context.Context, - changefeedID common.ChangeFeedID, - admin AdminClient, - options *options, - info TopicDetail, -) error { +func adjustExistingTopicOption(ctx context.Context, changefeedID common.ChangeFeedID, admin AdminClient, options *options, info TopicDetail) error { maxMessageBytes, found, err := getTopicMaxMessageBytes(ctx, admin, info.Name) if err != nil || !found { log.Warn("kafka topic `max.message.bytes` unavailable, using configured value", @@ -702,12 +690,7 @@ func adjustExistingTopicOption( return nil } -func adjustNewTopicOptions( - ctx context.Context, - admin AdminClient, - changefeedID common.ChangeFeedID, - options *options, -) { +func adjustNewTopicOptions(ctx context.Context, admin AdminClient, changefeedID common.ChangeFeedID, options *options) { // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. messageMaxBytes, found, err := getBrokerMaxMessageBytes(ctx, admin) @@ -725,16 +708,8 @@ func adjustNewTopicOptions( } } -func getTopicMaxMessageBytes( - ctx context.Context, - admin AdminClient, - topic string, -) (int, bool, error) { - raw, found, err := getTopicConfig( - ctx, admin, topic, - TopicMaxMessageBytesConfigName, - BrokerMessageMaxBytesConfigName, - ) +func getTopicMaxMessageBytes(ctx context.Context, admin AdminClient, topic string) (int, bool, error) { + raw, found, err := getTopicConfig(ctx, admin, topic, TopicMaxMessageBytesConfigName, BrokerMessageMaxBytesConfigName) if err != nil { return 0, false, err } @@ -767,13 +742,7 @@ func getBrokerMaxMessageBytes(ctx context.Context, admin AdminClient) (int, bool // If the topic does not have this configuration, // we will try to get it from the broker's configuration. // NOTICE: The configuration names of topic and broker may be different for the same configuration. -func getTopicConfig( - ctx context.Context, - admin AdminClient, - topicName string, - topicConfigName string, - brokerConfigName string, -) (string, bool, error) { +func getTopicConfig(ctx context.Context, admin AdminClient, topicName, topicConfigName, brokerConfigName string) (string, bool, error) { c, found, err := admin.GetTopicConfig(ctx, topicName, topicConfigName) if err == nil && found { return c, true, nil diff --git a/pkg/sink/kafka/sarama_sync_producer.go b/pkg/sink/kafka/sarama_sync_producer.go index dd482dcab8..628ea6b51e 100644 --- a/pkg/sink/kafka/sarama_sync_producer.go +++ b/pkg/sink/kafka/sarama_sync_producer.go @@ -44,9 +44,7 @@ type saramaSyncProducer struct { closed *atomic.Bool } -func (p *saramaSyncProducer) SendMessage( - _ context.Context, topic string, partitionNum int32, message *codecCommon.Message, -) error { +func (p *saramaSyncProducer) SendMessage(_ context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -69,9 +67,7 @@ func (p *saramaSyncProducer) SendMessage( return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *saramaSyncProducer) SendMessages( - _ context.Context, topic string, partitionNum int32, message *codecCommon.Message, -) error { +func (p *saramaSyncProducer) SendMessages(_ context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } From 7fc73be1b2fdf644cd3ec8bb79fb09e592d76087 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 16:05:37 +0800 Subject: [PATCH 52/61] add all code --- downstreamadapter/sink/kafka/helper.go | 3 + downstreamadapter/sink/kafka/sink.go | 3 + pkg/sink/codec/avro/encoder.go | 7 ++- pkg/sink/codec/canal/canal_json_encoder.go | 8 +-- .../codec/canal/canal_json_txn_encoder.go | 2 +- pkg/sink/codec/common/config.go | 26 ++++++++ pkg/sink/codec/common/message.go | 13 ++++ pkg/sink/codec/common/message_test.go | 35 +++++++++++ pkg/sink/codec/open/codec.go | 6 +- pkg/sink/codec/open/encoder.go | 12 +++- pkg/sink/codec/open/encoder_test.go | 17 ++++++ pkg/sink/codec/simple/encoder.go | 21 ++++--- pkg/sink/kafka/franz_admin.go | 22 +------ pkg/sink/kafka/franz_admin_test.go | 3 - pkg/sink/kafka/franz_async_producer.go | 9 +-- pkg/sink/kafka/franz_async_producer_test.go | 3 +- pkg/sink/kafka/franz_config.go | 60 +++++++++++++------ pkg/sink/kafka/franz_config_test.go | 28 +++++---- pkg/sink/kafka/franz_factory.go | 8 +-- 19 files changed, 198 insertions(+), 88 deletions(-) create mode 100644 pkg/sink/codec/common/message_test.go diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index a6fa184a15..73940bd552 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -109,6 +109,9 @@ func newKafkaSinkComponent( if err != nil { return comp, protocol, err } + if options.Client == kafka.KafkaClientFranz { + encoderConfig.WithKafkaRecordBatchSize() + } comp.claimCheck, err = claimcheck.New(ctx, encoderConfig.LargeMessageHandle, changefeedID) if err != nil { diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index b4471019e2..90fc087e8e 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -96,6 +96,9 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, if err != nil { return err } + if options.Client == kafka.KafkaClientFranz { + encoderConfig.WithKafkaRecordBatchSize() + } claimCheck, err := claimcheck.New(ctx, encoderConfig.LargeMessageHandle, changefeedID) if err != nil { diff --git a/pkg/sink/codec/avro/encoder.go b/pkg/sink/codec/avro/encoder.go index fdad6c7dbf..7d518b2c2b 100644 --- a/pkg/sink/codec/avro/encoder.go +++ b/pkg/sink/codec/avro/encoder.go @@ -88,12 +88,13 @@ func (a *BatchEncoder) AppendRowChangedEvent( message.Callback = e.Callback message.IncRowsCount() - if message.Length() > a.config.MaxMessageBytes { + length := a.config.MessageLength(message) + if length > a.config.MaxMessageBytes { log.Warn("Single message is too large for avro", zap.Int("maxMessageBytes", a.config.MaxMessageBytes), - zap.Int("length", message.Length()), + zap.Int("length", length), zap.Any("table", e.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), length, a.config.MaxMessageBytes) } a.result = append(a.result, message) diff --git a/pkg/sink/codec/canal/canal_json_encoder.go b/pkg/sink/codec/canal/canal_json_encoder.go index dc076f3ee0..0c914e2f91 100644 --- a/pkg/sink/codec/canal/canal_json_encoder.go +++ b/pkg/sink/codec/canal/canal_json_encoder.go @@ -463,8 +463,8 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( m.IncRowsCount() targetTable := e.TableInfo.GetTargetTableName() - originLength := m.Length() - if m.Length() > c.config.MaxMessageBytes { + originLength := c.config.MessageLength(m) + if originLength > c.config.MaxMessageBytes { // for single message that is longer than max-message-bytes, do not send it. if c.config.LargeMessageHandle.Disabled() { log.Error("Single message is too large for canal-json", @@ -487,7 +487,7 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( } m.Value = value - length := m.Length() + length := c.config.MessageLength(m) if length > c.config.MaxMessageBytes { log.Error("Single message is still too large for canal-json only encode handle-key columns", zap.Int("maxMessageBytes", c.config.MaxMessageBytes), @@ -540,7 +540,7 @@ func (c *JSONRowEventEncoder) newClaimCheckLocationMessage( result.Callback = event.Callback result.IncRowsCount() - length := result.Length() + length := c.config.MessageLength(result) if length > c.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json, when create the claim check location message", zap.Int("maxMessageBytes", c.config.MaxMessageBytes), diff --git a/pkg/sink/codec/canal/canal_json_txn_encoder.go b/pkg/sink/codec/canal/canal_json_txn_encoder.go index 61f20ab173..6c789b10c3 100644 --- a/pkg/sink/codec/canal/canal_json_txn_encoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_encoder.go @@ -50,7 +50,7 @@ func (j *JSONTxnEventEncoder) AppendTxnEvent(rowEvents []*commonEvent.RowEvent) if err != nil { return err } - length := len(value) + common.MaxRecordOverhead + length := j.config.MessageLengthForKeyValue(0, len(value)) if length > j.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json", zap.Int("maxMessageBytes", j.config.MaxMessageBytes), diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 58c34aaab9..10687af7db 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -48,6 +48,8 @@ type Config struct { // MaxBatchedBytes controls open-protocol encoder's maximum number of events for a batched message. MaxBatchSize int + useKafkaRecordBatchSize bool + // DeleteOnlyHandleKeyColumns is true, for the delete event only output the handle key columns. DeleteOnlyHandleKeyColumns bool @@ -382,6 +384,30 @@ func (c *Config) WithMaxBatchedBytes(bytes int) *Config { return c } +// WithKafkaRecordBatchSize applies encoder byte limits to the complete +// uncompressed Kafka record batch rather than a client-specific estimate. +func (c *Config) WithKafkaRecordBatchSize() *Config { + c.useKafkaRecordBatchSize = true + return c +} + +// MessageLength returns the size used by encoder byte-limit checks. +func (c *Config) MessageLength(message *Message) int { + if c.useKafkaRecordBatchSize { + return message.KafkaRecordBatchLength() + } + return message.Length() +} + +// MessageLengthForKeyValue returns the size used by encoder byte-limit checks +// before a Message has been constructed. +func (c *Config) MessageLengthForKeyValue(keyLength, valueLength int) int { + if c.useKafkaRecordBatchSize { + return keyLength + valueLength + kafkaRecordBatchOverhead + } + return keyLength + valueLength + MaxRecordOverhead +} + // WithChangefeedID set the `changefeedID` func (c *Config) WithChangefeedID(id common.ChangeFeedID) *Config { c.ChangefeedID = id diff --git a/pkg/sink/codec/common/message.go b/pkg/sink/codec/common/message.go index 5cfd56b010..869942ddd1 100644 --- a/pkg/sink/codec/common/message.go +++ b/pkg/sink/codec/common/message.go @@ -25,6 +25,13 @@ import ( // which will be treated as `version = 2` by sarama producer. const MaxRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 +// kafkaRecordBatchOverhead is the maximum framing added to one headerless +// record in a Kafka record batch. franz-go counts 65 bytes of fixed batch +// framing. The record adds at most 19 bytes: 5 for its length, 1 each for +// attributes, timestamp delta and offset delta, 5 each for key and value +// lengths, and 1 for the zero header count. +const kafkaRecordBatchOverhead = 65 + 19 + // MessageType is the type of message, which is used by MqSink and RedoLog. type MessageType int @@ -94,6 +101,12 @@ func (m *Message) Length() int { return len(m.Key) + len(m.Value) + MaxRecordOverhead } +// KafkaRecordBatchLength returns a conservative uncompressed size for a record +// batch containing only this message and no record headers. +func (m *Message) KafkaRecordBatchLength() int { + return len(m.Key) + len(m.Value) + kafkaRecordBatchOverhead +} + // GetRowsCount returns the number of rows batched in one Message func (m *Message) GetRowsCount() int { return m.rowsCount diff --git a/pkg/sink/codec/common/message_test.go b/pkg/sink/codec/common/message_test.go new file mode 100644 index 0000000000..886c5b76d0 --- /dev/null +++ b/pkg/sink/codec/common/message_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +func TestKafkaRecordBatchLength(t *testing.T) { + message := NewMsg([]byte("k"), []byte("v")) + require.Equal(t, len(message.Key)+len(message.Value)+84, message.KafkaRecordBatchLength()) +} + +func TestConfigMessageLength(t *testing.T) { + message := NewMsg([]byte("k"), []byte("v")) + codecConfig := NewConfig(config.ProtocolOpen) + require.Equal(t, message.Length(), codecConfig.MessageLength(message)) + + codecConfig.WithKafkaRecordBatchSize() + require.Equal(t, message.KafkaRecordBatchLength(), codecConfig.MessageLength(message)) +} diff --git a/pkg/sink/codec/open/codec.go b/pkg/sink/codec/open/codec.go index 454cb0d372..a9e3e80970 100644 --- a/pkg/sink/codec/open/codec.go +++ b/pkg/sink/codec/open/codec.go @@ -112,9 +112,9 @@ func encodeRowChangedEvent( return nil, nil, 0, err } - // for single message that is longer than max-message-bytes - // 16 is the length of `keyLenByte` and `valueLenByte`, 8 is the length of `versionHead` - length := len(key) + len(valueCompressed) + common.MaxRecordOverhead + 16 + 8 + // Open Protocol adds an 8-byte version, key length, and value length to + // the encoded row before it becomes one Kafka record. + length := config.MessageLengthForKeyValue(len(key)+16, len(valueCompressed)+8) return key, valueCompressed, length, nil } diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index b8e608a298..df12ad7664 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -155,8 +155,6 @@ func (d *batchEncoder) Build() (messages []*common.Message) { } func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { - length := len(key) + len(value) + 16 - var ( keyLenByte [8]byte valueLenByte [8]byte @@ -164,7 +162,15 @@ func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { binary.BigEndian.PutUint64(keyLenByte[:], uint64(len(key))) binary.BigEndian.PutUint64(valueLenByte[:], uint64(len(value))) - if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxBatchedBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { + var batchedMessageLength int + if len(d.messages) > 0 { + latestMessage := d.messages[len(d.messages)-1] + batchedMessageLength = d.config.MessageLengthForKeyValue( + len(latestMessage.Key)+len(keyLenByte)+len(key), + len(latestMessage.Value)+len(valueLenByte)+len(value), + ) + } + if len(d.messages) == 0 || batchedMessageLength > d.config.MaxBatchedBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { d.finalizeCallback() // create a new message versionHead := make([]byte, 8) diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index de88347061..0b1fc9909f 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -857,6 +857,23 @@ func TestEncoderMultipleMessage(t *testing.T) { common.CompareRow(t, insertEvents[2].Event, insertEvents[2].TableInfo, change, decoded.TableInfo) } +func TestEncoderUsesKafkaRecordBatchSize(t *testing.T) { + codecConfig := common.NewConfig(config.ProtocolOpen). + WithMaxMessageBytes(200). + WithMaxBatchedBytes(120). + WithKafkaRecordBatchSize() + encoder := &batchEncoder{config: codecConfig} + + encoder.pushMessage([]byte("k"), []byte("v"), nil) + require.Len(t, encoder.messages, 1) + require.LessOrEqual(t, encoder.messages[0].KafkaRecordBatchLength(), codecConfig.MaxBatchedBytes) + + encoder.pushMessage([]byte("k"), []byte("v"), nil) + require.Len(t, encoder.messages, 2) + require.Equal(t, 1, encoder.messages[0].GetRowsCount()) + require.Equal(t, 1, encoder.messages[1].GetRowsCount()) +} + func TestMessageTooLarge(t *testing.T) { ctx := context.Background() codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(100) diff --git a/pkg/sink/codec/simple/encoder.go b/pkg/sink/codec/simple/encoder.go index c8a4208f59..3b18aa44dd 100644 --- a/pkg/sink/codec/simple/encoder.go +++ b/pkg/sink/codec/simple/encoder.go @@ -62,7 +62,8 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co } result.IncRowsCount() - length := result.Length() + length := e.config.MessageLength(result) + originLength := length if length <= e.config.MaxMessageBytes { e.messages = append(e.messages, result) return nil @@ -95,11 +96,12 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co } result.Value = value - if result.Length() <= e.config.MaxMessageBytes { + length = e.config.MessageLength(result) + if length <= e.config.MaxMessageBytes { log.Warn("Single message is too large for simple, only encode handle key columns", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("originLength", length), - zap.Int("length", result.Length()), + zap.Int("originLength", originLength), + zap.Int("length", length), zap.Any("table", event.TableInfo.TableName)) e.messages = append(e.messages, result) return nil @@ -107,9 +109,9 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co log.Error("Single message is still too large for simple after only encode handle key columns", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("length", result.Length()), + zap.Int("length", length), zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) } // Build implement the RowEventEncoder interface @@ -148,12 +150,13 @@ func (e *Encoder) EncodeDDLEvent(event *commonEvent.DDLEvent) (*common.Message, } result := common.NewMsg(nil, value) - if result.Length() > e.config.MaxMessageBytes { + length := e.config.MessageLength(result) + if length > e.config.MaxMessageBytes { log.Error("DDL message is too large for simple", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("length", result.Length()), + zap.Int("length", length), zap.String("table", event.GetTargetTableName())) - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs(event.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs(event.GetTargetTableName(), length, e.config.MaxMessageBytes) } return result, nil } diff --git a/pkg/sink/kafka/franz_admin.go b/pkg/sink/kafka/franz_admin.go index 5ff644f108..1b998f4173 100644 --- a/pkg/sink/kafka/franz_admin.go +++ b/pkg/sink/kafka/franz_admin.go @@ -28,12 +28,10 @@ import ( type admin struct { changefeed common.ChangeFeedID - - admin *kadm.Client - timeout time.Duration + admin *kadm.Client } -func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt, timeout time.Duration) (*admin, error) { +func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt) (*admin, error) { opts := make([]kgo.Opt, 0, len(clientOpts)+3) opts = append(opts, clientOpts...) opts = append(opts, kgo.WithContext(ctx), kgo.WithLogger(newClientLogger(changefeedID, "admin"))) @@ -49,14 +47,10 @@ func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts return &admin{ changefeed: changefeedID, admin: kadm.NewClient(client), - timeout: timeout, }, nil } func (a *admin) GetAllBrokers(ctx context.Context) []Broker { - ctx, cancel := context.WithTimeout(ctx, a.timeout) - defer cancel() - meta, err := a.admin.BrokerMetadata(ctx) if err != nil { return nil @@ -70,9 +64,6 @@ func (a *admin) GetAllBrokers(ctx context.Context) []Broker { } func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, bool, error) { - ctx, cancel := context.WithTimeout(ctx, a.timeout) - defer cancel() - meta, err := a.admin.BrokerMetadata(ctx) if err != nil { if isAuthorizationFailed(err) { @@ -118,9 +109,6 @@ func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, } func (a *admin) GetTopicConfig(ctx context.Context, topicName string, configName string) (string, bool, error) { - ctx, cancel := context.WithTimeout(ctx, a.timeout) - defer cancel() - configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) if err != nil { if isAuthorizationFailed(err) { @@ -157,9 +145,6 @@ func (a *admin) GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicE return make(map[string]TopicDetail), nil } - ctx, cancel := context.WithTimeout(ctx, a.timeout) - defer cancel() - meta, err := a.admin.Metadata(ctx, topics...) if err != nil { resource := strings.Join(topics, ",") @@ -223,9 +208,6 @@ func (a *admin) GetTopicsPartitionsNum(ctx context.Context, topics []string) (ma } func (a *admin) CreateTopic(ctx context.Context, detail *TopicDetail) error { - ctx, cancel := context.WithTimeout(ctx, a.timeout) - defer cancel() - responses, err := a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) if err != nil { if isAuthorizationFailed(err) { diff --git a/pkg/sink/kafka/franz_admin_test.go b/pkg/sink/kafka/franz_admin_test.go index 9b44454c8f..d02d1f353a 100644 --- a/pkg/sink/kafka/franz_admin_test.go +++ b/pkg/sink/kafka/franz_admin_test.go @@ -182,7 +182,6 @@ func TestAdminHonorsCallContext(t *testing.T) { t.Context(), common.NewChangefeedID4Test(common.DefaultKeyspaceName, "context"), testClientOptions(t, o), - requestTimeout(o), ) require.NoError(t, err) t.Cleanup(admin.Close) @@ -205,7 +204,6 @@ func TestAdminOperations(t *testing.T) { ctx, common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), testClientOptions(t, o), - requestTimeout(o), ) require.NoError(t, err) defer admin.Close() @@ -264,7 +262,6 @@ func TestCreateTopicErrors(t *testing.T) { ctx, common.NewChangefeedID4Test(common.DefaultKeyspaceName, "create-errors"), testClientOptions(t, o), - requestTimeout(o), ) require.NoError(t, err) defer admin.Close() diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index 68e19428bf..f17f731ad0 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -30,16 +30,14 @@ type asyncProducer struct { client *kgo.Client changefeedID common.ChangeFeedID - closeStarted atomic.Bool - closed atomic.Bool - errCh chan error + closed atomic.Bool + errCh chan error } func (p *asyncProducer) Close() { - if !p.closeStarted.CompareAndSwap(false, true) { + if !p.closed.CompareAndSwap(false, true) { return } - p.closed.Store(true) start := time.Now() p.client.Close() @@ -97,7 +95,6 @@ func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition i } func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { - defer p.closed.Store(true) for { select { case <-ctx.Done(): diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index 535d98faa2..4a620489db 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -45,7 +45,7 @@ func TestAsyncSendCanceledContext(t *testing.T) { require.ErrorIs(t, producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{}), context.Canceled) } -func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { +func TestAsyncRunCallbackReturnsQueuedError(t *testing.T) { producer := &asyncProducer{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), errCh: make(chan error, 1), @@ -55,7 +55,6 @@ func TestAsyncRunCallbackReturnsQueuedErrorAndCloses(t *testing.T) { err := producer.AsyncRunCallback(context.Background()) require.ErrorIs(t, err, context.DeadlineExceeded) - require.True(t, producer.closed.Load()) } func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go index 0e3a307cac..973f4814db 100644 --- a/pkg/sink/kafka/franz_config.go +++ b/pkg/sink/kafka/franz_config.go @@ -20,7 +20,6 @@ import ( "net/http" "net/url" "strings" - "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" @@ -35,16 +34,15 @@ import ( "golang.org/x/oauth2/clientcredentials" ) -// producerMaxBufferedBytes bounds each producer client's buffered payload. -// Produce blocks when the buffer is full and resumes when records complete -// or its context is canceled. A larger single record fails with MessageTooLarge. -const producerMaxBufferedBytes = 64 << 20 - -// Kafka defaults socket.request.max.bytes to 100 MiB. Keeping franz-go at the -// same limit prevents a Topic's batch configuration from increasing request memory. -const franzDefaultMaxRequestBytes = 100 << 20 - -func requestTimeout(o *options) time.Duration { return max(o.ReadTimeout, o.WriteTimeout) } +// franz-go counts a record from Produce acceptance until its delivery callback +// returns, including metadata lookup, batching, sending, Broker response, and +// retries. A record larger than the byte limit fails immediately; otherwise, +// either limit blocks later Produce calls. Their ratio is 1 KiB per record, so +// bytes govern larger records while count bounds smaller record objects. +const ( + producerMaxBufferedBytes = 64 << 20 + producerMaxBufferedRecords = 1 << 16 +) // Admin and producer clients share connection options. Producer delivery and // resource limits stay separate so they cannot affect admin operations. @@ -53,7 +51,10 @@ func clientOptions(o *options) ([]kgo.Opt, error) { kgo.SeedBrokers(o.BrokerEndpoints...), kgo.ClientID(o.ClientID), kgo.DialTimeout(o.DialTimeout), - kgo.RequestTimeoutOverhead(requestTimeout(o)), + // franz-go does not expose an independent socket read timeout. This value + // sets the socket write deadline and is added to each request-specific + // Broker processing timeout to form the socket read deadline. + kgo.RequestTimeoutOverhead(o.WriteTimeout), } if o.EnableTLS { @@ -83,7 +84,6 @@ func clientOptions(o *options) ([]kgo.Opt, error) { } func producerOptions(o *options) []kgo.Opt { - maxMessageBytes := int32(min(o.MaxMessageBytes, franzDefaultMaxRequestBytes)) return []kgo.Opt{ kgo.RecordPartitioner(kgo.ManualPartitioner()), kgo.RequiredAcks(requiredAcks(o.RequiredAcks)), @@ -91,17 +91,36 @@ func producerOptions(o *options) []kgo.Opt { kgo.DisableIdempotentWrite(), // More than one in-flight request can reorder records when an earlier request is retried. kgo.MaxProduceRequestsInflightPerBroker(1), + // The default of five retries allows six Produce attempts. franz-go's + // default jittered backoff adds about 6.2s to 9.3s across five retries. kgo.RecordRetries(o.MaxRetry), kgo.UnknownTopicRetries(o.MaxRetry), // Limit each client to 64 MiB of buffered payload. The in-flight limit // applies per broker and does not bound records queued for other brokers, // metadata, or retries, so the producer needs a separate byte limit. // 64 MiB leaves room above TiCDC's default 10 MiB message limit while - // bounding the memory hidden by the 10,000-record default. + // bounding buffered payload memory. kgo.MaxBufferedBytes(producerMaxBufferedBytes), + kgo.MaxBufferedRecords(producerMaxBufferedRecords), // ProducerBatchMaxBytes is franz-go's name for Kafka's max.message.bytes limit. - kgo.ProducerBatchMaxBytes(maxMessageBytes), - kgo.ProduceRequestTimeout(requestTimeout(o)), + kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), + // This value limits how long the Broker may process a Produce request; + // read-timeout is therefore not an exact socket read deadline. Together + // with RequestTimeoutOverhead above, the socket write deadline is + // write-timeout, the Broker processing timeout is read-timeout, and the + // socket read deadline is read-timeout plus write-timeout. + // With the default 10s timeout, the Broker may process a Produce request + // for 10s, the socket write deadline is 10s, and the socket read deadline + // is 20s. Across six attempts, consecutive timeouts take about 66s-69s + // when the Broker returns on its processing deadline, 126s-129s when it + // never replies, or 186s-189s if every write and read reaches its deadline. + // Buffering, metadata lookup, connection setup, and Broker throttling are + // not included; the caller context is the end-to-end bound. + // A Broker processing timeout returns REQUEST_TIMED_OUT, which franz-go + // retries. The original record may already be stored, so retries may create + // duplicates while idempotent writes are disabled. Exhausting the retry + // budget fails the record and reports the error through its callback. + kgo.ProduceRequestTimeout(o.ReadTimeout), kgo.ProducerLinger(0), compressionOption(o.Compression), } @@ -164,10 +183,15 @@ func buildOAuthMechanism(cfg oauth2Config) (sasl.Mechanism, error) { }), nil } -func newProducerClient(ctx context.Context, changefeedID common.ChangeFeedID, role string, clientOpts []kgo.Opt, producerOpts []kgo.Opt) (*kgo.Client, error) { +func newProducerClient( + ctx context.Context, changefeedID common.ChangeFeedID, role string, clientOpts []kgo.Opt, producerOpts []kgo.Opt, +) (*kgo.Client, error) { opts := make([]kgo.Opt, 0, len(clientOpts)+len(producerOpts)+3) opts = append(opts, clientOpts...) - opts = append(opts, kgo.WithContext(ctx), kgo.WithLogger(newClientLogger(changefeedID, role)), kgo.WithHooks(newMetricsHook(changefeedID))) + opts = append(opts, + kgo.WithContext(ctx), + kgo.WithLogger(newClientLogger(changefeedID, role)), + kgo.WithHooks(newMetricsHook(changefeedID))) opts = append(opts, producerOpts...) client, err := kgo.NewClient(opts...) diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index be94c0385f..08b9e62f75 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -62,12 +62,18 @@ func TestFranzRequiredAcks(t *testing.T) { } } -func TestFranzRequestTimeoutUsesLargerTimeout(t *testing.T) { - o := &options{ReadTimeout: time.Second, WriteTimeout: 2 * time.Second} - require.Equal(t, 2*time.Second, requestTimeout(o)) - +func TestFranzProducerTimeouts(t *testing.T) { + o := testOptions([]string{"127.0.0.1:9092"}) o.ReadTimeout = 3 * time.Second - require.Equal(t, 3*time.Second, requestTimeout(o)) + o.WriteTimeout = 2 * time.Second + + opts := testClientOptions(t, o) + client, err := kgo.NewClient(append(opts, producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, 2*time.Second, client.OptValue(kgo.RequestTimeoutOverhead)) + require.Equal(t, 3*time.Second, client.OptValue(kgo.ProduceRequestTimeout)) } func TestFranzIgnoresConfiguredKafkaVersion(t *testing.T) { @@ -108,7 +114,7 @@ func TestProducerOptionsConfigureMessageAndBufferLimits(t *testing.T) { require.Equal(t, int32(maxMessageBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) - require.Equal(t, int64(10000), client.OptValue(kgo.MaxBufferedRecords)) + require.Equal(t, int64(producerMaxBufferedRecords), client.OptValue(kgo.MaxBufferedRecords)) require.Equal(t, int64(1), client.OptValue(kgo.RecordRetries)) require.Equal(t, int64(1), client.OptValue(kgo.UnknownTopicRetries)) require.Equal(t, int32(100<<20), client.OptValue(kgo.BrokerMaxWriteBytes)) @@ -152,18 +158,20 @@ func TestProducerOptionsDoNotClampSmallBatch(t *testing.T) { require.Error(t, err) } -func TestProducerOptionsCapMessageBytesAtRequestLimit(t *testing.T) { +func TestProducerOptionsUseKafkaBatchLimitDirectly(t *testing.T) { + const maxMessageBytes = 128 << 20 config := testOptions([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = franzDefaultMaxRequestBytes + 1 + config.MaxMessageBytes = maxMessageBytes producerOpts := producerOptions(config) + // Raise the request limit only to let franz-go validate this test client. + producerOpts = append(producerOpts, kgo.BrokerMaxWriteBytes(maxMessageBytes)) client, err := kgo.NewClient(producerOpts...) require.NoError(t, err) defer client.Close() - require.Equal(t, int32(franzDefaultMaxRequestBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) - require.Equal(t, int32(franzDefaultMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) + require.Equal(t, int32(maxMessageBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) } func TestCompressionOptions(t *testing.T) { diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 7586359be1..bbdbcd45ee 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -17,7 +17,6 @@ package kafka import ( "context" "strings" - "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" @@ -29,7 +28,6 @@ type franzFactory struct { changefeedID common.ChangeFeedID clientOpts []kgo.Opt producerOpts []kgo.Opt - timeout time.Duration } func newFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { @@ -37,8 +35,7 @@ func newFranzFactory(ctx context.Context, o *options, changefeedID common.Change if err != nil { return nil, err } - timeout := requestTimeout(o) - admin, err := newAdmin(ctx, changefeedID, clientOpts, timeout) + admin, err := newAdmin(ctx, changefeedID, clientOpts) if err != nil { return nil, err } @@ -72,12 +69,11 @@ func newFranzFactory(ctx context.Context, o *options, changefeedID common.Change changefeedID: changefeedID, clientOpts: clientOpts, producerOpts: producerOpts, - timeout: timeout, }, nil } func (f *franzFactory) AdminClient(ctx context.Context) (AdminClient, error) { - return newAdmin(ctx, f.changefeedID, f.clientOpts, f.timeout) + return newAdmin(ctx, f.changefeedID, f.clientOpts) } func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { From da6a1c42a141b16b8b9744ad420cb58887c6817b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 17:40:08 +0800 Subject: [PATCH 53/61] fix more code --- docs/franz-go/kafka-producer-size-limits.md | 249 -------------------- pkg/sink/kafka/factory.go | 6 +- pkg/sink/kafka/franz_admin.go | 12 +- pkg/sink/kafka/franz_admin_test.go | 44 +++- pkg/sink/kafka/franz_async_producer.go | 47 ++-- pkg/sink/kafka/franz_async_producer_test.go | 74 +++++- pkg/sink/kafka/franz_config.go | 22 +- pkg/sink/kafka/franz_config_test.go | 49 ++-- pkg/sink/kafka/franz_factory.go | 4 +- 9 files changed, 177 insertions(+), 330 deletions(-) delete mode 100644 docs/franz-go/kafka-producer-size-limits.md diff --git a/docs/franz-go/kafka-producer-size-limits.md b/docs/franz-go/kafka-producer-size-limits.md deleted file mode 100644 index 55ca8ce26e..0000000000 --- a/docs/franz-go/kafka-producer-size-limits.md +++ /dev/null @@ -1,249 +0,0 @@ -# Kafka Producer 大小限制 - -Last updated: 2026-09-03 -Status: Produce request 和 producer buffer 已确定；消息与 record batch 的边界仍在讨论 - -## 背景 - -Kafka producer 处理四种不同对象：单条消息、record batch、Produce request 和内存 buffer。它们的限制作用在不同阶段。复用一个值会产生两个问题：Kafka Topic 配置可能意外放大进程内存，单条消息也可能通过 encoder 后被 producer 拒绝。 - -本文先说明每个参数的作用范围，再用具体场景说明参数之间的关系。本文使用 MiB，因为代码以二进制移位表示容量：`1 MiB = 1 << 20 bytes`。MB 表示十进制容量：`1 MB = 1,000,000 bytes`。 - -## 参数及作用范围 - -- `max-message-bytes` - - 配置位置：Sink URI 或 SinkConfig。 - - 作用对象：encoder 生成的 `common.Message`。 - - 当前取值：默认 10 MiB。 - - 配置方式：用户可独立配置。 - -- Topic `max.message.bytes` - - 配置位置：Kafka Topic。 - - 作用对象：broker 接受的 record batch。 - - 当前取值：TiCDC 读取 Kafka 配置。 - - 配置方式：由 Kafka 管理员独立配置。 - -- Broker `message.max.bytes` - - 配置位置：Kafka Broker。 - - 作用对象：Topic 未覆盖时的 record batch。 - - 当前取值：TiCDC 读取 Kafka 配置。 - - 配置方式：由 Kafka 管理员独立配置。 - -- `ProducerBatchMaxBytes` - - 配置位置：franz-go client。 - - 作用对象：未压缩的完整 record batch。 - - 当前取值：Kafka batch 上限与 100 MiB 的较小值。 - - 配置方式：TiCDC 内部计算，不对用户开放。 - -- `BrokerMaxWriteBytes` - - 配置位置：franz-go client。 - - 作用对象：发给一个 broker 的完整 Produce request。 - - 当前取值：franz-go 默认 100 MiB。 - - 配置方式：不对用户开放。 - -- `MaxBufferedBytes` - - 配置位置：franz-go client。 - - 作用对象：一个 client 中尚未完成的 record payload 总量。 - - 当前取值：固定 64 MiB。 - - 配置方式：不对用户开放。 - -- `MaxBufferedRecords` - - 配置位置：franz-go client。 - - 作用对象：一个 client 中尚未完成的 record 数量。 - - 当前取值：franz-go 默认 10,000。 - - 配置方式：不对用户开放。 - -- `MaxProduceRequestsInflightPerBroker` - - 配置位置：franz-go client。 - - 作用对象：每个 broker 同时在途的 Produce request 数量。 - - 当前取值：固定 1。 - - 配置方式：不对用户开放。 - -`max-message-bytes` 是当前唯一由 TiCDC 用户直接配置的大小参数。Produce request 和 producer buffer 使用固定值。Kafka record batch 的服务端上限由 Kafka 管理员控制。 - -## 数据经过哪些限制 - -```text -row events - ↓ encoder -common.Message / Kafka record - ↓ 按 topic-partition 组 batch -record batch - ↓ 按目标 broker 组 request -Produce request -``` - -record 被 franz-go 接收后会计入 producer buffer，直到发送成功、发送失败或被取消。等待 metadata、等待组 batch、等待发送、等待 broker 响应和等待重试的 record 都占用 buffer。 - -## 已确定方案 - -### Produce request 固定为 100 MiB - -TiCDC 不再设置 `BrokerMaxWriteBytes`，沿用 franz-go 的 100 MiB 默认值。该值对应 Kafka `socket.request.max.bytes` 的常见默认值。 - -Kafka Topic 的 `max.message.bytes` 不再放大 Produce request。franz-go 会把发往同一 broker 的多个 record batches 放入一个 request；达到 100 MiB 后创建下一个 request。 - -TiCDC 将 `ProducerBatchMaxBytes` 设置为 Kafka batch 上限与 100 MiB 的较小值。franz-go 随后按实际 client ID、Topic 名称和协议字段扣除 request envelope。 - -TiCDC 代码继续使用 `maxMessageBytes` 命名，与 Kafka `max.message.bytes` 和原 Sarama 配置保持一致。`ProducerBatchMaxBytes` 只作为 franz-go API 名称出现。 - -`MaxProduceRequestsInflightPerBroker(1)` 保持不变。一个 broker 同时最多有一个在途 Produce request。多个 broker 可以各有一个在途 request。 - -### Producer buffer 固定为 64 MiB - -每个 franz-go client 设置： - -```go -kgo.MaxBufferedBytes(64 << 20) -``` - -不增加 `max-buffered-bytes` Sink URI 参数。64 MiB 是每个 client 的 payload 上限，实际 RSS 还包括 record 对象、batch 编码、压缩空间、request 和网络 buffer。 - -`MaxBufferedBytes` 与默认的 `MaxBufferedRecords=10000` 同时生效。任一上限先达到，`Produce` 就会等待已有 record 完成或 context 被取消。 - -64 MiB 高于 TiCDC 默认的 10 MiB `max-message-bytes`，同时限制 10,000 条 record 可能占用的 payload 内存。该值会限制单条 record：当 key、value 和 headers 的总长度超过 64 MiB 时，franz-go 直接返回 `kerr.MessageTooLarge`。 - -`MaxProduceRequestsInflightPerBroker(1)` 无法替代 buffer 限制。它只限制在途 request 数量；等待 metadata、等待重试及发往其他 broker 的 records 仍可留在 buffer 中。 - -## 配置场景 - -### 场景一：普通消息，Kafka batch 上限约 1 MiB - -配置示例： - -```text -Sink URI 不设置 max-message-bytes，使用默认值 10485760（10 MiB） -Kafka Topic max.message.bytes = 1048576（1 MiB） -Produce request = 100 MiB -Producer buffer = 64 MiB -``` - -结果： - -- encoder 的普通消息不能超过 Kafka record batch 实际可容纳的范围。 -- 一个 Produce request 可以携带多个约 1 MiB 的 record batches。 -- 如果平均 payload 为 1 MiB，buffer 大约容纳 64 条 record，随后开始反压。 - -这里的 `max-message-bytes`、Produce request 和 producer buffer 使用各自的值。Kafka record batch 上限仍会约束 encoder 的最终有效上限，具体 framing 预留量待确认。 - -### 场景二：普通消息按 1 MiB 聚合，允许 10 MiB 的单行大消息 - -配置示例： - -```text -Sink URI: kafka://broker/topic?protocol=open-protocol&max-message-bytes=1048576 -Kafka Topic max.message.bytes = 10485760 -Produce request = 100 MiB -Producer buffer = 64 MiB -``` - -`max-message-bytes=1048576` 表示 1 MiB，Kafka 的 `10485760` 表示 10 MiB。 - -预期结果： - -- Open Protocol 普通多行消息在约 1 MiB 时结束聚合。 -- 单行大消息可以继续使用 Kafka record batch 提供的空间。 -- franz-go 的 `ProducerBatchMaxBytes` 应接近 10 MiB，并为 batch framing 留出空间。 -- 100 MiB request 和 64 MiB buffer 都能容纳这类消息。 - -这个场景体现 `max-message-bytes` 与 Kafka `max.message.bytes` 的独立用途：前者控制普通消息聚合，后者控制单条大消息和 record batch 的最终上限。 - -### 场景三：Kafka 允许 80 MiB batch，单条 payload 为 70 MiB - -配置示例： - -```text -Kafka Topic max.message.bytes = 83886080（80 MiB） -单条 record payload = 73400320（70 MiB） -Produce request = 100 MiB -Producer buffer = 64 MiB -``` - -结果：franz-go 会因为单条 payload 超过 `MaxBufferedBytes` 而拒绝该 record。Kafka Topic 和 Produce request 虽然具备足够空间，固定的 64 MiB buffer 仍形成了单条 record 上限。 - -当前方案因此不支持超过 64 MiB 的单条 payload。后续需要决定 encoder 是否提前按该限制报错，保证错误发生在 producer 之前。 - -### 场景四：Kafka batch 上限超过 100 MiB - -配置示例： - -```text -Kafka Topic max.message.bytes = 134217728（128 MiB） -Produce request = 100 MiB -ProducerBatchMaxBytes = 100 MiB -``` - -Kafka 允许管理员将 `max.message.bytes` 调到 100 MiB 以上。broker 的 `socket.request.max.bytes` 也必须相应增大，才能接收这种 batch。TiCDC 当前固定使用 100 MiB Produce request，因此 franz-go 的 batch 配置会限制为 100 MiB；franz-go 还会扣除 request envelope。 - -### 场景五：Kafka 暂时不可用 - -假设平均 payload 为 1 MiB： - -- buffer 接受约 64 条 record 后达到 64 MiB。 -- 后续 `Produce` 阻塞，反压逐步传回上游。 -- Kafka 恢复后，已缓存 records 继续发送。 -- context 取消时，阻塞中的调用退出。 - -假设平均 payload 为 4 KiB，10,000 条 record 约占 39 MiB。此时 `MaxBufferedRecords=10000` 先达到，record 数量限制触发反压。 - -### 场景六：Topic 分区分布在三个 broker - -franz-go 最多可以同时存在三个在途 Produce requests，每个 broker 一个。所有等待中和在途的 records 共用同一个 64 MiB client buffer。 - -这个场景说明 `MaxProduceRequestsInflightPerBroker` 与 `MaxBufferedBytes` 可以分别设置。前者控制每个 broker 的请求并发，后者控制整个 client 的 payload 总量。 - -### 场景七：一个进程运行多个 Changefeed - -Kafka sink 会为一个 Changefeed 创建 async 和 sync 两个 producer clients。每个 client 的上限都是 64 MiB，因此一个 Changefeed 的 producer payload 理论上限为 128 MiB。sync producer 主要发送低频 DDL，通常不会长期占满。 - -例如，一个进程运行 10 个 Changefeed，producer payload 的理论上限为： - -```text -10 × 2 × 64 MiB = 1280 MiB -``` - -该数字仍未包含 encoder、event pipeline、batch、压缩和网络相关内存。 - -## 参数之间的关系 - -- `max-message-bytes` 与 Produce request 独立配置。修改 `max-message-bytes` 不会改变 100 MiB request。 -- Kafka `max.message.bytes` 与 Produce request 独立配置。一个完整 batch 加 request envelope 必须小于 100 MiB。 -- `ProducerBatchMaxBytes` 受 Kafka `max.message.bytes` 约束，不能超过 Kafka 接受的 batch。 -- `ProducerBatchMaxBytes` 受 Produce request 约束，必须能装入一个 100 MiB request。 -- 单条 record 受 `MaxBufferedBytes` 约束，payload 必须小于等于 64 MiB。 -- `MaxBufferedBytes` 与 Produce request 独立配置，两者限制不同对象，数值无需相等。 -- `MaxBufferedBytes` 与 `MaxBufferedRecords` 独立且同时生效，任一上限先达到就触发反压。 -- `MaxBufferedBytes` 与每 broker 在途请求数独立。buffer 统计整个 client，在途请求数按 broker 统计。 - -## 待讨论项 - -- encoder 应为 franz-go record batch framing 预留多少空间。 -- 单条 payload 超过固定 64 MiB 时，encoder 如何提前执行 claim-check、handle-key-only 或报错。 -- Admin API 无法读取 Topic/Broker 配置时，应该使用哪个 fallback 上限。 -- 动态 Topic 具有不同 `max.message.bytes` 时，是否需要 per-topic batch limit。 - -## 已落地的代码改动 - -`pkg/sink/kafka/franz_config.go`： - -- 删除显式 `BrokerMaxWriteBytes`，使用 franz-go 默认的 100 MiB。 -- 固定设置 `MaxBufferedBytes(64 << 20)`。 -- 删除本地的 512 bytes 和 1 GiB batch 范围常量，不再静默提升 batch 上限。 -- 将 Kafka batch 上限限制在 100 MiB 以内，再传给 franz-go。 -- 保留 `MaxBufferedRecords=10000` 默认值。 -- 保留 `MaxProduceRequestsInflightPerBroker(1)`。 -- 代码注释说明 100 MiB request 和 64 MiB buffer 的作用范围及取值原因。 - -本次没有增加 Sink URI 参数。encoder 的消息边界将在上述待讨论项确认后实现。 - -## 验证 - -当前单元测试应确认： - -- `BrokerMaxWriteBytes` 为 franz-go 默认的 100 MiB。 -- `MaxBufferedBytes` 固定为 64 MiB。 -- 修改 `max-message-bytes` 不会改变 request 或 buffer 上限。 -- Kafka batch 上限超过 100 MiB 时，`ProducerBatchMaxBytes` 限制为 100 MiB。 -- `MaxBufferedRecords` 仍为 10,000。 - -后续边界测试应覆盖 64 MiB 单条 payload、100 MiB request envelope、Kafka batch 上限超过 request 上限，以及 buffer 满后的阻塞和取消行为。 diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index dc394d4b7b..c8078f0d8e 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -74,8 +74,8 @@ type AsyncProducer interface { // wish to send. AsyncSend(ctx context.Context, topic string, partition int32, message *codecCommon.Message) error - // AsyncRunCallback process the messages that has sent to kafka, - // and run tha attached callback. the caller should call this - // method in a background goroutine + // AsyncRunCallback invokes callbacks for successfully delivered messages and + // returns the first terminal delivery error. The caller should run it in a + // background goroutine. AsyncRunCallback(ctx context.Context) error } diff --git a/pkg/sink/kafka/franz_admin.go b/pkg/sink/kafka/franz_admin.go index 1b998f4173..697f9d2bc2 100644 --- a/pkg/sink/kafka/franz_admin.go +++ b/pkg/sink/kafka/franz_admin.go @@ -19,11 +19,13 @@ import ( "strings" "time" + "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" ) type admin struct { @@ -148,6 +150,14 @@ func (a *admin) GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicE meta, err := a.admin.Metadata(ctx, topics...) if err != nil { resource := strings.Join(topics, ",") + if ignoreTopicError && errors.Is(err, kerr.TopicAuthorizationFailed) { + log.Warn("kafka topic metadata refresh failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("topic", resource), + zap.Error(err)) + return make(map[string]TopicDetail), nil + } if isAuthorizationFailed(err) { return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-topics", resource) } @@ -176,7 +186,7 @@ func topicDetailsFromMetadata(meta kadm.Metadata, topics []string, ignoreTopicEr continue } - if ignoreTopicError && errors.Is(detail.Err, kerr.UnknownTopicOrPartition) { + if ignoreTopicError { continue } diff --git a/pkg/sink/kafka/franz_admin_test.go b/pkg/sink/kafka/franz_admin_test.go index d02d1f353a..5102051ac2 100644 --- a/pkg/sink/kafka/franz_admin_test.go +++ b/pkg/sink/kafka/franz_admin_test.go @@ -116,22 +116,20 @@ func TestFranzTopicDetailsFromMetadata(t *testing.T) { expectedCause: kerr.TopicAuthorizationFailed, }, { - name: "do not ignore authorization failure", + name: "ignore authorization failure", metadata: kadm.Metadata{Topics: kadm.TopicDetails{ topic: {Topic: topic, Err: kerr.TopicAuthorizationFailed}, }}, ignoreTopicError: true, - expectedError: errors.ErrKafkaAuthorizationFailed, - expectedCause: kerr.TopicAuthorizationFailed, + expected: map[string]TopicDetail{}, }, { - name: "do not ignore general failure", + name: "ignore general failure", metadata: kadm.Metadata{Topics: kadm.TopicDetails{ topic: {Topic: topic, Err: kerr.InvalidTopicException}, }}, ignoreTopicError: true, - expectedError: errors.ErrKafkaAdminAPI, - expectedCause: kerr.InvalidTopicException, + expected: map[string]TopicDetail{}, }, } @@ -153,6 +151,40 @@ func TestFranzTopicDetailsFromMetadata(t *testing.T) { } } +func TestFranzGetTopicsMetaIgnoresTopicAuthorizationFailure(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1)) + defer cluster.Close() + + cluster.ControlKey(int16(kmsg.Metadata), func(req kmsg.Request) (kmsg.Response, error, bool) { + request := req.(*kmsg.MetadataRequest) + response := request.ResponseKind().(*kmsg.MetadataResponse) + for _, requestTopic := range request.Topics { + responseTopic := kmsg.NewMetadataResponseTopic() + responseTopic.Topic = requestTopic.Topic + responseTopic.ErrorCode = kerr.TopicAuthorizationFailed.Code + response.Topics = append(response.Topics, responseTopic) + } + return response, nil, true + }) + + o := testOptions(cluster.ListenAddrs()) + admin, err := newAdmin( + t.Context(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "ignore-topic-authorization"), + testClientOptions(t, o), + ) + require.NoError(t, err) + defer admin.Close() + + topics, err := admin.GetTopicsMeta(t.Context(), []string{"topic"}, true) + require.NoError(t, err) + require.Empty(t, topics) + + _, err = admin.GetTopicsMeta(t.Context(), []string{"topic"}, false) + require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) + require.ErrorIs(t, err, kerr.TopicAuthorizationFailed) +} + func TestFranzIsAuthorizationFailed(t *testing.T) { t.Parallel() diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index f17f731ad0..819e8c5184 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -30,8 +30,14 @@ type asyncProducer struct { client *kgo.Client changefeedID common.ChangeFeedID - closed atomic.Bool - errCh chan error + closed atomic.Bool + resultCh chan asyncProduceResult +} + +type asyncProduceResult struct { + callback func() + logInfo *codeccommon.MessageLogInfo + err error } func (p *asyncProducer) Close() { @@ -69,24 +75,10 @@ func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition i callback := message.Callback logInfo := message.LogInfo promise := func(_ *kgo.Record, err error) { - if err != nil { - log.Error("kafka message send failed", - zap.String("keyspace", p.changefeedID.Keyspace()), - zap.String("changefeed", p.changefeedID.Name()), - zap.String("eventContext", BuildEventLogContext( - p.changefeedID.Keyspace(), p.changefeedID.Name(), logInfo)), - zap.Error(err)) - - select { - case p.errCh <- errors.WrapError(errors.ErrKafkaSendMessage, err): - // Keep the first error until the dispatcher can recover from multiple errors. - default: - } - return - } - - if callback != nil { - callback() + p.resultCh <- asyncProduceResult{ + callback: callback, + logInfo: logInfo, + err: err, } } @@ -99,8 +91,19 @@ func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { select { case <-ctx.Done(): return context.Cause(ctx) - case err := <-p.errCh: - return err + case result := <-p.resultCh: + if result.err != nil { + log.Error("kafka message send failed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.String("eventContext", BuildEventLogContext( + p.changefeedID.Keyspace(), p.changefeedID.Name(), result.logInfo)), + zap.Error(result.err)) + return errors.WrapError(errors.ErrKafkaSendMessage, result.err) + } + if result.callback != nil { + result.callback() + } } } } diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index 4a620489db..b1db529108 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -48,9 +48,9 @@ func TestAsyncSendCanceledContext(t *testing.T) { func TestAsyncRunCallbackReturnsQueuedError(t *testing.T) { producer := &asyncProducer{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), - errCh: make(chan error, 1), + resultCh: make(chan asyncProduceResult, 1), } - producer.errCh <- context.DeadlineExceeded + producer.resultCh <- asyncProduceResult{err: context.DeadlineExceeded} err := producer.AsyncRunCallback(context.Background()) @@ -65,7 +65,7 @@ func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { producer := &asyncProducer{ client: client, changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), - errCh: make(chan error, 1), + resultCh: make(chan asyncProduceResult, 1), } err = producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{ Callback: func() { @@ -87,12 +87,13 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { defer cluster.Close() o := testOptions(cluster.ListenAddrs()) - producer, err := (&franzFactory{ + created, err := (&franzFactory{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), clientOpts: testClientOptions(t, o), producerOpts: producerOptions(o), }).AsyncProducer(context.Background()) require.NoError(t, err) + producer := created.(*asyncProducer) defer producer.Close() var calls atomic.Int32 @@ -105,6 +106,12 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { }, } require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, message)) + require.Eventually(t, func() bool { return producer.client.BufferedProduceRecords() == 0 }, time.Second, time.Millisecond) + require.Zero(t, calls.Load()) + + callbackCtx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- producer.AsyncRunCallback(callbackCtx) }() select { case <-called: @@ -114,6 +121,63 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { time.Sleep(20 * time.Millisecond) require.Equal(t, int32(1), calls.Load()) + cancel() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestAsyncProducerCallbackDoesNotBlockPromiseWorker(t *testing.T) { + const topic = "async-callback-isolation" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + + created, err := (&franzFactory{ + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback-isolation"), + clientOpts: testClientOptions(t, o), + producerOpts: producerOptions(o), + }).AsyncProducer(context.Background()) + require.NoError(t, err) + producer := created.(*asyncProducer) + defer producer.Close() + + callbackStarted := make(chan struct{}) + releaseCallback := make(chan struct{}) + secondCallback := make(chan struct{}) + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, &codeccommon.Message{ + Value: []byte("first"), + Callback: func() { + close(callbackStarted) + <-releaseCallback + }, + })) + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, &codeccommon.Message{ + Value: []byte("second"), + Callback: func() { close(secondCallback) }, + })) + + callbackCtx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- producer.AsyncRunCallback(callbackCtx) }() + select { + case <-callbackStarted: + case <-time.After(time.Second): + t.Fatal("produce callback was not called") + } + require.Eventually(t, func() bool { return producer.client.BufferedProduceRecords() == 0 }, time.Second, time.Millisecond) + select { + case <-secondCallback: + t.Fatal("second callback ran before the first callback completed") + case <-time.After(20 * time.Millisecond): + } + + close(releaseCallback) + select { + case <-secondCallback: + case <-time.After(time.Second): + t.Fatal("second callback was not called") + } + cancel() + require.ErrorIs(t, <-done, context.Canceled) } func TestAsyncProducerReportsProduceFailure(t *testing.T) { @@ -157,7 +221,7 @@ func TestBufferBackpressureCanBeCanceled(t *testing.T) { producer := &asyncProducer{ client: client, changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "backpressure"), - errCh: make(chan error, 1), + resultCh: make(chan asyncProduceResult, 2), } t.Cleanup(producer.Close) diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go index 973f4814db..2c7cc421f6 100644 --- a/pkg/sink/kafka/franz_config.go +++ b/pkg/sink/kafka/franz_config.go @@ -46,7 +46,7 @@ const ( // Admin and producer clients share connection options. Producer delivery and // resource limits stay separate so they cannot affect admin operations. -func clientOptions(o *options) ([]kgo.Opt, error) { +func clientOptions(ctx context.Context, o *options) ([]kgo.Opt, error) { opts := []kgo.Opt{ kgo.SeedBrokers(o.BrokerEndpoints...), kgo.ClientID(o.ClientID), @@ -74,7 +74,7 @@ func clientOptions(o *options) ([]kgo.Opt, error) { } if o.sasl != nil && o.sasl.mechanism != "" { - mechanism, err := buildSASLMechanism(o.sasl) + mechanism, err := buildSASLMechanism(ctx, o.sasl) if err != nil { return nil, err } @@ -126,7 +126,7 @@ func producerOptions(o *options) []kgo.Opt { } } -func buildSASLMechanism(cfg *saslConfig) (sasl.Mechanism, error) { +func buildSASLMechanism(ctx context.Context, cfg *saslConfig) (sasl.Mechanism, error) { switch cfg.mechanism { case plainMechanism: return plain.Auth{User: cfg.user, Pass: cfg.password}.AsMechanism(), nil @@ -135,7 +135,7 @@ func buildSASLMechanism(cfg *saslConfig) (sasl.Mechanism, error) { case scram512Mechanism: return scram.Auth{User: cfg.user, Pass: cfg.password}.AsSha512Mechanism(), nil case oauthMechanism: - return buildOAuthMechanism(cfg.oauth2) + return buildOAuthMechanism(ctx, cfg.oauth2) case gssapiMechanism: return buildGSSAPIMechanism(cfg.gssapi) default: @@ -143,7 +143,7 @@ func buildSASLMechanism(cfg *saslConfig) (sasl.Mechanism, error) { } } -func buildOAuthMechanism(cfg oauth2Config) (sasl.Mechanism, error) { +func buildOAuthMechanism(ctx context.Context, cfg oauth2Config) (sasl.Mechanism, error) { var httpClient *http.Client if cfg.caPath != "" { var err error @@ -171,11 +171,13 @@ func buildOAuthMechanism(cfg oauth2Config) (sasl.Mechanism, error) { EndpointParams: endpointParams, Scopes: cfg.scopes, } - return oauth.Oauth(func(ctx context.Context) (oauth.Auth, error) { - if httpClient != nil { - ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) - } - token, err := config.TokenSource(ctx).Token() + if httpClient != nil { + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) + } + // One token source shares cached credentials across broker connections and refreshes them on expiry. + tokenSource := config.TokenSource(ctx) + return oauth.Oauth(func(context.Context) (oauth.Auth, error) { + token, err := tokenSource.Token() if err != nil { return oauth.Auth{}, errors.WrapError(errors.ErrNewKafkaSink, err) } diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index 08b9e62f75..2093bc9b90 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -20,13 +20,12 @@ import ( "net/http" "net/http/httptest" "net/url" + "sync/atomic" "testing" "time" - "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" - "github.com/twmb/franz-go/pkg/kfake" "github.com/twmb/franz-go/pkg/kgo" ) @@ -43,7 +42,7 @@ func testOptions(brokers []string) *options { func testClientOptions(t *testing.T, o *options) []kgo.Opt { t.Helper() - opts, err := clientOptions(o) + opts, err := clientOptions(t.Context(), o) require.NoError(t, err) return opts } @@ -76,34 +75,12 @@ func TestFranzProducerTimeouts(t *testing.T) { require.Equal(t, 3*time.Second, client.OptValue(kgo.ProduceRequestTimeout)) } -func TestFranzIgnoresConfiguredKafkaVersion(t *testing.T) { - const topic = "version-negotiation" - cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) - defer cluster.Close() - - o := NewOptions() - o.ClientID = "ticdc-test" - o.BrokerEndpoints = cluster.ListenAddrs() - o.Topic = topic - o.Version = "invalid" - o.IsAssignedVersion = true - - factory, err := NewFactory( - context.Background(), - o, - common.NewChangefeedID4Test(common.DefaultKeyspaceName, "version-negotiation"), - ) - require.NoError(t, err) - require.IsType(t, &franzFactory{}, factory) - factory.CleanupMetrics() -} - func TestProducerOptionsConfigureMessageAndBufferLimits(t *testing.T) { const maxMessageBytes = 1048588 o := testOptions([]string{"127.0.0.1:9092"}) o.MaxMessageBytes = maxMessageBytes - opts, err := clientOptions(o) + opts, err := clientOptions(t.Context(), o) require.NoError(t, err) producerOpts := producerOptions(o) @@ -212,7 +189,7 @@ func TestBuildFranzGSSAPIMechanism(t *testing.T) { cfg.username = "alice" cfg.realm = "EXAMPLE.COM" - mechanism, err := buildSASLMechanism(&saslConfig{ + mechanism, err := buildSASLMechanism(t.Context(), &saslConfig{ mechanism: gssapiMechanism, gssapi: cfg, }) @@ -223,7 +200,7 @@ func TestBuildFranzGSSAPIMechanism(t *testing.T) { func TestBuildFranzSASLMechanisms(t *testing.T) { for _, mechanism := range []saslMechanism{plainMechanism, scram256Mechanism, scram512Mechanism} { - actual, err := buildSASLMechanism(&saslConfig{ + actual, err := buildSASLMechanism(t.Context(), &saslConfig{ mechanism: mechanism, user: "alice", password: "secret", @@ -232,11 +209,12 @@ func TestBuildFranzSASLMechanisms(t *testing.T) { require.Equal(t, string(mechanism), actual.Name()) } - _, err := buildSASLMechanism(&saslConfig{mechanism: "unknown"}) + _, err := buildSASLMechanism(t.Context(), &saslConfig{mechanism: "unknown"}) require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } func TestFranzOAuthTokenSource(t *testing.T) { + var tokenRequests atomic.Int32 request := make(chan url.Values, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { @@ -244,7 +222,11 @@ func TestFranzOAuthTokenSource(t *testing.T) { w.WriteHeader(http.StatusBadRequest) return } - request <- r.PostForm + tokenRequests.Add(1) + select { + case request <- r.PostForm: + default: + } w.Header().Set("Content-Type", "application/json") if _, err := io.WriteString(w, `{"access_token":"token","token_type":"bearer"}`); err != nil { @@ -253,7 +235,7 @@ func TestFranzOAuthTokenSource(t *testing.T) { })) defer server.Close() - mechanism, err := buildSASLMechanism(&saslConfig{ + mechanism, err := buildSASLMechanism(t.Context(), &saslConfig{ mechanism: oauthMechanism, oauth2: oauth2Config{ clientID: "client", @@ -266,17 +248,20 @@ func TestFranzOAuthTokenSource(t *testing.T) { }) require.NoError(t, err) + _, _, err = mechanism.Authenticate(context.Background(), "") + require.NoError(t, err) _, _, err = mechanism.Authenticate(context.Background(), "") require.NoError(t, err) form := <-request + require.Equal(t, int32(1), tokenRequests.Load()) require.Equal(t, "custom", form.Get("grant_type")) require.Equal(t, "audience", form.Get("audience")) require.Equal(t, "scope-a scope-b", form.Get("scope")) } func TestFranzOAuthTokenSourceRejectsInvalidURL(t *testing.T) { - _, err := buildSASLMechanism(&saslConfig{ + _, err := buildSASLMechanism(t.Context(), &saslConfig{ mechanism: oauthMechanism, oauth2: oauth2Config{tokenURL: "http://example.com/%%"}, }) diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index bbdbcd45ee..9302eac823 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -31,7 +31,7 @@ type franzFactory struct { } func newFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { - clientOpts, err := clientOptions(o) + clientOpts, err := clientOptions(ctx, o) if err != nil { return nil, err } @@ -95,7 +95,7 @@ func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) return &asyncProducer{ client: client, changefeedID: f.changefeedID, - errCh: make(chan error, 1), + resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), }, nil } From 9046a301459d11f1a0207e3a201b58df0b771842 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 18:25:41 +0800 Subject: [PATCH 54/61] fix more code --- pkg/sink/kafka/franz_config.go | 7 ++++-- pkg/sink/kafka/franz_config_test.go | 34 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go index 2c7cc421f6..479a7a681f 100644 --- a/pkg/sink/kafka/franz_config.go +++ b/pkg/sink/kafka/franz_config.go @@ -44,6 +44,9 @@ const ( producerMaxBufferedRecords = 1 << 16 ) +// producerMaxRequestBytes matches franz-go's default BrokerMaxWriteBytes and Kafka's default socket.request.max.bytes. +const producerMaxRequestBytes = 100 << 20 + // Admin and producer clients share connection options. Producer delivery and // resource limits stay separate so they cannot affect admin operations. func clientOptions(ctx context.Context, o *options) ([]kgo.Opt, error) { @@ -102,8 +105,8 @@ func producerOptions(o *options) []kgo.Opt { // bounding buffered payload memory. kgo.MaxBufferedBytes(producerMaxBufferedBytes), kgo.MaxBufferedRecords(producerMaxBufferedRecords), - // ProducerBatchMaxBytes is franz-go's name for Kafka's max.message.bytes limit. - kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)), + // A record batch must fit in the 100 MiB Produce request limit. + kgo.ProducerBatchMaxBytes(int32(min(o.MaxMessageBytes, producerMaxRequestBytes))), // This value limits how long the Broker may process a Produce request; // read-timeout is therefore not an exact socket read deadline. Together // with RequestTimeoutOverhead above, the socket write deadline is diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index 2093bc9b90..afbfe78f07 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -94,7 +94,7 @@ func TestProducerOptionsConfigureMessageAndBufferLimits(t *testing.T) { require.Equal(t, int64(producerMaxBufferedRecords), client.OptValue(kgo.MaxBufferedRecords)) require.Equal(t, int64(1), client.OptValue(kgo.RecordRetries)) require.Equal(t, int64(1), client.OptValue(kgo.UnknownTopicRetries)) - require.Equal(t, int32(100<<20), client.OptValue(kgo.BrokerMaxWriteBytes)) + require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) } func TestProducerOptionsUseSingleNonIdempotentRequest(t *testing.T) { @@ -122,7 +122,7 @@ func TestProducerLimitsDoNotScaleWithConfiguredMessage(t *testing.T) { defer client.Close() require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) - require.Equal(t, int32(100<<20), client.OptValue(kgo.BrokerMaxWriteBytes)) + require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) } func TestProducerOptionsDoNotClampSmallBatch(t *testing.T) { @@ -135,20 +135,26 @@ func TestProducerOptionsDoNotClampSmallBatch(t *testing.T) { require.Error(t, err) } -func TestProducerOptionsUseKafkaBatchLimitDirectly(t *testing.T) { - const maxMessageBytes = 128 << 20 - config := testOptions([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = maxMessageBytes - - producerOpts := producerOptions(config) - // Raise the request limit only to let franz-go validate this test client. - producerOpts = append(producerOpts, kgo.BrokerMaxWriteBytes(maxMessageBytes)) +func TestProducerOptionsLimitBatchToProduceRequest(t *testing.T) { + for _, test := range []struct { + name string + maxMessageBytes int + }{ + {name: "at request limit", maxMessageBytes: producerMaxRequestBytes}, + {name: "above request limit", maxMessageBytes: 128 << 20}, + } { + t.Run(test.name, func(t *testing.T) { + config := testOptions([]string{"127.0.0.1:9092"}) + config.MaxMessageBytes = test.maxMessageBytes - client, err := kgo.NewClient(producerOpts...) - require.NoError(t, err) - defer client.Close() + client, err := kgo.NewClient(producerOptions(config)...) + require.NoError(t, err) + defer client.Close() - require.Equal(t, int32(maxMessageBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) + }) + } } func TestCompressionOptions(t *testing.T) { From f6872ae92325a56451cd8851b43384e9477aa03a Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 18:39:21 +0800 Subject: [PATCH 55/61] fix more code --- pkg/sink/kafka/franz_config_test.go | 10 ---------- pkg/sink/kafka/franz_sync_producer.go | 12 ++++-------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index afbfe78f07..bab556ca07 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -125,16 +125,6 @@ func TestProducerLimitsDoNotScaleWithConfiguredMessage(t *testing.T) { require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) } -func TestProducerOptionsDoNotClampSmallBatch(t *testing.T) { - config := testOptions([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = 511 - - producerOpts := producerOptions(config) - - _, err := kgo.NewClient(producerOpts...) - require.Error(t, err) -} - func TestProducerOptionsLimitBatchToProduceRequest(t *testing.T) { for _, test := range []struct { name string diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go index b6349f6d9a..dfd6c6706c 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -34,10 +34,6 @@ type syncProducer struct { } func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { - if p.closed.Load() { - return errors.ErrKafkaSinkClosed.GenWithStackByArgs() - } - record := &kgo.Record{ Topic: topic, Partition: partitionNum, @@ -48,10 +44,6 @@ func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionN } func (p *syncProducer) SendMessages(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { - if p.closed.Load() { - return errors.ErrKafkaSinkClosed.GenWithStackByArgs() - } - records := make([]*kgo.Record, 0, partitionNum) for i := 0; i < int(partitionNum); i++ { records = append(records, &kgo.Record{ @@ -66,6 +58,10 @@ func (p *syncProducer) SendMessages(ctx context.Context, topic string, partition } func (p *syncProducer) sendRecords(ctx context.Context, message *codecCommon.Message, records ...*kgo.Record) error { + if p.closed.Load() { + return errors.ErrKafkaSinkClosed.GenWithStackByArgs() + } + err := p.client.ProduceSync(ctx, records...).FirstErr() if err == nil { return nil From 034885e4c3fae90832f73d806f64f74ad4fda5c2 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 20:48:33 +0800 Subject: [PATCH 56/61] kafka: refine franz-go producer lifecycle --- ...6-07-02-sarama-message-size-calculation.md | 674 ++++++++++++++++++ ...026-07-03-franz-go-replace-sarama-audit.md | 484 +++++++++++++ ...franz-go-replace-sarama-migration-steps.md | 672 +++++++++++++++++ docs/franz-go/franz-go-ga-test-plan.md | 87 +++ .../kafka-producer-idempotence-design.md | 152 ++++ docs/franz-go/milestone-1-todo-list.md | 10 + .../ticdc-kafka-franz-go-ga-test-record.md | 180 +++++ downstreamadapter/sink/kafka/helper.go | 3 + downstreamadapter/sink/kafka/sink.go | 4 +- downstreamadapter/sink/kafka/sink_test.go | 10 +- pkg/sink/kafka/factory.go | 16 +- pkg/sink/kafka/factory_mock.go | 12 +- pkg/sink/kafka/franz_admin.go | 21 +- pkg/sink/kafka/franz_async_producer.go | 9 +- pkg/sink/kafka/franz_async_producer_test.go | 84 ++- pkg/sink/kafka/franz_config.go | 34 +- pkg/sink/kafka/franz_factory.go | 65 +- pkg/sink/kafka/franz_factory_test.go | 75 ++ pkg/sink/kafka/franz_sync_producer.go | 2 - pkg/sink/kafka/franz_sync_producer_test.go | 57 +- pkg/sink/kafka/options_test.go | 2 +- pkg/sink/kafka/sarama_factory.go | 2 +- 22 files changed, 2509 insertions(+), 146 deletions(-) create mode 100644 docs/design/2026-07-02-sarama-message-size-calculation.md create mode 100644 docs/design/2026-07-03-franz-go-replace-sarama-audit.md create mode 100644 docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md create mode 100644 docs/franz-go/franz-go-ga-test-plan.md create mode 100644 docs/franz-go/kafka-producer-idempotence-design.md create mode 100644 docs/franz-go/milestone-1-todo-list.md create mode 100644 docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md create mode 100644 pkg/sink/kafka/franz_factory_test.go diff --git a/docs/design/2026-07-02-sarama-message-size-calculation.md b/docs/design/2026-07-02-sarama-message-size-calculation.md new file mode 100644 index 0000000000..09e4bacab4 --- /dev/null +++ b/docs/design/2026-07-02-sarama-message-size-calculation.md @@ -0,0 +1,674 @@ +# Kafka 消息大小检查口径说明 + +本文梳理 TiCDC Kafka sink 在 master 分支 Sarama 实现和当前 franz-go 实现里的 +消息大小检查链路。重点是区分这些对象： + +```text +TiCDC common.Message + -> Kafka record + -> Kafka record batch + -> Kafka ProduceRequest +``` + +一次 `ProduceRequest` 按 topic 再按 partition 组织。现代 Kafka 版本下，每个 +topic-partition 的 `Records` 字段承载一个 `RecordBatch`；一个 request 可以 +包含多个 topic、多个 partition 的多个 record batches。就 Sarama 和 franz-go +这两个实现而言，一次 request 中同一个 topic-partition 只放一个 batch。 + +## Kafka 原始配置语义 + +Kafka broker/topic 层与消息大小相关的两个原始配置是 +`message.max.bytes` 和 `max.message.bytes`。 + +- `message.max.bytes` 是 broker 级默认值。Apache Kafka 4.3 官方文档定义为 + Kafka 允许的最大 record batch size；如果启用了压缩，按压缩后的 batch 大小 + 判断。它可以被 topic 级的 `max.message.bytes` 覆盖。默认值是 `1048588`。 +- `max.message.bytes` 是 topic 级配置。它的语义同样是 Kafka 允许的最大 + record batch size；如果启用了压缩，按压缩后的 batch 大小判断。没有显式 + topic override 时，该 topic 使用 server default property，也就是 + broker 级的 `message.max.bytes`。默认值同样显示为 `1048588`。 + +所以，把它们口语化理解成“Kafka 单条消息大小上限”只在一个应用层 record 独占 +一个 record batch 时近似成立。Kafka protocol 的精确对象是 record batch： +一个 batch 可以包含多条 records；broker/topic 限制校验的是这个 batch，而不是 +单独某个应用层 key/value payload。 + +另一个容易混淆的 producer 侧参数是 `max.request.size`。Apache Kafka 4.3 官方 +文档把它定义为 producer request 的最大大小，同时也是最大未压缩 record batch +size 的有效上限。server 侧仍然有自己的 record batch 上限，也就是上面的 +`message.max.bytes` / `max.message.bytes`，并且这个 server 侧上限在启用压缩时 +按压缩后大小判断。 + +相关官方文档： + +- Apache Kafka 4.3 Broker Configs, `message.max.bytes`: + https://kafka.apache.org/43/configuration/broker-configs/ +- Apache Kafka 4.3 Topic Configs, `max.message.bytes`: + https://kafka.apache.org/43/configuration/topic-configs/ +- Apache Kafka 4.3 Producer Configs, `max.request.size`: + https://kafka.apache.org/43/configuration/producer-configs/ + +## master 分支 Sarama 实现 + +master 分支的大小检查链路是： + +```text +Kafka raw topic/broker limit + -> TiCDC options.MaxMessageBytes + -> encoder MaxMessageBytes + -> open-protocol 单行/claim-check 检查 + -> open-protocol 多行 common.Message batching 检查 + -> Sarama ProducerMessage.ByteSize 检查 + -> Sarama produceSet batch / request rollover 检查 + -> broker 按 Kafka record batch limit 最终校验 +``` + +### 1. TiCDC 从 Kafka raw config 折算 options.MaxMessageBytes + +master 分支 `pkg/sink/kafka/options.go` 里有： + +```go +maxMessageBytesOverhead = 128 +``` + +topic 已存在时，`adjustOptions` 读取 topic 的 `max.message.bytes`，如果没有 +topic override 则回退到 broker 的 `message.max.bytes`。随后使用： + +```text +effective MaxMessageBytes = min(configured max-message-bytes, source max bytes - 128) +``` + +topic 不存在、需要 TiCDC 创建 topic 时，`adjustOptions` 读取 broker 的 +`message.max.bytes`，也使用同样的 `source - 128` 折算。 + +因此，在 master 分支上，TiCDC 的 `options.MaxMessageBytes` 不是 Kafka +broker/topic raw value，而是一个扣掉 128 字节 safety margin 后的 TiCDC/Sarama +侧预算。 + +源码位置： + +- `master:pkg/sink/kafka/options.go`：`maxMessageBytesOverhead = 128` +- `master:pkg/sink/kafka/options.go`：topic path 使用 + `topicMaxMessageBytes - maxMessageBytesOverhead` +- `master:pkg/sink/kafka/options.go`：broker path 使用 + `brokerMessageMaxBytes - maxMessageBytesOverhead` + +### 2. Sarama producer 和 encoder 使用同一个 MaxMessageBytes + +master 分支 `newSaramaConfig` 将： + +```go +config.Producer.MaxMessageBytes = o.MaxMessageBytes +``` + +同时，`downstreamadapter/sink/helper/helper.go` 明确把 encoder 的 +`MaxMessageBytes` 设置成 producer 的 `MaxMessageBytes`： + +```go +encoderConfig = encoderConfig.WithMaxMessageBytes(maxMsgBytes) +``` + +这意味着 master 的意图是：encoder 不要生成超过 producer 预算的 +`common.Message`。 + +源码位置： + +- `master:pkg/sink/kafka/sarama_config.go` +- `master:downstreamadapter/sink/helper/helper.go` + +### 3. open-protocol 单行编码与 claim-check 检查 + +open-protocol `batchEncoder.AppendRowChangedEvent` 会先把单行 RowEvent 编码成 +key/value，并得到一个 `length`： + +```go +key, value, length, err := encodeRowChangedEvent(...) +if length > d.config.MaxMessageBytes { + ... +} +``` + +如果单行原始消息超过 `MaxMessageBytes`： + +- large message handle disabled：直接返回 `ErrMessageTooLarge`。 +- claim-check enabled：先把原始 key/value 写入外部存储，再重新编码一条 + claim-check location message。 +- claim-check location message 仍超过 `MaxMessageBytes`：返回 + `ErrMessageTooLarge`。 + +这个检查发生在 Kafka producer 之前。 + +源码位置： + +- `master:pkg/sink/codec/open/encoder.go` + +### 4. open-protocol 多行 common.Message batching 检查 + +claim-check location message 单条通常很小，但 open-protocol 会继续把多条 +row events 合并进一个 TiCDC `common.Message`。 + +`pushMessage` 里新加入一行时，计算： + +```go +length := len(key) + len(value) + 16 +``` + +然后用当前 TiCDC message 的 `Length()` 判断是否还能继续追加： + +```go +latestMessage.Length() + length > d.config.MaxMessageBytes +``` + +`common.Message.Length()` 在 master 分支是： + +```go +len(m.Key) + len(m.Value) + MaxRecordOverhead +``` + +其中： + +```text +MaxRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 = 36 +``` + +也就是说，encoder 的 batching 口径和 Sarama +`ProducerMessage.ByteSize(2)` 的无 headers 估算口径一致： + +```text +len(key) + len(value) + 36 +``` + +源码位置： + +- `master:pkg/sink/codec/open/encoder.go` +- `master:pkg/sink/codec/common/message.go` + +### 5. Sarama AsyncSend 本身不做大小检查 + +TiCDC Sarama producer 的 `AsyncSend` 只是把 `common.Message` 转成 +`sarama.ProducerMessage` 并写入 Sarama input channel： + +```go +msg := &sarama.ProducerMessage{ + Topic: topic, + Partition: partition, + Key: sarama.StringEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), +} +p.producer.Input() <- msg +``` + +TiCDC 这一层没有额外 size check。 + +源码位置： + +- `master:pkg/sink/kafka/sarama_async_producer.go` + +### 6. Sarama 单条 ProducerMessage 硬检查 + +Sarama 内部第一层硬检查在 `asyncProducer.dispatcher`： + +```go +size := msg.ByteSize(version) +if size > p.conf.Producer.MaxMessageBytes { + reject +} +``` + +Kafka `>= 0.11` 时，`version = 2`，`ProducerMessage.ByteSize(2)` 为： + +```text +len(key) + len(value) + maximumRecordOverhead + headers estimate +``` + +没有 headers 时： + +```text +len(key) + len(value) + 36 +``` + +这一层是本地拒绝条件。 + +Sarama 源码位置： + +- `/Users/edison/go/sarama/async_producer.go:365` +- `/Users/edison/go/sarama/async_producer.go:626` + +### 7. Sarama produceSet rollover 检查 + +消息进入 broker producer 后，Sarama 调用 `produceSet.wouldOverflow(msg)`。 +这里有三类检查： + +```text +1. 整个 produce request 估算： + ps.bufferBytes + msg.ByteSize(version) >= MaxRequestSize - 10KiB + +2. 已存在 topic-partition batch 估算： + set.bufferBytes + msg.ByteSize(version) >= Producer.MaxMessageBytes + +3. Flush.MaxMessages 条数限制 +``` + +这几类检查触发后，Sarama 会 `waitForSpace` / flush / rollover，而不是把当前 +消息作为 `MESSAGE_TOO_LARGE` 直接失败。 + +关键细节：第二类 partition batch 检查只有在当前 topic-partition 的 +`partitionSet` 已经存在时才执行。第一条 record 进入一个空的 partition batch +时，partition set 尚不存在，所以这个检查会跳过。 + +随后 `produceSet.add` 创建 batch： + +```text +recordBatchOverhead = 49 +``` + +并把本条消息加入 batch。此时 `partitionSet.bufferBytes` 会变成： + +```text +49 + len(key) + len(value) + 36 +``` + +即使这个值已经超过 `Producer.MaxMessageBytes`，第一条 record 也已经被接受。 + +Sarama 源码位置： + +- `/Users/edison/go/sarama/produce_set.go:39` +- `/Users/edison/go/sarama/produce_set.go:303` +- `/Users/edison/go/sarama/async_producer.go:1188` +- `/Users/edison/go/sarama/async_producer.go:1328` + +### 8. Sarama 例子：759 字节 claim-check batch + +假设： + +```text +Producer.MaxMessageBytes = 800 +len(key) + len(value) = 759 +headers = none +``` + +Sarama 单条硬检查： + +```text +759 + 36 = 795 <= 800 +``` + +所以能通过。 + +如果这是该 topic-partition 当前 batch 的第一条 record，partition batch +rollover 检查会跳过。加入后 Sarama 内部估算为： + +```text +49 + 795 = 844 +``` + +但这不是第一条 record 的拒绝条件。 + +如果具体 key/value 拆分为 `527 + 232`，实际 headerless record 编码是： + +```text +record body: + attributes 1 + timestamp delta 1 + offset delta 1 + key length varint 2 + key bytes 527 + value length varint 2 + value bytes 232 + headers count 1 + total 767 + +record length varint 2 +encoded record total 769 +``` + +不启用 producer compression 时，Sarama 实际 `RecordBatch.encode` 大小约为： + +```text +61 + 769 = 830 +``` + +Sarama 本地仍然不会因为这个完整 encoded record batch 大于 800 而拒绝这条 +空 batch 的第一条 record。 + +## 修正后的 franz-go 实现 + +修正后的当前分支大小检查链路是： + +```text +Kafka raw topic/broker limit + -> TiCDC options.ProducerBatchMaxBytes + -> franz-go ProducerBatchMaxBytes + -> broker 按 Kafka record batch limit 最终校验 + +用户配置 max-message-bytes / Kafka raw topic/broker limit + -> TiCDC options.MaxMessageBytes + -> encoder MaxMessageBytes payload 检查 + -> open-protocol 单行/claim-check 检查 + -> open-protocol 多行 common.Message batching 检查 + -> franz-go Produce(ctx, kgo.Record) + -> franz-go buffered bytes/backpressure 检查 + -> franz-go recBatch.tryBuffer record batch 大小检查 + -> franz-go produceRequest request 总大小检查 +``` + +这里刻意把两个值分开： + +- `options.MaxMessageBytes`：TiCDC encoder 的 payload 预算，用于 open protocol + 自身分包以及 large message handle。 +- `options.ProducerBatchMaxBytes`：franz-go producer 的 Kafka record batch 预算， + 直接来自 topic `max.message.bytes` 或 broker `message.max.bytes`。 + +### 1. 删除 maxMessageBytesOverhead，但不再混淆 producer batch 预算 + +当前分支删除 `maxMessageBytesOverhead`。`adjustOptions` 现在直接使用 Kafka raw +source limit 约束 encoder： + +```text +effective MaxMessageBytes = min(configured max-message-bytes, kafka raw source max bytes) +``` + +topic 已存在时，source 是 topic `max.message.bytes`；topic 不存在时，source 是 +broker `message.max.bytes`。 + +同时，`adjustOptions` 记录 Kafka raw source limit： + +```text +ProducerBatchMaxBytes = kafka raw source max bytes +``` + +例如 integration test 里的： + +```text +max-message-bytes=800 +``` + +在 topic/broker raw limit 是 Kafka 默认值 `1048588` 时： + +```text +options.MaxMessageBytes = 800 +options.ProducerBatchMaxBytes = 1048588 +``` + +这正是 claim-check 场景需要的语义：`800` 只控制 TiCDC 何时把原始大消息转成 +claim-check location message，不应该把 franz-go 的 record batch 上限也压成 +`800`。 + +源码位置： + +- `pkg/sink/kafka/options.go` +- `pkg/sink/kafka/options_test.go` + +### 2. encoder 侧检查改为 payload 口径 + +encoder 仍然必须使用 `MaxMessageBytes` 做检查，原因是它承担两个 Kafka producer +之前的应用层功能： + +- large message handle disabled 时，如果单行编码后的 payload 超过 + `MaxMessageBytes`，直接返回 `ErrMessageTooLarge`，不会等 producer/broker 拒绝。 +- claim-check enabled 时，同一个检查点触发 claim-check：原始 key/value 写入外部 + 存储，再生成一条 claim-check location message。 +- open-protocol 会把多条 row events 合并成一个 Kafka record 的 key/value + payload，因此 `pushMessage` 需要用同一预算决定是否开一个新的 TiCDC + `common.Message`。 + +修正点是：`common.Message.Length()` 不再包含 Sarama 的 `MaxRecordOverhead = 36`， +而是返回： + +```text +len(key) + len(value) +``` + +open-protocol 单行检查也不再额外加 `common.MaxRecordOverhead`。对 open protocol +来说，一条 row 在最终 Kafka record payload 里的真实应用层长度是： + +```text +len(row key) + len(compressed row value) + 8(version) + 8(key length) + 8(value length) +``` + +后续追加一条 row 到同一个 `common.Message` 时增加： + +```text +len(row key) + len(compressed row value) + 8(key length) + 8(value length) +``` + +因此，encoder 现在检查的是 TiCDC 实际生成的 key/value payload 大小，而不是 +Sarama `ProducerMessage.ByteSize` 估算大小。 + +源码位置： + +- `pkg/sink/codec/open/encoder.go` +- `pkg/sink/codec/open/codec.go` +- `pkg/sink/codec/common/message.go` + +### 3. TiCDC franz-go AsyncSend 不做大小检查 + +当前分支 `kafkaAsyncProducer.AsyncSend` 把 `common.Message` 直接转成 +`kgo.Record`： + +```go +record := &kgo.Record{ + Topic: topic, + Partition: partition, + Key: message.Key, + Value: message.Value, +} +p.client.Produce(ctx, record, promise) +``` + +TiCDC 这一层没有额外 size check。 + +源码位置： + +- `pkg/sink/kafka/async_producer.go` + +### 4. franz-go ProducerBatchMaxBytes 使用 Kafka raw source limit + +修正前，当前分支在构造 franz-go producer options 时设置： + +```go +kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)) +``` + +franz-go 文档注释说明 `ProducerBatchMaxBytes` 限制的是 record batch 大小， +并且它 mirrors Kafka `max.message.bytes`。注释还明确说：record batch 是 +topic-partition 维度，`ProduceRequest` 可以包含多个 topics 的多个 record +batches。 + +这一步是当前分支和 master/Sarama 行为不同的核心：master 的 +`o.MaxMessageBytes` 进入 Sarama 后首先用于 `ProducerMessage.ByteSize` 单条估算； +当前分支的同一个值进入 franz-go 后用于 `ProducerBatchMaxBytes` record batch +上限。 + +修正后，franz-go producer 使用： + +```go +kgo.ProducerBatchMaxBytes(int32(o.ProducerBatchMaxBytes)) +``` + +也就是 topic/broker 的原始 record-batch 上限。`o.MaxMessageBytes` 继续传给 +encoder，不再直接作为 franz-go record-batch 上限。 + +源码位置： + +- `pkg/sink/kafka/client_options.go` +- `pkg/sink/kafka/kafka_factory.go` +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/config.go:1255` + +### 5. franz-go buffered bytes/backpressure 检查 + +franz-go `Produce` 开始时先计算： + +```text +userSize = len(key) + len(value) + sum(header key/value) +``` + +如果配置了 `MaxBufferedBytes`： + +```go +if maxBufferedBytes > 0 && userSize > maxBufferedBytes { + MESSAGE_TOO_LARGE +} +``` + +随后还会检查客户端当前 buffered bytes 是否超过 `MaxBufferedBytes`。不过 TiCDC +当前没有设置 `kgo.MaxBufferedBytes`，所以这个检查通常不是本问题的来源。 + +franz-go 源码位置： + +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/record_and_fetch.go:157` +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/producer.go:599` + +### 6. franz-go recBatch.tryBuffer record batch 检查 + +franz-go 把 record 放入 topic-partition 的 `recBatch` 时，会先尝试加入当前最后 +一个 batch；放不进去就创建新 batch 再试。 + +关键检查在 `recBatch.tryBuffer`： + +```go +nums := b.calculateRecordNumbers(pr.Record) +batchWireLength, _, _ := b.wireLengthForProduceVersion(produceVersion) +newBatchLength := batchWireLength + nums.wireLength() + +if b.frozen || newBatchLength > maxBatchBytes { + return false, false +} +``` + +如果一个空的新 batch 也放不下这条 record，franz-go 会本地失败： + +```go +MESSAGE_TOO_LARGE (uncompressed_bytes=...) +``` + +这是当前 claim-check case 的直接失败点。 + +franz-go 源码位置： + +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:1640` +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:1958` +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2320` +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2413` + +### 7. franz-go produceRequest 总大小检查 + +batch 准备写入 produce request 时，franz-go 还有 request 层检查： + +```go +if p.wireLength + batchWireLength > p.wireLengthLimit { + return false +} +``` + +这里的 `wireLengthLimit` 来源于 `maxBrokerWriteBytes`，默认对应 Kafka +`socket.request.max.bytes` 级别，默认约 100 MiB。它不是本次 800 字节失败的来源。 + +franz-go 源码位置： + +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2102` +- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2250` + +### 8. franz-go 例子：759 字节 claim-check payload + +仍用同一个例子，sink URI 显式配置： + +```text +max-message-bytes = 800 +len(key) + len(value) = 759 +key = 527 +value = 232 +headers = none +``` + +如果 Kafka topic/broker raw limit 是默认值： + +```text +Kafka max.message.bytes / message.max.bytes = 1048588 +``` + +修正后： + +```text +options.MaxMessageBytes = 800 +options.ProducerBatchMaxBytes = 1048588 +``` + +franz-go 对单条 record 的编码大小： + +```text +record body: + attributes 1 + timestamp delta 1 + offset delta 1 + key length varint 2 + key bytes 527 + value length varint 2 + value bytes 232 + headers count 1 + total 767 + +record length varint 2 +encoded record total 769 +``` + +franz-go 新 batch 固定 wire length 是： + +```text +record batch overhead = 65 +``` + +所以空 batch 加入这条 record 后： + +```text +65 + 769 = 834 +``` + +修正前，因为代码设置： + +```text +ProducerBatchMaxBytes = options.MaxMessageBytes = 800 +``` + +于是： + +```text +834 > 800 +``` + +franz-go 在本地返回 `MESSAGE_TOO_LARGE (uncompressed_bytes=759)`。这里的 +`uncompressed_bytes=759` 是用户 key/value payload 大小，不是完整 record batch +wire size。 + +修正后： + +```text +834 <= ProducerBatchMaxBytes(1048588) +``` + +这条 claim-check location message 可以进入 producer，并交给 broker 按 Kafka +record batch 语义最终校验。 + +如果 Kafka topic 本身真的配置为： + +```text +max.message.bytes = 800 +``` + +那么完整 record batch wire size `834 > 800`，franz-go 本地拒绝是合理的。那表示 +Kafka topic record-batch 上限确实放不下这条 claim-check location record，而不是 +TiCDC 的 claim-check 阈值被误用为 producer batch 阈值。 + +## 当前结论 + +1. Kafka broker/topic 的 `message.max.bytes` / `max.message.bytes` 语义是 + record batch 上限，不是 TiCDC `common.Message` 上限。 +2. master/Sarama 链路里，TiCDC encoder 和 Sarama 单条硬检查都主要使用 + `len(key) + len(value) + 36` 这一估算口径；Sarama 不会在空 batch 第一条 + record 时用完整 record batch wire size 拒绝消息。 +3. franz-go 链路必须区分 TiCDC encoder payload 预算和 Kafka record batch 预算。 + `max-message-bytes=800` 应触发 open-protocol claim-check；Kafka producer 的 + `ProducerBatchMaxBytes` 应来自 topic/broker raw limit。 +4. `common.Message.Length()` 不能继续携带 Sarama 的 36 字节 record overhead。 + 修正后它表示 TiCDC 生成的 key/value payload 大小；Kafka record encoding 和 + record batch header 由 franz-go 在 producer 层按真实协议口径检查。 diff --git a/docs/design/2026-07-03-franz-go-replace-sarama-audit.md b/docs/design/2026-07-03-franz-go-replace-sarama-audit.md new file mode 100644 index 0000000000..f5d759e9fd --- /dev/null +++ b/docs/design/2026-07-03-franz-go-replace-sarama-audit.md @@ -0,0 +1,484 @@ +# 使用 franz-go 替换 Sarama 的 Kafka sink 功能审计清单 + +## 背景 + +本文基于 `master` 分支代码和 TiCDC Kafka sink 官方文档，枚举用 +`~/go/franz-go` 替换 Sarama 时必须保持、实现和验证的功能点。本文不是 +PRD，也不是最终实现方案；它的目标是把替换边界、兼容性风险和验收项列清楚， +避免只替换 producer API 后遗漏 Kafka sink 对正确性、性能、可靠性和运维的 +隐含约束。 + +代码阅读基准： + +- `master` revision: `d2da619279f877a9964facdabebdc608044523cd` +- 本地 franz-go: `/Users/edison/go/franz-go` + +官方文档阅读范围： + +- [TiCDC 同步数据到 Kafka](https://docs.pingcap.com/zh/tidb/stable/ticdc-sink-to-kafka/) +- [TiCDC Changefeed 配置参数](https://docs.pingcap.com/zh/tidb/stable/ticdc-changefeed-config/) +- [TiCDC Open Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-open-protocol/) +- [TiCDC Canal-JSON Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-canal-json/) +- [TiCDC Avro Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-avro-protocol/) +- [TiCDC Simple Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-simple-protocol/) +- [TiCDC Debezium Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-debezium/) +- [TiCDC 数据校验](https://docs.pingcap.com/zh/tidb/stable/ticdc-integrity-check/) +- [TiCDC 常见问题](https://docs.pingcap.com/zh/tidb/stable/ticdc-faq/) +- [TiCDC 故障处理](https://docs.pingcap.com/zh/tidb/stable/troubleshoot-ticdc/) + +## 当前 master 上的 Kafka sink 结构 + +`downstreamadapter/sink/kafka` 是 Kafka sink 的业务层： + +- `helper.go` 解析 sink URI、创建 encoder、event router、topic manager，并调用 + `kafka.NewSaramaFactory` 创建客户端层。 +- `sink.go` 把 DML、DDL、checkpoint 分成三条路径： + - DML 经过 event router 计算 topic/partition，encoder group 编码后用 + `AsyncProducer.AsyncSend` 发送。 + - DDL 用 `SyncProducer.SendMessage` 或 `SendMessages` 发送。Open Protocol 的 + DDL 需要广播到全部 partition，Canal-JSON 的 DDL 走 partition 0。 + - checkpoint/resolved message 广播到当前活跃 topic 的全部 partition。没有表时， + 发送到 default topic，以兼容旧行为。 + +`pkg/sink/kafka` 是客户端抽象层： + +- `factory.go` 定义 `Factory`、`ClusterAdminClient`、`AsyncProducer`、 + `SyncProducer`、`MetricsCollector` 这些上层依赖的接口。 +- `sarama_factory.go` 创建 Sarama admin、sync producer、async producer，并在 + producer 上挂 Sarama metrics registry。 +- `sarama_config.go` 把 TiCDC Kafka options 转为 Sarama config，包括版本探测、 + TLS/SASL、acks、compression、manual partitioner、retry、timeout 和 + `Net.MaxOpenRequests=1`。 +- `options.go` 合并 sink URI 和 changefeed config，并通过 admin 查询 topic/broker + 配置，调整 `max-message-bytes`、`partition-num` 和 `min.insync.replicas`。 +- `admin.go` 封装 topic metadata、topic/broker config 和 create topic。 +- `sarama_async_producer.go` 在 Sarama success channel 中执行 DML callback；遇到 + producer error 时返回错误，让 sink 重建。 +- `sarama_sync_producer.go` 用于 DDL/checkpoint 的同步发送。 +- `metrics_collector.go` 从 Sarama go-metrics registry 中采集并暴露 TiCDC 既有 + Prometheus 指标。 + +可替换边界相对清晰：优先保持 `pkg/sink/kafka/factory.go` 的接口稳定，在 +`pkg/sink/kafka` 内新增 franz-go 实现。真正需要谨慎处理的是“默认行为差异”， +例如 franz-go 默认启用 idempotent write、默认 compression preference 包含 +snappy、默认 linger 为 10ms、默认 record retries 近似无限，这些都不能直接沿用。 + +## 替换原则 + +1. 对用户可见的 sink URI、changefeed config、协议输出、错误语义和指标名称默认 + 保持兼容。 +2. 不因为换客户端而扩大官方支持矩阵。franz-go 自身支持更宽的 Kafka 版本，不代表 + TiCDC Kafka sink 的官方版本支持自动扩大。 +3. 不默认引入新的 ACL 要求。特别是 franz-go 默认 idempotent write 在 Kafka 3.0 + 以前通常需要 Cluster 级 `IDEMPOTENT_WRITE` 权限，而 TiCDC 文档当前最小 ACL + 没有列它。 +4. TiCDC 的 at-least-once 语义、单行更新顺序、DDL/checkpoint/resolved 广播语义 + 优先于吞吐优化。 +5. 性能结论必须来自 TiCDC 场景 A/B 实测。franz-go README 的 benchmark 只能说明 + 客户端潜力，不能直接作为 TiCDC 替换收益结论。 + +## 必须保持的用户配置面 + +以下配置在替换后必须继续支持，默认值、校验和配置文件/URI 覆盖关系也应保持： + +| 类别 | 配置项 | 要求 | +| --- | --- | --- | +| 基础 | broker endpoints、topic、`protocol` | 保持 URI 语法和协议名不变。 | +| 版本 | `kafka-version` | 保留用户显式指定能力；保留版本错误诊断和文档中的兼容要求。 | +| producer | `partition-num`、`replication-factor`、`max-message-bytes`、`max-retry`、`required-acks` | 行为必须和 `options.go` 的校验/自适应一致。 | +| topic | `auto-create-topic` | true 时 TiCDC 创建 topic；false 且 topic 不存在时仍报配置错误。 | +| 压缩 | `compression=none,gzip,snappy,lz4,zstd` | 默认必须是 `none`，不能继承 franz-go 默认 snappy preference。 | +| TLS | `enable-tls`、`ca`、`cert`、`key`、`insecure-skip-verify` | 保持三证书校验和“配置了证书即启用 TLS”的现有行为。 | +| SASL | PLAIN、SCRAM-SHA-256、SCRAM-SHA-512、GSSAPI、OAUTHBEARER | 都需要映射；GSSAPI 需要用 franz-go `pkg/sasl/kerberos` 做专项验证。 | +| 超时 | `dial-timeout`、`write-timeout`、`read-timeout` | franz-go 没有完全同名语义，需要显式设计等价映射和测试。 | +| 协议扩展 | `enable-tidb-extension`、Avro decimal/bigint 模式、Debezium schema 开关、Simple codec config | 客户端替换不能改变 encoder 行为。 | +| 大消息 | `large-message-handle-*`、`claim-check-*` | 客户端替换不能改变大消息判定、外部存储写入和 Kafka marker 格式。 | + +`enable-kafka-sink-v2` 在当前代码中已是 deprecated，并且仍使用默认 Kafka sink。 +不要把它偷偷复用成 franz-go 开关。若需要灰度，建议新增内部开关或新的明确配置， +并为兼容性变更写单独方案。 + +## Kafka 版本支持 + +官方文档列出的 TiCDC Kafka sink 最低 Kafka 版本是产品承诺，替换后仍应遵守： + +| TiCDC 版本 | Kafka 最低版本 | +| --- | --- | +| TiCDC >= v8.1.0 | Kafka >= 2.1.0 | +| v7.6.0 <= TiCDC < v8.1.0 | Kafka >= 2.4.0 | +| v7.5.2 <= TiCDC < v7.6.0 | Kafka >= 2.1.0 | +| v7.5.0 <= TiCDC < v7.5.2 | Kafka >= 2.4.0 | +| v6.5.0 <= TiCDC < v7.5.0 | Kafka >= 2.1.0 | +| v6.1.0 <= TiCDC < v6.5.0 | Kafka >= 2.0.0 | + +当前 Sarama 实现的实际行为： + +- `options.NewOptions` 默认 `Version` 为 `2.4.0`。 +- `sarama_config.go` 的 `defaultKafkaVersion` 是 `2.0.0.0`，`maxKafkaVersion` 是 + `2.8.0.0`。 +- 如果用户未指定 `kafka-version`，代码会通过 broker `ApiVersions` 中 Metadata + API 的 max version 推断版本；失败时退回 `2.0.0.0`。 +- 如果用户指定 `kafka-version`，解析失败返回 `ErrKafkaInvalidVersion`，并在 + 指定版本和探测版本不一致时告警。 + +franz-go 侧能力： + +- README 表示 franz-go 支持 Kafka 0.8.0 到 4.2+ 的协议范围。 +- franz-go 默认会使用 ApiVersions 协商，也可通过 `kgo.MaxVersions` 固定协议版本。 + +替换要求： + +1. 不能因为 franz-go 支持更高或更低版本就改变 TiCDC 文档承诺。 +2. `kafka-version` 必须继续生效，建议映射为 `kgo.MaxVersions(...)` 或等价能力。 +3. 当前“自动探测 + 指定版本告警”的诊断体验需要保留，至少不能退化为静默忽略。 +4. 需要覆盖 Kafka 2.1、2.4、2.8、3.x、Confluent Cloud，以及如果仍支持则覆盖 KOP。 +5. 若去掉 Sarama 的 `maxKafkaVersion=2.8.0` 上限，需要在 release note 中说明这只是 + 客户端协议协商变化，不代表 TiCDC 扩大官方最低版本矩阵。 + +## ACL 和权限要求 + +官方文档列出的 Kafka 最小权限： + +| Resource | Operation | 用途 | +| --- | --- | --- | +| Topic | Create | 自动创建 topic。 | +| Topic | Write | 写入变更事件。 | +| Topic | Describe | 启动和 topic metadata 查询。 | +| Cluster | DescribeConfigs | 读取 broker/topic 配置，如 `message.max.bytes`、`min.insync.replicas`。 | + +如果 topic 已存在，文档说明可省略 Topic `Create`，但代码仍会读取 topic metadata 和 +配置，因此 `Describe` / `DescribeConfigs` 的实际需求要按部署环境验证。 + +替换时的新增风险： + +- franz-go 默认启用 idempotent write。Kafka 3.0 以前通常需要 Cluster 级 + `IDEMPOTENT_WRITE` 权限。当前 TiCDC 文档没有要求这个 ACL，Sarama 实现也没有开启 + idempotent producer。因此默认必须使用 `kgo.DisableIdempotentWrite()`，除非另开 + 配置并同步更新文档、权限说明和回滚策略。 +- 如果未来选择启用 idempotency，需要说明它改变的是客户端内部重试去重能力，不改变 + TiCDC 对外的 at-least-once 语义；TiCDC 仍可能在重启、故障恢复或上游重放后发送 + 重复消息。 +- Schema Registry、AWS Glue Schema Registry、claim-check 外部存储权限不是 Kafka + ACL，但替换不能破坏其认证和错误诊断。 + +## Producer 行为映射 + +| TiCDC/Sarama 现状 | franz-go 替换要求 | +| --- | --- | +| 手动指定 partition，Sarama `NewManualPartitioner`。 | 使用 `kgo.RecordPartitioner(kgo.ManualPartitioner())`，所有 record 必须设置 `Topic` 和 `Partition`。 | +| `required-acks=-1/1/0` 映射到 Sarama RequiredAcks。 | 映射到 `kgo.AllISRAcks()`、`kgo.LeaderAck()`、`kgo.NoAck()`。 | +| 默认 `compression=none`。 | 显式 `kgo.ProducerBatchCompression(kgo.NoCompression())` 或等价配置。 | +| `Producer.Flush.*=0`，尽快 flush。 | 显式 `kgo.ProducerLinger(0)`，避免默认 10ms linger 改变延迟。 | +| `Producer.Retry.Max=o.MaxRetry`，默认 5，backoff 100ms。 | 显式 `kgo.RecordRetries(o.MaxRetry)`，并设置等价 backoff；不要继承 unlimited retries。 | +| `Net.MaxOpenRequests=1` 作为顺序保护。 | 禁用 idempotency 后保留 `MaxProduceRequestsInflightPerBroker(1)`；不要为吞吐随意调大。 | +| producer max message bytes 来自 `options.MaxMessageBytes`。 | 设置 `ProducerBatchMaxBytes`，并校准其“record batch pre-compression”语义和 TiCDC 大消息判定。 | +| async success 后执行 message callback。 | franz-go promise 只有 `err == nil` 才能执行 callback。promise 不得阻塞或调用可能阻塞的 Produce/Flush。 | +| async error 让 `AsyncRunCallback` 返回，sink 重建。 | 需要有中心错误通道/errgroup，把首个 produce error 转为 TiCDC error 并返回。 | +| sync DDL/checkpoint 用 `SendMessage` 或 `SendMessages`。 | 用 `ProduceSync` 实现；`SendMessages` 必须构造每个 partition 一条 record，并在任一失败时返回错误。 | + +`required-acks=0` 必须保留，但需要明确：这本来就没有 broker durable ack 保证。替换后 +不要在 callback 中假装获得了真实 broker ack；只能保持与现有“允许但风险自担”的语义。 + +## Admin 行为映射 + +当前 `ClusterAdminClient` 行为需要完整保留： + +| 接口 | 当前用途 | franz-go/kadm 替换注意点 | +| --- | --- | --- | +| `GetAllBrokers` | metrics collector 获取 broker label。 | 可用 metadata/broker metadata。 | +| `GetBrokerConfig` | 读取 `message.max.bytes`、`min.insync.replicas`。 | 当前 Sarama 通过 controller broker 的 DescribeConfig 读取；kadm 实现要确认是否等价。 | +| `GetTopicConfig` | 读取 topic `max.message.bytes` 和 topic 级 `min.insync.replicas`。 | 需要兼容 KOP/不同 broker 返回 config entries 的形式。 | +| `GetTopicsMeta` | 判断 topic 是否存在、读取 partition 数。 | `UnknownTopicOrPartition` 在 ignore 模式下要跳过；其他错误不能吞。 | +| `GetTopicsPartitionsNum` | topic manager 定时刷新动态 topic partition 数。 | 返回值必须和当前 map 语义一致。 | +| `CreateTopic` | 自动创建 topic。 | `TopicAlreadyExists` 继续按成功处理；其他 policy/rf/auth 错误要保留。 | +| `Close` | 释放 admin client。 | 不能阻塞 sink 关闭路径。 | + +`topicmanager/kafka_topic_manager.go` 的行为不要改： + +- default topic 已存在且实际 partition 更多时，只使用配置中的 partition 子集。 +- 用户指定 `partition-num` 大于实际 topic partition 数时返回错误，避免 dispatch 到 + 不存在的 partition。 +- 动态 topic 缓存刷新和 create-topic-then-wait-visible 逻辑继续由 topic manager 负责， + 不要转移到 producer 自动创建。 + +## Topic、partition 和顺序保证 + +官方文档和代码共同依赖以下顺序约束： + +1. `index-value`、`columns`、`table/default` 这类 dispatcher 必须保证同一行的多次更新 + 进入同一 Kafka partition。 +2. `ts` dispatcher 可能把同一行不同版本发到不同 partition，消费者必须按 commitTs + 排序；客户端替换不能额外提供或破坏这个语义。 +3. Open Protocol 的 DDL 和 Resolved Event 需要广播到所有 MQ partition，消费者用 + resolved ts 做多 partition 排序。 +4. Canal-JSON DDL 发送到 partition 0；WATERMARK 只有在 `enable-tidb-extension=true` + 时输出。 +5. Simple Protocol 的 WATERMARK 和 BOOTSTRAP 语义必须保持。客户端替换不能改变 + BOOTSTRAP 周期、发送分区和 DML/DDL 顺序关系。 +6. DML callback 只能在 Kafka client 认为该 record 成功后执行，否则上游可能提前推进 + checkpoint，造成数据丢失。 + +franz-go 文档说明成功写入的 records 会按 partition 保持顺序；同时 `RecordRetries` +耗尽时会失败同 partition buffered records，避免跳过失败 record 后继续成功写入后续 +record。替换实现必须利用这一点，而不是在 TiCDC 层自行绕过失败继续发送。 + +## 协议输出兼容性 + +客户端替换不应改 encoder，但实现和测试必须覆盖所有 Kafka 支持协议： + +| 协议 | 必须保持的行为 | +| --- | --- | +| Open Protocol | Row Changed、DDL、Resolved 事件；batch key/value 格式；DDL/Resolved broadcast；`max-batch-size`。 | +| Canal-JSON | 一行一条 DML；DDL partition 0；`_tidb.commitTs`、WATERMARK、`content-compatible`。 | +| Avro | Confluent Avro wire format；每个 topic 只对应一张表；delete value 为 nil；Schema Registry / Glue 注册和错误处理。 | +| Debezium | 只输出 Row Changed Event，不输出 DDL/WATERMARK；schema 开关；TiDB 扩展字段。 | +| Simple | DDL、DML、WATERMARK、BOOTSTRAP；JSON/Avro codec；消费者 schema cache 依赖 BOOTSTRAP。 | + +相关配置和限制也要覆盖： + +- `delete-only-output-handle-key-columns` +- `only-output-updated-columns` +- `column-selectors` +- `enable-tidb-extension` +- `schema-registry` / AWS Glue schema registry +- row-level checksum：Kafka + Simple/Avro；Avro 需 TiDB extension 和 decimal/bigint string + 模式。 + +## 大消息处理和消息大小估算 + +这是替换中的高风险点。 + +当前 `pkg/sink/codec/common/message.go` 的 `Message.Length()` 使用： + +```go +len(m.Key) + len(m.Value) + MaxRecordOverhead +``` + +其中 `MaxRecordOverhead` 的注释明确基于 Sarama 的 record batch 编码估算。这个值参与： + +- producer `max-message-bytes` 前置检查； +- Open Protocol batch 拆分； +- large message compression 后是否进入 `handle-key-only` 或 `claim-check`； +- 报错 `Message was too large` 前的客户端侧保护。 + +franz-go 的 `ProducerBatchMaxBytes` 限制的是未压缩 record batch 上限。如果继续使用 +Sarama overhead，可能出现两类问题： + +- TiCDC 认为没超限，franz-go 或 broker 拒绝，造成 changefeed 报错。 +- TiCDC 认为超限而提前 claim-check/handle-key-only，导致不必要的外部存储写入或消息降级。 + +替换要求： + +1. 重新校准 Kafka record batch overhead。优先使用 franz-go 可复用的编码/估算能力； + 如果无法直接复用，使用保守上界并写明依据。 +2. 覆盖 key/value 为空、key 大 value 小、value 大 key 小、header 为空、不同 compression + 的测试。 +3. `large-message-handle-compression` 是 TiCDC 在消息级别先压缩再判断大小；producer + `compression` 是 Kafka batch 压缩。两者不能混淆。 +4. `claim-check` 和 `claim-check-raw-value` 的 Kafka marker 格式、外部存储路径、清理 + 责任必须保持不变。 +5. `max-message-bytes` 仍要和 broker/topic `message.max.bytes` / `max.message.bytes` + 通过 admin 自适应，并保留当前 `128` bytes safety margin 或给出替代依据。 + +## TLS、SASL 和认证 + +TLS 替换要求： + +- 复用 `security.Credential.ToTLSConfig()`。 +- 保留 TLS 1.2 minimum、证书文件校验、`insecure-skip-verify` 行为。 +- franz-go 可用 `kgo.DialTLSConfig` 或自定义 dialer，具体选择要覆盖证书和系统 CA 两种路径。 + +SASL 替换要求： + +| 机制 | franz-go 映射 | 注意点 | +| --- | --- | --- | +| PLAIN | `pkg/sasl/plain` | 用户名/密码为空时的现有错误行为要保持。 | +| SCRAM-SHA-256 | `pkg/sasl/scram` | 使用 SHA-256 mechanism；保持大小写和错误信息。 | +| SCRAM-SHA-512 | `pkg/sasl/scram` | 使用 SHA-512 mechanism。 | +| OAUTHBEARER | `pkg/sasl/oauth` 或自定义 provider | 复用现有 OAuth2 token provider 行为，包括 base64 secret、scope、grant type、audience。 | +| GSSAPI | `pkg/sasl/kerberos` | 需要把 `sasl-gssapi-*` 字段映射到 Kerberos client；user auth/keytab 两种都要集成测试。 | + +`pkg/security/sasl.go` 当前直接引用 Sarama 常量。真正移除 Sarama 依赖时，需要先把这些 +公共安全常量改成 TiCDC 自己的字符串常量，否则 Sarama 依赖会继续被保留。 + +## Metrics、日志和观测性 + +当前 TiCDC 暴露的 Kafka producer 指标名称和 label 是外部运维契约： + +- `ticdc_sink_kafka_producer_in_flight_requests` +- `ticdc_sink_kafka_producer_outgoing_byte_rate` +- `ticdc_sink_kafka_producer_request_rate` +- `ticdc_sink_kafka_producer_request_latency` +- `ticdc_sink_kafka_producer_compression_ratio` +- `ticdc_sink_kafka_producer_records_per_request` +- `ticdc_sink_kafka_producer_response_rate` + +这些指标目前来自 Sarama go-metrics registry。franz-go 可用 hook 或 `plugin/kprom`，但 +`kprom` 的默认指标名不是 TiCDC 现有指标名。因此推荐实现 TiCDC 自己的 hook collector： + +- `HookBrokerWrite` / `HookBrokerRead` / E2E hook：请求数、响应数、latency、broker label。 +- `HookProduceBatchWritten`：records per request、compression ratio、outgoing bytes。 +- 需要自行维护 in-flight gauge，或明确一个等价口径。 +- collector cleanup 必须删除 `namespace/changefeed/broker/type` label，避免 changefeed + 删除后遗留时间序列。 + +日志相关替换点： + +- `pkg/logger/log.go` 目前有 `WithInitSaramaLogger` 和 `sarama.Logger` hack。franz-go + 需要接入 `kgo.WithLogger` 或等价 logger adapter。 +- `pkg/leakutil/leak_helper.go` 目前忽略 Sarama goroutine。替换后应删除或改成 + franz-go 相关 goroutine 的测试策略，不能永久掩盖泄漏。 +- producer/admin 错误必须继续通过 `logutil.go` 附加 `MessageLogInfo`，否则 + `kafka_log_info` 类测试会退化。 + +## 正确性风险清单 + +P0 必须解决： + +- DML callback 不得早于 Kafka 成功返回。 +- 手动 partition 不得退化为客户端默认 partitioner。 +- `required-acks`、`compression`、`max-retry`、`linger`、in-flight、idempotency 等默认值 + 必须显式设置，不得使用 franz-go 默认值。 +- DDL/checkpoint/resolved 的广播分区数必须来自 topic manager，而不是 producer metadata + 的临时结果。 +- 大消息大小估算必须重新校准。 +- `required-acks=-1` 时 `replication-factor >= min.insync.replicas` 的前置校验要保留。 +- `UnknownTopicOrPartition`、`TopicAlreadyExists`、auth、policy、message too large 等错误 + 要保持可诊断，不能被统一包装成无信息的 client error。 + +P1 需要验证： + +- `required-acks=0` 下 promise/callback 语义和 Sarama 一致。 +- broker idle connection 关闭后的 EOF/broken pipe 恢复行为。官方 FAQ 中曾提到 Sarama + broken pipe；当前 master 已使用 bounded retry 和 Sarama fork ordering fix。franz-go + 是否改善该场景，需要专门压测。 +- Confluent Cloud 中 `min.insync.replicas` 不可见时的容错告警仍然保留。 +- KOP 对 DescribeConfig 返回项的兼容性。 +- 动态 topic 表达式和多 topic checkpoint 广播。 + +## 可靠性和资源管理 + +需要明确设计： + +- async producer close 是否等待 flush。当前 Sarama close 为避免阻塞，会异步关闭并接受 + 可能重复数据。franz-go `Client.Close()` / cancel / `Flush` 的使用必须与这个策略一致， + 不能在 sink 关闭路径无限阻塞。 +- producer promise 串行执行，不能在 promise 中执行阻塞操作。 +- franz-go `MaxBufferedRecords` 默认 10000，`MaxBufferedBytes` 默认无限。TiCDC 上游已有 + unlimited channel，需要评估双重缓冲是否导致内存放大，并为高流量场景设置或暴露合理 + 上限。 +- admin、async producer、sync producer 是否共享同一个 `kgo.Client`。第一阶段建议和 + 现状一致，每个组件独立 client，降低 close 生命周期复杂度；共享 client 可作为后续优化。 +- request timeout、record delivery timeout、record retries 之间的关系必须有限制，避免 + transient error 下无限阻塞 changefeed。 + +## 性能对比和验收方法 + +不要在没有 TiCDC A/B 数据前宣称性能提升。建议基准如下： + +环境变量： + +- Go version、TiCDC commit、Kafka version、broker 数、topic partition 数、replication + factor、`min.insync.replicas`。 +- 是否启用 TLS/SASL。 +- `required-acks`、producer compression、large-message compression、`max-message-bytes`。 + +workload： + +- Canal-JSON 小行高吞吐。 +- Open Protocol batch encode，覆盖 `max-batch-size`。 +- Avro + Schema Registry / Glue。 +- Simple JSON/Avro，覆盖 BOOTSTRAP。 +- Debezium JSON/Avro。 +- 大消息：普通超限、message-level compression、handle-key-only、claim-check、 + claim-check-raw-value。 +- 多 topic 动态路由、单大表多 partition、高 partition 数。 +- 低流量长 idle，验证连接保活和 broker idle close。 + +指标： + +- rows/sec、bytes/sec。 +- Kafka produce latency p50/p95/p99。 +- CPU、heap、allocs/op、goroutine 数。 +- producer request rate、response rate、in-flight、records/request、compression ratio。 +- TiCDC changefeed checkpoint lag、resolved ts lag。 +- 错误率、重试次数、重建次数。 + +验收标准： + +- 功能正确性优先。性能不得显著退化；若有吞吐/延迟 trade-off，必须说明对应配置。 +- metrics 名称和 label 兼容，或明确给出 dashboard/alert 迁移方案。 +- A/B 报告要列出 Sarama 和 franz-go 的完整 producer 配置，避免比较默认值不同的结果。 + +## 测试计划 + +单元测试： + +- options 合并和默认值：URI 覆盖 config、非法 client ID、非法 acks、非法 partition。 +- franz-go option mapping：acks、compression、linger、retry、idempotency、manual partition、 + batch max bytes、buffer limits、timeouts。 +- TLS/SASL：PLAIN、SCRAM、OAuth、GSSAPI user/keytab。 +- admin wrapper：topic 存在/不存在、已存在 topic、invalid replication factor、policy + violation、config not found、Confluent Cloud fallback。 +- async producer：成功 callback、错误返回、context cancel、close、message log info。 +- sync producer：DDL partition 0、Open Protocol broadcast、checkpoint broadcast、partial failure。 +- message size：franz-go overhead 校准、large message option 触发点。 +- metrics collector：hook 数据转换、label cleanup。 + +集成测试： + +- 复用现有 Kafka integration cases，分别跑 Sarama 和 franz-go。 +- 增加 ACL 测试：仅官方最小权限时 franz-go 默认配置必须能写入；若打开 idempotency， + 缺少 `IDEMPOTENT_WRITE` 要有明确错误。 +- 增加 idle connection / broken pipe 场景。 +- 增加 Kafka 2.1、2.4、2.8、3.x、Confluent Cloud 或兼容环境。 +- 增加 TLS/SASL/GSSAPI/OAuth 覆盖。 +- 增加 topic 已存在且 partition 数大于配置、partition 数小于配置、auto-create=false。 + +建议命令按改动范围选择： + +- `make unit_test_pkg PKG=./pkg/sink/kafka/...` +- `make unit_test_pkg PKG=./downstreamadapter/sink/...` +- `make integration_test_kafka CASE=` +- 最终切默认前跑完整 `make unit_test` 和 Kafka integration suite。 + +## 实施拆分建议 + +1. 增加 franz-go 依赖和内部 factory 实现，但默认仍走 Sarama。 +2. 实现 `franzAdminClient`，让 `adjustOptions`、topic manager 测试先通过。 +3. 实现 `franzSyncProducer`，先覆盖 DDL/checkpoint。 +4. 实现 `franzAsyncProducer`，覆盖 callback、错误、close、backpressure。 +5. 完成配置映射和 SASL/TLS/OAuth/GSSAPI。 +6. 完成消息大小估算替换和大消息测试。 +7. 完成 metrics collector 和 logger/leakutil 清理。 +8. 增加灰度开关和 A/B 测试。 +9. 满足功能、性能、可靠性验收后，再决定是否切默认并保留 Sarama 回滚窗口。 +10. 最后移除 Sarama 依赖前，清理 `pkg/security/sasl.go`、logger、leak helper、mocks、 + go.mod/go.sum 和文档中的 Sarama 表述。 + +## 待决策项 + +- 默认是否禁用 idempotency：建议禁用，保持现有 ACL 和 Sarama 非幂等 producer 语义。 +- `kafka-version` 是完全固定 franz-go MaxVersions，还是继续自动探测并只在用户显式指定时 + 固定。 +- message size overhead 使用精确编码计算还是保守上界。 +- metrics 是自研 hook collector 还是迁移到 kprom 指标名。 +- 是否需要公开 `kafka-client=franz|sarama` 灰度参数；如果公开，需要 API 兼容评审和文档。 +- admin/sync/async 是否共享 `kgo.Client`。 + +## 结论 + +franz-go 替换 Sarama 的代码入口集中在 `pkg/sink/kafka`，但完整替换不是单纯把 +`sarama.ProducerMessage` 换成 `kgo.Record`。必须显式复刻 TiCDC Kafka sink 的产品契约： +配置兼容、最小 ACL、协议输出、大消息处理、顺序保证、at-least-once、DDL/checkpoint +广播、metrics 和错误诊断。 + +最容易被遗漏、也最可能影响线上正确性的点是： + +- franz-go 默认 idempotent write 带来的 ACL 和语义变化； +- franz-go 默认 snappy compression、10ms linger、unlimited record retries； +- Sarama record overhead 被用于 TiCDC 消息大小判断； +- 既有 Kafka producer Prometheus 指标来自 Sarama registry； +- GSSAPI、OAuth、Confluent Cloud、KOP 这类非本地单机 Kafka 场景。 + +这些点全部有测试和灰度证据后，才能考虑把 franz-go 设为默认实现。 diff --git a/docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md b/docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md new file mode 100644 index 0000000000..5899893cd4 --- /dev/null +++ b/docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md @@ -0,0 +1,672 @@ +# 使用 franz-go 替换 Sarama 的迁移步骤 + +## 目标 + +本文给出在 TiCDC Kafka sink 中用 `~/go/franz-go` 替换 Sarama 的迁移步骤。 +配套功能审计见： + +- `docs/design/2026-07-03-franz-go-replace-sarama-audit.md` + +本文关注“怎么迁移”：分阶段实现、验证、灰度、切默认、回滚和清理。 +迁移默认采用 expand-migrate-contract 方式：先新增 franz-go 实现并保持 Sarama 可用， +再灰度切流量，最后在兼容窗口结束后移除 Sarama。 + +## 迁移性质 + +| 项目 | 结论 | +| --- | --- | +| 状态源 | Kafka topic 中的 change event、TiCDC checkpoint/resolved 进度、changefeed config、Prometheus 指标。 | +| 外部可见面 | sink URI/config、Kafka 消息协议、topic/partition 顺序、ACL、metrics、日志、错误语义。 | +| 可逆性 | 分阶段可逆；切默认前可通过配置退回 Sarama；移除 Sarama 依赖后只剩版本回滚。 | +| 主要风险 | producer 默认行为差异、消息大小估算、callback 时机、metrics 兼容、SASL/GSSAPI、ACL 变化。 | +| 推荐策略 | 默认禁用 franz-go idempotency；保留 Sarama 回滚路径至少一个发布窗口。 | + +## 总体阶段 + +1. 准备阶段：冻结兼容契约，补足 baseline 测试和 A/B 工具。 +2. Expand：新增 franz-go 适配层和选择开关，默认仍使用 Sarama。 +3. 功能迁移：先 admin，再 sync producer，再 async producer，再 TLS/SASL/metrics。 +4. Shadow / 对照验证：同配置跑 Sarama 与 franz-go，确认功能、顺序、错误和指标。 +5. 小流量灰度：按 changefeed 逐步启用 franz-go，保留即时回滚。 +6. 切默认：把默认实现从 Sarama 切到 franz-go，但仍保留 `sarama` fallback。 +7. Contract：兼容窗口结束后移除 Sarama 代码、依赖和文档残留。 + +## 阶段 0：准备和基线 + +### 0.1 明确兼容契约 + +产出： + +- 一份确认过的兼容清单，至少覆盖： + - sink URI/config key 和默认值。 + - Kafka 版本支持矩阵。 + - 最小 ACL 要求。 + - protocol 输出格式。 + - DDL/checkpoint/resolved broadcast 规则。 + - DML callback 时机。 + - Kafka producer metrics 名称和 label。 + - 错误类型、错误码和关键日志字段。 + +进入下一步前必须确认： + +- 不公开改变 `required-acks`、`compression`、`max-retry`、`kafka-version`、 + `max-message-bytes` 等默认行为。 +- 不复用已 deprecated 的 `enable-kafka-sink-v2` 作为 franz-go 开关。 +- 如果要新增用户可见开关，例如 `kafka-client=franz|sarama`，需要单独做兼容评审。 + +### 0.2 建立 Sarama baseline + +建议先在未引入 franz-go 的 `master` 上记录 baseline： + +- unit tests： + - `make unit_test_pkg PKG=./pkg/sink/kafka/...` + - `make unit_test_pkg PKG=./downstreamadapter/sink/...` +- Kafka integration tests： + - canal-json basic。 + - open-protocol basic。 + - avro / schema registry。 + - simple protocol。 + - large message / claim-check / handle-key-only。 + - dispatcher / dynamic topic。 + - mq sink error resume。 +- 性能 baseline： + - 小行高吞吐。 + - 大消息。 + - 多 partition。 + - 低流量长 idle。 + +记录内容： + +- TiCDC commit、Go version、Kafka version、broker 配置。 +- sink URI 和 changefeed config。 +- rows/sec、bytes/sec、send latency p95/p99、CPU、heap、goroutine。 +- Kafka producer request rate、response rate、in-flight、records/request、compression ratio。 +- changefeed checkpoint lag / resolved ts lag。 + +退出条件： + +- Sarama baseline 本身稳定。 +- 已知 flaky test 单独记录，不和 franz-go 替换混在一起判断。 + +## 阶段 1：新增 franz-go 适配骨架 + +### 1.1 引入依赖 + +改动点： + +- `go.mod` / `go.sum` + - `github.com/twmb/franz-go/pkg/kgo` + - `github.com/twmb/franz-go/pkg/kadm` + - `github.com/twmb/franz-go/pkg/kmsg` + - `github.com/twmb/franz-go/pkg/sasl/plain` + - `github.com/twmb/franz-go/pkg/sasl/scram` + - `github.com/twmb/franz-go/pkg/sasl/oauth` + - `github.com/twmb/franz-go/pkg/sasl/kerberos` + +注意： + +- 不在这一阶段移除 Sarama。 +- 不把 franz-go metrics plugin 直接作为最终指标方案，除非已决定迁移指标名。 + +### 1.2 保持现有接口 + +优先不改上层 sink，只新增实现： + +- `pkg/sink/kafka/franz_factory.go` +- `pkg/sink/kafka/franz_config.go` +- `pkg/sink/kafka/franz_admin.go` +- `pkg/sink/kafka/franz_sync_producer.go` +- `pkg/sink/kafka/franz_async_producer.go` +- `pkg/sink/kafka/franz_metrics_collector.go` + +保留接口： + +- `Factory` +- `ClusterAdminClient` +- `AsyncProducer` +- `SyncProducer` +- `MetricsCollector` + +退出条件： + +- 新代码可编译。 +- 默认路径仍是 Sarama。 +- 没有任何用户在未显式启用时走 franz-go。 + +### 1.3 增加选择开关 + +推荐先用内部灰度开关，不立即变成公开文档承诺： + +- 方案 A：内部 config/env/build tag，用于 CI 和受控灰度。 +- 方案 B：公开 sink URI 参数 `kafka-client=sarama|franz`。 + +推荐顺序： + +1. 第一阶段用内部开关验证。 +2. 如果需要用户级灰度，再公开 `kafka-client`，并补充文档、测试和 release note。 + +实现要求： + +- 默认值必须是 `sarama`。 +- 无效值返回明确配置错误。 +- 切换只影响 Kafka client 层，不影响 encoder、event router、topic manager。 + +回滚： + +- 将开关改回 `sarama`。 +- 如果开关在 changefeed config 中，回滚不应要求删除 changefeed。 + +## 阶段 2：迁移 admin client + +先迁移 admin，是因为 `adjustOptions` 和 topic manager 依赖它，且不触碰数据写入。 + +### 2.1 实现 `franzAdminClient` + +需要实现： + +- `GetAllBrokers` +- `GetBrokerConfig` +- `GetTopicConfig` +- `GetTopicsMeta` +- `GetTopicsPartitionsNum` +- `CreateTopic` +- `Close` + +映射建议： + +- 使用 `kadm.Client` 做 topic metadata、create topic、describe config。 +- `GetBrokerConfig` 要确认是否等价于当前 Sarama 从 controller broker 读取 config 的行为。 +- `TopicAlreadyExists` 继续按成功处理。 +- `UnknownTopicOrPartition` 在 `ignoreTopicError=true` 时跳过。 +- `ErrKafkaConfigNotFound` 语义保持。 + +### 2.2 admin 单元测试 + +覆盖： + +- topic 存在，partition 数读取正确。 +- topic 不存在，ignore true/false 行为正确。 +- create topic 成功。 +- create topic 时 topic 已存在。 +- invalid replication factor / policy violation / authorization error 保留原始诊断。 +- broker/topic config 找不到时保留现有 fallback 和告警语义。 +- Confluent Cloud 下 `min.insync.replicas` 不可见时保持允许启动但告警。 + +退出条件： + +- `adjustOptions` 测试可在 franz admin wrapper 下通过。 +- `topicmanager` 测试可在 franz admin wrapper 下通过。 +- 默认 Sarama 测试未回归。 + +## 阶段 3：迁移 sync producer + +sync producer 负责 DDL 和 checkpoint，流量低但正确性要求高。 + +### 3.1 实现 `franzSyncProducer` + +要求： + +- `SendMessage(topic, partitionNum, message)` 发送单条 record 到指定 partition。 +- `SendMessages(topic, partitionNum, message)` 发送 `0..partitionNum-1` 每个 partition 一条。 +- 使用 `ProduceSync(ctx, records...)` 或等价同步路径。 +- 任一 partition 失败时返回错误。 +- 错误必须通过 `AnnotateEventError` 附加 DDL/checkpoint log info。 + +配置必须显式设置： + +- `RecordPartitioner(kgo.ManualPartitioner())` +- `RequiredAcks(...)` +- `ProducerBatchCompression(...)` +- `ProducerLinger(0)` +- `RecordRetries(options.MaxRetry)` +- `DisableIdempotentWrite()` +- `MaxProduceRequestsInflightPerBroker(1)` +- `ProducerBatchMaxBytes(...)` + +### 3.2 sync producer 测试 + +覆盖： + +- Canal-JSON DDL 到 partition 0。 +- Open Protocol DDL broadcast 到全部 partition。 +- checkpoint/resolved broadcast 到全部 partition。 +- no table 时 checkpoint 发 default topic。 +- 部分 partition 发送失败时返回错误。 +- `required-acks=0/1/-1` 配置映射。 + +退出条件： + +- DDL/checkpoint 单元测试通过。 +- Kafka integration 中 DDL、checkpoint、resolved 语义和 Sarama 对齐。 + +回滚： + +- `kafka-client=sarama`。 +- 因为没有改变 Kafka 消息格式，已经写入的 DDL/checkpoint 可继续被消费者按原协议读取。 + +## 阶段 4：迁移 async producer + +async producer 是 DML 主路径，必须最后接入，并且先在可回滚模式下运行。 + +### 4.1 实现 `franzAsyncProducer` + +要求： + +- `AsyncSend(ctx, topic, partition, message)`： + - 构造 `kgo.Record{Topic, Partition, Key, Value}`。 + - record partition 必须来自 TiCDC event router，不能让 franz-go 重新 hash。 + - 发送前设置 message partition key，保持日志和统计语义。 +- promise / callback： + - `err == nil` 时执行 `message.Callback`。 + - `err != nil` 时不执行 callback。 + - promise 不做阻塞操作，不调用 `Flush` 或可能阻塞的 `Produce`。 +- `AsyncRunCallback(ctx)`： + - 等待首个 produce error 或 ctx done。 + - 首个 produce error 返回给 sink，让 sink 重建。 + - 返回错误带 TiCDC stack 和 `MessageLogInfo`。 +- `Close()`： + - 不在 sink 关闭路径无限等待 flush。 + - 明确是否允许未 ack record 后续由上游重放造成重复。 + +### 4.2 async producer 测试 + +覆盖： + +- 成功后 callback 恰好执行一次。 +- producer error 时 callback 不执行，`AsyncRunCallback` 返回错误。 +- context cancel 时可退出。 +- close 不阻塞。 +- manual partition 生效。 +- `required-acks=0` 下 callback 语义和 Sarama 对齐。 +- `RecordRetries` 耗尽时同 partition 不越过失败 record。 + +退出条件： + +- DML 单元测试通过。 +- Kafka integration 中 DML 顺序、重复和恢复语义与 Sarama 对齐。 +- `mq_sink_error_resume` 类场景通过。 + +## 阶段 5：配置、TLS、SASL 和版本映射 + +这一阶段不要改变用户配置名，只做映射。 + +### 5.1 producer option 映射 + +必须显式覆盖 franz-go 默认值： + +| 配置 | franz-go 设置 | +| --- | --- | +| brokers | `kgo.SeedBrokers(...)` | +| client id | `kgo.ClientID(...)` | +| kafka version | `kgo.MaxVersions(...)` 或等价固定版本能力 | +| required acks | `kgo.RequiredAcks(...)` | +| compression | `kgo.ProducerBatchCompression(...)` | +| max message bytes | `kgo.ProducerBatchMaxBytes(...)` | +| retry | `kgo.RecordRetries(options.MaxRetry)` + backoff | +| linger | `kgo.ProducerLinger(0)` | +| partition | `kgo.RecordPartitioner(kgo.ManualPartitioner())` | +| idempotency | 默认 `kgo.DisableIdempotentWrite()` | +| inflight | `kgo.MaxProduceRequestsInflightPerBroker(1)` | +| buffer | 明确 `MaxBufferedRecords` / `MaxBufferedBytes` 策略 | + +### 5.2 TLS 映射 + +覆盖： + +- 系统 CA + `enable-tls=true`。 +- 自签 CA + cert/key。 +- cert/key/ca 不完整时仍报配置错误。 +- `enable-tls=false` 但配置证书时仍报配置错误。 +- `insecure-skip-verify` 只在 TLS 开启时生效。 + +### 5.3 SASL 映射 + +覆盖： + +- PLAIN。 +- SCRAM-SHA-256。 +- SCRAM-SHA-512。 +- OAUTHBEARER： + - base64 secret 解码。 + - token URL。 + - scopes。 + - grant type。 + - audience。 +- GSSAPI： + - user/password auth。 + - keytab auth。 + - service name。 + - realm。 + - kerberos config path。 + - disable PAFXFAST。 + +注意： + +- `pkg/security/sasl.go` 当前引用 Sarama 常量。只要 Sarama 还没移除，可以先保留； + contract 阶段必须改成 TiCDC 自有常量。 +- GSSAPI 不能只靠单元测试，至少需要一个可运行的 Kerberos/Kafka 集成验证或明确记录 + 未覆盖风险。 + +### 5.4 Kafka version 映射 + +要求： + +- 用户显式 `kafka-version` 必须生效。 +- 无法解析版本仍返回 `ErrKafkaInvalidVersion` 或等价 TiCDC 错误。 +- 未指定版本时可继续自动协商，但不能扩大产品支持承诺。 +- 版本不匹配的告警体验尽量保留。 + +退出条件： + +- 所有配置映射测试通过。 +- 旧 sink URI/config 不修改即可加载。 +- 官方最小 ACL 下可以启动并写入。 + +## 阶段 6：消息大小和大消息路径 + +这一步是切 DML 流量前的硬门槛。 + +### 6.1 替换 size accounting + +当前 `Message.Length()` 使用 Sarama `MaxRecordOverhead`。迁移步骤： + +1. 写一个 franz-go record batch size 估算 helper。 +2. 用测试对照 franz-go 实际编码或 producer 拒绝边界。 +3. 将 `Message.Length()` 或其调用方切到新的估算方式。 +4. 保留或重新论证 `maxMessageBytesOverhead=128` safety margin。 +5. 对 open-protocol batch splitter、large-message compression、claim-check 都加测试。 + +### 6.2 大消息测试 + +覆盖： + +- 普通消息接近 `max-message-bytes`。 +- 单行大于限制。 +- Open Protocol batch 被拆分。 +- message-level lz4/snappy compression 后不过限。 +- `handle-key-only`。 +- `claim-check`。 +- `claim-check-raw-value`。 +- broker/topic `message.max.bytes` 小于用户配置。 + +退出条件： + +- 不出现“TiCDC 判定可发送但 franz-go/broker 拒绝”的边界误差。 +- 不出现“TiCDC 过早 claim-check”的明显误差。 +- `Message was too large` 错误仍可诊断。 + +## 阶段 7：metrics、日志和泄漏检查 + +### 7.1 metrics 兼容 + +默认要求保留现有指标名和 label： + +- `ticdc_sink_kafka_producer_in_flight_requests` +- `ticdc_sink_kafka_producer_outgoing_byte_rate` +- `ticdc_sink_kafka_producer_request_rate` +- `ticdc_sink_kafka_producer_request_latency` +- `ticdc_sink_kafka_producer_compression_ratio` +- `ticdc_sink_kafka_producer_records_per_request` +- `ticdc_sink_kafka_producer_response_rate` + +实现步骤： + +1. 基于 franz-go hooks 实现 TiCDC collector。 +2. 对齐 Sarama collector 的 label：`namespace`、`changefeed`、`broker`、`type`。 +3. 明确 in-flight 的等价口径。 +4. 在 changefeed stop/delete 后清理 label。 +5. A/B 对比指标是否在同一数量级。 + +如果决定改指标名： + +- 必须提供 dashboard / alert 迁移方案。 +- 需要至少一个版本同时暴露新旧指标。 +- release note 必须说明。 + +### 7.2 logger 和 leakutil + +步骤： + +- 给 franz-go 接 `kgo.WithLogger`。 +- 保留 Kafka client 日志中的 keyspace/changefeed 上下文。 +- 清理 `WithInitSaramaLogger` 的依赖路径，但不要在 Sarama fallback 存在期间破坏 Sarama。 +- 更新 `pkg/leakutil/leak_helper.go`，不要继续用 Sarama goroutine ignore 掩盖新泄漏。 + +退出条件： + +- `kafka_log_info` 类测试通过。 +- goroutine leak 测试不需要新增宽泛 ignore。 +- metrics cleanup 测试通过。 + +## 阶段 8：Shadow 和 A/B 验证 + +目标是确认 franz-go 在相同 TiCDC/Kafka 配置下不改变语义。 + +### 8.1 本地/CI 对照 + +对每组 case 跑两次： + +- `kafka-client=sarama` +- `kafka-client=franz` + +比较： + +- 下游行数。 +- DDL 顺序。 +- partition 分布。 +- row-level checksum。 +- checkpoint/resolved 推进。 +- 错误恢复后的重复消息是否仍可由 protocol 语义处理。 +- Kafka producer metrics。 + +### 8.2 性能对照 + +至少覆盖： + +- 无压缩。 +- gzip/snappy/lz4/zstd。 +- `required-acks=-1`。 +- `required-acks=1`。 +- TLS/SASL。 +- 高 partition 数。 +- 大消息。 + +退出条件： + +- 正确性无差异。 +- 性能无不可解释显著退化。 +- 内存和 goroutine 无明显泄漏。 +- 失败场景的恢复方式可解释。 + +## 阶段 9：小流量灰度 + +### 9.1 灰度前置条件 + +必须满足： + +- 默认仍是 Sarama。 +- 每个灰度 changefeed 可单独切回 Sarama。 +- operator 知道回滚命令。 +- dashboard 同时能看 Kafka sink lag、producer error、request latency、resource usage。 +- Kafka ACL 是官方最小权限时，franz-go 已验证可写入。 + +### 9.2 灰度顺序 + +推荐顺序： + +1. 内部测试环境，单 changefeed，单 topic，低流量。 +2. 内部测试环境，多 topic / 动态 topic。 +3. 预发环境，真实 schema，低写入。 +4. 生产 canary，低风险 changefeed。 +5. 生产扩大到 5%。 +6. 生产扩大到 25%。 +7. 生产扩大到 50%。 +8. 切默认前维持观察窗口。 + +每一档观察： + +- checkpoint lag / resolved lag。 +- Kafka producer error rate。 +- request latency p99。 +- DML callback backlog。 +- broker request/response rate。 +- CPU、heap、goroutine。 +- topic partition 写入分布。 +- DDL 和 checkpoint 是否正常推进。 + +### 9.3 回滚动作 + +可回滚点： + +- 切默认前：把 changefeed 的 Kafka client 选择改回 Sarama。 +- 切默认后但保留 fallback：显式设置 `kafka-client=sarama` 或回滚默认配置。 +- 移除 Sarama 后：只能回滚二进制版本。 + +回滚后验证： + +- changefeed 恢复 running。 +- checkpoint/resolved 继续推进。 +- 下游消费者能处理可能重复的 at-least-once 消息。 +- 大消息 claim-check 外部存储没有新增不可读 marker。 +- metrics 回到 Sarama collector。 + +## 阶段 10：切默认 + +切默认的进入条件： + +- franz-go 路径完成至少一个发布候选版本或一个充分观察窗口。 +- 所有 P0 风险关闭。 +- A/B 性能报告已归档。 +- metrics 和日志兼容。 +- 回滚路径演练过。 +- 官方文档和 release note 已准备。 + +切默认步骤： + +1. 将默认 Kafka client 从 Sarama 改为 franz-go。 +2. 保留显式 `kafka-client=sarama` fallback。 +3. release note 说明： + - 默认 Kafka client 改变。 + - 配置兼容。 + - ACL 不需要新增 `IDEMPOTENT_WRITE`，因为默认禁用 idempotency。 + - 已知差异或调优建议。 +4. 灰度发布。 +5. 观察至少一个完整业务周期。 + +切默认后监控： + +- Kafka sink error rate。 +- changefeed restart count。 +- Kafka produce latency。 +- checkpoint lag。 +- broker throttle。 +- message too large。 +- auth failures。 +- metrics cardinality。 + +回滚： + +- 优先配置回滚到 Sarama。 +- 如果默认切换导致启动期失败，可回滚二进制。 +- 不需要迁移 Kafka topic 数据，因为消息协议未改变。 + +## 阶段 11：Contract 和移除 Sarama + +只有在兼容窗口结束后执行。 + +前置条件： + +- 没有线上 changefeed 仍配置 `kafka-client=sarama`。 +- 至少一个稳定版本周期内 franz-go 是默认实现。 +- 没有未关闭的 franz-go P0/P1 correctness issue。 +- 运营 dashboard 和 alert 不再依赖 Sarama-only 指标来源。 + +清理项： + +- 删除 `sarama_factory.go`。 +- 删除 `sarama_config.go`。 +- 删除 `sarama_async_producer.go`。 +- 删除 `sarama_sync_producer.go`。 +- 删除或更新 Sarama 专属 mocks/tests。 +- 将 `pkg/security/sasl.go` 中的 Sarama 常量改为 TiCDC 自有常量。 +- 移除 `pkg/logger/log.go` 中 Sarama logger 初始化。 +- 移除 `pkg/leakutil/leak_helper.go` 中 Sarama goroutine ignore。 +- 删除 `go.mod` / `go.sum` 中不再需要的 Sarama 依赖。 +- 更新 Kafka sink 文档中 Sarama client id 或 Sarama 行为描述。 + +Contract 阶段测试： + +- `make unit_test_pkg PKG=./pkg/sink/kafka/...` +- `make unit_test_pkg PKG=./downstreamadapter/sink/...` +- `make unit_test_pkg PKG=./pkg/security/...` +- `make cdc` +- Kafka integration suite。 +- `make check`，用于确认 go.mod、format、codegen 等。 + +回滚： + +- Contract 后不能配置回滚到 Sarama，只能回滚二进制版本。 +- 如果需要保留更强回滚能力，不要执行 Contract。 + +## 关键验收门槛 + +以下任一项未满足，不应切默认： + +- DML callback 时机未被测试证明。 +- 手动 partition 未被测试证明。 +- 大消息 size accounting 未完成。 +- 官方最小 Kafka ACL 下未验证。 +- TLS/SASL/GSSAPI/OAuth 未覆盖。 +- `required-acks=0/1/-1` 未覆盖。 +- 既有 Kafka producer metrics 未兼容或未提供迁移方案。 +- error resume / broken pipe / idle connection 场景未覆盖。 +- 没有 Sarama fallback。 +- 没有回滚演练。 + +## 推荐 PR 拆分 + +1. PR 1：新增 franz-go dependency、config builder skeleton、默认不启用。 +2. PR 2：franz admin client + admin/topic manager tests。 +3. PR 3：franz sync producer + DDL/checkpoint tests。 +4. PR 4：franz async producer + callback/error/close tests。 +5. PR 5：TLS/SASL/OAuth/GSSAPI mapping tests。 +6. PR 6：message size accounting + large message tests。 +7. PR 7：franz metrics collector + logger/leakutil。 +8. PR 8：integration tests and A/B scripts。 +9. PR 9：controlled gray switch documentation / release note。 +10. PR 10：切默认，保留 Sarama fallback。 +11. PR 11：Contract 移除 Sarama，需等兼容窗口结束。 + +## 运行手册摘要 + +启用 franz-go 前： + +1. 确认 Kafka ACL 没有依赖 franz-go idempotency。 +2. 确认 topic `max.message.bytes` 和 sink `max-message-bytes`。 +3. 确认 TLS/SASL 配置在 franz-go 路径验证过。 +4. 确认 dashboard 已能看 franz-go collector。 +5. 确认回滚命令。 + +启用后观察： + +1. 10 分钟内无 producer error spike。 +2. checkpoint lag 不持续增长。 +3. produce latency p99 不异常。 +4. broker throttle 不异常。 +5. consumer 没有解析错误。 + +触发回滚： + +- Kafka auth/ACL error。 +- message too large 明显增加。 +- checkpoint lag 持续增长。 +- DDL/checkpoint/resolved 不推进。 +- producer goroutine 或 heap 持续增长。 +- 下游消费者出现协议解析错误。 + +回滚后： + +- 确认 changefeed running。 +- 确认 checkpoint 继续推进。 +- 确认消费者可处理重复消息。 +- 保留 franz-go 错误日志、metrics 和 Kafka broker logs 供根因分析。 diff --git a/docs/franz-go/franz-go-ga-test-plan.md b/docs/franz-go/franz-go-ga-test-plan.md new file mode 100644 index 0000000000..ece35d2980 --- /dev/null +++ b/docs/franz-go/franz-go-ga-test-plan.md @@ -0,0 +1,87 @@ +# TiCDC Kafka Sink franz-go GA 测试计划 + +Last updated: 2026-09-02 +Status: 评审稿 +Scope: franz-go Kafka Sink 的正确性、故障恢复、性能和可观测性验证 +Related documents: + +- [执行计划](https://pingcap.feishu.cn/wiki/YK6UwCWn0iNvlfkAGDncAOK2nWh) +- [Milestone 1 TODO List](./milestone-1-todo-list.md) + +## 1. 测试原则 + +- 测试以最终行为为准，不限定 SDK、caselib 或 Test Plan 的具体实现。 +- 现有 testcase 能覆盖的场景，使用指定的 franz-go TiCDC image 直接运行，不改造 test-infra。 +- 只有现有 test-infra 无法构造或验证的场景才新增代码。 +- correctness testcase 负责消费和数据一致性校验；专项 testcase 只验证对应能力，不重复完整业务流程。 + +## 2. 正确性 + +执行方式：使用指定的 franz-go TiCDC image 运行现有 Kafka testcase，不需要修改 test-infra。 + +- [ ] 覆盖初始同步、增量同步、支持的协议和 dispatcher。 +- [ ] 覆盖 message-size 边界、large message、claim-check 和 handle-key-only。 +- [ ] 覆盖 Topic 自动创建、已有 Topic、partition 变化、Topic 配置和 Schema Registry 正常与异常路径。 +- [ ] 覆盖多 Changefeed、扩缩容和 HA，并校验 callback、checkpoint、消息顺序和最终消费结果。 + +通过标准：所有 correctness testcase 通过，不存在数据丢失、重复 callback 或 checkpoint 提前推进。 + +## 3. 鲁棒性与故障恢复 + +执行方式：复用现有 Kafka chaos testcase；只为缺失场景新增 test-infra 代码。 + +- [ ] 补充多 broker 故障、滚动升级、metadata 变化、request timeout、retry、idle connection 和 broken pipe 场景。 +- [ ] 补充 controller 和 broker 的网络延迟、丢包及更多网络分区组合。 +- [ ] 验证 broker 长时间不可用时内存有界，取消和关闭能够解除等待。 +- [ ] 验证恢复期间 partition 内消息顺序、callback 和 retry 行为。 +- [ ] 使用 Kafka 集群状态、checkpoint 和数据一致性判断恢复结果，不使用固定 sleep。 +- [ ] 失败时保存 TiCDC、Kafka、consumer 日志、关键 metrics、集群状态和测试时间范围。 + +通过标准：每个故障用例都能命中目标节点；故障解除后 Kafka 集群恢复可用，checkpoint 最终追平，消费结果与上游一致，资源使用保持有界。 + +## 4. 性能 + +执行方式：新增三 broker 性能 testcase。 + +- [ ] Kafka Topic replication factor 为 3，`min.insync.replicas` 为 2。 +- [ ] 固定 Kafka 集群、Topic、partition、TiCDC 规格、workload、预热方式、数据规模和重复次数。 +- [ ] 覆盖 sysbench、bank、jitu，单表和多表，以及 table 和 index-value dispatcher。 +- [ ] 执行 Changefeed create、pause、workload、resume、catch-up 和一致性校验。 +- [ ] 记录吞吐、catch-up 时间、p99、TiCDC/Kafka CPU 和内存、GC、goroutine、heap、batch、buffer、retry、error 和 consumer lag。 +- [ ] 保存测试参数、代码版本、资源规格和原始时间序列，并复验超出阈值的结果。 + +通过标准：所有场景完成一致性校验，吞吐、延迟、CPU 和内存满足确定的阈值。 + +## 5. 可观测性 + +执行方式：新增 Prometheus 查询和 Dashboard 断言 testcase。 + +- [ ] 验证 franz-go logger、metrics collector 和敏感信息脱敏。 +- [ ] 覆盖吞吐、request/response rate、latency、retry、error、batch、buffer 和 broker 指标。 +- [ ] 验证 metrics label 不包含非预期高基数值，Changefeed 关闭后对应 series 被清理。 +- [ ] 分别产生正常写入、retry、broker 故障和恢复流量，验证指标随场景变化。 +- [ ] 验证 Dashboard PromQL 可执行，并保存 Prometheus 原始结果、Dashboard 定义和对应日志。 + +通过标准：每个 franz-go Dashboard panel 都有自动查询断言，日志和指标足以诊断正常写入、重试、错误和恢复。 + +## 6. 代码变更验证 + +仅在新增或修改 test-infra 代码时执行： + +- [ ] 在仓库根目录运行 `go test ./common/model/resource/...`。 +- [ ] 在 `sdk` 模块运行 `go test ./resource/impl/k8s/...`。 +- [ ] 在 `caselib` 模块运行 `go test ./pkg/model/kafka/... ./pkg/steps/...`。 +- [ ] 运行新增端到端 Test Plan，并保存资源状态和诊断产物。 + +## 7. 阻塞输入 + +- [ ] 确定性能场景的数据量、partition 数、重复次数以及吞吐、延迟、CPU 和内存阈值。 +- [ ] 确定 franz-go 指标名、label 和 Dashboard 定义。 + +## 8. 执行阶段 + +- 开发与 PR：运行改动涉及的单元测试和集成测试。 +- Nightly：运行完整 correctness regression、故障恢复和可观测性测试。 +- Release：在 Nightly 范围上增加完整性能和稳定性验证。 + +GA 前不得遗留阻塞发布的正确性、稳定性、资源、性能或可观测性问题。 diff --git a/docs/franz-go/kafka-producer-idempotence-design.md b/docs/franz-go/kafka-producer-idempotence-design.md new file mode 100644 index 0000000000..8f37591914 --- /dev/null +++ b/docs/franz-go/kafka-producer-idempotence-design.md @@ -0,0 +1,152 @@ +# Kafka Producer Idempotence 设计 + +Last updated: 2026-09-03 +Status: 待讨论 +Scope: franz-go producer 的幂等写入 +Related documents: + +- [Milestone 1 TODO List](./milestone-1-todo-list.md) + +## Background + +TiCDC producer 会重试可恢复的 Kafka 写入错误。下面的故障会产生重复消息： + +1. Producer 向 Broker 发送一个 record batch。 +2. Broker 已经把 record batch 写入日志。 +3. Broker 响应在网络中丢失，或者 Producer 等待响应超时。 +4. Producer 无法判断 Broker 是否已经完成写入，因此重新发送相同数据。 +5. Broker 把重试请求作为新数据再次写入。 + +当前 Sarama producer 没有启用幂等写入。franz-go 路径也显式配置了 +[`DisableIdempotentWrite`](../../pkg/sink/kafka/franz_config.go)，因此两条路径都存在上述风险。 + +Kafka 幂等 producer 为每个 producer 分配 producer ID 和 epoch，并为每个 Topic partition +维护 sequence number。Broker 可以根据这些信息识别同一个 producer 重发的 record batch， +避免网络错误触发的 client 内部重试写入重复数据。Kafka 从 0.11 开始提供该能力，协议原理见 +[Kafka Design](https://kafka.apache.org/41/design/design/)。 + +支持幂等写入可以减少 TiCDC 正常运行期间由 franz-go 内部重试产生的重复消息。TiCDC 的 +对外投递语义继续保持 at-least-once。Producer 重建、TiCDC 重启以及从 checkpoint 重放的 +消息会使用新的 sequence number，Kafka 无法把这些消息识别为同一次发送。 + +## Proposed Behavior + +- 新增 `enable-idempotence` Kafka Sink 配置，默认值为 `false`。 +- `enable-idempotence=false` 保持当前 producer 行为。 +- `enable-idempotence=true` 只支持 franz-go，并要求 `required-acks=-1`。 +- 用户同时配置 `enable-idempotence=true` 和 `required-acks=0` 或 `1` 时，TiCDC 在创建 + Changefeed 时返回配置错误。Kafka 要求幂等 producer 使用 `acks=all`，详见 + [Kafka Producer Configs](https://kafka.apache.org/41/configuration/producer-configs/)。 +- TiCDC 不会在初始化失败后自动关闭幂等写入。权限、Broker 版本或消息格式不满足要求时， + Changefeed 返回明确错误。 +- TiCDC 启动日志记录最终是否启用幂等写入。 + +显式配置可以避免升级后自动增加 Kafka 权限要求。完成兼容性和故障测试后，可以单独讨论 +是否修改默认值。 + +## Producer Configuration + +启用幂等写入时： + +- 不配置 `DisableIdempotentWrite()`。 +- 不配置 `MaxProduceRequestsInflightPerBroker(1)`。franz-go 在幂等模式下自行选择在途请求 + 数量；支持相应 Produce API 的 Broker 最多允许 5 个在途请求，并使用 sequence number + 保证同一 partition 的消息顺序。 +- 保留 `RecordRetries(max-retry)` 和现有退避配置。 +- 配置 `AllowIdempotentProduceCancellation()`，使 `max-retry` 耗尽和调用 context 取消仍能 + 结束发送并释放 buffer。 + +关闭幂等写入时： + +- 配置 `DisableIdempotentWrite()`。 +- 保留 `MaxProduceRequestsInflightPerBroker(1)`，避免非幂等重试导致同一 partition 乱序。 + +`AllowIdempotentProduceCancellation()` 保留 franz-go 正常内部重试的 Broker 去重能力。 +发送结果仍不确定时，如果 TiCDC 在收到最终错误后重新发送相同消息，Kafka 仍可能写入重复 +数据。该行为符合 TiCDC 现有的 at-least-once 语义。franz-go 对取消行为的说明见 +[`AllowIdempotentProduceCancellation`](https://github.com/twmb/franz-go/blob/v1.21.6/pkg/kgo/config.go#L1191-L1219)。 + +## Producer ID Initialization + +一个 Changefeed 创建两个 franz-go producer client： + +- async producer 发送 DML。 +- sync producer 发送 DDL 和 checkpoint。 + +两个 client 分别持有 producer ID 和 sequence number。创建每个 producer client 后，TiCDC +调用 `client.ProducerID(ctx)`，在发送业务消息之前执行 `InitProducerID`： + +- 返回错误时，producer 创建失败。 +- producer ID 小于 0 时，producer 创建失败。 +- producer ID 有效时，producer 创建成功。 + +franz-go 遇到不支持 `InitProducerID` 的旧 Broker 时，可能返回 `producer ID = -1` 和空错误， +然后以非幂等方式继续发送。TiCDC 必须同时检查错误和 producer ID，避免静默降低用户要求的 +投递保证。 + +初始化需要使用有界 context。达到初始化 deadline 后，TiCDC 关闭该 client 并返回创建失败， +避免 Changefeed 创建过程无限等待 Kafka。 + +## Kafka Permissions + +启用幂等写入会增加 Kafka 权限要求： + +- Kafka 2.8 之前通常要求 producer principal 具有 Cluster 级 `IDEMPOTENT_WRITE`，同时具有 + 目标 Topic 的 `WRITE` 权限。 +- Kafka 2.8 及以上把 `InitProducerID` 权限放宽为对任意 Topic 具有 `WRITE` 权限。目标 Topic + 仍需要各自的 `WRITE` 权限。 +- 自定义 Authorizer 需要正确实现 Kafka 2.8 引入的权限检查接口，否则升级后的 Broker 仍可能 + 拒绝 `InitProducerID`。 + +权限变化和兼容场景见 +[KIP-679](https://cwiki.apache.org/confluence/spaces/KAFKA/pages/165221843/KIP-679%2BProducer%2Bwill%2Benable%2Bthe%2Bstrongest%2Bdelivery%2Bguarantee%2Bby%2Bdefault)。 + +TiCDC 不通过 Admin API 推测权限。`ProducerID(ctx)` 发出的真实 `InitProducerID` 请求作为权限 +检查依据。`CLUSTER_AUTHORIZATION_FAILED` 等错误应保留 Kafka 错误原因，方便用户补充 ACL。 + +## Guarantee Boundaries + +幂等写入只对同一个 producer ID、epoch 和 Topic partition 上的 client 内部重试去重。下面的 +情况仍可能产生重复消息： + +- TiCDC 在 Broker 写入成功但 callback 执行前退出，重启后从 checkpoint 重放消息。 +- franz-go client 被关闭并重新创建，新 client 获得新的 producer ID。 +- 发送结果不确定并达到取消或重试上限后，TiCDC 重新发送消息。 +- sync producer 向多个 partition 发送 DDL 或 checkpoint，其中部分 partition 成功后整体操作 + 返回失败。 + +幂等写入不提供跨 partition 原子性，也不为两个 producer client 提供共同的去重范围。实现不应 +把该功能描述为 TiCDC 到 Kafka 的 exactly-once 投递。 + +## Risks + +- 旧集群权限不足：现有 Topic 写入权限可能不足以执行 `InitProducerID`。 +- Broker 或消息格式过旧：Kafka 0.11 之前的 Broker，以及使用 v2 之前消息格式的 Topic，不能 + 接受幂等 record batch。 +- 请求并发变化：幂等模式下 franz-go 可能把每个 Broker 的在途 Produce request 增加到 5， + 从而改变吞吐和故障期间的在途数据量。 +- 取消后的重复：`AllowIdempotentProduceCancellation()` 保证有界退出，但取消后重新发送不能 + 使用原来的 sequence number 去重。 +- 部分成功：幂等写入不解决跨 partition 发送的部分成功。 +- 首次发送延迟：两个 producer client 都需要执行一次 `InitProducerID`。 + +## Verification + +- 覆盖 `enable-idempotence` 与 `required-acks=-1/1/0` 的配置组合。 +- 验证关闭幂等写入时保留 `MaxProduceRequestsInflightPerBroker(1)`。 +- 验证启用幂等写入时 sync 和 async producer 都取得有效 producer ID。 +- 模拟 Broker 已写入但首次响应丢失，确认 franz-go 内部重试后 Kafka 中只有一份记录。 +- 验证 `max-retry` 耗尽、调用 context 取消和 client 关闭都能结束 callback 并释放 buffer。 +- 验证缺少 `IDEMPOTENT_WRITE` 权限时，Changefeed 初始化返回包含 Kafka 原因的错误。 +- 覆盖 Kafka 0.11、Kafka 2.7、Kafka 2.8 及更高版本的版本和权限边界。 +- 覆盖旧 Topic 消息格式被 Broker 拒绝的错误路径。 +- 验证多个在途 batch 发生重试后，同一 partition 的消息顺序不变。 +- 验证多 partition 发送部分成功时，TiCDC 仍按 at-least-once 语义处理。 + +## Open Questions + +- `enable-idempotence` 是否只加入 Sink URI，还是同时加入 `SinkConfig`。 +- 初始化 producer ID 使用哪个 deadline。 +- 是否接受 `AllowIdempotentProduceCancellation()` 带来的取消后重复风险，或者选择严格幂等并 + 接受单次发送可能超过 `max-retry` 和调用 context deadline。 +- 完成兼容性测试后，是否把 `enable-idempotence` 的默认值改为 `true`。 diff --git a/docs/franz-go/milestone-1-todo-list.md b/docs/franz-go/milestone-1-todo-list.md new file mode 100644 index 0000000000..a072c580fc --- /dev/null +++ b/docs/franz-go/milestone-1-todo-list.md @@ -0,0 +1,10 @@ +# franz-go 待验证事项 + +## P2｜Kerberos 性能 + +- [ ] 验证每个 `kgo.Client` 复用 Kerberos client 是否安全且有实际收益。 + - 代码：[每次 SASL 认证创建 Kerberos client](../../pkg/sink/kafka/franz_gssapi.go#L33)。 + - 当前行为：每个 broker 连接进行 SASL 认证时都会重新加载 Kerberos 配置和 keytab，并在没有可复用 TGT 的新 client 上执行 AS exchange。 + - 约束：franz-go 支持持久 Kerberos client，但要求其生命周期归属于一个 `kgo.Client`。当前 client options 会先用于临时 admin client，再用于长期共享 client，不能让二者共同关闭同一个认证状态。 + - 验证：比较连接稳定和反复重连场景的认证延迟、KDC 请求数、CPU 和分配；覆盖 password、keytab、TGT 续期、并发连接、client 关闭和 race test。 + - 完成条件：确认收益显著且生命周期隔离方案通过上述验证后再实现；否则保留当前按认证创建的行为。 diff --git a/docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md b/docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md new file mode 100644 index 0000000000..a8044b28ce --- /dev/null +++ b/docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md @@ -0,0 +1,180 @@ +# TiCDC franz-go Kafka GA 本地执行记录 + +更新时间：2026-09-02 + +本文记录本地 `test-plan` 中可用于 TiCDC franz-go Kafka GA 验证的测试资产及实际 execution。测试资产以当前生效的 `caseName` 为准，不统计已注释的 case。 + +## 1. GA Plan 清单 + +### 1.1 标准环境 + +共 63 个非 EKS、非 TiDB-X Plan。这里不表示执行状态，实际结果以第 3、4 节为准。 + +- 协议、数据与工作负载： + - `cdc_newarch_airbnb_simple_titan` + - `cdc-newarch-kafka-debezium-basic` + - `cdc_newarch_kafka_large_msg_claim_check` + - `cdc_newarch_kafka_large_message_handle` + - `cdc-newarch-kafka-multiple-topic` + - `cdc-newarch-kafka-realtime` + - `cdc_newarch_kafka_scale_big_table_longrun` + - `cdc_newarch_kafka_scale_big_table_ops` + - `cdc_newarch_kafka_simple_ops` + - `cdc_newarch_kafka_simple_ops_titan_off` + - `cdc_newarch_kafka_simple_protocol` + - `cdc_newarch_kafka_simple_protocol_misc_workloads` + - `cdc_newarch_kafka_simple_misc_workloads_dispatcher_index` +- Kafka 安全： + - `cdc-newarch-kafka-security` +- Kafka 与 TiCDC 故障恢复： + - `cdc-newarch-kafka-all-2-owner-network-partition` + - `cdc-newarch-kafka-broker-failure` + - `cdc-newarch-kafka-controller-2-cdc-random-network-partition` + - `cdc-newarch-kafka-controller-failure` + - `cdc-newarch-kafka-random-2_cdc-random-network-partition` + - `cdc-newarch-kafka-random-2-owner-network-partition` + - `cdc-newarch-kafka-controller-2-owner-network-partition` +- Release dailyrun： + - `cdc_newarch_kafka_basic_functionality` + - `cdc-newarch-kafka-airbnb-scenario` + - `cdc-newarch-kafka-avro` + - `cdc-newarch-kafka-avro-2-workloads` + - `cdc-newarch-kafka-debezium-avro` + - `cdc-newarch-kafka-debezium-avro-2-workloads` + - `cdc-newarch-kafka-mysql-sync` + - `cdc-newarch-kafka-mysql-sync-gcttl` + - `cdc-newarch-kafka-random-node-down` + - `cdc_newarch_kafka_scale_big_table_cdc_scale` + - `cdc_newarch_lightning_comp_kafka` + - `cdc_newarch_sarama_no_broken_pipe` + - `cdc-newarch-upstream-chaos-kafka-sync` +- Kafka 版本覆盖： + - `cdc-newarch-kafka-version-0.11.0-0-r0` + - `cdc-newarch-kafka-version-0.11.0-1-r0` + - `cdc-newarch-kafka-version-1.0.0-r0` + - `cdc-newarch-kafka-version-1.0.1-r0` + - `cdc-newarch-kafka-version-1.1.0` + - `cdc-newarch-kafka-version-1.1.1` + - `cdc-newarch-kafka-version-2.0.0` + - `cdc-newarch-kafka-version-2.0.1` + - `cdc-newarch-kafka-version-2.1.0` + - `cdc-newarch-kafka-version-2.2.0` + - `cdc-newarch-kafka-version-2.3.0` + - `cdc-newarch-kafka-version-2.4.0` + - `cdc-newarch-kafka-version-2.5.0` + - `cdc-newarch-kafka-version-2.6.0` + - `cdc-newarch-kafka-version-2.7.0` + - `cdc-newarch-kafka-version-2.8.0` + - `cdc-newarch-kafka-version-3.0.0` + - `cdc-newarch-kafka-version-3.1.0` + - `cdc-newarch-kafka-version-3.2.0` + - `cdc-newarch-kafka-version-3.4.0` + - `cdc-newarch-kafka-version-3.5.0` + - `cdc-newarch-kafka-version-3.6.0` + - `cdc-newarch-kafka-version-3.7.0` + - `cdc-newarch-kafka-version-3.8.0` + - `cdc-newarch-kafka-version-3.9.0` + - `cdc-newarch-kafka-version-4.0.0` + - `cdc-newarch-kafka-version-4.1.0` + - `cdc-newarch-kafka-version-4.2.0` + - `cdc-newarch-kafka-version-4.3.0` + +### 1.2 EKS 与 TiDB-X 补充环境 + +这些 Plan 用于补充环境兼容性验证，不阻塞标准 TiCDC franz-go GA： + +- EKS：`cdc_newarch_kafka_simple_protocol-eks`，验证基本 DDL/DML、全数据类型和端到端一致性。 +- EKS：`cdc-newarch-kafka-broker-failure-eks`，验证 Chaos Mesh、Kafka Pod 故障和恢复。 +- EKS：`cdc-newarch-kafka-avro-eks`，验证 Schema Registry、consumer 和跨组件网络访问。 +- TiDB-X：`tidbx_cdc_newarch_kafka_basic_functionality`，验证 realtime、incremental 和 TiCDC scale。 +- TiDB-X：`tidbx-cdc-newarch-kafka-broker-failure`，验证 Kafka broker 故障后的恢复。 + +执行前统一配置： + +- TiCDC 使用同一个 franz-go 构建产物。 +- Kafka 使用 Apache Kafka `4.1.2` KRaft。 +- case image 使用当前 GA 测试版本。 +- sdkserver 使用 `hub.pingcap.net/qa/sdkserver:kafka-auth-amd64` 或目标环境中的同一镜像。 +- EKS 所需镜像先同步到 EKS 可访问的 registry。 + +## 2. 补充说明 + +### 2.1 范围 + +共找到 109 个 TiCDC New Architecture Kafka YAML/Jsonnet 文件： + +- 标准 TiCDC：`data-platform/cdc_newarch/kafka/` 37 个。 +- 标准 TiCDC dailyrun：`release/dailyrun/data-platform/cdc_newarch/` 21 个。 +- TiDBX：`data-platform/tidbx_cdc_newarch/kafka/` 32 个。 +- TiDBX dailyrun：`release/dailyrun/data-platform/tidbx_cdc_newarch/` 19 个。 + +EKS 和 TiDBX 变体复用对应标准 Plan 的 case 与验证意图，下面不重复展开相同 case，但它们不是完全等价的重复执行： + +- EKS 变体主要切换 resource pool、镜像仓库、存储、节点规格和调度配置；多数 Plan 沿用相同 TiCDC version 参数，但部分 Plan 固定使用 `master`，Kafka 通常固定为 `3.9.0`。 +- TiDBX 变体切换为 TiDB-X 集群拓扑和配置，并使用 `mirrors/tidbx/pingcap/ticdc/image:master-nextgen`。它可能来自同一 TiCDC 代码库，但不是标准 Plan 使用的同一个镜像产物。 +- 验证指定 franz-go TiCDC binary 时，只有显式使用目标 TiCDC image 的 execution 才计入结果；EKS/TiDBX Plan 只是可复用的测试覆盖入口。 + +### 2.2 Kafka 安全能力 + +`cdc-newarch-kafka-security` 包含 15 个 case： + +- GSSAPI：用户名密码、keytab、TLS + 用户名密码、TLS + keytab + ACL。 +- TLS：单向 TLS、mTLS + ACL。 +- SASL/PLAIN：PLAIN、TLS + PLAIN。 +- SASL/SCRAM：SHA-256 + ACL、SHA-512、TLS + SHA-256、TLS + SHA-512 + ACL。 +- OAuth：HTTP token、HTTP compatibility、TLS + HTTPS token 私有 CA + ACL。 + +对应 case： + +`cdc_kafka_auth_sasl_gssapi_user`、`cdc_kafka_auth_sasl_gssapi_keytab`、`cdc_kafka_auth_tls_sasl_gssapi_user`、`cdc_kafka_auth_tls_sasl_gssapi_keytab_acl`、`cdc_kafka_auth_tls`、`cdc_kafka_auth_mtls_acl`、`cdc_kafka_auth_sasl_plain`、`cdc_kafka_auth_sasl_scram_sha_256_acl`、`cdc_kafka_auth_sasl_scram_sha_512`、`cdc_kafka_auth_tls_sasl_plain`、`cdc_kafka_auth_tls_sasl_scram_sha_256`、`cdc_kafka_auth_tls_sasl_scram_sha_512_acl`、`cdc_kafka_auth_sasl_oauthbearer`、`cdc_kafka_auth_sasl_oauthbearer_http_compatibility`、`cdc_kafka_auth_tls_sasl_oauthbearer_acl`。 + +### 2.3 Kafka 版本覆盖 + +`cdc_kafka_version.tpl.jsonnet` 为每个版本生成 `cdc-newarch-kafka-version-`，执行 `kafka_realtime`。基础版本集合为： + +`0.11.0-0-r0`、`0.11.0-1-r0`、`1.0.0-r0`、`1.0.1-r0`、`1.1.0`、`1.1.1`、`2.0.0`、`2.0.1`、`2.1.0`、`2.2.0`、`2.3.0`、`2.4.0`、`2.5.0`、`2.6.0`、`2.7.0`、`2.8.0`、`3.0.0`、`3.1.0`、`3.2.0`、`3.4.0`、`3.5.0`、`3.6.0`、`3.7.0`、`3.8.0`、`3.9.0`。 + +- 标准环境：基础集合加 `4.0.0`、`4.1.0`、`4.2.0`、`4.3.0`，共 29 个版本。 +- EKS：基础集合加 `4.0.0`，共 26 个版本。 +- TiDBX 与 TiDBX EKS：使用基础集合，各 25 个版本。 + +当前版本集合不包含 Kafka `3.3.x`。 + +### 2.4 TiDBX 变体 + +TiDBX 当前复用以下标准 Plan 的 case，但使用 TiDB-X 上游集群和 TiDBX TiCDC image： + +- 基础与协议：Airbnb simple titan、Debezium、large message claim check、large message handle、multiple topic、realtime、scale big table、simple ops、simple protocol、misc workloads、dispatcher index。 +- Kafka 故障：all broker to owner、broker failure、controller to CDC、controller failure、random broker to CDC、random broker to owner、controller to owner。 +- Release dailyrun：basic functionality、Airbnb scenario、Avro、Avro 2 workloads、MySQL sync、GC TTL、random CDC node down、scale big table、Lightning compatibility、断线重连、upstream chaos。 +- Kafka 版本：`tidbx-cdc-newarch-kafka-version-` 及 EKS 变体。 + +TiDBX 暂无对应的 Kafka security、Debezium Avro 和 Debezium Avro 2 workloads Plan。 + +## 3. 已完成 execution + +以下 execution 使用 franz-go TiCDC image `hub-zot.pingcap.net/mirrors/dev/pingcap/ticdc/image:pull-4167-31d4137_linux_amd64`。 + +### 3.1 Kafka security + +- [8204473](https://tcms.pingcap.net/dashboard/executions/plan/8204473)：15/15 SUCCESS。 + +### 3.2 Kafka chaos + +- [8229205](https://tcms.pingcap.net/dashboard/executions/plan/8229205)：broker 故障，SUCCESS。 +- [8204474](https://tcms.pingcap.net/dashboard/executions/plan/8204474)：controller 故障，SUCCESS。 +- [8204475](https://tcms.pingcap.net/dashboard/executions/plan/8204475)：所有 Kafka broker 到 TiCDC owner 网络分区，SUCCESS。 +- [8204476](https://tcms.pingcap.net/dashboard/executions/plan/8204476)：Kafka controller 到随机 TiCDC 节点网络分区，SUCCESS。 + +### 3.3 Kafka 版本 + +- Kafka 2.x：[2.0.0](https://tcms.pingcap.net/dashboard/executions/plan/8229191)、[2.0.1](https://tcms.pingcap.net/dashboard/executions/plan/8229192)、[2.1.0](https://tcms.pingcap.net/dashboard/executions/plan/8204465)、[2.2.0](https://tcms.pingcap.net/dashboard/executions/plan/8204466)、[2.3.0](https://tcms.pingcap.net/dashboard/executions/plan/8204467)、[2.4.0](https://tcms.pingcap.net/dashboard/executions/plan/8204468)、[2.5.0](https://tcms.pingcap.net/dashboard/executions/plan/8204469)、[2.7.0](https://tcms.pingcap.net/dashboard/executions/plan/8181348)，均 SUCCESS。 +- Kafka 3.x：[3.0.0](https://tcms.pingcap.net/dashboard/executions/plan/8229193)、[3.5.0](https://tcms.pingcap.net/dashboard/executions/plan/8229196)、[3.7.0](https://tcms.pingcap.net/dashboard/executions/plan/8229198)、[3.9.0](https://tcms.pingcap.net/dashboard/executions/plan/8229200)，均 SUCCESS。 +- Kafka 4.x：[4.0.0](https://tcms.pingcap.net/dashboard/executions/plan/8229201)、[4.1.0](https://tcms.pingcap.net/dashboard/executions/plan/8229202)、[4.2.0](https://tcms.pingcap.net/dashboard/executions/plan/8229203)、[4.3.0](https://tcms.pingcap.net/dashboard/executions/plan/8229204)，均 SUCCESS。 + +### 3.4 Avro 2 workloads + +`cdc-newarch-kafka-avro-2-workloads` 使用同一 franz-go TiCDC image，分别验证 Kafka 3.1 和 Kafka 4.1.2。上游 TiDB、PD、TiKV 和 BR 固定为 `v8.5.8`，sdkserver 固定为 `hub.pingcap.net/qa/sdkserver:kafka-auth-amd64`，资源池使用 `ksyun-scenario-and-system-test`。 + +- Kafka 3.1：[8229286](https://tcms.pingcap.net/dashboard/executions/plan/8229286)、[8181445](https://tcms.pingcap.net/dashboard/executions/plan/8181445)、[8204540](https://tcms.pingcap.net/dashboard/executions/plan/8204540)，均 SUCCESS。 +- Kafka 4.1.2：[8204541](https://tcms.pingcap.net/dashboard/executions/plan/8204541)、[8229287](https://tcms.pingcap.net/dashboard/executions/plan/8229287)、[8229288](https://tcms.pingcap.net/dashboard/executions/plan/8229288)，均 SUCCESS。 diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index 73940bd552..6f555bdb61 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -51,6 +51,9 @@ func (c components) close() { if c.claimCheck != nil { c.claimCheck.Close() } + if c.factory != nil { + c.factory.Close() + } } func newKafkaSinkComponent( diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 90fc087e8e..4f6302ccc3 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -119,6 +119,7 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, if err != nil { return err } + defer factory.Close() adminClient, err := factory.AdminClient(ctx) if err != nil { @@ -173,7 +174,6 @@ func newWithComponents( } comp.close() statistics.Close() - comp.factory.CleanupMetrics() }() asyncProducer, err = comp.factory.AsyncProducer(ctx) @@ -425,7 +425,6 @@ func (s *sink) sendMessages(ctx context.Context) error { for _, message := range future.Messages { start := time.Now() if err = s.statistics.RecordBatchExecution(func() (int, int64, error) { - message.SetPartitionKey(future.Key.PartitionKey) if err = s.dmlProducer.AsyncSend( ctx, future.Key.Topic, @@ -579,7 +578,6 @@ func (s *sink) Close() { s.dmlProducer.Close() s.comp.close() s.statistics.Close() - s.comp.factory.CleanupMetrics() } func (s *sink) BatchCount() int { diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 01630f545d..0b3ecfe725 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -234,7 +234,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { gomock.InOrder( adminClient.EXPECT().Close(), topicManager.EXPECT().Close(), - factory.EXPECT().CleanupMetrics(), + factory.EXPECT().Close(), ) kafkaSink, err := newWithComponents( @@ -263,7 +263,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { asyncProducer.EXPECT().Close(), adminClient.EXPECT().Close(), topicManager.EXPECT().Close(), - factory.EXPECT().CleanupMetrics(), + factory.EXPECT().Close(), ) kafkaSink, err := newWithComponents( @@ -295,7 +295,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { asyncProducer.EXPECT().Close().Do(func() { closeCount.Add(1) }), adminClient.EXPECT().Close().Do(func() { closeCount.Add(1) }), topicManager.EXPECT().Close().Do(func() { closeCount.Add(1) }), - factory.EXPECT().CleanupMetrics(), + factory.EXPECT().Close().Do(func() { closeCount.Add(1) }), ) kafkaSink, err := newWithComponents( @@ -311,7 +311,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { require.True(t, kafkaSink.IsNormal()) kafkaSink.Close() - require.Equal(t, int64(4), closeCount.Load()) + require.Equal(t, int64(5), closeCount.Load()) require.False(t, kafkaSink.IsNormal()) kafkaSink.AddDMLEvent(&commonEvent.DMLEvent{}) require.Zero(t, kafkaSink.eventChan.Len()) @@ -612,7 +612,7 @@ func newKafkaSinkForTest( factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(syncProducer, nil) factory.EXPECT().MetricsCollector(nil).Return(noopMetricsCollector{}) - factory.EXPECT().CleanupMetrics().AnyTimes() + factory.EXPECT().Close().AnyTimes() kafkaSink, err := newWithComponents(ctx, changefeedID, common.DefaultKeyspaceID, protocol, components{ encoderGroup: encoderGroup, diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index c8078f0d8e..4ca1912569 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -28,8 +28,10 @@ func NewFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedI return newFranzFactory(ctx, o, changefeedID) } -// Factory is used to produce all kafka components. +// Factory creates Kafka components for one changefeed. type Factory interface { + // Close releases factory-owned resources after all components are closed. + Close() // AdminClient return a kafka cluster admin client AdminClient(ctx context.Context) (AdminClient, error) // SyncProducer creates a sync producer to writer message to kafka @@ -38,8 +40,6 @@ type Factory interface { AsyncProducer(ctx context.Context) (AsyncProducer, error) // MetricsCollector returns the kafka metrics collector MetricsCollector(adminClient AdminClient) MetricsCollector - // CleanupMetrics removes metrics owned directly by the factory. - CleanupMetrics() } // SyncProducer is the kafka sync producer @@ -55,19 +55,13 @@ type SyncProducer interface { // SendMessages will return an error. SendMessages(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error - // Close shuts down the producer; you must call this function before a producer - // object passes out of scope, as it may otherwise leak memory. - // You must call this before calling Close on the underlying client. + // Close stops the producer. It must be called before Factory.Close. Close() } // AsyncProducer is the kafka async producer type AsyncProducer interface { - // Close shuts down the producer and waits for any buffered messages to be - // flushed. You must call this function before a producer object passes out of - // scope, as it may otherwise leak memory. You must call this before process - // shutting down, or you may lose messages. You must call this before calling - // Close on the underlying client. + // Close stops the producer. It must be called before Factory.Close. Close() // AsyncSend is the input channel for the user to write messages to that they diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index e1d892fb00..a1f76e6f5a 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -65,16 +65,16 @@ func (mr *MockFactoryMockRecorder) AsyncProducer(ctx interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), ctx) } -// CleanupMetrics mocks base method. -func (m *MockFactory) CleanupMetrics() { +// Close mocks base method. +func (m *MockFactory) Close() { m.ctrl.T.Helper() - m.ctrl.Call(m, "CleanupMetrics") + m.ctrl.Call(m, "Close") } -// CleanupMetrics indicates an expected call of CleanupMetrics. -func (mr *MockFactoryMockRecorder) CleanupMetrics() *gomock.Call { +// Close indicates an expected call of Close. +func (mr *MockFactoryMockRecorder) Close() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanupMetrics", reflect.TypeOf((*MockFactory)(nil).CleanupMetrics)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockFactory)(nil).Close)) } // MetricsCollector mocks base method. diff --git a/pkg/sink/kafka/franz_admin.go b/pkg/sink/kafka/franz_admin.go index 697f9d2bc2..c37b07b85f 100644 --- a/pkg/sink/kafka/franz_admin.go +++ b/pkg/sink/kafka/franz_admin.go @@ -29,17 +29,19 @@ import ( ) type admin struct { - changefeed common.ChangeFeedID - admin *kadm.Client + changefeed common.ChangeFeedID + admin *kadm.Client + closeClient func() } +const adminMetadataMinAge = 100 * time.Millisecond + func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt) (*admin, error) { opts := make([]kgo.Opt, 0, len(clientOpts)+3) opts = append(opts, clientOpts...) opts = append(opts, kgo.WithContext(ctx), kgo.WithLogger(newClientLogger(changefeedID, "admin"))) - // MetadataMinAge is the minimum interval between metadata requests. // It must stay below the visibility retry interval to avoid retrying a cached topic-not-found result. - opts = append(opts, kgo.MetadataMinAge(100*time.Millisecond)) + opts = append(opts, kgo.MetadataMinAge(adminMetadataMinAge)) client, err := kgo.NewClient(opts...) if err != nil { @@ -47,8 +49,9 @@ func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts } return &admin{ - changefeed: changefeedID, - admin: kadm.NewClient(client), + changefeed: changefeedID, + admin: kadm.NewClient(client), + closeClient: client.Close, }, nil } @@ -249,4 +252,8 @@ func (a *admin) CreateTopic(ctx context.Context, detail *TopicDetail) error { return errors.WrapError(errors.ErrKafkaAdminAPI, resp.Err, "create-topic", detail.Name) } -func (a *admin) Close() { a.admin.Close() } +func (a *admin) Close() { + if a.closeClient != nil { + a.closeClient() + } +} diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index 819e8c5184..bceae2df05 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -46,8 +46,6 @@ func (p *asyncProducer) Close() { } start := time.Now() - p.client.Close() - log.Info("kafka async producer closed", zap.String("keyspace", p.changefeedID.Keyspace()), zap.String("changefeed", p.changefeedID.Name()), @@ -75,11 +73,16 @@ func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition i callback := message.Callback logInfo := message.LogInfo promise := func(_ *kgo.Record, err error) { - p.resultCh <- asyncProduceResult{ + result := asyncProduceResult{ callback: callback, logInfo: logInfo, err: err, } + select { + case p.resultCh <- result: + case <-ctx.Done(): + case <-p.client.Context().Done(): + } } p.client.Produce(ctx, record, promise) diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index b1db529108..2206b63d0e 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -29,7 +29,7 @@ import ( "github.com/twmb/franz-go/pkg/kmsg" ) -func TestAsyncSendClosedProducer(t *testing.T) { +func TestAsyncSendClosed(t *testing.T) { producer := &asyncProducer{} producer.closed.Store(true) @@ -38,14 +38,14 @@ func TestAsyncSendClosedProducer(t *testing.T) { require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) } -func TestAsyncSendCanceledContext(t *testing.T) { +func TestAsyncSendCanceled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() producer := &asyncProducer{} require.ErrorIs(t, producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{}), context.Canceled) } -func TestAsyncRunCallbackReturnsQueuedError(t *testing.T) { +func TestAsyncCallbackError(t *testing.T) { producer := &asyncProducer{ changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), resultCh: make(chan asyncProduceResult, 1), @@ -57,7 +57,7 @@ func TestAsyncRunCallbackReturnsQueuedError(t *testing.T) { require.ErrorIs(t, err, context.DeadlineExceeded) } -func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { +func TestCloseDoesNotAcknowledge(t *testing.T) { client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) require.NoError(t, err) @@ -67,6 +67,7 @@ func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), resultCh: make(chan asyncProduceResult, 1), } + defer client.Close() err = producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{ Callback: func() { callbackCalled.Store(true) @@ -77,23 +78,44 @@ func TestCloseDoesNotAcknowledgeBufferedMessage(t *testing.T) { producer.Close() require.False(t, callbackCalled.Load()) - err = producer.AsyncRunCallback(context.Background()) - require.ErrorIs(t, err, kgo.ErrClientClosed) } -func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { +func TestFactoryCloseUnblocksPromise(t *testing.T) { + client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) + require.NoError(t, err) + factory := &franzFactory{client: client} + defer factory.Close() + + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close-full-result-channel"), + resultCh: make(chan asyncProduceResult, 1), + } + producer.resultCh <- asyncProduceResult{} + require.NoError(t, producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{})) + require.Equal(t, int64(1), client.BufferedProduceRecords()) + + factory.Close() + + require.Eventually(t, func() bool { + return client.BufferedProduceRecords() == 0 + }, time.Second, time.Millisecond) +} + +func TestAsyncCallbackOnce(t *testing.T) { const topic = "async-topic" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) defer cluster.Close() o := testOptions(cluster.ListenAddrs()) - created, err := (&franzFactory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), - clientOpts: testClientOptions(t, o), - producerOpts: producerOptions(o), - }).AsyncProducer(context.Background()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) require.NoError(t, err) - producer := created.(*asyncProducer) + defer client.Close() + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), + resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), + } defer producer.Close() var calls atomic.Int32 @@ -125,19 +147,20 @@ func TestAsyncProducerCallbackExactlyOnce(t *testing.T) { require.ErrorIs(t, <-done, context.Canceled) } -func TestAsyncProducerCallbackDoesNotBlockPromiseWorker(t *testing.T) { +func TestAsyncCallbackIsolation(t *testing.T) { const topic = "async-callback-isolation" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) defer cluster.Close() o := testOptions(cluster.ListenAddrs()) - created, err := (&franzFactory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback-isolation"), - clientOpts: testClientOptions(t, o), - producerOpts: producerOptions(o), - }).AsyncProducer(context.Background()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) require.NoError(t, err) - producer := created.(*asyncProducer) + defer client.Close() + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback-isolation"), + resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), + } defer producer.Close() callbackStarted := make(chan struct{}) @@ -180,7 +203,7 @@ func TestAsyncProducerCallbackDoesNotBlockPromiseWorker(t *testing.T) { require.ErrorIs(t, <-done, context.Canceled) } -func TestAsyncProducerReportsProduceFailure(t *testing.T) { +func TestAsyncProduceFailure(t *testing.T) { const topic = "async-error" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) defer cluster.Close() @@ -190,12 +213,14 @@ func TestAsyncProducerReportsProduceFailure(t *testing.T) { }) o := testOptions(cluster.ListenAddrs()) - producer, err := (&franzFactory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), - clientOpts: testClientOptions(t, o), - producerOpts: producerOptions(o), - }).AsyncProducer(context.Background()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) require.NoError(t, err) + defer client.Close() + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), + resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), + } defer producer.Close() var callbackCalled atomic.Bool @@ -214,7 +239,7 @@ func TestAsyncProducerReportsProduceFailure(t *testing.T) { require.False(t, callbackCalled.Load()) } -func TestBufferBackpressureCanBeCanceled(t *testing.T) { +func TestBufferBackpressure(t *testing.T) { client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1"), kgo.MaxBufferedBytes(10), kgo.RecordRetries(100)) require.NoError(t, err) @@ -223,6 +248,7 @@ func TestBufferBackpressureCanBeCanceled(t *testing.T) { changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "backpressure"), resultCh: make(chan asyncProduceResult, 2), } + t.Cleanup(client.Close) t.Cleanup(producer.Close) require.NoError(t, producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{Value: make([]byte, 10)})) @@ -246,8 +272,4 @@ func TestBufferBackpressureCanBeCanceled(t *testing.T) { case <-time.After(time.Second): t.Fatal("canceled send remained blocked") } - - callbackCtx, callbackCancel := context.WithTimeout(context.Background(), time.Second) - defer callbackCancel() - require.ErrorIs(t, producer.AsyncRunCallback(callbackCtx), context.Canceled) } diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go index 479a7a681f..f6c8b1da69 100644 --- a/pkg/sink/kafka/franz_config.go +++ b/pkg/sink/kafka/franz_config.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" "github.com/twmb/franz-go/pkg/kgo" "github.com/twmb/franz-go/pkg/sasl" @@ -37,18 +36,18 @@ import ( // franz-go counts a record from Produce acceptance until its delivery callback // returns, including metadata lookup, batching, sending, Broker response, and // retries. A record larger than the byte limit fails immediately; otherwise, -// either limit blocks later Produce calls. Their ratio is 1 KiB per record, so +// either limit blocks later Produce calls. Their ratio is 2 KiB per record, so // bytes govern larger records while count bounds smaller record objects. const ( - producerMaxBufferedBytes = 64 << 20 + producerMaxBufferedBytes = 128 << 20 producerMaxBufferedRecords = 1 << 16 ) // producerMaxRequestBytes matches franz-go's default BrokerMaxWriteBytes and Kafka's default socket.request.max.bytes. const producerMaxRequestBytes = 100 << 20 -// Admin and producer clients share connection options. Producer delivery and -// resource limits stay separate so they cannot affect admin operations. +// The shared client uses these options for all Kafka requests. Producer options +// below apply only to Produce requests and buffered records. func clientOptions(ctx context.Context, o *options) ([]kgo.Opt, error) { opts := []kgo.Opt{ kgo.SeedBrokers(o.BrokerEndpoints...), @@ -63,7 +62,6 @@ func clientOptions(ctx context.Context, o *options) ([]kgo.Opt, error) { if o.EnableTLS { tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, - NextProtos: []string{"h2", "http/1.1"}, } if o.Credential != nil && o.Credential.IsTLSEnabled() { var err error @@ -98,11 +96,11 @@ func producerOptions(o *options) []kgo.Opt { // default jittered backoff adds about 6.2s to 9.3s across five retries. kgo.RecordRetries(o.MaxRetry), kgo.UnknownTopicRetries(o.MaxRetry), - // Limit each client to 64 MiB of buffered payload. The in-flight limit + // Limit each client to 128 MiB of buffered payload. The in-flight limit // applies per broker and does not bound records queued for other brokers, // metadata, or retries, so the producer needs a separate byte limit. - // 64 MiB leaves room above TiCDC's default 10 MiB message limit while - // bounding buffered payload memory. + // 128 MiB exceeds the 100 MiB record batch limit, so a valid single + // record is not rejected by the buffer limit. kgo.MaxBufferedBytes(producerMaxBufferedBytes), kgo.MaxBufferedRecords(producerMaxBufferedRecords), // A record batch must fit in the 100 MiB Produce request limit. @@ -188,24 +186,6 @@ func buildOAuthMechanism(ctx context.Context, cfg oauth2Config) (sasl.Mechanism, }), nil } -func newProducerClient( - ctx context.Context, changefeedID common.ChangeFeedID, role string, clientOpts []kgo.Opt, producerOpts []kgo.Opt, -) (*kgo.Client, error) { - opts := make([]kgo.Opt, 0, len(clientOpts)+len(producerOpts)+3) - opts = append(opts, clientOpts...) - opts = append(opts, - kgo.WithContext(ctx), - kgo.WithLogger(newClientLogger(changefeedID, role)), - kgo.WithHooks(newMetricsHook(changefeedID))) - opts = append(opts, producerOpts...) - - client, err := kgo.NewClient(opts...) - if err != nil { - return nil, errors.WrapError(errors.ErrNewKafkaSink, err) - } - return client, nil -} - func requiredAcks(required RequiredAcks) kgo.Acks { switch required { case WaitForAll: diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index 9302eac823..c1bd0ee4fa 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -17,17 +17,20 @@ package kafka import ( "context" "strings" + "sync" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kgo" "go.uber.org/zap" ) type franzFactory struct { changefeedID common.ChangeFeedID - clientOpts []kgo.Opt - producerOpts []kgo.Opt + client *kgo.Client + closeOnce sync.Once } func newFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { @@ -39,12 +42,27 @@ func newFranzFactory(ctx context.Context, o *options, changefeedID common.Change if err != nil { return nil, err } - defer admin.Close() - - if err := adjustOptions(ctx, changefeedID, admin, o, o.Topic); err != nil { + err = adjustOptions(ctx, changefeedID, admin, o, o.Topic) + admin.Close() + if err != nil { return nil, err } producerOpts := producerOptions(o) + metricsHook := newMetricsHook(changefeedID) + opts := make([]kgo.Opt, 0, len(clientOpts)+len(producerOpts)+4) + opts = append(opts, clientOpts...) + opts = append(opts, + kgo.WithContext(ctx), + kgo.WithLogger(newClientLogger(changefeedID, "shared")), + kgo.WithHooks(metricsHook), + kgo.MetadataMinAge(adminMetadataMinAge)) + opts = append(opts, producerOpts...) + + client, err := kgo.NewClient(opts...) + if err != nil { + cleanupMetrics(changefeedID) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } compression := strings.ToLower(strings.TrimSpace(o.Compression)) if compression == "" { @@ -67,44 +85,45 @@ func newFranzFactory(ctx context.Context, o *options, changefeedID common.Change zap.Duration("writeTimeout", o.WriteTimeout)) return &franzFactory{ changefeedID: changefeedID, - clientOpts: clientOpts, - producerOpts: producerOpts, + client: client, }, nil } -func (f *franzFactory) AdminClient(ctx context.Context) (AdminClient, error) { - return newAdmin(ctx, f.changefeedID, f.clientOpts) +func (f *franzFactory) AdminClient(context.Context) (AdminClient, error) { + return &admin{ + changefeed: f.changefeedID, + admin: kadm.NewClient(f.client), + }, nil } -func (f *franzFactory) SyncProducer(ctx context.Context) (SyncProducer, error) { - client, err := newProducerClient(ctx, f.changefeedID, "sync-producer", f.clientOpts, f.producerOpts) - if err != nil { - return nil, err - } +func (f *franzFactory) SyncProducer(context.Context) (SyncProducer, error) { return &syncProducer{ id: f.changefeedID, - client: client, + client: f.client, }, nil } -func (f *franzFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error) { - client, err := newProducerClient(ctx, f.changefeedID, "async-producer", f.clientOpts, f.producerOpts) - if err != nil { - return nil, err - } +func (f *franzFactory) AsyncProducer(context.Context) (AsyncProducer, error) { return &asyncProducer{ - client: client, + client: f.client, changefeedID: f.changefeedID, resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), }, nil } +func (f *franzFactory) Close() { + f.closeOnce.Do(func() { + if f.client != nil { + f.client.Close() + } + cleanupMetrics(f.changefeedID) + }) +} + func (f *franzFactory) MetricsCollector(AdminClient) MetricsCollector { return noopMetricsCollector{} } -func (f *franzFactory) CleanupMetrics() { cleanupMetrics(f.changefeedID) } - type noopMetricsCollector struct{} func (noopMetricsCollector) Run(ctx context.Context) { <-ctx.Done() } diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go new file mode 100644 index 0000000000..64c6364a60 --- /dev/null +++ b/pkg/sink/kafka/franz_factory_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" +) + +func TestSharedClientLifecycle(t *testing.T) { + const topic = "shared-client" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + o := testOptions(cluster.ListenAddrs()) + o.Topic = topic + created, err := newFranzFactory( + t.Context(), + o, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "shared-client"), + ) + require.NoError(t, err) + factory := created.(*franzFactory) + defer factory.Close() + + adminClient, err := factory.AdminClient(t.Context()) + require.NoError(t, err) + + asyncClient, err := factory.AsyncProducer(t.Context()) + require.NoError(t, err) + asyncProducer := asyncClient.(*asyncProducer) + + syncClient, err := factory.SyncProducer(t.Context()) + require.NoError(t, err) + syncProducer := syncClient.(*syncProducer) + + require.Same(t, factory.client, asyncProducer.client) + require.Same(t, factory.client, syncProducer.client) + require.NoError(t, syncProducer.SendMessage(t.Context(), topic, 0, &codeccommon.Message{Value: []byte("sync")})) + + syncProducer.Close() + require.NoError(t, asyncProducer.AsyncSend(t.Context(), topic, 0, &codeccommon.Message{Value: []byte("value")})) + require.Eventually(t, func() bool { + return asyncProducer.client.BufferedProduceRecords() == 0 + }, time.Second, time.Millisecond) + + asyncProducer.Close() + topics, err := adminClient.GetTopicsMeta(context.Background(), []string{topic}, false) + require.NoError(t, err) + require.Contains(t, topics, topic) + + adminClient.Close() + require.NoError(t, factory.client.Context().Err()) + factory.Close() + require.ErrorIs(t, asyncProducer.client.Context().Err(), context.Canceled) + factory.Close() +} diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go index dfd6c6706c..cfc341ea14 100644 --- a/pkg/sink/kafka/franz_sync_producer.go +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -82,8 +82,6 @@ func (p *syncProducer) Close() { } start := time.Now() - p.client.Close() - log.Info("kafka ddl producer closed", zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), zap.Duration("duration", time.Since(start))) diff --git a/pkg/sink/kafka/franz_sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go index a1905d07e0..01558a74e6 100644 --- a/pkg/sink/kafka/franz_sync_producer_test.go +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -23,10 +23,11 @@ import ( "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kgo" "github.com/twmb/franz-go/pkg/kmsg" ) -func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { +func TestSyncProducerClosed(t *testing.T) { producer := &syncProducer{} producer.closed.Store(true) @@ -37,25 +38,26 @@ func TestSyncProducerClosedReturnsProducerClosed(t *testing.T) { require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) } -func TestSyncProducerSendsToRequestedPartitions(t *testing.T) { +func TestSyncProducerPartitions(t *testing.T) { const topic = "sync-topic" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) defer cluster.Close() o := testOptions(cluster.ListenAddrs()) - producer, err := (&franzFactory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), - clientOpts: testClientOptions(t, o), - producerOpts: producerOptions(o), - }).SyncProducer(context.Background()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) require.NoError(t, err) + defer client.Close() + producer := &syncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), + client: client, + } defer producer.Close() require.NoError(t, producer.SendMessage(t.Context(), topic, 2, &codeccommon.Message{Key: []byte("key"), Value: []byte("value")})) require.NoError(t, producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("all")})) } -func TestSyncProducerReturnsPartialFailure(t *testing.T) { +func TestSyncProducerPartialFailure(t *testing.T) { const topic = "partial-failure" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) defer cluster.Close() @@ -65,12 +67,13 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { }) o := testOptions(cluster.ListenAddrs()) - producer, err := (&franzFactory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), - clientOpts: testClientOptions(t, o), - producerOpts: producerOptions(o), - }).SyncProducer(context.Background()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) require.NoError(t, err) + defer client.Close() + producer := &syncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), + client: client, + } defer producer.Close() err = producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("value")}) @@ -78,14 +81,15 @@ func TestSyncProducerReturnsPartialFailure(t *testing.T) { require.ErrorIs(t, err, kerr.InvalidTopicException) } -func TestSyncProducerUsesSendContext(t *testing.T) { +func TestSyncProducerContext(t *testing.T) { o := testOptions([]string{"127.0.0.1:1"}) - producer, err := (&franzFactory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), - clientOpts: testClientOptions(t, o), - producerOpts: producerOptions(o), - }).SyncProducer(t.Context()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) require.NoError(t, err) + defer client.Close() + producer := &syncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), + client: client, + } defer producer.Close() ctx, cancel := context.WithCancel(t.Context()) @@ -98,15 +102,16 @@ func TestSyncProducerUsesSendContext(t *testing.T) { func TestSyncProducerCloseIsIdempotent(t *testing.T) { o := testOptions([]string{"127.0.0.1:1"}) - client, err := (&franzFactory{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), - clientOpts: testClientOptions(t, o), - producerOpts: producerOptions(o), - }).SyncProducer(context.Background()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) require.NoError(t, err) + defer client.Close() + producer := &syncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), + client: client, + } - client.Close() - client.Close() + producer.Close() + producer.Close() } func produceResponseWithError(req kmsg.Request, failedPartition int32, errorCode int16) (kmsg.Response, error, bool) { diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 4674340737..c742ce1b80 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -107,7 +107,7 @@ func TestFactorySelection(t *testing.T) { require.NoError(t, err) require.IsType(t, test.expected, factory) - factory.CleanupMetrics() + factory.Close() }) } } diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index ecbab26407..1fcf601793 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -32,7 +32,7 @@ type saramaFactory struct { metricRegistry metrics.Registry } -func (*saramaFactory) CleanupMetrics() {} +func (*saramaFactory) Close() {} // newSaramaFactory constructs a Factory with sarama implementation. func newSaramaFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { From 01f445e0a45daf360235a5099e65bf217894ada1 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 20:54:55 +0800 Subject: [PATCH 57/61] kafka: reuse kerberos client per franz client --- docs/franz-go/milestone-1-todo-list.md | 9 +++---- pkg/sink/kafka/franz_config_test.go | 37 +++++++++++++++++++++++--- pkg/sink/kafka/franz_factory.go | 9 +++++-- pkg/sink/kafka/franz_gssapi.go | 14 ++++------ pkg/sink/kafka/franz_gssapi_test.go | 7 ++--- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/docs/franz-go/milestone-1-todo-list.md b/docs/franz-go/milestone-1-todo-list.md index a072c580fc..4f3f78a073 100644 --- a/docs/franz-go/milestone-1-todo-list.md +++ b/docs/franz-go/milestone-1-todo-list.md @@ -2,9 +2,8 @@ ## P2｜Kerberos 性能 -- [ ] 验证每个 `kgo.Client` 复用 Kerberos client 是否安全且有实际收益。 - - 代码：[每次 SASL 认证创建 Kerberos client](../../pkg/sink/kafka/franz_gssapi.go#L33)。 - - 当前行为：每个 broker 连接进行 SASL 认证时都会重新加载 Kerberos 配置和 keytab，并在没有可复用 TGT 的新 client 上执行 AS exchange。 - - 约束：franz-go 支持持久 Kerberos client，但要求其生命周期归属于一个 `kgo.Client`。当前 client options 会先用于临时 admin client，再用于长期共享 client，不能让二者共同关闭同一个认证状态。 +- [ ] 验证长期 `kgo.Client` 复用 Kerberos client 的运行行为和收益。 + - 代码：[Kerberos client 生命周期](../../pkg/sink/kafka/franz_gssapi.go#L26)。 + - 当前行为：临时 admin 和长期共享 client 分别持有 Kerberos client；一个 `kgo.Client` 的 Broker 建连和重连共享认证状态，并在 `kgo.Client.Close` 时销毁。 - 验证：比较连接稳定和反复重连场景的认证延迟、KDC 请求数、CPU 和分配；覆盖 password、keytab、TGT 续期、并发连接、client 关闭和 race test。 - - 完成条件：确认收益显著且生命周期隔离方案通过上述验证后再实现；否则保留当前按认证创建的行为。 + - 完成条件：真实 Kerberos 集群的功能、并发、重连、续期、关闭和性能验证通过。 diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index bab556ca07..2e91d1ed84 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -20,13 +20,18 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "sync/atomic" "testing" "time" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" + "github.com/jcmturner/gokrb5/v8/keytab" "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/sasl" ) func testOptions(brokers []string) *options { @@ -175,12 +180,28 @@ func TestCompressionOptions(t *testing.T) { } } -func TestBuildFranzGSSAPIMechanism(t *testing.T) { +func TestGSSAPIMechanism(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "krb5.conf") + require.NoError(t, os.WriteFile(configPath, []byte(`[libdefaults] + default_realm = EXAMPLE.COM +[realms] + EXAMPLE.COM = { + kdc = localhost:88 + } +`), 0o600)) + keytabPath := filepath.Join(dir, "client.keytab") + kt := keytab.New() + require.NoError(t, kt.AddEntry("alice", "EXAMPLE.COM", "pwd", time.Now(), 1, etypeID.AES256_CTS_HMAC_SHA1_96)) + keytabBytes, err := kt.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(keytabPath, keytabBytes, 0o600)) + for _, cfg := range []gssapiConfig{ {authType: userAuth, password: "pwd"}, - {authType: keyTabAuth, keyTabPath: "/tmp/a.keytab"}, + {authType: keyTabAuth, keyTabPath: keytabPath}, } { - cfg.kerberosConfigPath = "/etc/krb5.conf" + cfg.kerberosConfigPath = configPath cfg.serviceName = "kafka" cfg.username = "alice" cfg.realm = "EXAMPLE.COM" @@ -191,6 +212,16 @@ func TestBuildFranzGSSAPIMechanism(t *testing.T) { }) require.NoError(t, err) require.Equal(t, "GSSAPI", mechanism.Name()) + closing, ok := mechanism.(sasl.ClosingMechanism) + require.True(t, ok) + closing.Close() + + next, err := buildGSSAPIMechanism(cfg) + require.NoError(t, err) + require.NotSame(t, mechanism, next) + nextClosing, ok := next.(sasl.ClosingMechanism) + require.True(t, ok) + nextClosing.Close() } } diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index c1bd0ee4fa..e3a11451a2 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -34,11 +34,11 @@ type franzFactory struct { } func newFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { - clientOpts, err := clientOptions(ctx, o) + adminOpts, err := clientOptions(ctx, o) if err != nil { return nil, err } - admin, err := newAdmin(ctx, changefeedID, clientOpts) + admin, err := newAdmin(ctx, changefeedID, adminOpts) if err != nil { return nil, err } @@ -47,6 +47,11 @@ func newFranzFactory(ctx context.Context, o *options, changefeedID common.Change if err != nil { return nil, err } + // The temporary admin and shared client must not own the same SASL authentication state. + clientOpts, err := clientOptions(ctx, o) + if err != nil { + return nil, err + } producerOpts := producerOptions(o) metricsHook := newMetricsHook(changefeedID) opts := make([]kgo.Opt, 0, len(clientOpts)+len(producerOpts)+4) diff --git a/pkg/sink/kafka/franz_gssapi.go b/pkg/sink/kafka/franz_gssapi.go index 0961265be8..a053230e79 100644 --- a/pkg/sink/kafka/franz_gssapi.go +++ b/pkg/sink/kafka/franz_gssapi.go @@ -15,8 +15,6 @@ package kafka import ( - "context" - "github.com/jcmturner/gokrb5/v8/client" "github.com/jcmturner/gokrb5/v8/config" "github.com/jcmturner/gokrb5/v8/keytab" @@ -30,13 +28,11 @@ func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { return nil, err } - return kerberos.Kerberos(func(context.Context) (kerberos.Auth, error) { - krbClient, err := newKerberosClient(g) - if err != nil { - return kerberos.Auth{}, err - } - return kerberos.Auth{Client: krbClient, Service: g.serviceName}, nil - }), nil + krbClient, err := newKerberosClient(g) + if err != nil { + return nil, err + } + return kerberos.Auth{Client: krbClient, Service: g.serviceName}.AsMechanismWithClose(), nil } func validateGSSAPIConfig(g gssapiConfig) error { diff --git a/pkg/sink/kafka/franz_gssapi_test.go b/pkg/sink/kafka/franz_gssapi_test.go index fbd1975438..0326d3cd2b 100644 --- a/pkg/sink/kafka/franz_gssapi_test.go +++ b/pkg/sink/kafka/franz_gssapi_test.go @@ -15,7 +15,6 @@ package kafka import ( - "context" "testing" "github.com/pingcap/ticdc/pkg/errors" @@ -49,7 +48,7 @@ func TestGSSAPIConfigValidation(t *testing.T) { } } -func TestGSSAPIRejectsMissingKerberosConfig(t *testing.T) { +func TestGSSAPIMissingConfig(t *testing.T) { mechanism, err := buildGSSAPIMechanism(gssapiConfig{ authType: userAuth, kerberosConfigPath: "/path/that/does/not/exist", @@ -58,8 +57,6 @@ func TestGSSAPIRejectsMissingKerberosConfig(t *testing.T) { password: "secret", realm: "EXAMPLE.COM", }) - require.NoError(t, err) - - _, _, err = mechanism.Authenticate(context.Background(), "broker:9092") require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.Nil(t, mechanism) } From 45d82da09ce5669eadf175ead54cf774b71a825f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 21:44:21 +0800 Subject: [PATCH 58/61] fix all code --- pkg/sink/kafka/franz_admin_test.go | 48 ++++ pkg/sink/kafka/franz_async_producer.go | 2 + pkg/sink/kafka/franz_async_producer_test.go | 51 ++-- pkg/sink/kafka/franz_config.go | 35 +-- pkg/sink/kafka/franz_config_test.go | 251 ++++++++++-------- pkg/sink/kafka/franz_metrics_hook_test.go | 79 +++--- pkg/sink/kafka/franz_sync_producer_test.go | 39 ++- pkg/sink/kafka/oauth2.go | 88 ++++++ pkg/sink/kafka/oauth2_test.go | 109 ++++++++ pkg/sink/kafka/producer_test.go | 31 +++ .../kafka/sarama_oauth2_token_provider.go | 74 +----- .../sarama_oauth2_token_provider_test.go | 108 -------- pkg/sink/kafka/sarama_sync_producer_test.go | 9 - 13 files changed, 523 insertions(+), 401 deletions(-) create mode 100644 pkg/sink/kafka/oauth2.go create mode 100644 pkg/sink/kafka/oauth2_test.go create mode 100644 pkg/sink/kafka/producer_test.go diff --git a/pkg/sink/kafka/franz_admin_test.go b/pkg/sink/kafka/franz_admin_test.go index 5102051ac2..a9860ad221 100644 --- a/pkg/sink/kafka/franz_admin_test.go +++ b/pkg/sink/kafka/franz_admin_test.go @@ -284,6 +284,54 @@ func TestAdminOperations(t *testing.T) { require.NoError(t, admin.CreateTopic(ctx, &TopicDetail{Name: topic, NumPartitions: 3, ReplicationFactor: 1})) } +func TestAdminConfigErrors(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, "topic")) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + admin, err := newAdmin( + t.Context(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "config-errors"), + testClientOptions(t, o), + ) + require.NoError(t, err) + defer admin.Close() + + for _, test := range []struct { + name string + broker bool + responseError *kerr.Error + expectedError error + }{ + {name: "broker authorization", broker: true, responseError: kerr.ClusterAuthorizationFailed, expectedError: errors.ErrKafkaAuthorizationFailed}, + {name: "broker error", broker: true, responseError: kerr.InvalidRequest, expectedError: errors.ErrKafkaAdminAPI}, + {name: "topic authorization", responseError: kerr.TopicAuthorizationFailed, expectedError: errors.ErrKafkaAuthorizationFailed}, + {name: "topic error", responseError: kerr.InvalidTopicException, expectedError: errors.ErrKafkaAdminAPI}, + } { + t.Run(test.name, func(t *testing.T) { + cluster.ControlKey(int16(kmsg.DescribeConfigs), func(req kmsg.Request) (kmsg.Response, error, bool) { + request := req.(*kmsg.DescribeConfigsRequest) + response := req.ResponseKind().(*kmsg.DescribeConfigsResponse) + for _, requested := range request.Resources { + resource := kmsg.NewDescribeConfigsResponseResource() + resource.ResourceType = requested.ResourceType + resource.ResourceName = requested.ResourceName + resource.ErrorCode = test.responseError.Code + response.Resources = append(response.Resources, resource) + } + return response, nil, true + }) + + if test.broker { + _, _, err = admin.GetBrokerConfig(t.Context(), "message.max.bytes") + } else { + _, _, err = admin.GetTopicConfig(t.Context(), "topic", "max.message.bytes") + } + require.ErrorIs(t, err, test.expectedError) + require.ErrorIs(t, err, test.responseError) + }) + } +} + func TestCreateTopicErrors(t *testing.T) { ctx := t.Context() cluster := kfake.MustCluster(kfake.NumBrokers(1)) diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index bceae2df05..602759a80f 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -94,6 +94,8 @@ func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { select { case <-ctx.Done(): return context.Cause(ctx) + case <-p.client.Context().Done(): + return context.Cause(p.client.Context()) case result := <-p.resultCh: if result.err != nil { log.Error("kafka message send failed", diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index 2206b63d0e..15338dc8cb 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -45,39 +45,49 @@ func TestAsyncSendCanceled(t *testing.T) { require.ErrorIs(t, producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{}), context.Canceled) } -func TestAsyncCallbackError(t *testing.T) { +func TestAsyncPartition(t *testing.T) { + const topic = "async-partition" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) + defer cluster.Close() + partition := make(chan int32, 1) + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + partition <- req.(*kmsg.ProduceRequest).Topics[0].Partitions[0].Partition + return produceResponseWithError(req, -1, 0) + }) + o := testOptions(cluster.ListenAddrs()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() producer := &asyncProducer{ - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback"), + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-partition"), resultCh: make(chan asyncProduceResult, 1), } - producer.resultCh <- asyncProduceResult{err: context.DeadlineExceeded} - - err := producer.AsyncRunCallback(context.Background()) - require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, producer.AsyncSend(t.Context(), topic, 2, &codeccommon.Message{Value: []byte("value")})) + require.Eventually(t, func() bool { return client.BufferedProduceRecords() == 0 }, time.Second, time.Millisecond) + require.Equal(t, int32(2), <-partition) } -func TestCloseDoesNotAcknowledge(t *testing.T) { +func TestAsyncCallbackStopsWithClient(t *testing.T) { client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) require.NoError(t, err) - - var callbackCalled atomic.Bool producer := &asyncProducer{ client: client, - changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close"), + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback-close"), resultCh: make(chan asyncProduceResult, 1), } - defer client.Close() - err = producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{ - Callback: func() { - callbackCalled.Store(true) - }, - }) - require.NoError(t, err) - producer.Close() + done := make(chan error, 1) + go func() { done <- producer.AsyncRunCallback(context.Background()) }() + client.Close() - require.False(t, callbackCalled.Load()) + select { + case err = <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("callback runner did not stop with the client") + } } func TestFactoryCloseUnblocksPromise(t *testing.T) { @@ -234,8 +244,7 @@ func TestAsyncProduceFailure(t *testing.T) { defer cancel() err = producer.AsyncRunCallback(callbackCtx) - require.ErrorIs(t, err, errors.ErrKafkaSendMessage) - require.ErrorIs(t, err, kerr.InvalidTopicException) + requireKafkaSendError(t, err, kerr.InvalidTopicException) require.False(t, callbackCalled.Load()) } diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go index f6c8b1da69..14c02faed3 100644 --- a/pkg/sink/kafka/franz_config.go +++ b/pkg/sink/kafka/franz_config.go @@ -17,8 +17,6 @@ package kafka import ( "context" "crypto/tls" - "net/http" - "net/url" "strings" "github.com/pingcap/log" @@ -29,8 +27,6 @@ import ( "github.com/twmb/franz-go/pkg/sasl/plain" "github.com/twmb/franz-go/pkg/sasl/scram" "go.uber.org/zap" - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" ) // franz-go counts a record from Produce acceptance until its delivery callback @@ -145,38 +141,11 @@ func buildSASLMechanism(ctx context.Context, cfg *saslConfig) (sasl.Mechanism, e } func buildOAuthMechanism(ctx context.Context, cfg oauth2Config) (sasl.Mechanism, error) { - var httpClient *http.Client - if cfg.caPath != "" { - var err error - httpClient, err = oauthHTTPClient(cfg.caPath) - if err != nil { - return nil, err - } - } - - endpointParams := url.Values{} - if cfg.grantType != "" { - endpointParams.Set("grant_type", cfg.grantType) - } - if cfg.audience != "" { - endpointParams.Set("audience", cfg.audience) - } - tokenURL, err := url.Parse(cfg.tokenURL) + tokenSource, err := newOAuthTokenSource(ctx, cfg) if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - config := &clientcredentials.Config{ - ClientID: cfg.clientID, - ClientSecret: cfg.clientSecret, - TokenURL: tokenURL.String(), - EndpointParams: endpointParams, - Scopes: cfg.scopes, - } - if httpClient != nil { - ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) + return nil, err } // One token source shares cached credentials across broker connections and refreshes them on expiry. - tokenSource := config.TokenSource(ctx) return oauth.Oauth(func(context.Context) (oauth.Auth, error) { token, err := tokenSource.Token() if err != nil { diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go index 2e91d1ed84..4d451fdefa 100644 --- a/pkg/sink/kafka/franz_config_test.go +++ b/pkg/sink/kafka/franz_config_test.go @@ -16,6 +16,7 @@ package kafka import ( "context" + "crypto/tls" "io" "net/http" "net/http/httptest" @@ -29,9 +30,13 @@ import ( "github.com/jcmturner/gokrb5/v8/iana/etypeID" "github.com/jcmturner/gokrb5/v8/keytab" "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kfake" "github.com/twmb/franz-go/pkg/kgo" "github.com/twmb/franz-go/pkg/sasl" + "golang.org/x/oauth2" ) func testOptions(brokers []string) *options { @@ -66,88 +71,95 @@ func TestFranzRequiredAcks(t *testing.T) { } } -func TestFranzProducerTimeouts(t *testing.T) { - o := testOptions([]string{"127.0.0.1:9092"}) - o.ReadTimeout = 3 * time.Second - o.WriteTimeout = 2 * time.Second - - opts := testClientOptions(t, o) - client, err := kgo.NewClient(append(opts, producerOptions(o)...)...) - require.NoError(t, err) - defer client.Close() - - require.Equal(t, 2*time.Second, client.OptValue(kgo.RequestTimeoutOverhead)) - require.Equal(t, 3*time.Second, client.OptValue(kgo.ProduceRequestTimeout)) -} - -func TestProducerOptionsConfigureMessageAndBufferLimits(t *testing.T) { - const maxMessageBytes = 1048588 - o := testOptions([]string{"127.0.0.1:9092"}) - o.MaxMessageBytes = maxMessageBytes - - opts, err := clientOptions(t.Context(), o) - require.NoError(t, err) - - producerOpts := producerOptions(o) - - client, err := kgo.NewClient(append(opts, producerOpts...)...) - require.NoError(t, err) - defer client.Close() - - require.Equal(t, int32(maxMessageBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) - require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) - require.Equal(t, int64(producerMaxBufferedRecords), client.OptValue(kgo.MaxBufferedRecords)) - require.Equal(t, int64(1), client.OptValue(kgo.RecordRetries)) - require.Equal(t, int64(1), client.OptValue(kgo.UnknownTopicRetries)) - require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) -} - -func TestProducerOptionsUseSingleNonIdempotentRequest(t *testing.T) { - config := testOptions([]string{"127.0.0.1:9092"}) - - producerOpts := producerOptions(config) - - client, err := kgo.NewClient(producerOpts...) - require.NoError(t, err) - defer client.Close() +func TestClientOptions(t *testing.T) { + t.Run("timeouts", func(t *testing.T) { + o := testOptions([]string{"127.0.0.1:9092"}) + o.ReadTimeout = 3 * time.Second + o.WriteTimeout = 2 * time.Second + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() - require.Equal(t, true, client.OptValue(kgo.DisableIdempotentWrite)) - require.Equal(t, 1, client.OptValue(kgo.MaxProduceRequestsInflightPerBroker)) -} + require.Equal(t, 2*time.Second, client.OptValue(kgo.RequestTimeoutOverhead)) + require.Equal(t, 3*time.Second, client.OptValue(kgo.ProduceRequestTimeout)) + }) -func TestProducerLimitsDoNotScaleWithConfiguredMessage(t *testing.T) { - maxMessageBytes := 32 << 20 - config := testOptions([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = maxMessageBytes + t.Run("TLS", func(t *testing.T) { + ca, err := security.NewCA() + require.NoError(t, err) + certPEM, keyPEM, err := ca.GenerateCerts("localhost") + require.NoError(t, err) + certificate, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.TLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{certificate}, + })) + defer cluster.Close() + + dir := t.TempDir() + caPath := filepath.Join(dir, "ca.pem") + certPath := filepath.Join(dir, "cert.pem") + keyPath := filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(caPath, ca.CAPEM, 0o600)) + require.NoError(t, os.WriteFile(certPath, certPEM, 0o600)) + require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600)) + o := testOptions(cluster.ListenAddrs()) + o.EnableTLS = true + o.Credential = &security.Credential{CAPath: caPath, CertPath: certPath, KeyPath: keyPath} + client, err := kgo.NewClient(testClientOptions(t, o)...) + require.NoError(t, err) + defer client.Close() - producerOpts := producerOptions(config) + metadata, err := kadm.NewClient(client).Metadata(t.Context()) + require.NoError(t, err) + require.Len(t, metadata.Brokers, 1) + }) - client, err := kgo.NewClient(producerOpts...) - require.NoError(t, err) - defer client.Close() + t.Run("SASL", func(t *testing.T) { + cluster := kfake.MustCluster( + kfake.NumBrokers(1), + kfake.EnableSASL(), + kfake.Superuser("PLAIN", "alice", "secret"), + ) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + o.sasl = &saslConfig{mechanism: plainMechanism, user: "alice", password: "secret"} + client, err := kgo.NewClient(testClientOptions(t, o)...) + require.NoError(t, err) + defer client.Close() - require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) - require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) + metadata, err := kadm.NewClient(client).Metadata(t.Context()) + require.NoError(t, err) + require.Len(t, metadata.Brokers, 1) + }) } -func TestProducerOptionsLimitBatchToProduceRequest(t *testing.T) { +func TestProducerLimits(t *testing.T) { for _, test := range []struct { name string maxMessageBytes int + expectedBatch int32 }{ - {name: "at request limit", maxMessageBytes: producerMaxRequestBytes}, - {name: "above request limit", maxMessageBytes: 128 << 20}, + {name: "configured", maxMessageBytes: 1048588, expectedBatch: 1048588}, + {name: "at request limit", maxMessageBytes: producerMaxRequestBytes, expectedBatch: producerMaxRequestBytes}, + {name: "above request limit", maxMessageBytes: 128 << 20, expectedBatch: producerMaxRequestBytes}, } { t.Run(test.name, func(t *testing.T) { - config := testOptions([]string{"127.0.0.1:9092"}) - config.MaxMessageBytes = test.maxMessageBytes + o := testOptions([]string{"127.0.0.1:9092"}) + o.MaxMessageBytes = test.maxMessageBytes - client, err := kgo.NewClient(producerOptions(config)...) + client, err := kgo.NewClient(producerOptions(o)...) require.NoError(t, err) defer client.Close() - require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, test.expectedBatch, client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) + require.Equal(t, int64(producerMaxBufferedRecords), client.OptValue(kgo.MaxBufferedRecords)) + require.Equal(t, int64(o.MaxRetry), client.OptValue(kgo.RecordRetries)) + require.Equal(t, int64(o.MaxRetry), client.OptValue(kgo.UnknownTopicRetries)) require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) + require.Equal(t, time.Duration(0), client.OptValue(kgo.ProducerLinger)) }) } } @@ -240,57 +252,72 @@ func TestBuildFranzSASLMechanisms(t *testing.T) { require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } -func TestFranzOAuthTokenSource(t *testing.T) { - var tokenRequests atomic.Int32 - request := make(chan url.Values, 1) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := r.ParseForm(); err != nil { - t.Errorf("parse token request: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - tokenRequests.Add(1) - select { - case request <- r.PostForm: - default: - } - - w.Header().Set("Content-Type", "application/json") - if _, err := io.WriteString(w, `{"access_token":"token","token_type":"bearer"}`); err != nil { - t.Errorf("write token response: %v", err) - } - })) - defer server.Close() - - mechanism, err := buildSASLMechanism(t.Context(), &saslConfig{ - mechanism: oauthMechanism, - oauth2: oauth2Config{ - clientID: "client", - clientSecret: "secret", - tokenURL: server.URL, - scopes: []string{"scope-a", "scope-b"}, - grantType: "custom", - audience: "audience", - }, +func TestFranzOAuth(t *testing.T) { + t.Run("token reuse", func(t *testing.T) { + var tokenRequests atomic.Int32 + request := make(chan url.Values, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Errorf("parse token request: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + tokenRequests.Add(1) + select { + case request <- r.PostForm: + default: + } + w.Header().Set("Content-Type", "application/json") + if _, err := io.WriteString(w, `{"access_token":"token","token_type":"bearer"}`); err != nil { + t.Errorf("write token response: %v", err) + } + })) + defer server.Close() + + mechanism, err := buildSASLMechanism(t.Context(), &saslConfig{ + mechanism: oauthMechanism, + oauth2: oauth2Config{ + clientID: "client", + clientSecret: "secret", + tokenURL: server.URL, + scopes: []string{"scope-a", "scope-b"}, + grantType: "custom", + audience: "audience", + }, + }) + require.NoError(t, err) + _, _, err = mechanism.Authenticate(context.Background(), "") + require.NoError(t, err) + _, _, err = mechanism.Authenticate(context.Background(), "") + require.NoError(t, err) + + form := <-request + require.Equal(t, int32(1), tokenRequests.Load()) + require.Equal(t, "custom", form.Get("grant_type")) + require.Equal(t, "audience", form.Get("audience")) + require.Equal(t, "scope-a scope-b", form.Get("scope")) }) - require.NoError(t, err) - _, _, err = mechanism.Authenticate(context.Background(), "") - require.NoError(t, err) - _, _, err = mechanism.Authenticate(context.Background(), "") - require.NoError(t, err) + t.Run("endpoint error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + if _, err := io.WriteString(w, `{"error":"invalid_client"}`); err != nil { + t.Errorf("write token error response: %v", err) + } + })) + defer server.Close() + mechanism, err := buildOAuthMechanism(t.Context(), oauth2Config{tokenURL: server.URL}) + require.NoError(t, err) - form := <-request - require.Equal(t, int32(1), tokenRequests.Load()) - require.Equal(t, "custom", form.Get("grant_type")) - require.Equal(t, "audience", form.Get("audience")) - require.Equal(t, "scope-a scope-b", form.Get("scope")) -} + _, _, err = mechanism.Authenticate(context.Background(), "") + require.ErrorIs(t, err, errors.ErrNewKafkaSink) + var retrieveErr *oauth2.RetrieveError + require.ErrorAs(t, err, &retrieveErr) + }) -func TestFranzOAuthTokenSourceRejectsInvalidURL(t *testing.T) { - _, err := buildSASLMechanism(t.Context(), &saslConfig{ - mechanism: oauthMechanism, - oauth2: oauth2Config{tokenURL: "http://example.com/%%"}, + t.Run("invalid URL", func(t *testing.T) { + _, err := buildOAuthMechanism(t.Context(), oauth2Config{tokenURL: "http://example.com/%%"}) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) }) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) } diff --git a/pkg/sink/kafka/franz_metrics_hook_test.go b/pkg/sink/kafka/franz_metrics_hook_test.go index 7033a4ffff..1f9c2d51f0 100644 --- a/pkg/sink/kafka/franz_metrics_hook_test.go +++ b/pkg/sink/kafka/franz_metrics_hook_test.go @@ -16,6 +16,7 @@ package kafka import ( "context" + "strings" "testing" "time" @@ -27,42 +28,7 @@ import ( "github.com/twmb/franz-go/pkg/kgo" ) -func TestInitMetrics(t *testing.T) { - changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "metrics-registration") - cleanupMetrics(changefeedID) - t.Cleanup(func() { cleanupMetrics(changefeedID) }) - - hook := newMetricsHook(changefeedID) - hook.OnProduceBatchWritten( - kgo.BrokerMetadata{}, - "topic", - 0, - kgo.ProduceBatchMetrics{ - NumRecords: 1, - UncompressedBytes: 2, - CompressedBytes: 1, - }, - ) - hook.OnBrokerThrottle(kgo.BrokerMetadata{NodeID: 1}, time.Millisecond, true) - - registry := prometheus.NewRegistry() - InitMetrics(registry) - - metricFamilies, err := registry.Gather() - require.NoError(t, err) - - names := make([]string, 0, len(metricFamilies)) - for _, family := range metricFamilies { - names = append(names, family.GetName()) - } - - require.Contains(t, names, "ticdc_sink_kafka_franz_producer_records_per_batch") - require.Contains(t, names, "ticdc_sink_kafka_franz_producer_uncompressed_bytes_total") - require.Contains(t, names, "ticdc_sink_kafka_franz_producer_compressed_bytes_total") - require.Contains(t, names, "ticdc_sink_kafka_franz_producer_throttle_time_seconds") -} - -func TestMetricsHookRecordsRawValues(t *testing.T) { +func TestMetricsHook(t *testing.T) { changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "metrics-hook") cleanupMetrics(changefeedID) t.Cleanup(func() { cleanupMetrics(changefeedID) }) @@ -93,6 +59,34 @@ func TestMetricsHookRecordsRawValues(t *testing.T) { require.Equal(t, float64(5), testutil.ToFloat64( compressedBytesTotal.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name()), )) + batchMetric, ok := hook.recordsPerBatch.(prometheus.Metric) + require.True(t, ok) + batchHistogram := &dto.Metric{} + require.NoError(t, batchMetric.Write(batchHistogram)) + require.Equal(t, uint64(1), batchHistogram.GetHistogram().GetSampleCount()) + require.Equal(t, float64(3), batchHistogram.GetHistogram().GetSampleSum()) + + registry := prometheus.NewRegistry() + InitMetrics(registry) + metricFamilies, err := registry.Gather() + require.NoError(t, err) + names := make([]string, 0, len(metricFamilies)) + for _, family := range metricFamilies { + if strings.HasPrefix(family.GetName(), "ticdc_sink_kafka_franz_producer_") { + names = append(names, family.GetName()) + } + } + require.ElementsMatch(t, []string{ + "ticdc_sink_kafka_franz_producer_compressed_bytes_total", + "ticdc_sink_kafka_franz_producer_in_flight_requests", + "ticdc_sink_kafka_franz_producer_outgoing_bytes_total", + "ticdc_sink_kafka_franz_producer_records_per_batch", + "ticdc_sink_kafka_franz_producer_request_duration_seconds", + "ticdc_sink_kafka_franz_producer_requests_total", + "ticdc_sink_kafka_franz_producer_responses_total", + "ticdc_sink_kafka_franz_producer_throttle_time_seconds", + "ticdc_sink_kafka_franz_producer_uncompressed_bytes_total", + }, names) hook.OnBrokerWrite(meta, 0, 0, 0, 0, nil) hook.OnBrokerE2E(meta, 0, kgo.BrokerE2E{ @@ -132,4 +126,17 @@ func TestMetricsHookRecordsRawValues(t *testing.T) { require.NoError(t, throttleMetric.Write(throttleHistogram)) require.Equal(t, uint64(2), throttleHistogram.GetHistogram().GetSampleCount()) require.InDelta(t, 0.06, throttleHistogram.GetHistogram().GetSampleSum(), 0.000001) + keyspace, changefeed, broker := changefeedID.Keyspace(), changefeedID.Name(), "1" + cleanupMetrics(changefeedID) + require.False(t, outgoingBytesTotal.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, requestsTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultSuccess)) + require.False(t, requestsTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultWriteError)) + require.False(t, responsesTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultSuccess)) + require.False(t, responsesTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultReadError)) + require.False(t, requestsInFlight.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, requestDuration.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, throttleTime.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, recordsPerBatch.DeleteLabelValues(keyspace, changefeed)) + require.False(t, uncompressedBytesTotal.DeleteLabelValues(keyspace, changefeed)) + require.False(t, compressedBytesTotal.DeleteLabelValues(keyspace, changefeed)) } diff --git a/pkg/sink/kafka/franz_sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go index 01558a74e6..2bfd09399d 100644 --- a/pkg/sink/kafka/franz_sync_producer_test.go +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -15,6 +15,8 @@ package kafka import ( "context" + "slices" + "sync" "testing" "github.com/pingcap/ticdc/pkg/common" @@ -42,6 +44,20 @@ func TestSyncProducerPartitions(t *testing.T) { const topic = "sync-topic" cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) defer cluster.Close() + var mu sync.Mutex + var partitions []int32 + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + cluster.KeepControl() + request := req.(*kmsg.ProduceRequest) + mu.Lock() + defer mu.Unlock() + for _, requestTopic := range request.Topics { + for _, partition := range requestTopic.Partitions { + partitions = append(partitions, partition.Partition) + } + } + return produceResponseWithError(req, -1, 0) + }) o := testOptions(cluster.ListenAddrs()) client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) @@ -54,7 +70,15 @@ func TestSyncProducerPartitions(t *testing.T) { defer producer.Close() require.NoError(t, producer.SendMessage(t.Context(), topic, 2, &codeccommon.Message{Key: []byte("key"), Value: []byte("value")})) + mu.Lock() + require.Equal(t, []int32{2}, partitions) + partitions = nil + mu.Unlock() require.NoError(t, producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("all")})) + mu.Lock() + slices.Sort(partitions) + require.Equal(t, []int32{0, 1, 2}, partitions) + mu.Unlock() } func TestSyncProducerPartialFailure(t *testing.T) { @@ -77,8 +101,7 @@ func TestSyncProducerPartialFailure(t *testing.T) { defer producer.Close() err = producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("value")}) - require.ErrorIs(t, err, errors.ErrKafkaSendMessage) - require.ErrorIs(t, err, kerr.InvalidTopicException) + requireKafkaSendError(t, err, kerr.InvalidTopicException) } func TestSyncProducerContext(t *testing.T) { @@ -100,18 +123,12 @@ func TestSyncProducerContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } -func TestSyncProducerCloseIsIdempotent(t *testing.T) { - o := testOptions([]string{"127.0.0.1:1"}) - client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) - require.NoError(t, err) - defer client.Close() - producer := &syncProducer{ - id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close"), - client: client, - } +func TestFranzSyncProducerClose(t *testing.T) { + producer := &syncProducer{id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close")} producer.Close() producer.Close() + require.True(t, producer.closed.Load()) } func produceResponseWithError(req kmsg.Request, failedPartition int32, errorCode int16) (kmsg.Response, error, bool) { diff --git a/pkg/sink/kafka/oauth2.go b/pkg/sink/kafka/oauth2.go new file mode 100644 index 0000000000..ae53e01290 --- /dev/null +++ b/pkg/sink/kafka/oauth2.go @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net/http" + "net/url" + "os" + + "github.com/pingcap/ticdc/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +func newOAuthTokenSource(ctx context.Context, cfg oauth2Config) (oauth2.TokenSource, error) { + endpointParams := url.Values{} + if cfg.grantType != "" { + endpointParams.Set("grant_type", cfg.grantType) + } + if cfg.audience != "" { + endpointParams.Set("audience", cfg.audience) + } + + tokenURL, err := url.Parse(cfg.tokenURL) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if cfg.caPath != "" { + httpClient, err := oauthHTTPClient(cfg.caPath) + if err != nil { + return nil, err + } + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) + } + + config := clientcredentials.Config{ + ClientID: cfg.clientID, + ClientSecret: cfg.clientSecret, + TokenURL: tokenURL.String(), + EndpointParams: endpointParams, + Scopes: cfg.scopes, + } + return config.TokenSource(ctx), nil +} + +func oauthHTTPClient(caPath string) (*http.Client, error) { + caPEM, err := os.ReadFile(caPath) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if !rootCAs.AppendCertsFromPEM(caPEM) { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "OAuth2 CA file %q does not contain a valid certificate", caPath) + } + + defaultTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "cannot configure OAuth2 CA file %q with HTTP transport type %T", + caPath, http.DefaultTransport) + } + transport := defaultTransport.Clone() + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = rootCAs + return &http.Client{Transport: transport}, nil +} diff --git a/pkg/sink/kafka/oauth2_test.go b/pkg/sink/kafka/oauth2_test.go new file mode 100644 index 0000000000..814ca89eff --- /dev/null +++ b/pkg/sink/kafka/oauth2_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" + "github.com/stretchr/testify/require" + "golang.org/x/oauth2" +) + +func TestOAuthCA(t *testing.T) { + server := newTLSTokenServer(t) + tokenSource, err := newOAuthTokenSource(t.Context(), newOAuthConfig(server.URL, writeServerCA(t, server))) + require.NoError(t, err) + token, err := tokenSource.Token() + require.NoError(t, err) + require.Equal(t, "access-token", token.AccessToken) +} + +func TestOAuthCAErrors(t *testing.T) { + t.Run("missing file", func(t *testing.T) { + caPath := filepath.Join(t.TempDir(), "missing-ca.pem") + _, err := newOAuthTokenSource(t.Context(), newOAuthConfig("https://example.com/token", caPath)) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.ErrorContains(t, err, caPath) + }) + + t.Run("invalid PEM", func(t *testing.T) { + caPath := filepath.Join(t.TempDir(), "invalid-ca.pem") + require.NoError(t, os.WriteFile(caPath, []byte("not a certificate"), 0o600)) + _, err := newOAuthTokenSource(t.Context(), newOAuthConfig("https://example.com/token", caPath)) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.ErrorContains(t, err, "does not contain a valid certificate") + }) + + t.Run("mismatched certificate", func(t *testing.T) { + server := newTLSTokenServer(t) + unrelatedCA, err := security.NewCA() + require.NoError(t, err) + caPath := filepath.Join(t.TempDir(), "unrelated-ca.pem") + require.NoError(t, os.WriteFile(caPath, unrelatedCA.CAPEM, 0o600)) + tokenSource, err := newOAuthTokenSource(t.Context(), newOAuthConfig(server.URL, caPath)) + require.NoError(t, err) + _, err = tokenSource.Token() + require.ErrorContains(t, err, "certificate signed by unknown authority") + }) +} + +func TestOAuthContextClient(t *testing.T) { + server := newTLSTokenServer(t) + ctx := context.WithValue(t.Context(), oauth2.HTTPClient, server.Client()) + tokenSource, err := newOAuthTokenSource(ctx, newOAuthConfig(server.URL, "")) + require.NoError(t, err) + token, err := tokenSource.Token() + require.NoError(t, err) + require.Equal(t, "access-token", token.AccessToken) +} + +func newTLSTokenServer(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if _, err := io.WriteString(w, `{"access_token":"access-token","token_type":"bearer"}`); err != nil { + t.Errorf("write token response: %v", err) + } + })) + t.Cleanup(server.Close) + return server +} + +func writeServerCA(t *testing.T, server *httptest.Server) string { + t.Helper() + caPath := filepath.Join(t.TempDir(), "ca.pem") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + require.NotNil(t, caPEM) + require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) + return caPath +} + +func newOAuthConfig(tokenURL, caPath string) oauth2Config { + return oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: tokenURL, + caPath: caPath, + } +} diff --git a/pkg/sink/kafka/producer_test.go b/pkg/sink/kafka/producer_test.go new file mode 100644 index 0000000000..53259d3c60 --- /dev/null +++ b/pkg/sink/kafka/producer_test.go @@ -0,0 +1,31 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "strings" + "testing" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func requireKafkaSendError(t *testing.T, err, cause error) { + t.Helper() + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) + require.ErrorIs(t, err, cause) + require.Equal(t, 1, strings.Count(err.Error(), string(errors.ErrKafkaSendMessage.RFCCode()))) + require.NotContains(t, err.Error(), "keyspace=test") +} diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider.go b/pkg/sink/kafka/sarama_oauth2_token_provider.go index 3b5d6fed2e..0c8d80eaf6 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider.go @@ -15,16 +15,9 @@ package kafka import ( "context" - "crypto/tls" - "crypto/x509" - "net/http" - "net/url" - "os" "github.com/IBM/sarama" - "github.com/pingcap/ticdc/pkg/errors" "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" ) // tokenProvider is a user-defined callback for generating @@ -56,70 +49,9 @@ func (t *tokenProvider) Token() (*sarama.AccessToken, error) { } func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvider, error) { - // grant_type is by default going to be set to 'client_credentials' by the - // client credentials library as defined by the spec, however non-compliant - // auth server implementations may want a custom type - endpointParams := url.Values{} - if o.sasl.oauth2.grantType != "" { - endpointParams.Set("grant_type", o.sasl.oauth2.grantType) - } - - // audience is an optional parameter that can be used to specify the - // intended audience of the token. - if o.sasl.oauth2.audience != "" { - endpointParams.Set("audience", o.sasl.oauth2.audience) - } - - tokenURL, err := url.Parse(o.sasl.oauth2.tokenURL) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - - if o.sasl.oauth2.caPath != "" { - httpClient, err := oauthHTTPClient(o.sasl.oauth2.caPath) - if err != nil { - return nil, err - } - ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) - } - - cfg := clientcredentials.Config{ - ClientID: o.sasl.oauth2.clientID, - ClientSecret: o.sasl.oauth2.clientSecret, - TokenURL: tokenURL.String(), - EndpointParams: endpointParams, - Scopes: o.sasl.oauth2.scopes, - } - return &tokenProvider{ - tokenSource: cfg.TokenSource(ctx), - }, nil -} - -func oauthHTTPClient(caPath string) (*http.Client, error) { - caPEM, err := os.ReadFile(caPath) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - - rootCAs, err := x509.SystemCertPool() + tokenSource, err := newOAuthTokenSource(ctx, o.sasl.oauth2) if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - if !rootCAs.AppendCertsFromPEM(caPEM) { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "OAuth2 CA file %q does not contain a valid certificate", caPath) - } - - defaultTransport, ok := http.DefaultTransport.(*http.Transport) - if !ok { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "cannot configure OAuth2 CA file %q with HTTP transport type %T", - caPath, http.DefaultTransport) - } - transport := defaultTransport.Clone() - if transport.TLSClientConfig == nil { - transport.TLSClientConfig = &tls.Config{} + return nil, err } - transport.TLSClientConfig.RootCAs = rootCAs - return &http.Client{Transport: transport}, nil + return &tokenProvider{tokenSource: tokenSource}, nil } diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go index fc2dffa6a2..d7d0473c32 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go @@ -14,18 +14,13 @@ package kafka import ( - "context" - "encoding/pem" "io" "net/http" "net/http/httptest" "net/url" - "os" - "path/filepath" "testing" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" "golang.org/x/oauth2" ) @@ -137,106 +132,3 @@ func TestTokenProviderPropagatesEndpointError(t *testing.T) { require.Equal(t, "invalid_client", retrieveErr.ErrorCode) require.Equal(t, "bad credentials", retrieveErr.ErrorDescription) } - -func TestTokenProviderUsesOAuthCA(t *testing.T) { - t.Parallel() - - server := newTLSTokenServer(t) - caPath := writeServerCA(t, server) - options := newOAuthOptions(server.URL, caPath) - - provider, err := newTokenProvider(t.Context(), options) - require.NoError(t, err) - token, err := provider.Token() - require.NoError(t, err) - require.Equal(t, "access-token", token.Token) -} - -func TestTokenProviderRejectsInvalidOAuthCA(t *testing.T) { - t.Parallel() - - t.Run("missing file", func(t *testing.T) { - t.Parallel() - - caPath := filepath.Join(t.TempDir(), "missing-ca.pem") - _, err := newTokenProvider(t.Context(), newOAuthOptions("https://example.com/token", caPath)) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) - require.ErrorContains(t, err, caPath) - require.ErrorContains(t, err, "no such file") - }) - - t.Run("invalid PEM", func(t *testing.T) { - t.Parallel() - - caPath := filepath.Join(t.TempDir(), "invalid-ca.pem") - require.NoError(t, os.WriteFile(caPath, []byte("not a certificate"), 0o600)) - _, err := newTokenProvider(t.Context(), newOAuthOptions("https://example.com/token", caPath)) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) - require.ErrorContains(t, err, caPath) - require.ErrorContains(t, err, "does not contain a valid certificate") - }) -} - -func TestTokenProviderRejectsMismatchedOAuthCA(t *testing.T) { - t.Parallel() - - tokenServer := newTLSTokenServer(t) - unrelatedCA, err := security.NewCA() - require.NoError(t, err) - caPath := filepath.Join(t.TempDir(), "unrelated-ca.pem") - require.NoError(t, os.WriteFile(caPath, unrelatedCA.CAPEM, 0o600)) - options := newOAuthOptions(tokenServer.URL, caPath) - - provider, err := newTokenProvider(t.Context(), options) - require.NoError(t, err) - _, err = provider.Token() - require.ErrorContains(t, err, "certificate signed by unknown authority") -} - -func TestTokenProviderWithoutOAuthCAKeepsContextHTTPClient(t *testing.T) { - t.Parallel() - - server := newTLSTokenServer(t) - ctx := context.WithValue(t.Context(), oauth2.HTTPClient, server.Client()) - provider, err := newTokenProvider(ctx, newOAuthOptions(server.URL, "")) - require.NoError(t, err) - token, err := provider.Token() - require.NoError(t, err) - require.Equal(t, "access-token", token.Token) -} - -func newTLSTokenServer(t *testing.T) *httptest.Server { - t.Helper() - - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - if _, err := io.WriteString(w, `{"access_token":"access-token","token_type":"bearer"}`); err != nil { - t.Errorf("write token response: %v", err) - } - })) - t.Cleanup(server.Close) - return server -} - -func writeServerCA(t *testing.T, server *httptest.Server) string { - t.Helper() - - caPath := filepath.Join(t.TempDir(), "ca.pem") - caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) - require.NotNil(t, caPEM) - require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) - return caPath -} - -func newOAuthOptions(tokenURL, caPath string) *options { - return &options{ - sasl: &saslConfig{ - oauth2: oauth2Config{ - clientID: "client-id", - clientSecret: "client-secret", - tokenURL: tokenURL, - caPath: caPath, - }, - }, - } -} diff --git a/pkg/sink/kafka/sarama_sync_producer_test.go b/pkg/sink/kafka/sarama_sync_producer_test.go index 92e93c8a8c..27f1888826 100644 --- a/pkg/sink/kafka/sarama_sync_producer_test.go +++ b/pkg/sink/kafka/sarama_sync_producer_test.go @@ -16,7 +16,6 @@ package kafka import ( "context" "io" - "strings" "testing" "github.com/golang/mock/gomock" @@ -131,11 +130,3 @@ func TestAsyncProducerErrorWrappedOnce(t *testing.T) { requireKafkaSendError(t, err, cause) } - -func requireKafkaSendError(t *testing.T, err, cause error) { - t.Helper() - require.ErrorIs(t, err, errors.ErrKafkaSendMessage) - require.ErrorIs(t, err, cause) - require.Equal(t, 1, strings.Count(err.Error(), string(errors.ErrKafkaSendMessage.RFCCode()))) - require.NotContains(t, err.Error(), "keyspace=test") -} From 56865843c5c83f9975aa1ef6ce3b604619a56802 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 3 Sep 2026 21:50:47 +0800 Subject: [PATCH 59/61] docs: remove branch-local documents --- ...6-07-02-sarama-message-size-calculation.md | 674 ------------------ ...026-07-03-franz-go-replace-sarama-audit.md | 484 ------------- ...franz-go-replace-sarama-migration-steps.md | 672 ----------------- docs/franz-go/franz-go-ga-test-plan.md | 87 --- .../kafka-producer-idempotence-design.md | 152 ---- docs/franz-go/milestone-1-todo-list.md | 9 - .../ticdc-kafka-franz-go-ga-test-record.md | 180 ----- 7 files changed, 2258 deletions(-) delete mode 100644 docs/design/2026-07-02-sarama-message-size-calculation.md delete mode 100644 docs/design/2026-07-03-franz-go-replace-sarama-audit.md delete mode 100644 docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md delete mode 100644 docs/franz-go/franz-go-ga-test-plan.md delete mode 100644 docs/franz-go/kafka-producer-idempotence-design.md delete mode 100644 docs/franz-go/milestone-1-todo-list.md delete mode 100644 docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md diff --git a/docs/design/2026-07-02-sarama-message-size-calculation.md b/docs/design/2026-07-02-sarama-message-size-calculation.md deleted file mode 100644 index 09e4bacab4..0000000000 --- a/docs/design/2026-07-02-sarama-message-size-calculation.md +++ /dev/null @@ -1,674 +0,0 @@ -# Kafka 消息大小检查口径说明 - -本文梳理 TiCDC Kafka sink 在 master 分支 Sarama 实现和当前 franz-go 实现里的 -消息大小检查链路。重点是区分这些对象： - -```text -TiCDC common.Message - -> Kafka record - -> Kafka record batch - -> Kafka ProduceRequest -``` - -一次 `ProduceRequest` 按 topic 再按 partition 组织。现代 Kafka 版本下，每个 -topic-partition 的 `Records` 字段承载一个 `RecordBatch`；一个 request 可以 -包含多个 topic、多个 partition 的多个 record batches。就 Sarama 和 franz-go -这两个实现而言，一次 request 中同一个 topic-partition 只放一个 batch。 - -## Kafka 原始配置语义 - -Kafka broker/topic 层与消息大小相关的两个原始配置是 -`message.max.bytes` 和 `max.message.bytes`。 - -- `message.max.bytes` 是 broker 级默认值。Apache Kafka 4.3 官方文档定义为 - Kafka 允许的最大 record batch size；如果启用了压缩，按压缩后的 batch 大小 - 判断。它可以被 topic 级的 `max.message.bytes` 覆盖。默认值是 `1048588`。 -- `max.message.bytes` 是 topic 级配置。它的语义同样是 Kafka 允许的最大 - record batch size；如果启用了压缩，按压缩后的 batch 大小判断。没有显式 - topic override 时，该 topic 使用 server default property，也就是 - broker 级的 `message.max.bytes`。默认值同样显示为 `1048588`。 - -所以，把它们口语化理解成“Kafka 单条消息大小上限”只在一个应用层 record 独占 -一个 record batch 时近似成立。Kafka protocol 的精确对象是 record batch： -一个 batch 可以包含多条 records；broker/topic 限制校验的是这个 batch，而不是 -单独某个应用层 key/value payload。 - -另一个容易混淆的 producer 侧参数是 `max.request.size`。Apache Kafka 4.3 官方 -文档把它定义为 producer request 的最大大小，同时也是最大未压缩 record batch -size 的有效上限。server 侧仍然有自己的 record batch 上限，也就是上面的 -`message.max.bytes` / `max.message.bytes`，并且这个 server 侧上限在启用压缩时 -按压缩后大小判断。 - -相关官方文档： - -- Apache Kafka 4.3 Broker Configs, `message.max.bytes`: - https://kafka.apache.org/43/configuration/broker-configs/ -- Apache Kafka 4.3 Topic Configs, `max.message.bytes`: - https://kafka.apache.org/43/configuration/topic-configs/ -- Apache Kafka 4.3 Producer Configs, `max.request.size`: - https://kafka.apache.org/43/configuration/producer-configs/ - -## master 分支 Sarama 实现 - -master 分支的大小检查链路是： - -```text -Kafka raw topic/broker limit - -> TiCDC options.MaxMessageBytes - -> encoder MaxMessageBytes - -> open-protocol 单行/claim-check 检查 - -> open-protocol 多行 common.Message batching 检查 - -> Sarama ProducerMessage.ByteSize 检查 - -> Sarama produceSet batch / request rollover 检查 - -> broker 按 Kafka record batch limit 最终校验 -``` - -### 1. TiCDC 从 Kafka raw config 折算 options.MaxMessageBytes - -master 分支 `pkg/sink/kafka/options.go` 里有： - -```go -maxMessageBytesOverhead = 128 -``` - -topic 已存在时，`adjustOptions` 读取 topic 的 `max.message.bytes`，如果没有 -topic override 则回退到 broker 的 `message.max.bytes`。随后使用： - -```text -effective MaxMessageBytes = min(configured max-message-bytes, source max bytes - 128) -``` - -topic 不存在、需要 TiCDC 创建 topic 时，`adjustOptions` 读取 broker 的 -`message.max.bytes`，也使用同样的 `source - 128` 折算。 - -因此，在 master 分支上，TiCDC 的 `options.MaxMessageBytes` 不是 Kafka -broker/topic raw value，而是一个扣掉 128 字节 safety margin 后的 TiCDC/Sarama -侧预算。 - -源码位置： - -- `master:pkg/sink/kafka/options.go`：`maxMessageBytesOverhead = 128` -- `master:pkg/sink/kafka/options.go`：topic path 使用 - `topicMaxMessageBytes - maxMessageBytesOverhead` -- `master:pkg/sink/kafka/options.go`：broker path 使用 - `brokerMessageMaxBytes - maxMessageBytesOverhead` - -### 2. Sarama producer 和 encoder 使用同一个 MaxMessageBytes - -master 分支 `newSaramaConfig` 将： - -```go -config.Producer.MaxMessageBytes = o.MaxMessageBytes -``` - -同时，`downstreamadapter/sink/helper/helper.go` 明确把 encoder 的 -`MaxMessageBytes` 设置成 producer 的 `MaxMessageBytes`： - -```go -encoderConfig = encoderConfig.WithMaxMessageBytes(maxMsgBytes) -``` - -这意味着 master 的意图是：encoder 不要生成超过 producer 预算的 -`common.Message`。 - -源码位置： - -- `master:pkg/sink/kafka/sarama_config.go` -- `master:downstreamadapter/sink/helper/helper.go` - -### 3. open-protocol 单行编码与 claim-check 检查 - -open-protocol `batchEncoder.AppendRowChangedEvent` 会先把单行 RowEvent 编码成 -key/value，并得到一个 `length`： - -```go -key, value, length, err := encodeRowChangedEvent(...) -if length > d.config.MaxMessageBytes { - ... -} -``` - -如果单行原始消息超过 `MaxMessageBytes`： - -- large message handle disabled：直接返回 `ErrMessageTooLarge`。 -- claim-check enabled：先把原始 key/value 写入外部存储，再重新编码一条 - claim-check location message。 -- claim-check location message 仍超过 `MaxMessageBytes`：返回 - `ErrMessageTooLarge`。 - -这个检查发生在 Kafka producer 之前。 - -源码位置： - -- `master:pkg/sink/codec/open/encoder.go` - -### 4. open-protocol 多行 common.Message batching 检查 - -claim-check location message 单条通常很小，但 open-protocol 会继续把多条 -row events 合并进一个 TiCDC `common.Message`。 - -`pushMessage` 里新加入一行时，计算： - -```go -length := len(key) + len(value) + 16 -``` - -然后用当前 TiCDC message 的 `Length()` 判断是否还能继续追加： - -```go -latestMessage.Length() + length > d.config.MaxMessageBytes -``` - -`common.Message.Length()` 在 master 分支是： - -```go -len(m.Key) + len(m.Value) + MaxRecordOverhead -``` - -其中： - -```text -MaxRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 = 36 -``` - -也就是说，encoder 的 batching 口径和 Sarama -`ProducerMessage.ByteSize(2)` 的无 headers 估算口径一致： - -```text -len(key) + len(value) + 36 -``` - -源码位置： - -- `master:pkg/sink/codec/open/encoder.go` -- `master:pkg/sink/codec/common/message.go` - -### 5. Sarama AsyncSend 本身不做大小检查 - -TiCDC Sarama producer 的 `AsyncSend` 只是把 `common.Message` 转成 -`sarama.ProducerMessage` 并写入 Sarama input channel： - -```go -msg := &sarama.ProducerMessage{ - Topic: topic, - Partition: partition, - Key: sarama.StringEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), -} -p.producer.Input() <- msg -``` - -TiCDC 这一层没有额外 size check。 - -源码位置： - -- `master:pkg/sink/kafka/sarama_async_producer.go` - -### 6. Sarama 单条 ProducerMessage 硬检查 - -Sarama 内部第一层硬检查在 `asyncProducer.dispatcher`： - -```go -size := msg.ByteSize(version) -if size > p.conf.Producer.MaxMessageBytes { - reject -} -``` - -Kafka `>= 0.11` 时，`version = 2`，`ProducerMessage.ByteSize(2)` 为： - -```text -len(key) + len(value) + maximumRecordOverhead + headers estimate -``` - -没有 headers 时： - -```text -len(key) + len(value) + 36 -``` - -这一层是本地拒绝条件。 - -Sarama 源码位置： - -- `/Users/edison/go/sarama/async_producer.go:365` -- `/Users/edison/go/sarama/async_producer.go:626` - -### 7. Sarama produceSet rollover 检查 - -消息进入 broker producer 后，Sarama 调用 `produceSet.wouldOverflow(msg)`。 -这里有三类检查： - -```text -1. 整个 produce request 估算： - ps.bufferBytes + msg.ByteSize(version) >= MaxRequestSize - 10KiB - -2. 已存在 topic-partition batch 估算： - set.bufferBytes + msg.ByteSize(version) >= Producer.MaxMessageBytes - -3. Flush.MaxMessages 条数限制 -``` - -这几类检查触发后，Sarama 会 `waitForSpace` / flush / rollover，而不是把当前 -消息作为 `MESSAGE_TOO_LARGE` 直接失败。 - -关键细节：第二类 partition batch 检查只有在当前 topic-partition 的 -`partitionSet` 已经存在时才执行。第一条 record 进入一个空的 partition batch -时，partition set 尚不存在，所以这个检查会跳过。 - -随后 `produceSet.add` 创建 batch： - -```text -recordBatchOverhead = 49 -``` - -并把本条消息加入 batch。此时 `partitionSet.bufferBytes` 会变成： - -```text -49 + len(key) + len(value) + 36 -``` - -即使这个值已经超过 `Producer.MaxMessageBytes`，第一条 record 也已经被接受。 - -Sarama 源码位置： - -- `/Users/edison/go/sarama/produce_set.go:39` -- `/Users/edison/go/sarama/produce_set.go:303` -- `/Users/edison/go/sarama/async_producer.go:1188` -- `/Users/edison/go/sarama/async_producer.go:1328` - -### 8. Sarama 例子：759 字节 claim-check batch - -假设： - -```text -Producer.MaxMessageBytes = 800 -len(key) + len(value) = 759 -headers = none -``` - -Sarama 单条硬检查： - -```text -759 + 36 = 795 <= 800 -``` - -所以能通过。 - -如果这是该 topic-partition 当前 batch 的第一条 record，partition batch -rollover 检查会跳过。加入后 Sarama 内部估算为： - -```text -49 + 795 = 844 -``` - -但这不是第一条 record 的拒绝条件。 - -如果具体 key/value 拆分为 `527 + 232`，实际 headerless record 编码是： - -```text -record body: - attributes 1 - timestamp delta 1 - offset delta 1 - key length varint 2 - key bytes 527 - value length varint 2 - value bytes 232 - headers count 1 - total 767 - -record length varint 2 -encoded record total 769 -``` - -不启用 producer compression 时，Sarama 实际 `RecordBatch.encode` 大小约为： - -```text -61 + 769 = 830 -``` - -Sarama 本地仍然不会因为这个完整 encoded record batch 大于 800 而拒绝这条 -空 batch 的第一条 record。 - -## 修正后的 franz-go 实现 - -修正后的当前分支大小检查链路是： - -```text -Kafka raw topic/broker limit - -> TiCDC options.ProducerBatchMaxBytes - -> franz-go ProducerBatchMaxBytes - -> broker 按 Kafka record batch limit 最终校验 - -用户配置 max-message-bytes / Kafka raw topic/broker limit - -> TiCDC options.MaxMessageBytes - -> encoder MaxMessageBytes payload 检查 - -> open-protocol 单行/claim-check 检查 - -> open-protocol 多行 common.Message batching 检查 - -> franz-go Produce(ctx, kgo.Record) - -> franz-go buffered bytes/backpressure 检查 - -> franz-go recBatch.tryBuffer record batch 大小检查 - -> franz-go produceRequest request 总大小检查 -``` - -这里刻意把两个值分开： - -- `options.MaxMessageBytes`：TiCDC encoder 的 payload 预算，用于 open protocol - 自身分包以及 large message handle。 -- `options.ProducerBatchMaxBytes`：franz-go producer 的 Kafka record batch 预算， - 直接来自 topic `max.message.bytes` 或 broker `message.max.bytes`。 - -### 1. 删除 maxMessageBytesOverhead，但不再混淆 producer batch 预算 - -当前分支删除 `maxMessageBytesOverhead`。`adjustOptions` 现在直接使用 Kafka raw -source limit 约束 encoder： - -```text -effective MaxMessageBytes = min(configured max-message-bytes, kafka raw source max bytes) -``` - -topic 已存在时，source 是 topic `max.message.bytes`；topic 不存在时，source 是 -broker `message.max.bytes`。 - -同时，`adjustOptions` 记录 Kafka raw source limit： - -```text -ProducerBatchMaxBytes = kafka raw source max bytes -``` - -例如 integration test 里的： - -```text -max-message-bytes=800 -``` - -在 topic/broker raw limit 是 Kafka 默认值 `1048588` 时： - -```text -options.MaxMessageBytes = 800 -options.ProducerBatchMaxBytes = 1048588 -``` - -这正是 claim-check 场景需要的语义：`800` 只控制 TiCDC 何时把原始大消息转成 -claim-check location message，不应该把 franz-go 的 record batch 上限也压成 -`800`。 - -源码位置： - -- `pkg/sink/kafka/options.go` -- `pkg/sink/kafka/options_test.go` - -### 2. encoder 侧检查改为 payload 口径 - -encoder 仍然必须使用 `MaxMessageBytes` 做检查，原因是它承担两个 Kafka producer -之前的应用层功能： - -- large message handle disabled 时，如果单行编码后的 payload 超过 - `MaxMessageBytes`，直接返回 `ErrMessageTooLarge`，不会等 producer/broker 拒绝。 -- claim-check enabled 时，同一个检查点触发 claim-check：原始 key/value 写入外部 - 存储，再生成一条 claim-check location message。 -- open-protocol 会把多条 row events 合并成一个 Kafka record 的 key/value - payload，因此 `pushMessage` 需要用同一预算决定是否开一个新的 TiCDC - `common.Message`。 - -修正点是：`common.Message.Length()` 不再包含 Sarama 的 `MaxRecordOverhead = 36`， -而是返回： - -```text -len(key) + len(value) -``` - -open-protocol 单行检查也不再额外加 `common.MaxRecordOverhead`。对 open protocol -来说，一条 row 在最终 Kafka record payload 里的真实应用层长度是： - -```text -len(row key) + len(compressed row value) + 8(version) + 8(key length) + 8(value length) -``` - -后续追加一条 row 到同一个 `common.Message` 时增加： - -```text -len(row key) + len(compressed row value) + 8(key length) + 8(value length) -``` - -因此，encoder 现在检查的是 TiCDC 实际生成的 key/value payload 大小，而不是 -Sarama `ProducerMessage.ByteSize` 估算大小。 - -源码位置： - -- `pkg/sink/codec/open/encoder.go` -- `pkg/sink/codec/open/codec.go` -- `pkg/sink/codec/common/message.go` - -### 3. TiCDC franz-go AsyncSend 不做大小检查 - -当前分支 `kafkaAsyncProducer.AsyncSend` 把 `common.Message` 直接转成 -`kgo.Record`： - -```go -record := &kgo.Record{ - Topic: topic, - Partition: partition, - Key: message.Key, - Value: message.Value, -} -p.client.Produce(ctx, record, promise) -``` - -TiCDC 这一层没有额外 size check。 - -源码位置： - -- `pkg/sink/kafka/async_producer.go` - -### 4. franz-go ProducerBatchMaxBytes 使用 Kafka raw source limit - -修正前，当前分支在构造 franz-go producer options 时设置： - -```go -kgo.ProducerBatchMaxBytes(int32(o.MaxMessageBytes)) -``` - -franz-go 文档注释说明 `ProducerBatchMaxBytes` 限制的是 record batch 大小， -并且它 mirrors Kafka `max.message.bytes`。注释还明确说：record batch 是 -topic-partition 维度，`ProduceRequest` 可以包含多个 topics 的多个 record -batches。 - -这一步是当前分支和 master/Sarama 行为不同的核心：master 的 -`o.MaxMessageBytes` 进入 Sarama 后首先用于 `ProducerMessage.ByteSize` 单条估算； -当前分支的同一个值进入 franz-go 后用于 `ProducerBatchMaxBytes` record batch -上限。 - -修正后，franz-go producer 使用： - -```go -kgo.ProducerBatchMaxBytes(int32(o.ProducerBatchMaxBytes)) -``` - -也就是 topic/broker 的原始 record-batch 上限。`o.MaxMessageBytes` 继续传给 -encoder，不再直接作为 franz-go record-batch 上限。 - -源码位置： - -- `pkg/sink/kafka/client_options.go` -- `pkg/sink/kafka/kafka_factory.go` -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/config.go:1255` - -### 5. franz-go buffered bytes/backpressure 检查 - -franz-go `Produce` 开始时先计算： - -```text -userSize = len(key) + len(value) + sum(header key/value) -``` - -如果配置了 `MaxBufferedBytes`： - -```go -if maxBufferedBytes > 0 && userSize > maxBufferedBytes { - MESSAGE_TOO_LARGE -} -``` - -随后还会检查客户端当前 buffered bytes 是否超过 `MaxBufferedBytes`。不过 TiCDC -当前没有设置 `kgo.MaxBufferedBytes`，所以这个检查通常不是本问题的来源。 - -franz-go 源码位置： - -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/record_and_fetch.go:157` -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/producer.go:599` - -### 6. franz-go recBatch.tryBuffer record batch 检查 - -franz-go 把 record 放入 topic-partition 的 `recBatch` 时，会先尝试加入当前最后 -一个 batch；放不进去就创建新 batch 再试。 - -关键检查在 `recBatch.tryBuffer`： - -```go -nums := b.calculateRecordNumbers(pr.Record) -batchWireLength, _, _ := b.wireLengthForProduceVersion(produceVersion) -newBatchLength := batchWireLength + nums.wireLength() - -if b.frozen || newBatchLength > maxBatchBytes { - return false, false -} -``` - -如果一个空的新 batch 也放不下这条 record，franz-go 会本地失败： - -```go -MESSAGE_TOO_LARGE (uncompressed_bytes=...) -``` - -这是当前 claim-check case 的直接失败点。 - -franz-go 源码位置： - -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:1640` -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:1958` -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2320` -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2413` - -### 7. franz-go produceRequest 总大小检查 - -batch 准备写入 produce request 时，franz-go 还有 request 层检查： - -```go -if p.wireLength + batchWireLength > p.wireLengthLimit { - return false -} -``` - -这里的 `wireLengthLimit` 来源于 `maxBrokerWriteBytes`，默认对应 Kafka -`socket.request.max.bytes` 级别，默认约 100 MiB。它不是本次 800 字节失败的来源。 - -franz-go 源码位置： - -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2102` -- `/Users/edison/go/pkg/mod/github.com/twmb/franz-go@v1.21.4/pkg/kgo/sink.go:2250` - -### 8. franz-go 例子：759 字节 claim-check payload - -仍用同一个例子，sink URI 显式配置： - -```text -max-message-bytes = 800 -len(key) + len(value) = 759 -key = 527 -value = 232 -headers = none -``` - -如果 Kafka topic/broker raw limit 是默认值： - -```text -Kafka max.message.bytes / message.max.bytes = 1048588 -``` - -修正后： - -```text -options.MaxMessageBytes = 800 -options.ProducerBatchMaxBytes = 1048588 -``` - -franz-go 对单条 record 的编码大小： - -```text -record body: - attributes 1 - timestamp delta 1 - offset delta 1 - key length varint 2 - key bytes 527 - value length varint 2 - value bytes 232 - headers count 1 - total 767 - -record length varint 2 -encoded record total 769 -``` - -franz-go 新 batch 固定 wire length 是： - -```text -record batch overhead = 65 -``` - -所以空 batch 加入这条 record 后： - -```text -65 + 769 = 834 -``` - -修正前，因为代码设置： - -```text -ProducerBatchMaxBytes = options.MaxMessageBytes = 800 -``` - -于是： - -```text -834 > 800 -``` - -franz-go 在本地返回 `MESSAGE_TOO_LARGE (uncompressed_bytes=759)`。这里的 -`uncompressed_bytes=759` 是用户 key/value payload 大小，不是完整 record batch -wire size。 - -修正后： - -```text -834 <= ProducerBatchMaxBytes(1048588) -``` - -这条 claim-check location message 可以进入 producer，并交给 broker 按 Kafka -record batch 语义最终校验。 - -如果 Kafka topic 本身真的配置为： - -```text -max.message.bytes = 800 -``` - -那么完整 record batch wire size `834 > 800`，franz-go 本地拒绝是合理的。那表示 -Kafka topic record-batch 上限确实放不下这条 claim-check location record，而不是 -TiCDC 的 claim-check 阈值被误用为 producer batch 阈值。 - -## 当前结论 - -1. Kafka broker/topic 的 `message.max.bytes` / `max.message.bytes` 语义是 - record batch 上限，不是 TiCDC `common.Message` 上限。 -2. master/Sarama 链路里，TiCDC encoder 和 Sarama 单条硬检查都主要使用 - `len(key) + len(value) + 36` 这一估算口径；Sarama 不会在空 batch 第一条 - record 时用完整 record batch wire size 拒绝消息。 -3. franz-go 链路必须区分 TiCDC encoder payload 预算和 Kafka record batch 预算。 - `max-message-bytes=800` 应触发 open-protocol claim-check；Kafka producer 的 - `ProducerBatchMaxBytes` 应来自 topic/broker raw limit。 -4. `common.Message.Length()` 不能继续携带 Sarama 的 36 字节 record overhead。 - 修正后它表示 TiCDC 生成的 key/value payload 大小；Kafka record encoding 和 - record batch header 由 franz-go 在 producer 层按真实协议口径检查。 diff --git a/docs/design/2026-07-03-franz-go-replace-sarama-audit.md b/docs/design/2026-07-03-franz-go-replace-sarama-audit.md deleted file mode 100644 index f5d759e9fd..0000000000 --- a/docs/design/2026-07-03-franz-go-replace-sarama-audit.md +++ /dev/null @@ -1,484 +0,0 @@ -# 使用 franz-go 替换 Sarama 的 Kafka sink 功能审计清单 - -## 背景 - -本文基于 `master` 分支代码和 TiCDC Kafka sink 官方文档，枚举用 -`~/go/franz-go` 替换 Sarama 时必须保持、实现和验证的功能点。本文不是 -PRD，也不是最终实现方案；它的目标是把替换边界、兼容性风险和验收项列清楚， -避免只替换 producer API 后遗漏 Kafka sink 对正确性、性能、可靠性和运维的 -隐含约束。 - -代码阅读基准： - -- `master` revision: `d2da619279f877a9964facdabebdc608044523cd` -- 本地 franz-go: `/Users/edison/go/franz-go` - -官方文档阅读范围： - -- [TiCDC 同步数据到 Kafka](https://docs.pingcap.com/zh/tidb/stable/ticdc-sink-to-kafka/) -- [TiCDC Changefeed 配置参数](https://docs.pingcap.com/zh/tidb/stable/ticdc-changefeed-config/) -- [TiCDC Open Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-open-protocol/) -- [TiCDC Canal-JSON Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-canal-json/) -- [TiCDC Avro Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-avro-protocol/) -- [TiCDC Simple Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-simple-protocol/) -- [TiCDC Debezium Protocol](https://docs.pingcap.com/zh/tidb/stable/ticdc-debezium/) -- [TiCDC 数据校验](https://docs.pingcap.com/zh/tidb/stable/ticdc-integrity-check/) -- [TiCDC 常见问题](https://docs.pingcap.com/zh/tidb/stable/ticdc-faq/) -- [TiCDC 故障处理](https://docs.pingcap.com/zh/tidb/stable/troubleshoot-ticdc/) - -## 当前 master 上的 Kafka sink 结构 - -`downstreamadapter/sink/kafka` 是 Kafka sink 的业务层： - -- `helper.go` 解析 sink URI、创建 encoder、event router、topic manager，并调用 - `kafka.NewSaramaFactory` 创建客户端层。 -- `sink.go` 把 DML、DDL、checkpoint 分成三条路径： - - DML 经过 event router 计算 topic/partition，encoder group 编码后用 - `AsyncProducer.AsyncSend` 发送。 - - DDL 用 `SyncProducer.SendMessage` 或 `SendMessages` 发送。Open Protocol 的 - DDL 需要广播到全部 partition，Canal-JSON 的 DDL 走 partition 0。 - - checkpoint/resolved message 广播到当前活跃 topic 的全部 partition。没有表时， - 发送到 default topic，以兼容旧行为。 - -`pkg/sink/kafka` 是客户端抽象层： - -- `factory.go` 定义 `Factory`、`ClusterAdminClient`、`AsyncProducer`、 - `SyncProducer`、`MetricsCollector` 这些上层依赖的接口。 -- `sarama_factory.go` 创建 Sarama admin、sync producer、async producer，并在 - producer 上挂 Sarama metrics registry。 -- `sarama_config.go` 把 TiCDC Kafka options 转为 Sarama config，包括版本探测、 - TLS/SASL、acks、compression、manual partitioner、retry、timeout 和 - `Net.MaxOpenRequests=1`。 -- `options.go` 合并 sink URI 和 changefeed config，并通过 admin 查询 topic/broker - 配置，调整 `max-message-bytes`、`partition-num` 和 `min.insync.replicas`。 -- `admin.go` 封装 topic metadata、topic/broker config 和 create topic。 -- `sarama_async_producer.go` 在 Sarama success channel 中执行 DML callback；遇到 - producer error 时返回错误，让 sink 重建。 -- `sarama_sync_producer.go` 用于 DDL/checkpoint 的同步发送。 -- `metrics_collector.go` 从 Sarama go-metrics registry 中采集并暴露 TiCDC 既有 - Prometheus 指标。 - -可替换边界相对清晰：优先保持 `pkg/sink/kafka/factory.go` 的接口稳定，在 -`pkg/sink/kafka` 内新增 franz-go 实现。真正需要谨慎处理的是“默认行为差异”， -例如 franz-go 默认启用 idempotent write、默认 compression preference 包含 -snappy、默认 linger 为 10ms、默认 record retries 近似无限，这些都不能直接沿用。 - -## 替换原则 - -1. 对用户可见的 sink URI、changefeed config、协议输出、错误语义和指标名称默认 - 保持兼容。 -2. 不因为换客户端而扩大官方支持矩阵。franz-go 自身支持更宽的 Kafka 版本，不代表 - TiCDC Kafka sink 的官方版本支持自动扩大。 -3. 不默认引入新的 ACL 要求。特别是 franz-go 默认 idempotent write 在 Kafka 3.0 - 以前通常需要 Cluster 级 `IDEMPOTENT_WRITE` 权限，而 TiCDC 文档当前最小 ACL - 没有列它。 -4. TiCDC 的 at-least-once 语义、单行更新顺序、DDL/checkpoint/resolved 广播语义 - 优先于吞吐优化。 -5. 性能结论必须来自 TiCDC 场景 A/B 实测。franz-go README 的 benchmark 只能说明 - 客户端潜力，不能直接作为 TiCDC 替换收益结论。 - -## 必须保持的用户配置面 - -以下配置在替换后必须继续支持，默认值、校验和配置文件/URI 覆盖关系也应保持： - -| 类别 | 配置项 | 要求 | -| --- | --- | --- | -| 基础 | broker endpoints、topic、`protocol` | 保持 URI 语法和协议名不变。 | -| 版本 | `kafka-version` | 保留用户显式指定能力；保留版本错误诊断和文档中的兼容要求。 | -| producer | `partition-num`、`replication-factor`、`max-message-bytes`、`max-retry`、`required-acks` | 行为必须和 `options.go` 的校验/自适应一致。 | -| topic | `auto-create-topic` | true 时 TiCDC 创建 topic；false 且 topic 不存在时仍报配置错误。 | -| 压缩 | `compression=none,gzip,snappy,lz4,zstd` | 默认必须是 `none`，不能继承 franz-go 默认 snappy preference。 | -| TLS | `enable-tls`、`ca`、`cert`、`key`、`insecure-skip-verify` | 保持三证书校验和“配置了证书即启用 TLS”的现有行为。 | -| SASL | PLAIN、SCRAM-SHA-256、SCRAM-SHA-512、GSSAPI、OAUTHBEARER | 都需要映射；GSSAPI 需要用 franz-go `pkg/sasl/kerberos` 做专项验证。 | -| 超时 | `dial-timeout`、`write-timeout`、`read-timeout` | franz-go 没有完全同名语义，需要显式设计等价映射和测试。 | -| 协议扩展 | `enable-tidb-extension`、Avro decimal/bigint 模式、Debezium schema 开关、Simple codec config | 客户端替换不能改变 encoder 行为。 | -| 大消息 | `large-message-handle-*`、`claim-check-*` | 客户端替换不能改变大消息判定、外部存储写入和 Kafka marker 格式。 | - -`enable-kafka-sink-v2` 在当前代码中已是 deprecated，并且仍使用默认 Kafka sink。 -不要把它偷偷复用成 franz-go 开关。若需要灰度，建议新增内部开关或新的明确配置， -并为兼容性变更写单独方案。 - -## Kafka 版本支持 - -官方文档列出的 TiCDC Kafka sink 最低 Kafka 版本是产品承诺，替换后仍应遵守： - -| TiCDC 版本 | Kafka 最低版本 | -| --- | --- | -| TiCDC >= v8.1.0 | Kafka >= 2.1.0 | -| v7.6.0 <= TiCDC < v8.1.0 | Kafka >= 2.4.0 | -| v7.5.2 <= TiCDC < v7.6.0 | Kafka >= 2.1.0 | -| v7.5.0 <= TiCDC < v7.5.2 | Kafka >= 2.4.0 | -| v6.5.0 <= TiCDC < v7.5.0 | Kafka >= 2.1.0 | -| v6.1.0 <= TiCDC < v6.5.0 | Kafka >= 2.0.0 | - -当前 Sarama 实现的实际行为： - -- `options.NewOptions` 默认 `Version` 为 `2.4.0`。 -- `sarama_config.go` 的 `defaultKafkaVersion` 是 `2.0.0.0`，`maxKafkaVersion` 是 - `2.8.0.0`。 -- 如果用户未指定 `kafka-version`，代码会通过 broker `ApiVersions` 中 Metadata - API 的 max version 推断版本；失败时退回 `2.0.0.0`。 -- 如果用户指定 `kafka-version`，解析失败返回 `ErrKafkaInvalidVersion`，并在 - 指定版本和探测版本不一致时告警。 - -franz-go 侧能力： - -- README 表示 franz-go 支持 Kafka 0.8.0 到 4.2+ 的协议范围。 -- franz-go 默认会使用 ApiVersions 协商，也可通过 `kgo.MaxVersions` 固定协议版本。 - -替换要求： - -1. 不能因为 franz-go 支持更高或更低版本就改变 TiCDC 文档承诺。 -2. `kafka-version` 必须继续生效，建议映射为 `kgo.MaxVersions(...)` 或等价能力。 -3. 当前“自动探测 + 指定版本告警”的诊断体验需要保留，至少不能退化为静默忽略。 -4. 需要覆盖 Kafka 2.1、2.4、2.8、3.x、Confluent Cloud，以及如果仍支持则覆盖 KOP。 -5. 若去掉 Sarama 的 `maxKafkaVersion=2.8.0` 上限，需要在 release note 中说明这只是 - 客户端协议协商变化，不代表 TiCDC 扩大官方最低版本矩阵。 - -## ACL 和权限要求 - -官方文档列出的 Kafka 最小权限： - -| Resource | Operation | 用途 | -| --- | --- | --- | -| Topic | Create | 自动创建 topic。 | -| Topic | Write | 写入变更事件。 | -| Topic | Describe | 启动和 topic metadata 查询。 | -| Cluster | DescribeConfigs | 读取 broker/topic 配置，如 `message.max.bytes`、`min.insync.replicas`。 | - -如果 topic 已存在，文档说明可省略 Topic `Create`，但代码仍会读取 topic metadata 和 -配置，因此 `Describe` / `DescribeConfigs` 的实际需求要按部署环境验证。 - -替换时的新增风险： - -- franz-go 默认启用 idempotent write。Kafka 3.0 以前通常需要 Cluster 级 - `IDEMPOTENT_WRITE` 权限。当前 TiCDC 文档没有要求这个 ACL，Sarama 实现也没有开启 - idempotent producer。因此默认必须使用 `kgo.DisableIdempotentWrite()`，除非另开 - 配置并同步更新文档、权限说明和回滚策略。 -- 如果未来选择启用 idempotency，需要说明它改变的是客户端内部重试去重能力，不改变 - TiCDC 对外的 at-least-once 语义；TiCDC 仍可能在重启、故障恢复或上游重放后发送 - 重复消息。 -- Schema Registry、AWS Glue Schema Registry、claim-check 外部存储权限不是 Kafka - ACL，但替换不能破坏其认证和错误诊断。 - -## Producer 行为映射 - -| TiCDC/Sarama 现状 | franz-go 替换要求 | -| --- | --- | -| 手动指定 partition，Sarama `NewManualPartitioner`。 | 使用 `kgo.RecordPartitioner(kgo.ManualPartitioner())`，所有 record 必须设置 `Topic` 和 `Partition`。 | -| `required-acks=-1/1/0` 映射到 Sarama RequiredAcks。 | 映射到 `kgo.AllISRAcks()`、`kgo.LeaderAck()`、`kgo.NoAck()`。 | -| 默认 `compression=none`。 | 显式 `kgo.ProducerBatchCompression(kgo.NoCompression())` 或等价配置。 | -| `Producer.Flush.*=0`，尽快 flush。 | 显式 `kgo.ProducerLinger(0)`，避免默认 10ms linger 改变延迟。 | -| `Producer.Retry.Max=o.MaxRetry`，默认 5，backoff 100ms。 | 显式 `kgo.RecordRetries(o.MaxRetry)`，并设置等价 backoff；不要继承 unlimited retries。 | -| `Net.MaxOpenRequests=1` 作为顺序保护。 | 禁用 idempotency 后保留 `MaxProduceRequestsInflightPerBroker(1)`；不要为吞吐随意调大。 | -| producer max message bytes 来自 `options.MaxMessageBytes`。 | 设置 `ProducerBatchMaxBytes`，并校准其“record batch pre-compression”语义和 TiCDC 大消息判定。 | -| async success 后执行 message callback。 | franz-go promise 只有 `err == nil` 才能执行 callback。promise 不得阻塞或调用可能阻塞的 Produce/Flush。 | -| async error 让 `AsyncRunCallback` 返回，sink 重建。 | 需要有中心错误通道/errgroup，把首个 produce error 转为 TiCDC error 并返回。 | -| sync DDL/checkpoint 用 `SendMessage` 或 `SendMessages`。 | 用 `ProduceSync` 实现；`SendMessages` 必须构造每个 partition 一条 record，并在任一失败时返回错误。 | - -`required-acks=0` 必须保留，但需要明确：这本来就没有 broker durable ack 保证。替换后 -不要在 callback 中假装获得了真实 broker ack；只能保持与现有“允许但风险自担”的语义。 - -## Admin 行为映射 - -当前 `ClusterAdminClient` 行为需要完整保留： - -| 接口 | 当前用途 | franz-go/kadm 替换注意点 | -| --- | --- | --- | -| `GetAllBrokers` | metrics collector 获取 broker label。 | 可用 metadata/broker metadata。 | -| `GetBrokerConfig` | 读取 `message.max.bytes`、`min.insync.replicas`。 | 当前 Sarama 通过 controller broker 的 DescribeConfig 读取；kadm 实现要确认是否等价。 | -| `GetTopicConfig` | 读取 topic `max.message.bytes` 和 topic 级 `min.insync.replicas`。 | 需要兼容 KOP/不同 broker 返回 config entries 的形式。 | -| `GetTopicsMeta` | 判断 topic 是否存在、读取 partition 数。 | `UnknownTopicOrPartition` 在 ignore 模式下要跳过；其他错误不能吞。 | -| `GetTopicsPartitionsNum` | topic manager 定时刷新动态 topic partition 数。 | 返回值必须和当前 map 语义一致。 | -| `CreateTopic` | 自动创建 topic。 | `TopicAlreadyExists` 继续按成功处理；其他 policy/rf/auth 错误要保留。 | -| `Close` | 释放 admin client。 | 不能阻塞 sink 关闭路径。 | - -`topicmanager/kafka_topic_manager.go` 的行为不要改： - -- default topic 已存在且实际 partition 更多时，只使用配置中的 partition 子集。 -- 用户指定 `partition-num` 大于实际 topic partition 数时返回错误，避免 dispatch 到 - 不存在的 partition。 -- 动态 topic 缓存刷新和 create-topic-then-wait-visible 逻辑继续由 topic manager 负责， - 不要转移到 producer 自动创建。 - -## Topic、partition 和顺序保证 - -官方文档和代码共同依赖以下顺序约束： - -1. `index-value`、`columns`、`table/default` 这类 dispatcher 必须保证同一行的多次更新 - 进入同一 Kafka partition。 -2. `ts` dispatcher 可能把同一行不同版本发到不同 partition，消费者必须按 commitTs - 排序；客户端替换不能额外提供或破坏这个语义。 -3. Open Protocol 的 DDL 和 Resolved Event 需要广播到所有 MQ partition，消费者用 - resolved ts 做多 partition 排序。 -4. Canal-JSON DDL 发送到 partition 0；WATERMARK 只有在 `enable-tidb-extension=true` - 时输出。 -5. Simple Protocol 的 WATERMARK 和 BOOTSTRAP 语义必须保持。客户端替换不能改变 - BOOTSTRAP 周期、发送分区和 DML/DDL 顺序关系。 -6. DML callback 只能在 Kafka client 认为该 record 成功后执行，否则上游可能提前推进 - checkpoint，造成数据丢失。 - -franz-go 文档说明成功写入的 records 会按 partition 保持顺序；同时 `RecordRetries` -耗尽时会失败同 partition buffered records，避免跳过失败 record 后继续成功写入后续 -record。替换实现必须利用这一点，而不是在 TiCDC 层自行绕过失败继续发送。 - -## 协议输出兼容性 - -客户端替换不应改 encoder，但实现和测试必须覆盖所有 Kafka 支持协议： - -| 协议 | 必须保持的行为 | -| --- | --- | -| Open Protocol | Row Changed、DDL、Resolved 事件；batch key/value 格式；DDL/Resolved broadcast；`max-batch-size`。 | -| Canal-JSON | 一行一条 DML；DDL partition 0；`_tidb.commitTs`、WATERMARK、`content-compatible`。 | -| Avro | Confluent Avro wire format；每个 topic 只对应一张表；delete value 为 nil；Schema Registry / Glue 注册和错误处理。 | -| Debezium | 只输出 Row Changed Event，不输出 DDL/WATERMARK；schema 开关；TiDB 扩展字段。 | -| Simple | DDL、DML、WATERMARK、BOOTSTRAP；JSON/Avro codec；消费者 schema cache 依赖 BOOTSTRAP。 | - -相关配置和限制也要覆盖： - -- `delete-only-output-handle-key-columns` -- `only-output-updated-columns` -- `column-selectors` -- `enable-tidb-extension` -- `schema-registry` / AWS Glue schema registry -- row-level checksum：Kafka + Simple/Avro；Avro 需 TiDB extension 和 decimal/bigint string - 模式。 - -## 大消息处理和消息大小估算 - -这是替换中的高风险点。 - -当前 `pkg/sink/codec/common/message.go` 的 `Message.Length()` 使用： - -```go -len(m.Key) + len(m.Value) + MaxRecordOverhead -``` - -其中 `MaxRecordOverhead` 的注释明确基于 Sarama 的 record batch 编码估算。这个值参与： - -- producer `max-message-bytes` 前置检查； -- Open Protocol batch 拆分； -- large message compression 后是否进入 `handle-key-only` 或 `claim-check`； -- 报错 `Message was too large` 前的客户端侧保护。 - -franz-go 的 `ProducerBatchMaxBytes` 限制的是未压缩 record batch 上限。如果继续使用 -Sarama overhead，可能出现两类问题： - -- TiCDC 认为没超限，franz-go 或 broker 拒绝，造成 changefeed 报错。 -- TiCDC 认为超限而提前 claim-check/handle-key-only，导致不必要的外部存储写入或消息降级。 - -替换要求： - -1. 重新校准 Kafka record batch overhead。优先使用 franz-go 可复用的编码/估算能力； - 如果无法直接复用，使用保守上界并写明依据。 -2. 覆盖 key/value 为空、key 大 value 小、value 大 key 小、header 为空、不同 compression - 的测试。 -3. `large-message-handle-compression` 是 TiCDC 在消息级别先压缩再判断大小；producer - `compression` 是 Kafka batch 压缩。两者不能混淆。 -4. `claim-check` 和 `claim-check-raw-value` 的 Kafka marker 格式、外部存储路径、清理 - 责任必须保持不变。 -5. `max-message-bytes` 仍要和 broker/topic `message.max.bytes` / `max.message.bytes` - 通过 admin 自适应，并保留当前 `128` bytes safety margin 或给出替代依据。 - -## TLS、SASL 和认证 - -TLS 替换要求： - -- 复用 `security.Credential.ToTLSConfig()`。 -- 保留 TLS 1.2 minimum、证书文件校验、`insecure-skip-verify` 行为。 -- franz-go 可用 `kgo.DialTLSConfig` 或自定义 dialer，具体选择要覆盖证书和系统 CA 两种路径。 - -SASL 替换要求： - -| 机制 | franz-go 映射 | 注意点 | -| --- | --- | --- | -| PLAIN | `pkg/sasl/plain` | 用户名/密码为空时的现有错误行为要保持。 | -| SCRAM-SHA-256 | `pkg/sasl/scram` | 使用 SHA-256 mechanism；保持大小写和错误信息。 | -| SCRAM-SHA-512 | `pkg/sasl/scram` | 使用 SHA-512 mechanism。 | -| OAUTHBEARER | `pkg/sasl/oauth` 或自定义 provider | 复用现有 OAuth2 token provider 行为，包括 base64 secret、scope、grant type、audience。 | -| GSSAPI | `pkg/sasl/kerberos` | 需要把 `sasl-gssapi-*` 字段映射到 Kerberos client；user auth/keytab 两种都要集成测试。 | - -`pkg/security/sasl.go` 当前直接引用 Sarama 常量。真正移除 Sarama 依赖时，需要先把这些 -公共安全常量改成 TiCDC 自己的字符串常量，否则 Sarama 依赖会继续被保留。 - -## Metrics、日志和观测性 - -当前 TiCDC 暴露的 Kafka producer 指标名称和 label 是外部运维契约： - -- `ticdc_sink_kafka_producer_in_flight_requests` -- `ticdc_sink_kafka_producer_outgoing_byte_rate` -- `ticdc_sink_kafka_producer_request_rate` -- `ticdc_sink_kafka_producer_request_latency` -- `ticdc_sink_kafka_producer_compression_ratio` -- `ticdc_sink_kafka_producer_records_per_request` -- `ticdc_sink_kafka_producer_response_rate` - -这些指标目前来自 Sarama go-metrics registry。franz-go 可用 hook 或 `plugin/kprom`，但 -`kprom` 的默认指标名不是 TiCDC 现有指标名。因此推荐实现 TiCDC 自己的 hook collector： - -- `HookBrokerWrite` / `HookBrokerRead` / E2E hook：请求数、响应数、latency、broker label。 -- `HookProduceBatchWritten`：records per request、compression ratio、outgoing bytes。 -- 需要自行维护 in-flight gauge，或明确一个等价口径。 -- collector cleanup 必须删除 `namespace/changefeed/broker/type` label，避免 changefeed - 删除后遗留时间序列。 - -日志相关替换点： - -- `pkg/logger/log.go` 目前有 `WithInitSaramaLogger` 和 `sarama.Logger` hack。franz-go - 需要接入 `kgo.WithLogger` 或等价 logger adapter。 -- `pkg/leakutil/leak_helper.go` 目前忽略 Sarama goroutine。替换后应删除或改成 - franz-go 相关 goroutine 的测试策略，不能永久掩盖泄漏。 -- producer/admin 错误必须继续通过 `logutil.go` 附加 `MessageLogInfo`，否则 - `kafka_log_info` 类测试会退化。 - -## 正确性风险清单 - -P0 必须解决： - -- DML callback 不得早于 Kafka 成功返回。 -- 手动 partition 不得退化为客户端默认 partitioner。 -- `required-acks`、`compression`、`max-retry`、`linger`、in-flight、idempotency 等默认值 - 必须显式设置，不得使用 franz-go 默认值。 -- DDL/checkpoint/resolved 的广播分区数必须来自 topic manager，而不是 producer metadata - 的临时结果。 -- 大消息大小估算必须重新校准。 -- `required-acks=-1` 时 `replication-factor >= min.insync.replicas` 的前置校验要保留。 -- `UnknownTopicOrPartition`、`TopicAlreadyExists`、auth、policy、message too large 等错误 - 要保持可诊断，不能被统一包装成无信息的 client error。 - -P1 需要验证： - -- `required-acks=0` 下 promise/callback 语义和 Sarama 一致。 -- broker idle connection 关闭后的 EOF/broken pipe 恢复行为。官方 FAQ 中曾提到 Sarama - broken pipe；当前 master 已使用 bounded retry 和 Sarama fork ordering fix。franz-go - 是否改善该场景，需要专门压测。 -- Confluent Cloud 中 `min.insync.replicas` 不可见时的容错告警仍然保留。 -- KOP 对 DescribeConfig 返回项的兼容性。 -- 动态 topic 表达式和多 topic checkpoint 广播。 - -## 可靠性和资源管理 - -需要明确设计： - -- async producer close 是否等待 flush。当前 Sarama close 为避免阻塞，会异步关闭并接受 - 可能重复数据。franz-go `Client.Close()` / cancel / `Flush` 的使用必须与这个策略一致， - 不能在 sink 关闭路径无限阻塞。 -- producer promise 串行执行，不能在 promise 中执行阻塞操作。 -- franz-go `MaxBufferedRecords` 默认 10000，`MaxBufferedBytes` 默认无限。TiCDC 上游已有 - unlimited channel，需要评估双重缓冲是否导致内存放大，并为高流量场景设置或暴露合理 - 上限。 -- admin、async producer、sync producer 是否共享同一个 `kgo.Client`。第一阶段建议和 - 现状一致，每个组件独立 client，降低 close 生命周期复杂度；共享 client 可作为后续优化。 -- request timeout、record delivery timeout、record retries 之间的关系必须有限制，避免 - transient error 下无限阻塞 changefeed。 - -## 性能对比和验收方法 - -不要在没有 TiCDC A/B 数据前宣称性能提升。建议基准如下： - -环境变量： - -- Go version、TiCDC commit、Kafka version、broker 数、topic partition 数、replication - factor、`min.insync.replicas`。 -- 是否启用 TLS/SASL。 -- `required-acks`、producer compression、large-message compression、`max-message-bytes`。 - -workload： - -- Canal-JSON 小行高吞吐。 -- Open Protocol batch encode，覆盖 `max-batch-size`。 -- Avro + Schema Registry / Glue。 -- Simple JSON/Avro，覆盖 BOOTSTRAP。 -- Debezium JSON/Avro。 -- 大消息：普通超限、message-level compression、handle-key-only、claim-check、 - claim-check-raw-value。 -- 多 topic 动态路由、单大表多 partition、高 partition 数。 -- 低流量长 idle，验证连接保活和 broker idle close。 - -指标： - -- rows/sec、bytes/sec。 -- Kafka produce latency p50/p95/p99。 -- CPU、heap、allocs/op、goroutine 数。 -- producer request rate、response rate、in-flight、records/request、compression ratio。 -- TiCDC changefeed checkpoint lag、resolved ts lag。 -- 错误率、重试次数、重建次数。 - -验收标准： - -- 功能正确性优先。性能不得显著退化；若有吞吐/延迟 trade-off，必须说明对应配置。 -- metrics 名称和 label 兼容，或明确给出 dashboard/alert 迁移方案。 -- A/B 报告要列出 Sarama 和 franz-go 的完整 producer 配置，避免比较默认值不同的结果。 - -## 测试计划 - -单元测试： - -- options 合并和默认值：URI 覆盖 config、非法 client ID、非法 acks、非法 partition。 -- franz-go option mapping：acks、compression、linger、retry、idempotency、manual partition、 - batch max bytes、buffer limits、timeouts。 -- TLS/SASL：PLAIN、SCRAM、OAuth、GSSAPI user/keytab。 -- admin wrapper：topic 存在/不存在、已存在 topic、invalid replication factor、policy - violation、config not found、Confluent Cloud fallback。 -- async producer：成功 callback、错误返回、context cancel、close、message log info。 -- sync producer：DDL partition 0、Open Protocol broadcast、checkpoint broadcast、partial failure。 -- message size：franz-go overhead 校准、large message option 触发点。 -- metrics collector：hook 数据转换、label cleanup。 - -集成测试： - -- 复用现有 Kafka integration cases，分别跑 Sarama 和 franz-go。 -- 增加 ACL 测试：仅官方最小权限时 franz-go 默认配置必须能写入；若打开 idempotency， - 缺少 `IDEMPOTENT_WRITE` 要有明确错误。 -- 增加 idle connection / broken pipe 场景。 -- 增加 Kafka 2.1、2.4、2.8、3.x、Confluent Cloud 或兼容环境。 -- 增加 TLS/SASL/GSSAPI/OAuth 覆盖。 -- 增加 topic 已存在且 partition 数大于配置、partition 数小于配置、auto-create=false。 - -建议命令按改动范围选择： - -- `make unit_test_pkg PKG=./pkg/sink/kafka/...` -- `make unit_test_pkg PKG=./downstreamadapter/sink/...` -- `make integration_test_kafka CASE=` -- 最终切默认前跑完整 `make unit_test` 和 Kafka integration suite。 - -## 实施拆分建议 - -1. 增加 franz-go 依赖和内部 factory 实现，但默认仍走 Sarama。 -2. 实现 `franzAdminClient`，让 `adjustOptions`、topic manager 测试先通过。 -3. 实现 `franzSyncProducer`，先覆盖 DDL/checkpoint。 -4. 实现 `franzAsyncProducer`，覆盖 callback、错误、close、backpressure。 -5. 完成配置映射和 SASL/TLS/OAuth/GSSAPI。 -6. 完成消息大小估算替换和大消息测试。 -7. 完成 metrics collector 和 logger/leakutil 清理。 -8. 增加灰度开关和 A/B 测试。 -9. 满足功能、性能、可靠性验收后，再决定是否切默认并保留 Sarama 回滚窗口。 -10. 最后移除 Sarama 依赖前，清理 `pkg/security/sasl.go`、logger、leak helper、mocks、 - go.mod/go.sum 和文档中的 Sarama 表述。 - -## 待决策项 - -- 默认是否禁用 idempotency：建议禁用，保持现有 ACL 和 Sarama 非幂等 producer 语义。 -- `kafka-version` 是完全固定 franz-go MaxVersions，还是继续自动探测并只在用户显式指定时 - 固定。 -- message size overhead 使用精确编码计算还是保守上界。 -- metrics 是自研 hook collector 还是迁移到 kprom 指标名。 -- 是否需要公开 `kafka-client=franz|sarama` 灰度参数；如果公开，需要 API 兼容评审和文档。 -- admin/sync/async 是否共享 `kgo.Client`。 - -## 结论 - -franz-go 替换 Sarama 的代码入口集中在 `pkg/sink/kafka`，但完整替换不是单纯把 -`sarama.ProducerMessage` 换成 `kgo.Record`。必须显式复刻 TiCDC Kafka sink 的产品契约： -配置兼容、最小 ACL、协议输出、大消息处理、顺序保证、at-least-once、DDL/checkpoint -广播、metrics 和错误诊断。 - -最容易被遗漏、也最可能影响线上正确性的点是： - -- franz-go 默认 idempotent write 带来的 ACL 和语义变化； -- franz-go 默认 snappy compression、10ms linger、unlimited record retries； -- Sarama record overhead 被用于 TiCDC 消息大小判断； -- 既有 Kafka producer Prometheus 指标来自 Sarama registry； -- GSSAPI、OAuth、Confluent Cloud、KOP 这类非本地单机 Kafka 场景。 - -这些点全部有测试和灰度证据后，才能考虑把 franz-go 设为默认实现。 diff --git a/docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md b/docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md deleted file mode 100644 index 5899893cd4..0000000000 --- a/docs/design/2026-07-03-franz-go-replace-sarama-migration-steps.md +++ /dev/null @@ -1,672 +0,0 @@ -# 使用 franz-go 替换 Sarama 的迁移步骤 - -## 目标 - -本文给出在 TiCDC Kafka sink 中用 `~/go/franz-go` 替换 Sarama 的迁移步骤。 -配套功能审计见： - -- `docs/design/2026-07-03-franz-go-replace-sarama-audit.md` - -本文关注“怎么迁移”：分阶段实现、验证、灰度、切默认、回滚和清理。 -迁移默认采用 expand-migrate-contract 方式：先新增 franz-go 实现并保持 Sarama 可用， -再灰度切流量，最后在兼容窗口结束后移除 Sarama。 - -## 迁移性质 - -| 项目 | 结论 | -| --- | --- | -| 状态源 | Kafka topic 中的 change event、TiCDC checkpoint/resolved 进度、changefeed config、Prometheus 指标。 | -| 外部可见面 | sink URI/config、Kafka 消息协议、topic/partition 顺序、ACL、metrics、日志、错误语义。 | -| 可逆性 | 分阶段可逆；切默认前可通过配置退回 Sarama；移除 Sarama 依赖后只剩版本回滚。 | -| 主要风险 | producer 默认行为差异、消息大小估算、callback 时机、metrics 兼容、SASL/GSSAPI、ACL 变化。 | -| 推荐策略 | 默认禁用 franz-go idempotency；保留 Sarama 回滚路径至少一个发布窗口。 | - -## 总体阶段 - -1. 准备阶段：冻结兼容契约，补足 baseline 测试和 A/B 工具。 -2. Expand：新增 franz-go 适配层和选择开关，默认仍使用 Sarama。 -3. 功能迁移：先 admin，再 sync producer，再 async producer，再 TLS/SASL/metrics。 -4. Shadow / 对照验证：同配置跑 Sarama 与 franz-go，确认功能、顺序、错误和指标。 -5. 小流量灰度：按 changefeed 逐步启用 franz-go，保留即时回滚。 -6. 切默认：把默认实现从 Sarama 切到 franz-go，但仍保留 `sarama` fallback。 -7. Contract：兼容窗口结束后移除 Sarama 代码、依赖和文档残留。 - -## 阶段 0：准备和基线 - -### 0.1 明确兼容契约 - -产出： - -- 一份确认过的兼容清单，至少覆盖： - - sink URI/config key 和默认值。 - - Kafka 版本支持矩阵。 - - 最小 ACL 要求。 - - protocol 输出格式。 - - DDL/checkpoint/resolved broadcast 规则。 - - DML callback 时机。 - - Kafka producer metrics 名称和 label。 - - 错误类型、错误码和关键日志字段。 - -进入下一步前必须确认： - -- 不公开改变 `required-acks`、`compression`、`max-retry`、`kafka-version`、 - `max-message-bytes` 等默认行为。 -- 不复用已 deprecated 的 `enable-kafka-sink-v2` 作为 franz-go 开关。 -- 如果要新增用户可见开关，例如 `kafka-client=franz|sarama`，需要单独做兼容评审。 - -### 0.2 建立 Sarama baseline - -建议先在未引入 franz-go 的 `master` 上记录 baseline： - -- unit tests： - - `make unit_test_pkg PKG=./pkg/sink/kafka/...` - - `make unit_test_pkg PKG=./downstreamadapter/sink/...` -- Kafka integration tests： - - canal-json basic。 - - open-protocol basic。 - - avro / schema registry。 - - simple protocol。 - - large message / claim-check / handle-key-only。 - - dispatcher / dynamic topic。 - - mq sink error resume。 -- 性能 baseline： - - 小行高吞吐。 - - 大消息。 - - 多 partition。 - - 低流量长 idle。 - -记录内容： - -- TiCDC commit、Go version、Kafka version、broker 配置。 -- sink URI 和 changefeed config。 -- rows/sec、bytes/sec、send latency p95/p99、CPU、heap、goroutine。 -- Kafka producer request rate、response rate、in-flight、records/request、compression ratio。 -- changefeed checkpoint lag / resolved ts lag。 - -退出条件： - -- Sarama baseline 本身稳定。 -- 已知 flaky test 单独记录，不和 franz-go 替换混在一起判断。 - -## 阶段 1：新增 franz-go 适配骨架 - -### 1.1 引入依赖 - -改动点： - -- `go.mod` / `go.sum` - - `github.com/twmb/franz-go/pkg/kgo` - - `github.com/twmb/franz-go/pkg/kadm` - - `github.com/twmb/franz-go/pkg/kmsg` - - `github.com/twmb/franz-go/pkg/sasl/plain` - - `github.com/twmb/franz-go/pkg/sasl/scram` - - `github.com/twmb/franz-go/pkg/sasl/oauth` - - `github.com/twmb/franz-go/pkg/sasl/kerberos` - -注意： - -- 不在这一阶段移除 Sarama。 -- 不把 franz-go metrics plugin 直接作为最终指标方案，除非已决定迁移指标名。 - -### 1.2 保持现有接口 - -优先不改上层 sink，只新增实现： - -- `pkg/sink/kafka/franz_factory.go` -- `pkg/sink/kafka/franz_config.go` -- `pkg/sink/kafka/franz_admin.go` -- `pkg/sink/kafka/franz_sync_producer.go` -- `pkg/sink/kafka/franz_async_producer.go` -- `pkg/sink/kafka/franz_metrics_collector.go` - -保留接口： - -- `Factory` -- `ClusterAdminClient` -- `AsyncProducer` -- `SyncProducer` -- `MetricsCollector` - -退出条件： - -- 新代码可编译。 -- 默认路径仍是 Sarama。 -- 没有任何用户在未显式启用时走 franz-go。 - -### 1.3 增加选择开关 - -推荐先用内部灰度开关，不立即变成公开文档承诺： - -- 方案 A：内部 config/env/build tag，用于 CI 和受控灰度。 -- 方案 B：公开 sink URI 参数 `kafka-client=sarama|franz`。 - -推荐顺序： - -1. 第一阶段用内部开关验证。 -2. 如果需要用户级灰度，再公开 `kafka-client`，并补充文档、测试和 release note。 - -实现要求： - -- 默认值必须是 `sarama`。 -- 无效值返回明确配置错误。 -- 切换只影响 Kafka client 层，不影响 encoder、event router、topic manager。 - -回滚： - -- 将开关改回 `sarama`。 -- 如果开关在 changefeed config 中，回滚不应要求删除 changefeed。 - -## 阶段 2：迁移 admin client - -先迁移 admin，是因为 `adjustOptions` 和 topic manager 依赖它，且不触碰数据写入。 - -### 2.1 实现 `franzAdminClient` - -需要实现： - -- `GetAllBrokers` -- `GetBrokerConfig` -- `GetTopicConfig` -- `GetTopicsMeta` -- `GetTopicsPartitionsNum` -- `CreateTopic` -- `Close` - -映射建议： - -- 使用 `kadm.Client` 做 topic metadata、create topic、describe config。 -- `GetBrokerConfig` 要确认是否等价于当前 Sarama 从 controller broker 读取 config 的行为。 -- `TopicAlreadyExists` 继续按成功处理。 -- `UnknownTopicOrPartition` 在 `ignoreTopicError=true` 时跳过。 -- `ErrKafkaConfigNotFound` 语义保持。 - -### 2.2 admin 单元测试 - -覆盖： - -- topic 存在，partition 数读取正确。 -- topic 不存在，ignore true/false 行为正确。 -- create topic 成功。 -- create topic 时 topic 已存在。 -- invalid replication factor / policy violation / authorization error 保留原始诊断。 -- broker/topic config 找不到时保留现有 fallback 和告警语义。 -- Confluent Cloud 下 `min.insync.replicas` 不可见时保持允许启动但告警。 - -退出条件： - -- `adjustOptions` 测试可在 franz admin wrapper 下通过。 -- `topicmanager` 测试可在 franz admin wrapper 下通过。 -- 默认 Sarama 测试未回归。 - -## 阶段 3：迁移 sync producer - -sync producer 负责 DDL 和 checkpoint，流量低但正确性要求高。 - -### 3.1 实现 `franzSyncProducer` - -要求： - -- `SendMessage(topic, partitionNum, message)` 发送单条 record 到指定 partition。 -- `SendMessages(topic, partitionNum, message)` 发送 `0..partitionNum-1` 每个 partition 一条。 -- 使用 `ProduceSync(ctx, records...)` 或等价同步路径。 -- 任一 partition 失败时返回错误。 -- 错误必须通过 `AnnotateEventError` 附加 DDL/checkpoint log info。 - -配置必须显式设置： - -- `RecordPartitioner(kgo.ManualPartitioner())` -- `RequiredAcks(...)` -- `ProducerBatchCompression(...)` -- `ProducerLinger(0)` -- `RecordRetries(options.MaxRetry)` -- `DisableIdempotentWrite()` -- `MaxProduceRequestsInflightPerBroker(1)` -- `ProducerBatchMaxBytes(...)` - -### 3.2 sync producer 测试 - -覆盖： - -- Canal-JSON DDL 到 partition 0。 -- Open Protocol DDL broadcast 到全部 partition。 -- checkpoint/resolved broadcast 到全部 partition。 -- no table 时 checkpoint 发 default topic。 -- 部分 partition 发送失败时返回错误。 -- `required-acks=0/1/-1` 配置映射。 - -退出条件： - -- DDL/checkpoint 单元测试通过。 -- Kafka integration 中 DDL、checkpoint、resolved 语义和 Sarama 对齐。 - -回滚： - -- `kafka-client=sarama`。 -- 因为没有改变 Kafka 消息格式，已经写入的 DDL/checkpoint 可继续被消费者按原协议读取。 - -## 阶段 4：迁移 async producer - -async producer 是 DML 主路径，必须最后接入，并且先在可回滚模式下运行。 - -### 4.1 实现 `franzAsyncProducer` - -要求： - -- `AsyncSend(ctx, topic, partition, message)`： - - 构造 `kgo.Record{Topic, Partition, Key, Value}`。 - - record partition 必须来自 TiCDC event router，不能让 franz-go 重新 hash。 - - 发送前设置 message partition key，保持日志和统计语义。 -- promise / callback： - - `err == nil` 时执行 `message.Callback`。 - - `err != nil` 时不执行 callback。 - - promise 不做阻塞操作，不调用 `Flush` 或可能阻塞的 `Produce`。 -- `AsyncRunCallback(ctx)`： - - 等待首个 produce error 或 ctx done。 - - 首个 produce error 返回给 sink，让 sink 重建。 - - 返回错误带 TiCDC stack 和 `MessageLogInfo`。 -- `Close()`： - - 不在 sink 关闭路径无限等待 flush。 - - 明确是否允许未 ack record 后续由上游重放造成重复。 - -### 4.2 async producer 测试 - -覆盖： - -- 成功后 callback 恰好执行一次。 -- producer error 时 callback 不执行，`AsyncRunCallback` 返回错误。 -- context cancel 时可退出。 -- close 不阻塞。 -- manual partition 生效。 -- `required-acks=0` 下 callback 语义和 Sarama 对齐。 -- `RecordRetries` 耗尽时同 partition 不越过失败 record。 - -退出条件： - -- DML 单元测试通过。 -- Kafka integration 中 DML 顺序、重复和恢复语义与 Sarama 对齐。 -- `mq_sink_error_resume` 类场景通过。 - -## 阶段 5：配置、TLS、SASL 和版本映射 - -这一阶段不要改变用户配置名，只做映射。 - -### 5.1 producer option 映射 - -必须显式覆盖 franz-go 默认值： - -| 配置 | franz-go 设置 | -| --- | --- | -| brokers | `kgo.SeedBrokers(...)` | -| client id | `kgo.ClientID(...)` | -| kafka version | `kgo.MaxVersions(...)` 或等价固定版本能力 | -| required acks | `kgo.RequiredAcks(...)` | -| compression | `kgo.ProducerBatchCompression(...)` | -| max message bytes | `kgo.ProducerBatchMaxBytes(...)` | -| retry | `kgo.RecordRetries(options.MaxRetry)` + backoff | -| linger | `kgo.ProducerLinger(0)` | -| partition | `kgo.RecordPartitioner(kgo.ManualPartitioner())` | -| idempotency | 默认 `kgo.DisableIdempotentWrite()` | -| inflight | `kgo.MaxProduceRequestsInflightPerBroker(1)` | -| buffer | 明确 `MaxBufferedRecords` / `MaxBufferedBytes` 策略 | - -### 5.2 TLS 映射 - -覆盖： - -- 系统 CA + `enable-tls=true`。 -- 自签 CA + cert/key。 -- cert/key/ca 不完整时仍报配置错误。 -- `enable-tls=false` 但配置证书时仍报配置错误。 -- `insecure-skip-verify` 只在 TLS 开启时生效。 - -### 5.3 SASL 映射 - -覆盖： - -- PLAIN。 -- SCRAM-SHA-256。 -- SCRAM-SHA-512。 -- OAUTHBEARER： - - base64 secret 解码。 - - token URL。 - - scopes。 - - grant type。 - - audience。 -- GSSAPI： - - user/password auth。 - - keytab auth。 - - service name。 - - realm。 - - kerberos config path。 - - disable PAFXFAST。 - -注意： - -- `pkg/security/sasl.go` 当前引用 Sarama 常量。只要 Sarama 还没移除，可以先保留； - contract 阶段必须改成 TiCDC 自有常量。 -- GSSAPI 不能只靠单元测试，至少需要一个可运行的 Kerberos/Kafka 集成验证或明确记录 - 未覆盖风险。 - -### 5.4 Kafka version 映射 - -要求： - -- 用户显式 `kafka-version` 必须生效。 -- 无法解析版本仍返回 `ErrKafkaInvalidVersion` 或等价 TiCDC 错误。 -- 未指定版本时可继续自动协商，但不能扩大产品支持承诺。 -- 版本不匹配的告警体验尽量保留。 - -退出条件： - -- 所有配置映射测试通过。 -- 旧 sink URI/config 不修改即可加载。 -- 官方最小 ACL 下可以启动并写入。 - -## 阶段 6：消息大小和大消息路径 - -这一步是切 DML 流量前的硬门槛。 - -### 6.1 替换 size accounting - -当前 `Message.Length()` 使用 Sarama `MaxRecordOverhead`。迁移步骤： - -1. 写一个 franz-go record batch size 估算 helper。 -2. 用测试对照 franz-go 实际编码或 producer 拒绝边界。 -3. 将 `Message.Length()` 或其调用方切到新的估算方式。 -4. 保留或重新论证 `maxMessageBytesOverhead=128` safety margin。 -5. 对 open-protocol batch splitter、large-message compression、claim-check 都加测试。 - -### 6.2 大消息测试 - -覆盖： - -- 普通消息接近 `max-message-bytes`。 -- 单行大于限制。 -- Open Protocol batch 被拆分。 -- message-level lz4/snappy compression 后不过限。 -- `handle-key-only`。 -- `claim-check`。 -- `claim-check-raw-value`。 -- broker/topic `message.max.bytes` 小于用户配置。 - -退出条件： - -- 不出现“TiCDC 判定可发送但 franz-go/broker 拒绝”的边界误差。 -- 不出现“TiCDC 过早 claim-check”的明显误差。 -- `Message was too large` 错误仍可诊断。 - -## 阶段 7：metrics、日志和泄漏检查 - -### 7.1 metrics 兼容 - -默认要求保留现有指标名和 label： - -- `ticdc_sink_kafka_producer_in_flight_requests` -- `ticdc_sink_kafka_producer_outgoing_byte_rate` -- `ticdc_sink_kafka_producer_request_rate` -- `ticdc_sink_kafka_producer_request_latency` -- `ticdc_sink_kafka_producer_compression_ratio` -- `ticdc_sink_kafka_producer_records_per_request` -- `ticdc_sink_kafka_producer_response_rate` - -实现步骤： - -1. 基于 franz-go hooks 实现 TiCDC collector。 -2. 对齐 Sarama collector 的 label：`namespace`、`changefeed`、`broker`、`type`。 -3. 明确 in-flight 的等价口径。 -4. 在 changefeed stop/delete 后清理 label。 -5. A/B 对比指标是否在同一数量级。 - -如果决定改指标名： - -- 必须提供 dashboard / alert 迁移方案。 -- 需要至少一个版本同时暴露新旧指标。 -- release note 必须说明。 - -### 7.2 logger 和 leakutil - -步骤： - -- 给 franz-go 接 `kgo.WithLogger`。 -- 保留 Kafka client 日志中的 keyspace/changefeed 上下文。 -- 清理 `WithInitSaramaLogger` 的依赖路径，但不要在 Sarama fallback 存在期间破坏 Sarama。 -- 更新 `pkg/leakutil/leak_helper.go`，不要继续用 Sarama goroutine ignore 掩盖新泄漏。 - -退出条件： - -- `kafka_log_info` 类测试通过。 -- goroutine leak 测试不需要新增宽泛 ignore。 -- metrics cleanup 测试通过。 - -## 阶段 8：Shadow 和 A/B 验证 - -目标是确认 franz-go 在相同 TiCDC/Kafka 配置下不改变语义。 - -### 8.1 本地/CI 对照 - -对每组 case 跑两次： - -- `kafka-client=sarama` -- `kafka-client=franz` - -比较： - -- 下游行数。 -- DDL 顺序。 -- partition 分布。 -- row-level checksum。 -- checkpoint/resolved 推进。 -- 错误恢复后的重复消息是否仍可由 protocol 语义处理。 -- Kafka producer metrics。 - -### 8.2 性能对照 - -至少覆盖： - -- 无压缩。 -- gzip/snappy/lz4/zstd。 -- `required-acks=-1`。 -- `required-acks=1`。 -- TLS/SASL。 -- 高 partition 数。 -- 大消息。 - -退出条件： - -- 正确性无差异。 -- 性能无不可解释显著退化。 -- 内存和 goroutine 无明显泄漏。 -- 失败场景的恢复方式可解释。 - -## 阶段 9：小流量灰度 - -### 9.1 灰度前置条件 - -必须满足： - -- 默认仍是 Sarama。 -- 每个灰度 changefeed 可单独切回 Sarama。 -- operator 知道回滚命令。 -- dashboard 同时能看 Kafka sink lag、producer error、request latency、resource usage。 -- Kafka ACL 是官方最小权限时，franz-go 已验证可写入。 - -### 9.2 灰度顺序 - -推荐顺序： - -1. 内部测试环境，单 changefeed，单 topic，低流量。 -2. 内部测试环境，多 topic / 动态 topic。 -3. 预发环境，真实 schema，低写入。 -4. 生产 canary，低风险 changefeed。 -5. 生产扩大到 5%。 -6. 生产扩大到 25%。 -7. 生产扩大到 50%。 -8. 切默认前维持观察窗口。 - -每一档观察： - -- checkpoint lag / resolved lag。 -- Kafka producer error rate。 -- request latency p99。 -- DML callback backlog。 -- broker request/response rate。 -- CPU、heap、goroutine。 -- topic partition 写入分布。 -- DDL 和 checkpoint 是否正常推进。 - -### 9.3 回滚动作 - -可回滚点： - -- 切默认前：把 changefeed 的 Kafka client 选择改回 Sarama。 -- 切默认后但保留 fallback：显式设置 `kafka-client=sarama` 或回滚默认配置。 -- 移除 Sarama 后：只能回滚二进制版本。 - -回滚后验证： - -- changefeed 恢复 running。 -- checkpoint/resolved 继续推进。 -- 下游消费者能处理可能重复的 at-least-once 消息。 -- 大消息 claim-check 外部存储没有新增不可读 marker。 -- metrics 回到 Sarama collector。 - -## 阶段 10：切默认 - -切默认的进入条件： - -- franz-go 路径完成至少一个发布候选版本或一个充分观察窗口。 -- 所有 P0 风险关闭。 -- A/B 性能报告已归档。 -- metrics 和日志兼容。 -- 回滚路径演练过。 -- 官方文档和 release note 已准备。 - -切默认步骤： - -1. 将默认 Kafka client 从 Sarama 改为 franz-go。 -2. 保留显式 `kafka-client=sarama` fallback。 -3. release note 说明： - - 默认 Kafka client 改变。 - - 配置兼容。 - - ACL 不需要新增 `IDEMPOTENT_WRITE`，因为默认禁用 idempotency。 - - 已知差异或调优建议。 -4. 灰度发布。 -5. 观察至少一个完整业务周期。 - -切默认后监控： - -- Kafka sink error rate。 -- changefeed restart count。 -- Kafka produce latency。 -- checkpoint lag。 -- broker throttle。 -- message too large。 -- auth failures。 -- metrics cardinality。 - -回滚： - -- 优先配置回滚到 Sarama。 -- 如果默认切换导致启动期失败，可回滚二进制。 -- 不需要迁移 Kafka topic 数据，因为消息协议未改变。 - -## 阶段 11：Contract 和移除 Sarama - -只有在兼容窗口结束后执行。 - -前置条件： - -- 没有线上 changefeed 仍配置 `kafka-client=sarama`。 -- 至少一个稳定版本周期内 franz-go 是默认实现。 -- 没有未关闭的 franz-go P0/P1 correctness issue。 -- 运营 dashboard 和 alert 不再依赖 Sarama-only 指标来源。 - -清理项： - -- 删除 `sarama_factory.go`。 -- 删除 `sarama_config.go`。 -- 删除 `sarama_async_producer.go`。 -- 删除 `sarama_sync_producer.go`。 -- 删除或更新 Sarama 专属 mocks/tests。 -- 将 `pkg/security/sasl.go` 中的 Sarama 常量改为 TiCDC 自有常量。 -- 移除 `pkg/logger/log.go` 中 Sarama logger 初始化。 -- 移除 `pkg/leakutil/leak_helper.go` 中 Sarama goroutine ignore。 -- 删除 `go.mod` / `go.sum` 中不再需要的 Sarama 依赖。 -- 更新 Kafka sink 文档中 Sarama client id 或 Sarama 行为描述。 - -Contract 阶段测试： - -- `make unit_test_pkg PKG=./pkg/sink/kafka/...` -- `make unit_test_pkg PKG=./downstreamadapter/sink/...` -- `make unit_test_pkg PKG=./pkg/security/...` -- `make cdc` -- Kafka integration suite。 -- `make check`，用于确认 go.mod、format、codegen 等。 - -回滚： - -- Contract 后不能配置回滚到 Sarama，只能回滚二进制版本。 -- 如果需要保留更强回滚能力，不要执行 Contract。 - -## 关键验收门槛 - -以下任一项未满足，不应切默认： - -- DML callback 时机未被测试证明。 -- 手动 partition 未被测试证明。 -- 大消息 size accounting 未完成。 -- 官方最小 Kafka ACL 下未验证。 -- TLS/SASL/GSSAPI/OAuth 未覆盖。 -- `required-acks=0/1/-1` 未覆盖。 -- 既有 Kafka producer metrics 未兼容或未提供迁移方案。 -- error resume / broken pipe / idle connection 场景未覆盖。 -- 没有 Sarama fallback。 -- 没有回滚演练。 - -## 推荐 PR 拆分 - -1. PR 1：新增 franz-go dependency、config builder skeleton、默认不启用。 -2. PR 2：franz admin client + admin/topic manager tests。 -3. PR 3：franz sync producer + DDL/checkpoint tests。 -4. PR 4：franz async producer + callback/error/close tests。 -5. PR 5：TLS/SASL/OAuth/GSSAPI mapping tests。 -6. PR 6：message size accounting + large message tests。 -7. PR 7：franz metrics collector + logger/leakutil。 -8. PR 8：integration tests and A/B scripts。 -9. PR 9：controlled gray switch documentation / release note。 -10. PR 10：切默认，保留 Sarama fallback。 -11. PR 11：Contract 移除 Sarama，需等兼容窗口结束。 - -## 运行手册摘要 - -启用 franz-go 前： - -1. 确认 Kafka ACL 没有依赖 franz-go idempotency。 -2. 确认 topic `max.message.bytes` 和 sink `max-message-bytes`。 -3. 确认 TLS/SASL 配置在 franz-go 路径验证过。 -4. 确认 dashboard 已能看 franz-go collector。 -5. 确认回滚命令。 - -启用后观察： - -1. 10 分钟内无 producer error spike。 -2. checkpoint lag 不持续增长。 -3. produce latency p99 不异常。 -4. broker throttle 不异常。 -5. consumer 没有解析错误。 - -触发回滚： - -- Kafka auth/ACL error。 -- message too large 明显增加。 -- checkpoint lag 持续增长。 -- DDL/checkpoint/resolved 不推进。 -- producer goroutine 或 heap 持续增长。 -- 下游消费者出现协议解析错误。 - -回滚后： - -- 确认 changefeed running。 -- 确认 checkpoint 继续推进。 -- 确认消费者可处理重复消息。 -- 保留 franz-go 错误日志、metrics 和 Kafka broker logs 供根因分析。 diff --git a/docs/franz-go/franz-go-ga-test-plan.md b/docs/franz-go/franz-go-ga-test-plan.md deleted file mode 100644 index ece35d2980..0000000000 --- a/docs/franz-go/franz-go-ga-test-plan.md +++ /dev/null @@ -1,87 +0,0 @@ -# TiCDC Kafka Sink franz-go GA 测试计划 - -Last updated: 2026-09-02 -Status: 评审稿 -Scope: franz-go Kafka Sink 的正确性、故障恢复、性能和可观测性验证 -Related documents: - -- [执行计划](https://pingcap.feishu.cn/wiki/YK6UwCWn0iNvlfkAGDncAOK2nWh) -- [Milestone 1 TODO List](./milestone-1-todo-list.md) - -## 1. 测试原则 - -- 测试以最终行为为准，不限定 SDK、caselib 或 Test Plan 的具体实现。 -- 现有 testcase 能覆盖的场景，使用指定的 franz-go TiCDC image 直接运行，不改造 test-infra。 -- 只有现有 test-infra 无法构造或验证的场景才新增代码。 -- correctness testcase 负责消费和数据一致性校验；专项 testcase 只验证对应能力，不重复完整业务流程。 - -## 2. 正确性 - -执行方式：使用指定的 franz-go TiCDC image 运行现有 Kafka testcase，不需要修改 test-infra。 - -- [ ] 覆盖初始同步、增量同步、支持的协议和 dispatcher。 -- [ ] 覆盖 message-size 边界、large message、claim-check 和 handle-key-only。 -- [ ] 覆盖 Topic 自动创建、已有 Topic、partition 变化、Topic 配置和 Schema Registry 正常与异常路径。 -- [ ] 覆盖多 Changefeed、扩缩容和 HA，并校验 callback、checkpoint、消息顺序和最终消费结果。 - -通过标准：所有 correctness testcase 通过，不存在数据丢失、重复 callback 或 checkpoint 提前推进。 - -## 3. 鲁棒性与故障恢复 - -执行方式：复用现有 Kafka chaos testcase；只为缺失场景新增 test-infra 代码。 - -- [ ] 补充多 broker 故障、滚动升级、metadata 变化、request timeout、retry、idle connection 和 broken pipe 场景。 -- [ ] 补充 controller 和 broker 的网络延迟、丢包及更多网络分区组合。 -- [ ] 验证 broker 长时间不可用时内存有界，取消和关闭能够解除等待。 -- [ ] 验证恢复期间 partition 内消息顺序、callback 和 retry 行为。 -- [ ] 使用 Kafka 集群状态、checkpoint 和数据一致性判断恢复结果，不使用固定 sleep。 -- [ ] 失败时保存 TiCDC、Kafka、consumer 日志、关键 metrics、集群状态和测试时间范围。 - -通过标准：每个故障用例都能命中目标节点；故障解除后 Kafka 集群恢复可用，checkpoint 最终追平，消费结果与上游一致，资源使用保持有界。 - -## 4. 性能 - -执行方式：新增三 broker 性能 testcase。 - -- [ ] Kafka Topic replication factor 为 3，`min.insync.replicas` 为 2。 -- [ ] 固定 Kafka 集群、Topic、partition、TiCDC 规格、workload、预热方式、数据规模和重复次数。 -- [ ] 覆盖 sysbench、bank、jitu，单表和多表，以及 table 和 index-value dispatcher。 -- [ ] 执行 Changefeed create、pause、workload、resume、catch-up 和一致性校验。 -- [ ] 记录吞吐、catch-up 时间、p99、TiCDC/Kafka CPU 和内存、GC、goroutine、heap、batch、buffer、retry、error 和 consumer lag。 -- [ ] 保存测试参数、代码版本、资源规格和原始时间序列，并复验超出阈值的结果。 - -通过标准：所有场景完成一致性校验，吞吐、延迟、CPU 和内存满足确定的阈值。 - -## 5. 可观测性 - -执行方式：新增 Prometheus 查询和 Dashboard 断言 testcase。 - -- [ ] 验证 franz-go logger、metrics collector 和敏感信息脱敏。 -- [ ] 覆盖吞吐、request/response rate、latency、retry、error、batch、buffer 和 broker 指标。 -- [ ] 验证 metrics label 不包含非预期高基数值，Changefeed 关闭后对应 series 被清理。 -- [ ] 分别产生正常写入、retry、broker 故障和恢复流量，验证指标随场景变化。 -- [ ] 验证 Dashboard PromQL 可执行，并保存 Prometheus 原始结果、Dashboard 定义和对应日志。 - -通过标准：每个 franz-go Dashboard panel 都有自动查询断言，日志和指标足以诊断正常写入、重试、错误和恢复。 - -## 6. 代码变更验证 - -仅在新增或修改 test-infra 代码时执行： - -- [ ] 在仓库根目录运行 `go test ./common/model/resource/...`。 -- [ ] 在 `sdk` 模块运行 `go test ./resource/impl/k8s/...`。 -- [ ] 在 `caselib` 模块运行 `go test ./pkg/model/kafka/... ./pkg/steps/...`。 -- [ ] 运行新增端到端 Test Plan，并保存资源状态和诊断产物。 - -## 7. 阻塞输入 - -- [ ] 确定性能场景的数据量、partition 数、重复次数以及吞吐、延迟、CPU 和内存阈值。 -- [ ] 确定 franz-go 指标名、label 和 Dashboard 定义。 - -## 8. 执行阶段 - -- 开发与 PR：运行改动涉及的单元测试和集成测试。 -- Nightly：运行完整 correctness regression、故障恢复和可观测性测试。 -- Release：在 Nightly 范围上增加完整性能和稳定性验证。 - -GA 前不得遗留阻塞发布的正确性、稳定性、资源、性能或可观测性问题。 diff --git a/docs/franz-go/kafka-producer-idempotence-design.md b/docs/franz-go/kafka-producer-idempotence-design.md deleted file mode 100644 index 8f37591914..0000000000 --- a/docs/franz-go/kafka-producer-idempotence-design.md +++ /dev/null @@ -1,152 +0,0 @@ -# Kafka Producer Idempotence 设计 - -Last updated: 2026-09-03 -Status: 待讨论 -Scope: franz-go producer 的幂等写入 -Related documents: - -- [Milestone 1 TODO List](./milestone-1-todo-list.md) - -## Background - -TiCDC producer 会重试可恢复的 Kafka 写入错误。下面的故障会产生重复消息： - -1. Producer 向 Broker 发送一个 record batch。 -2. Broker 已经把 record batch 写入日志。 -3. Broker 响应在网络中丢失，或者 Producer 等待响应超时。 -4. Producer 无法判断 Broker 是否已经完成写入，因此重新发送相同数据。 -5. Broker 把重试请求作为新数据再次写入。 - -当前 Sarama producer 没有启用幂等写入。franz-go 路径也显式配置了 -[`DisableIdempotentWrite`](../../pkg/sink/kafka/franz_config.go)，因此两条路径都存在上述风险。 - -Kafka 幂等 producer 为每个 producer 分配 producer ID 和 epoch，并为每个 Topic partition -维护 sequence number。Broker 可以根据这些信息识别同一个 producer 重发的 record batch， -避免网络错误触发的 client 内部重试写入重复数据。Kafka 从 0.11 开始提供该能力，协议原理见 -[Kafka Design](https://kafka.apache.org/41/design/design/)。 - -支持幂等写入可以减少 TiCDC 正常运行期间由 franz-go 内部重试产生的重复消息。TiCDC 的 -对外投递语义继续保持 at-least-once。Producer 重建、TiCDC 重启以及从 checkpoint 重放的 -消息会使用新的 sequence number，Kafka 无法把这些消息识别为同一次发送。 - -## Proposed Behavior - -- 新增 `enable-idempotence` Kafka Sink 配置，默认值为 `false`。 -- `enable-idempotence=false` 保持当前 producer 行为。 -- `enable-idempotence=true` 只支持 franz-go，并要求 `required-acks=-1`。 -- 用户同时配置 `enable-idempotence=true` 和 `required-acks=0` 或 `1` 时，TiCDC 在创建 - Changefeed 时返回配置错误。Kafka 要求幂等 producer 使用 `acks=all`，详见 - [Kafka Producer Configs](https://kafka.apache.org/41/configuration/producer-configs/)。 -- TiCDC 不会在初始化失败后自动关闭幂等写入。权限、Broker 版本或消息格式不满足要求时， - Changefeed 返回明确错误。 -- TiCDC 启动日志记录最终是否启用幂等写入。 - -显式配置可以避免升级后自动增加 Kafka 权限要求。完成兼容性和故障测试后，可以单独讨论 -是否修改默认值。 - -## Producer Configuration - -启用幂等写入时： - -- 不配置 `DisableIdempotentWrite()`。 -- 不配置 `MaxProduceRequestsInflightPerBroker(1)`。franz-go 在幂等模式下自行选择在途请求 - 数量；支持相应 Produce API 的 Broker 最多允许 5 个在途请求，并使用 sequence number - 保证同一 partition 的消息顺序。 -- 保留 `RecordRetries(max-retry)` 和现有退避配置。 -- 配置 `AllowIdempotentProduceCancellation()`，使 `max-retry` 耗尽和调用 context 取消仍能 - 结束发送并释放 buffer。 - -关闭幂等写入时： - -- 配置 `DisableIdempotentWrite()`。 -- 保留 `MaxProduceRequestsInflightPerBroker(1)`，避免非幂等重试导致同一 partition 乱序。 - -`AllowIdempotentProduceCancellation()` 保留 franz-go 正常内部重试的 Broker 去重能力。 -发送结果仍不确定时，如果 TiCDC 在收到最终错误后重新发送相同消息，Kafka 仍可能写入重复 -数据。该行为符合 TiCDC 现有的 at-least-once 语义。franz-go 对取消行为的说明见 -[`AllowIdempotentProduceCancellation`](https://github.com/twmb/franz-go/blob/v1.21.6/pkg/kgo/config.go#L1191-L1219)。 - -## Producer ID Initialization - -一个 Changefeed 创建两个 franz-go producer client： - -- async producer 发送 DML。 -- sync producer 发送 DDL 和 checkpoint。 - -两个 client 分别持有 producer ID 和 sequence number。创建每个 producer client 后，TiCDC -调用 `client.ProducerID(ctx)`，在发送业务消息之前执行 `InitProducerID`： - -- 返回错误时，producer 创建失败。 -- producer ID 小于 0 时，producer 创建失败。 -- producer ID 有效时，producer 创建成功。 - -franz-go 遇到不支持 `InitProducerID` 的旧 Broker 时，可能返回 `producer ID = -1` 和空错误， -然后以非幂等方式继续发送。TiCDC 必须同时检查错误和 producer ID，避免静默降低用户要求的 -投递保证。 - -初始化需要使用有界 context。达到初始化 deadline 后，TiCDC 关闭该 client 并返回创建失败， -避免 Changefeed 创建过程无限等待 Kafka。 - -## Kafka Permissions - -启用幂等写入会增加 Kafka 权限要求： - -- Kafka 2.8 之前通常要求 producer principal 具有 Cluster 级 `IDEMPOTENT_WRITE`，同时具有 - 目标 Topic 的 `WRITE` 权限。 -- Kafka 2.8 及以上把 `InitProducerID` 权限放宽为对任意 Topic 具有 `WRITE` 权限。目标 Topic - 仍需要各自的 `WRITE` 权限。 -- 自定义 Authorizer 需要正确实现 Kafka 2.8 引入的权限检查接口，否则升级后的 Broker 仍可能 - 拒绝 `InitProducerID`。 - -权限变化和兼容场景见 -[KIP-679](https://cwiki.apache.org/confluence/spaces/KAFKA/pages/165221843/KIP-679%2BProducer%2Bwill%2Benable%2Bthe%2Bstrongest%2Bdelivery%2Bguarantee%2Bby%2Bdefault)。 - -TiCDC 不通过 Admin API 推测权限。`ProducerID(ctx)` 发出的真实 `InitProducerID` 请求作为权限 -检查依据。`CLUSTER_AUTHORIZATION_FAILED` 等错误应保留 Kafka 错误原因，方便用户补充 ACL。 - -## Guarantee Boundaries - -幂等写入只对同一个 producer ID、epoch 和 Topic partition 上的 client 内部重试去重。下面的 -情况仍可能产生重复消息： - -- TiCDC 在 Broker 写入成功但 callback 执行前退出，重启后从 checkpoint 重放消息。 -- franz-go client 被关闭并重新创建，新 client 获得新的 producer ID。 -- 发送结果不确定并达到取消或重试上限后，TiCDC 重新发送消息。 -- sync producer 向多个 partition 发送 DDL 或 checkpoint，其中部分 partition 成功后整体操作 - 返回失败。 - -幂等写入不提供跨 partition 原子性，也不为两个 producer client 提供共同的去重范围。实现不应 -把该功能描述为 TiCDC 到 Kafka 的 exactly-once 投递。 - -## Risks - -- 旧集群权限不足：现有 Topic 写入权限可能不足以执行 `InitProducerID`。 -- Broker 或消息格式过旧：Kafka 0.11 之前的 Broker，以及使用 v2 之前消息格式的 Topic，不能 - 接受幂等 record batch。 -- 请求并发变化：幂等模式下 franz-go 可能把每个 Broker 的在途 Produce request 增加到 5， - 从而改变吞吐和故障期间的在途数据量。 -- 取消后的重复：`AllowIdempotentProduceCancellation()` 保证有界退出，但取消后重新发送不能 - 使用原来的 sequence number 去重。 -- 部分成功：幂等写入不解决跨 partition 发送的部分成功。 -- 首次发送延迟：两个 producer client 都需要执行一次 `InitProducerID`。 - -## Verification - -- 覆盖 `enable-idempotence` 与 `required-acks=-1/1/0` 的配置组合。 -- 验证关闭幂等写入时保留 `MaxProduceRequestsInflightPerBroker(1)`。 -- 验证启用幂等写入时 sync 和 async producer 都取得有效 producer ID。 -- 模拟 Broker 已写入但首次响应丢失，确认 franz-go 内部重试后 Kafka 中只有一份记录。 -- 验证 `max-retry` 耗尽、调用 context 取消和 client 关闭都能结束 callback 并释放 buffer。 -- 验证缺少 `IDEMPOTENT_WRITE` 权限时，Changefeed 初始化返回包含 Kafka 原因的错误。 -- 覆盖 Kafka 0.11、Kafka 2.7、Kafka 2.8 及更高版本的版本和权限边界。 -- 覆盖旧 Topic 消息格式被 Broker 拒绝的错误路径。 -- 验证多个在途 batch 发生重试后，同一 partition 的消息顺序不变。 -- 验证多 partition 发送部分成功时，TiCDC 仍按 at-least-once 语义处理。 - -## Open Questions - -- `enable-idempotence` 是否只加入 Sink URI，还是同时加入 `SinkConfig`。 -- 初始化 producer ID 使用哪个 deadline。 -- 是否接受 `AllowIdempotentProduceCancellation()` 带来的取消后重复风险，或者选择严格幂等并 - 接受单次发送可能超过 `max-retry` 和调用 context deadline。 -- 完成兼容性测试后，是否把 `enable-idempotence` 的默认值改为 `true`。 diff --git a/docs/franz-go/milestone-1-todo-list.md b/docs/franz-go/milestone-1-todo-list.md deleted file mode 100644 index 4f3f78a073..0000000000 --- a/docs/franz-go/milestone-1-todo-list.md +++ /dev/null @@ -1,9 +0,0 @@ -# franz-go 待验证事项 - -## P2｜Kerberos 性能 - -- [ ] 验证长期 `kgo.Client` 复用 Kerberos client 的运行行为和收益。 - - 代码：[Kerberos client 生命周期](../../pkg/sink/kafka/franz_gssapi.go#L26)。 - - 当前行为：临时 admin 和长期共享 client 分别持有 Kerberos client；一个 `kgo.Client` 的 Broker 建连和重连共享认证状态，并在 `kgo.Client.Close` 时销毁。 - - 验证：比较连接稳定和反复重连场景的认证延迟、KDC 请求数、CPU 和分配；覆盖 password、keytab、TGT 续期、并发连接、client 关闭和 race test。 - - 完成条件：真实 Kerberos 集群的功能、并发、重连、续期、关闭和性能验证通过。 diff --git a/docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md b/docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md deleted file mode 100644 index a8044b28ce..0000000000 --- a/docs/franz-go/ticdc-kafka-franz-go-ga-test-record.md +++ /dev/null @@ -1,180 +0,0 @@ -# TiCDC franz-go Kafka GA 本地执行记录 - -更新时间：2026-09-02 - -本文记录本地 `test-plan` 中可用于 TiCDC franz-go Kafka GA 验证的测试资产及实际 execution。测试资产以当前生效的 `caseName` 为准，不统计已注释的 case。 - -## 1. GA Plan 清单 - -### 1.1 标准环境 - -共 63 个非 EKS、非 TiDB-X Plan。这里不表示执行状态，实际结果以第 3、4 节为准。 - -- 协议、数据与工作负载： - - `cdc_newarch_airbnb_simple_titan` - - `cdc-newarch-kafka-debezium-basic` - - `cdc_newarch_kafka_large_msg_claim_check` - - `cdc_newarch_kafka_large_message_handle` - - `cdc-newarch-kafka-multiple-topic` - - `cdc-newarch-kafka-realtime` - - `cdc_newarch_kafka_scale_big_table_longrun` - - `cdc_newarch_kafka_scale_big_table_ops` - - `cdc_newarch_kafka_simple_ops` - - `cdc_newarch_kafka_simple_ops_titan_off` - - `cdc_newarch_kafka_simple_protocol` - - `cdc_newarch_kafka_simple_protocol_misc_workloads` - - `cdc_newarch_kafka_simple_misc_workloads_dispatcher_index` -- Kafka 安全： - - `cdc-newarch-kafka-security` -- Kafka 与 TiCDC 故障恢复： - - `cdc-newarch-kafka-all-2-owner-network-partition` - - `cdc-newarch-kafka-broker-failure` - - `cdc-newarch-kafka-controller-2-cdc-random-network-partition` - - `cdc-newarch-kafka-controller-failure` - - `cdc-newarch-kafka-random-2_cdc-random-network-partition` - - `cdc-newarch-kafka-random-2-owner-network-partition` - - `cdc-newarch-kafka-controller-2-owner-network-partition` -- Release dailyrun： - - `cdc_newarch_kafka_basic_functionality` - - `cdc-newarch-kafka-airbnb-scenario` - - `cdc-newarch-kafka-avro` - - `cdc-newarch-kafka-avro-2-workloads` - - `cdc-newarch-kafka-debezium-avro` - - `cdc-newarch-kafka-debezium-avro-2-workloads` - - `cdc-newarch-kafka-mysql-sync` - - `cdc-newarch-kafka-mysql-sync-gcttl` - - `cdc-newarch-kafka-random-node-down` - - `cdc_newarch_kafka_scale_big_table_cdc_scale` - - `cdc_newarch_lightning_comp_kafka` - - `cdc_newarch_sarama_no_broken_pipe` - - `cdc-newarch-upstream-chaos-kafka-sync` -- Kafka 版本覆盖： - - `cdc-newarch-kafka-version-0.11.0-0-r0` - - `cdc-newarch-kafka-version-0.11.0-1-r0` - - `cdc-newarch-kafka-version-1.0.0-r0` - - `cdc-newarch-kafka-version-1.0.1-r0` - - `cdc-newarch-kafka-version-1.1.0` - - `cdc-newarch-kafka-version-1.1.1` - - `cdc-newarch-kafka-version-2.0.0` - - `cdc-newarch-kafka-version-2.0.1` - - `cdc-newarch-kafka-version-2.1.0` - - `cdc-newarch-kafka-version-2.2.0` - - `cdc-newarch-kafka-version-2.3.0` - - `cdc-newarch-kafka-version-2.4.0` - - `cdc-newarch-kafka-version-2.5.0` - - `cdc-newarch-kafka-version-2.6.0` - - `cdc-newarch-kafka-version-2.7.0` - - `cdc-newarch-kafka-version-2.8.0` - - `cdc-newarch-kafka-version-3.0.0` - - `cdc-newarch-kafka-version-3.1.0` - - `cdc-newarch-kafka-version-3.2.0` - - `cdc-newarch-kafka-version-3.4.0` - - `cdc-newarch-kafka-version-3.5.0` - - `cdc-newarch-kafka-version-3.6.0` - - `cdc-newarch-kafka-version-3.7.0` - - `cdc-newarch-kafka-version-3.8.0` - - `cdc-newarch-kafka-version-3.9.0` - - `cdc-newarch-kafka-version-4.0.0` - - `cdc-newarch-kafka-version-4.1.0` - - `cdc-newarch-kafka-version-4.2.0` - - `cdc-newarch-kafka-version-4.3.0` - -### 1.2 EKS 与 TiDB-X 补充环境 - -这些 Plan 用于补充环境兼容性验证，不阻塞标准 TiCDC franz-go GA： - -- EKS：`cdc_newarch_kafka_simple_protocol-eks`，验证基本 DDL/DML、全数据类型和端到端一致性。 -- EKS：`cdc-newarch-kafka-broker-failure-eks`，验证 Chaos Mesh、Kafka Pod 故障和恢复。 -- EKS：`cdc-newarch-kafka-avro-eks`，验证 Schema Registry、consumer 和跨组件网络访问。 -- TiDB-X：`tidbx_cdc_newarch_kafka_basic_functionality`，验证 realtime、incremental 和 TiCDC scale。 -- TiDB-X：`tidbx-cdc-newarch-kafka-broker-failure`，验证 Kafka broker 故障后的恢复。 - -执行前统一配置： - -- TiCDC 使用同一个 franz-go 构建产物。 -- Kafka 使用 Apache Kafka `4.1.2` KRaft。 -- case image 使用当前 GA 测试版本。 -- sdkserver 使用 `hub.pingcap.net/qa/sdkserver:kafka-auth-amd64` 或目标环境中的同一镜像。 -- EKS 所需镜像先同步到 EKS 可访问的 registry。 - -## 2. 补充说明 - -### 2.1 范围 - -共找到 109 个 TiCDC New Architecture Kafka YAML/Jsonnet 文件： - -- 标准 TiCDC：`data-platform/cdc_newarch/kafka/` 37 个。 -- 标准 TiCDC dailyrun：`release/dailyrun/data-platform/cdc_newarch/` 21 个。 -- TiDBX：`data-platform/tidbx_cdc_newarch/kafka/` 32 个。 -- TiDBX dailyrun：`release/dailyrun/data-platform/tidbx_cdc_newarch/` 19 个。 - -EKS 和 TiDBX 变体复用对应标准 Plan 的 case 与验证意图，下面不重复展开相同 case，但它们不是完全等价的重复执行： - -- EKS 变体主要切换 resource pool、镜像仓库、存储、节点规格和调度配置；多数 Plan 沿用相同 TiCDC version 参数，但部分 Plan 固定使用 `master`，Kafka 通常固定为 `3.9.0`。 -- TiDBX 变体切换为 TiDB-X 集群拓扑和配置，并使用 `mirrors/tidbx/pingcap/ticdc/image:master-nextgen`。它可能来自同一 TiCDC 代码库，但不是标准 Plan 使用的同一个镜像产物。 -- 验证指定 franz-go TiCDC binary 时，只有显式使用目标 TiCDC image 的 execution 才计入结果；EKS/TiDBX Plan 只是可复用的测试覆盖入口。 - -### 2.2 Kafka 安全能力 - -`cdc-newarch-kafka-security` 包含 15 个 case： - -- GSSAPI：用户名密码、keytab、TLS + 用户名密码、TLS + keytab + ACL。 -- TLS：单向 TLS、mTLS + ACL。 -- SASL/PLAIN：PLAIN、TLS + PLAIN。 -- SASL/SCRAM：SHA-256 + ACL、SHA-512、TLS + SHA-256、TLS + SHA-512 + ACL。 -- OAuth：HTTP token、HTTP compatibility、TLS + HTTPS token 私有 CA + ACL。 - -对应 case： - -`cdc_kafka_auth_sasl_gssapi_user`、`cdc_kafka_auth_sasl_gssapi_keytab`、`cdc_kafka_auth_tls_sasl_gssapi_user`、`cdc_kafka_auth_tls_sasl_gssapi_keytab_acl`、`cdc_kafka_auth_tls`、`cdc_kafka_auth_mtls_acl`、`cdc_kafka_auth_sasl_plain`、`cdc_kafka_auth_sasl_scram_sha_256_acl`、`cdc_kafka_auth_sasl_scram_sha_512`、`cdc_kafka_auth_tls_sasl_plain`、`cdc_kafka_auth_tls_sasl_scram_sha_256`、`cdc_kafka_auth_tls_sasl_scram_sha_512_acl`、`cdc_kafka_auth_sasl_oauthbearer`、`cdc_kafka_auth_sasl_oauthbearer_http_compatibility`、`cdc_kafka_auth_tls_sasl_oauthbearer_acl`。 - -### 2.3 Kafka 版本覆盖 - -`cdc_kafka_version.tpl.jsonnet` 为每个版本生成 `cdc-newarch-kafka-version-`，执行 `kafka_realtime`。基础版本集合为： - -`0.11.0-0-r0`、`0.11.0-1-r0`、`1.0.0-r0`、`1.0.1-r0`、`1.1.0`、`1.1.1`、`2.0.0`、`2.0.1`、`2.1.0`、`2.2.0`、`2.3.0`、`2.4.0`、`2.5.0`、`2.6.0`、`2.7.0`、`2.8.0`、`3.0.0`、`3.1.0`、`3.2.0`、`3.4.0`、`3.5.0`、`3.6.0`、`3.7.0`、`3.8.0`、`3.9.0`。 - -- 标准环境：基础集合加 `4.0.0`、`4.1.0`、`4.2.0`、`4.3.0`，共 29 个版本。 -- EKS：基础集合加 `4.0.0`，共 26 个版本。 -- TiDBX 与 TiDBX EKS：使用基础集合，各 25 个版本。 - -当前版本集合不包含 Kafka `3.3.x`。 - -### 2.4 TiDBX 变体 - -TiDBX 当前复用以下标准 Plan 的 case，但使用 TiDB-X 上游集群和 TiDBX TiCDC image： - -- 基础与协议：Airbnb simple titan、Debezium、large message claim check、large message handle、multiple topic、realtime、scale big table、simple ops、simple protocol、misc workloads、dispatcher index。 -- Kafka 故障：all broker to owner、broker failure、controller to CDC、controller failure、random broker to CDC、random broker to owner、controller to owner。 -- Release dailyrun：basic functionality、Airbnb scenario、Avro、Avro 2 workloads、MySQL sync、GC TTL、random CDC node down、scale big table、Lightning compatibility、断线重连、upstream chaos。 -- Kafka 版本：`tidbx-cdc-newarch-kafka-version-` 及 EKS 变体。 - -TiDBX 暂无对应的 Kafka security、Debezium Avro 和 Debezium Avro 2 workloads Plan。 - -## 3. 已完成 execution - -以下 execution 使用 franz-go TiCDC image `hub-zot.pingcap.net/mirrors/dev/pingcap/ticdc/image:pull-4167-31d4137_linux_amd64`。 - -### 3.1 Kafka security - -- [8204473](https://tcms.pingcap.net/dashboard/executions/plan/8204473)：15/15 SUCCESS。 - -### 3.2 Kafka chaos - -- [8229205](https://tcms.pingcap.net/dashboard/executions/plan/8229205)：broker 故障，SUCCESS。 -- [8204474](https://tcms.pingcap.net/dashboard/executions/plan/8204474)：controller 故障，SUCCESS。 -- [8204475](https://tcms.pingcap.net/dashboard/executions/plan/8204475)：所有 Kafka broker 到 TiCDC owner 网络分区，SUCCESS。 -- [8204476](https://tcms.pingcap.net/dashboard/executions/plan/8204476)：Kafka controller 到随机 TiCDC 节点网络分区，SUCCESS。 - -### 3.3 Kafka 版本 - -- Kafka 2.x：[2.0.0](https://tcms.pingcap.net/dashboard/executions/plan/8229191)、[2.0.1](https://tcms.pingcap.net/dashboard/executions/plan/8229192)、[2.1.0](https://tcms.pingcap.net/dashboard/executions/plan/8204465)、[2.2.0](https://tcms.pingcap.net/dashboard/executions/plan/8204466)、[2.3.0](https://tcms.pingcap.net/dashboard/executions/plan/8204467)、[2.4.0](https://tcms.pingcap.net/dashboard/executions/plan/8204468)、[2.5.0](https://tcms.pingcap.net/dashboard/executions/plan/8204469)、[2.7.0](https://tcms.pingcap.net/dashboard/executions/plan/8181348)，均 SUCCESS。 -- Kafka 3.x：[3.0.0](https://tcms.pingcap.net/dashboard/executions/plan/8229193)、[3.5.0](https://tcms.pingcap.net/dashboard/executions/plan/8229196)、[3.7.0](https://tcms.pingcap.net/dashboard/executions/plan/8229198)、[3.9.0](https://tcms.pingcap.net/dashboard/executions/plan/8229200)，均 SUCCESS。 -- Kafka 4.x：[4.0.0](https://tcms.pingcap.net/dashboard/executions/plan/8229201)、[4.1.0](https://tcms.pingcap.net/dashboard/executions/plan/8229202)、[4.2.0](https://tcms.pingcap.net/dashboard/executions/plan/8229203)、[4.3.0](https://tcms.pingcap.net/dashboard/executions/plan/8229204)，均 SUCCESS。 - -### 3.4 Avro 2 workloads - -`cdc-newarch-kafka-avro-2-workloads` 使用同一 franz-go TiCDC image，分别验证 Kafka 3.1 和 Kafka 4.1.2。上游 TiDB、PD、TiKV 和 BR 固定为 `v8.5.8`，sdkserver 固定为 `hub.pingcap.net/qa/sdkserver:kafka-auth-amd64`，资源池使用 `ksyun-scenario-and-system-test`。 - -- Kafka 3.1：[8229286](https://tcms.pingcap.net/dashboard/executions/plan/8229286)、[8181445](https://tcms.pingcap.net/dashboard/executions/plan/8181445)、[8204540](https://tcms.pingcap.net/dashboard/executions/plan/8204540)，均 SUCCESS。 -- Kafka 4.1.2：[8204541](https://tcms.pingcap.net/dashboard/executions/plan/8204541)、[8229287](https://tcms.pingcap.net/dashboard/executions/plan/8229287)、[8229288](https://tcms.pingcap.net/dashboard/executions/plan/8229288)，均 SUCCESS。 From 1fc9658ceb7a2283588a1ab897552d16e82fa61e Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 4 Sep 2026 16:38:43 +0800 Subject: [PATCH 60/61] kafka: report canceled franz async sends --- pkg/sink/kafka/franz_async_producer.go | 15 ++++++++------- pkg/sink/kafka/franz_async_producer_test.go | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go index 602759a80f..2966c10568 100644 --- a/pkg/sink/kafka/franz_async_producer.go +++ b/pkg/sink/kafka/franz_async_producer.go @@ -57,12 +57,6 @@ func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition i return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } - select { - case <-ctx.Done(): - return context.Cause(ctx) - default: - } - record := &kgo.Record{ Topic: topic, Partition: partition, @@ -85,8 +79,15 @@ func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition i } } + // Produce can buffer a record without checking ctx, so reject prior cancellation. + // If it waits for buffer space, canceling ctx makes it return; propagate the cause below. + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } p.client.Produce(ctx, record, promise) - return nil + return context.Cause(ctx) } func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go index 15338dc8cb..067f0241fd 100644 --- a/pkg/sink/kafka/franz_async_producer_test.go +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -277,7 +277,7 @@ func TestBufferBackpressure(t *testing.T) { select { case err = <-done: - require.NoError(t, err) + require.ErrorIs(t, err, context.Canceled) case <-time.After(time.Second): t.Fatal("canceled send remained blocked") } From c6ec5d1b89844e37754d68758663e6b66f0dd584 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 4 Sep 2026 17:42:31 +0800 Subject: [PATCH 61/61] fix --- pkg/sink/kafka/franz_factory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go index e3a11451a2..bdd0dea2b6 100644 --- a/pkg/sink/kafka/franz_factory.go +++ b/pkg/sink/kafka/franz_factory.go @@ -112,7 +112,7 @@ func (f *franzFactory) AsyncProducer(context.Context) (AsyncProducer, error) { return &asyncProducer{ client: f.client, changefeedID: f.changefeedID, - resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), + resultCh: make(chan asyncProduceResult, 1), }, nil }