diff --git a/coordinator/capture_write_lease.go b/coordinator/capture_write_lease.go
new file mode 100644
index 0000000000..df7271d426
--- /dev/null
+++ b/coordinator/capture_write_lease.go
@@ -0,0 +1,270 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package coordinator
+
+import (
+ "bytes"
+ "crypto/rand"
+ "slices"
+ "time"
+
+ "github.com/pingcap/ticdc/heartbeatpb"
+ "github.com/pingcap/ticdc/pkg/messaging"
+ "github.com/pingcap/ticdc/pkg/metrics"
+ "github.com/pingcap/ticdc/pkg/node"
+ "github.com/pingcap/ticdc/pkg/writelease"
+)
+
+const (
+ witnessNonceSize = 16
+ witnessChallengeTimeout = time.Second
+)
+
+type captureLeaseNodeState struct {
+ nodeEpoch uint64
+ lastRequestSeq uint64
+}
+
+type pendingWitnessChallenge struct {
+ selfNodeEpoch uint64
+ selfRequestSeq uint64
+ witnessNodeID node.ID
+ witnessNodeEpoch uint64
+ nonce []byte
+ expiresAt time.Time
+}
+
+// captureWriteLeaseController is owned by Controller's single event-handling
+// goroutine after construction. Its mutable state is not safe for concurrent use.
+type captureWriteLeaseController struct {
+ coordinatorVersion int64
+ selfNodeID node.ID
+ now func() time.Time
+ nonce func([]byte) (int, error)
+
+ nodes map[node.ID]*captureLeaseNodeState
+ p2pCapableNodes map[node.ID]struct{}
+ p2pLeaseEnabled bool
+ pendingWitness *pendingWitnessChallenge
+ nextWitnessIndex int
+}
+
+func newCaptureWriteLeaseController(version int64, selfNodeID node.ID) *captureWriteLeaseController {
+ return &captureWriteLeaseController{
+ coordinatorVersion: version,
+ selfNodeID: selfNodeID,
+ now: time.Now,
+ nonce: rand.Read,
+ nodes: make(map[node.ID]*captureLeaseNodeState),
+ p2pCapableNodes: make(map[node.ID]struct{}),
+ }
+}
+
+func (c *captureWriteLeaseController) observeNodeCapability(id node.ID, version uint32) {
+ if version == heartbeatpb.CurrentWriteLeaseProtocolVersion {
+ c.p2pCapableNodes[id] = struct{}{}
+ } else {
+ delete(c.p2pCapableNodes, id)
+ }
+}
+
+// updateClusterMode enables P2P only when every active capture has reported
+// support for the current protocol. A missing capability is treated as legacy.
+func (c *captureWriteLeaseController) updateClusterMode(activeNodes []node.ID) {
+ p2pEnabled := len(activeNodes) > 0
+ for _, id := range activeNodes {
+ if _, ok := c.p2pCapableNodes[id]; !ok {
+ p2pEnabled = false
+ break
+ }
+ }
+ if c.p2pLeaseEnabled && !p2pEnabled {
+ c.pendingWitness = nil
+ }
+ c.p2pLeaseEnabled = p2pEnabled
+}
+
+func (c *captureWriteLeaseController) handleHeartbeat(
+ from node.ID,
+ heartbeat *heartbeatpb.NodeHeartbeat,
+ initializedNodes []node.ID,
+) []*messaging.TargetMessage {
+ if heartbeat.GetWriteLeaseProtocolVersion() != heartbeatpb.CurrentWriteLeaseProtocolVersion ||
+ heartbeat.GetWriteLeaseRequestSeq() == 0 || heartbeat.GetNodeEpoch() == 0 ||
+ heartbeat.GetLiveness() == heartbeatpb.NodeLiveness_STOPPING {
+ return nil
+ }
+
+ state := c.nodes[from]
+ if state == nil {
+ state = &captureLeaseNodeState{nodeEpoch: heartbeat.GetNodeEpoch()}
+ c.nodes[from] = state
+ } else if state.nodeEpoch != heartbeat.GetNodeEpoch() {
+ // A process epoch is immutable while the same capture ID remains in the
+ // bootstrapper. Accepting an epoch change here would let a delayed old
+ // process heartbeat roll the coordinator's fencing state backwards.
+ return nil
+ }
+ if heartbeat.GetWriteLeaseRequestSeq() <= state.lastRequestSeq {
+ return nil
+ }
+ state.lastRequestSeq = heartbeat.GetWriteLeaseRequestSeq()
+
+ messages := c.handleWitnessAck(from, heartbeat)
+ if from != c.selfNodeID {
+ return append(messages, c.newGrant(from, heartbeat.GetNodeEpoch(), heartbeat.GetWriteLeaseRequestSeq()))
+ }
+
+ return append(messages, c.handleSelfHeartbeat(heartbeat, initializedNodes)...)
+}
+
+func (c *captureWriteLeaseController) handleWitnessAck(
+ from node.ID,
+ heartbeat *heartbeatpb.NodeHeartbeat,
+) []*messaging.TargetMessage {
+ ack := heartbeat.GetWriteLeaseWitnessAck()
+ pending := c.pendingWitness
+ if ack == nil || pending == nil {
+ return nil
+ }
+ if !c.now().Before(pending.expiresAt) {
+ c.pendingWitness = nil
+ return nil
+ }
+ if from != pending.witnessNodeID ||
+ heartbeat.GetNodeEpoch() != pending.witnessNodeEpoch ||
+ ack.GetCoordinatorVersion() != c.coordinatorVersion ||
+ ack.GetCoordinatorNodeEpoch() != pending.selfNodeEpoch ||
+ ack.GetSelfRequestSeq() != pending.selfRequestSeq ||
+ ack.GetWitnessNodeEpoch() != pending.witnessNodeEpoch ||
+ !bytes.Equal(ack.GetNonce(), pending.nonce) {
+ return nil
+ }
+
+ selfState := c.nodes[c.selfNodeID]
+ if selfState == nil || selfState.nodeEpoch != pending.selfNodeEpoch {
+ c.pendingWitness = nil
+ return nil
+ }
+ c.pendingWitness = nil
+ return []*messaging.TargetMessage{
+ c.newGrant(c.selfNodeID, pending.selfNodeEpoch, pending.selfRequestSeq),
+ }
+}
+
+func (c *captureWriteLeaseController) handleSelfHeartbeat(
+ heartbeat *heartbeatpb.NodeHeartbeat,
+ initializedNodes []node.ID,
+) []*messaging.TargetMessage {
+ if !c.p2pLeaseEnabled {
+ metrics.CaptureP2PWitnessAvailable.Set(0)
+ return []*messaging.TargetMessage{
+ c.newGrant(c.selfNodeID, heartbeat.GetNodeEpoch(), heartbeat.GetWriteLeaseRequestSeq()),
+ }
+ }
+
+ remoteExists := false
+ witnesses := make([]node.ID, 0, len(initializedNodes))
+ for _, id := range initializedNodes {
+ if id == c.selfNodeID {
+ continue
+ }
+ remoteExists = true
+ if state := c.nodes[id]; state != nil && state.nodeEpoch != 0 {
+ witnesses = append(witnesses, id)
+ }
+ }
+ if !remoteExists {
+ metrics.CaptureP2PWitnessAvailable.Set(0)
+ return []*messaging.TargetMessage{
+ c.newGrant(c.selfNodeID, heartbeat.GetNodeEpoch(), heartbeat.GetWriteLeaseRequestSeq()),
+ }
+ }
+ if len(witnesses) == 0 {
+ metrics.CaptureP2PWitnessAvailable.Set(0)
+ return nil
+ }
+ metrics.CaptureP2PWitnessAvailable.Set(1)
+
+ if c.pendingWitness != nil {
+ if c.now().Before(c.pendingWitness.expiresAt) {
+ return nil
+ }
+ c.pendingWitness = nil
+ }
+
+ slices.Sort(witnesses)
+ witness := witnesses[c.nextWitnessIndex%len(witnesses)]
+ c.nextWitnessIndex++
+ witnessEpoch := c.nodes[witness].nodeEpoch
+ nonce := make([]byte, witnessNonceSize)
+ if _, err := c.nonce(nonce); err != nil {
+ return nil
+ }
+
+ c.pendingWitness = &pendingWitnessChallenge{
+ selfNodeEpoch: heartbeat.GetNodeEpoch(),
+ selfRequestSeq: heartbeat.GetWriteLeaseRequestSeq(),
+ witnessNodeID: witness,
+ witnessNodeEpoch: witnessEpoch,
+ nonce: nonce,
+ expiresAt: c.now().Add(witnessChallengeTimeout),
+ }
+ response := &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: c.coordinatorVersion,
+ TargetNodeEpoch: witnessEpoch,
+ WitnessChallenge: &heartbeatpb.WriteLeaseWitnessChallenge{
+ CoordinatorVersion: c.coordinatorVersion,
+ CoordinatorNodeEpoch: heartbeat.GetNodeEpoch(),
+ SelfRequestSeq: heartbeat.GetWriteLeaseRequestSeq(),
+ WitnessNodeEpoch: witnessEpoch,
+ Nonce: append([]byte(nil), nonce...),
+ },
+ }
+ return []*messaging.TargetMessage{
+ messaging.NewSingleTargetMessage(witness, messaging.MaintainerManagerTopic, response),
+ }
+}
+
+func (c *captureWriteLeaseController) newGrant(
+ target node.ID,
+ targetNodeEpoch uint64,
+ requestSeq uint64,
+) *messaging.TargetMessage {
+ // A zero duration is an authenticated, sequence-checked signal that the
+ // cluster is in mixed-version mode and P2P enforcement must stay disabled.
+ leaseDurationMs := uint64(0)
+ if c.p2pLeaseEnabled {
+ leaseDurationMs = uint64(writelease.P2PLeaseDuration.Milliseconds())
+ }
+ return messaging.NewSingleTargetMessage(
+ target,
+ messaging.MaintainerManagerTopic,
+ &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: c.coordinatorVersion,
+ TargetNodeEpoch: targetNodeEpoch,
+ RequestSeq: requestSeq,
+ LeaseDurationMs: leaseDurationMs,
+ },
+ )
+}
+
+func (c *captureWriteLeaseController) removeNode(id node.ID) {
+ delete(c.nodes, id)
+ delete(c.p2pCapableNodes, id)
+ if c.pendingWitness != nil &&
+ (c.pendingWitness.witnessNodeID == id || id == c.selfNodeID) {
+ c.pendingWitness = nil
+ }
+}
diff --git a/coordinator/capture_write_lease_test.go b/coordinator/capture_write_lease_test.go
new file mode 100644
index 0000000000..09a94e1881
--- /dev/null
+++ b/coordinator/capture_write_lease_test.go
@@ -0,0 +1,267 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package coordinator
+
+import (
+ "testing"
+ "time"
+
+ "github.com/pingcap/ticdc/heartbeatpb"
+ "github.com/pingcap/ticdc/pkg/messaging"
+ "github.com/pingcap/ticdc/pkg/node"
+ "github.com/pingcap/ticdc/pkg/writelease"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCaptureWriteLeaseGrantsRemoteNode(t *testing.T) {
+ controller := newCaptureWriteLeaseController(10, node.ID("coordinator"))
+ enableP2PForNodes(controller, node.ID("capture-1"))
+ heartbeat := newWriteLeaseHeartbeat(11, 1)
+
+ messages := controller.handleHeartbeat(node.ID("capture-1"), heartbeat, nil)
+ require.Len(t, messages, 1)
+ grant := requireWriteLeaseResponse(t, messages[0])
+ require.Equal(t, int64(10), grant.CoordinatorVersion)
+ require.Equal(t, uint64(11), grant.TargetNodeEpoch)
+ require.Equal(t, uint64(1), grant.RequestSeq)
+ require.Equal(t, uint64(writelease.P2PLeaseDuration.Milliseconds()), grant.LeaseDurationMs)
+
+ require.Empty(t, controller.handleHeartbeat(node.ID("capture-1"), heartbeat, nil))
+
+ // A different process epoch cannot replace the epoch already associated
+ // with a tracked capture ID.
+ messages = controller.handleHeartbeat(node.ID("capture-1"), newWriteLeaseHeartbeat(12, 1), nil)
+ require.Empty(t, messages)
+
+ // Removing and re-adding the capture resets its fencing state.
+ controller.removeNode(node.ID("capture-1"))
+ messages = controller.handleHeartbeat(node.ID("capture-1"), newWriteLeaseHeartbeat(12, 1), nil)
+ require.Len(t, messages, 1)
+ require.Equal(t, uint64(12), requireWriteLeaseResponse(t, messages[0]).TargetNodeEpoch)
+}
+
+func TestCaptureWriteLeaseRequiresRemoteWitnessForCoordinatorNode(t *testing.T) {
+ now := time.Unix(100, 0)
+ controller := newCaptureWriteLeaseController(10, node.ID("coordinator"))
+ controller.now = func() time.Time { return now }
+ controller.nonce = func(nonce []byte) (int, error) {
+ for i := range nonce {
+ nonce[i] = byte(i + 1)
+ }
+ return len(nonce), nil
+ }
+ enableP2PForNodes(controller, node.ID("coordinator"), node.ID("capture-1"))
+
+ // Observe the remote node epoch first so it can serve as a witness.
+ remoteMessages := controller.handleHeartbeat(
+ node.ID("capture-1"),
+ newWriteLeaseHeartbeat(21, 1),
+ nil,
+ )
+ require.Len(t, remoteMessages, 1)
+
+ messages := controller.handleHeartbeat(
+ node.ID("coordinator"),
+ newWriteLeaseHeartbeat(11, 1),
+ []node.ID{"coordinator", "capture-1"},
+ )
+ require.Len(t, messages, 1)
+ require.Equal(t, node.ID("capture-1"), messages[0].To)
+ challengeResponse := requireWriteLeaseResponse(t, messages[0])
+ require.Zero(t, challengeResponse.RequestSeq)
+ challenge := challengeResponse.GetWitnessChallenge()
+ require.NotNil(t, challenge)
+ require.Equal(t, uint64(11), challenge.CoordinatorNodeEpoch)
+ require.Equal(t, uint64(21), challenge.WitnessNodeEpoch)
+
+ ackHeartbeat := newWriteLeaseHeartbeat(21, 2)
+ ackHeartbeat.WriteLeaseWitnessAck = &heartbeatpb.WriteLeaseWitnessAck{
+ CoordinatorVersion: challenge.CoordinatorVersion,
+ CoordinatorNodeEpoch: challenge.CoordinatorNodeEpoch,
+ SelfRequestSeq: challenge.SelfRequestSeq,
+ WitnessNodeEpoch: challenge.WitnessNodeEpoch,
+ Nonce: append([]byte(nil), challenge.Nonce...),
+ }
+ messages = controller.handleHeartbeat(
+ node.ID("capture-1"),
+ ackHeartbeat,
+ nil,
+ )
+ require.Len(t, messages, 2)
+ selfGrant := requireWriteLeaseResponse(t, messages[0])
+ require.Equal(t, node.ID("coordinator"), messages[0].To)
+ require.Equal(t, uint64(1), selfGrant.RequestSeq)
+ require.Equal(t, uint64(11), selfGrant.TargetNodeEpoch)
+
+ // The ack is one-shot. Replaying it only receives the witness's own grant.
+ ackHeartbeat.WriteLeaseRequestSeq = 3
+ messages = controller.handleHeartbeat(
+ node.ID("capture-1"),
+ ackHeartbeat,
+ nil,
+ )
+ require.Len(t, messages, 1)
+ require.Equal(t, node.ID("capture-1"), messages[0].To)
+}
+
+func TestCaptureWriteLeaseSingleNodeFallback(t *testing.T) {
+ controller := newCaptureWriteLeaseController(10, node.ID("coordinator"))
+ enableP2PForNodes(controller, node.ID("coordinator"))
+ messages := controller.handleHeartbeat(
+ node.ID("coordinator"),
+ newWriteLeaseHeartbeat(11, 1),
+ []node.ID{"coordinator"},
+ )
+
+ require.Len(t, messages, 1)
+ require.Equal(t, node.ID("coordinator"), messages[0].To)
+ require.Equal(t, uint64(1), requireWriteLeaseResponse(t, messages[0]).RequestSeq)
+}
+
+func TestCaptureWriteLeaseRetriesAnotherWitnessBeforeLeaseExpires(t *testing.T) {
+ now := time.Unix(100, 0)
+ controller := newCaptureWriteLeaseController(10, node.ID("coordinator"))
+ controller.now = func() time.Time { return now }
+ enableP2PForNodes(controller, node.ID("coordinator"), node.ID("capture-1"), node.ID("capture-2"))
+
+ controller.handleHeartbeat(node.ID("capture-1"), newWriteLeaseHeartbeat(21, 1), nil)
+ controller.handleHeartbeat(node.ID("capture-2"), newWriteLeaseHeartbeat(31, 1), nil)
+ initializedNodes := []node.ID{"coordinator", "capture-1", "capture-2"}
+
+ messages := controller.handleHeartbeat(
+ node.ID("coordinator"),
+ newWriteLeaseHeartbeat(11, 1),
+ initializedNodes,
+ )
+ require.Len(t, messages, 1)
+ require.Equal(t, node.ID("capture-1"), messages[0].To)
+
+ now = now.Add(writelease.NodeHeartbeatInterval)
+ require.Empty(t, controller.handleHeartbeat(
+ node.ID("coordinator"),
+ newWriteLeaseHeartbeat(11, 2),
+ initializedNodes,
+ ))
+
+ now = now.Add(writelease.NodeHeartbeatInterval)
+ messages = controller.handleHeartbeat(
+ node.ID("coordinator"),
+ newWriteLeaseHeartbeat(11, 3),
+ initializedNodes,
+ )
+ require.Len(t, messages, 1)
+ require.Equal(t, node.ID("capture-2"), messages[0].To)
+ require.Less(t, witnessChallengeTimeout, writelease.P2PLeaseDuration)
+}
+
+func TestCaptureWriteLeaseClusterModeFollowsNodeCapabilities(t *testing.T) {
+ controller := newCaptureWriteLeaseController(10, node.ID("coordinator"))
+ coordinatorID := node.ID("coordinator")
+ legacyID := node.ID("legacy")
+ currentID := node.ID("current")
+
+ controller.observeNodeCapability(coordinatorID, heartbeatpb.CurrentWriteLeaseProtocolVersion)
+ controller.observeNodeCapability(legacyID, heartbeatpb.LegacyWriteLeaseProtocolVersion)
+ controller.updateClusterMode([]node.ID{coordinatorID, legacyID})
+ require.False(t, controller.p2pLeaseEnabled)
+
+ messages := controller.handleHeartbeat(
+ coordinatorID,
+ newWriteLeaseHeartbeat(11, 1),
+ []node.ID{coordinatorID, legacyID},
+ )
+ require.Len(t, messages, 1)
+ require.Zero(t, requireWriteLeaseResponse(t, messages[0]).LeaseDurationMs)
+
+ // Replacing the legacy node first introduces an unknown capability. P2P is
+ // enabled only after the replacement reports the current protocol.
+ controller.removeNode(legacyID)
+ controller.updateClusterMode([]node.ID{coordinatorID, currentID})
+ require.False(t, controller.p2pLeaseEnabled)
+
+ controller.observeNodeCapability(currentID, heartbeatpb.CurrentWriteLeaseProtocolVersion)
+ controller.updateClusterMode([]node.ID{coordinatorID, currentID})
+ require.True(t, controller.p2pLeaseEnabled)
+
+ // Any newly added node disables P2P until its capability is known. Removing
+ // that unknown node restores the all-current cluster mode.
+ unknownID := node.ID("unknown")
+ controller.updateClusterMode([]node.ID{coordinatorID, currentID, unknownID})
+ require.False(t, controller.p2pLeaseEnabled)
+ controller.removeNode(unknownID)
+ controller.updateClusterMode([]node.ID{coordinatorID, currentID})
+ require.True(t, controller.p2pLeaseEnabled)
+}
+
+func TestCaptureWriteLeaseRejectsInvalidHeartbeatAndLateWitness(t *testing.T) {
+ now := time.Unix(100, 0)
+ controller := newCaptureWriteLeaseController(10, node.ID("coordinator"))
+ controller.now = func() time.Time { return now }
+ enableP2PForNodes(controller, node.ID("coordinator"), node.ID("capture-1"))
+
+ legacy := newWriteLeaseHeartbeat(21, 1)
+ legacy.WriteLeaseProtocolVersion = heartbeatpb.LegacyWriteLeaseProtocolVersion
+ require.Empty(t, controller.handleHeartbeat(node.ID("capture-1"), legacy, []node.ID{"capture-1"}))
+
+ stopping := newWriteLeaseHeartbeat(21, 2)
+ stopping.Liveness = heartbeatpb.NodeLiveness_STOPPING
+ require.Empty(t, controller.handleHeartbeat(node.ID("capture-1"), stopping, []node.ID{"capture-1"}))
+
+ controller.handleHeartbeat(node.ID("capture-1"), newWriteLeaseHeartbeat(21, 3), []node.ID{"coordinator", "capture-1"})
+ challengeMessages := controller.handleHeartbeat(
+ node.ID("coordinator"),
+ newWriteLeaseHeartbeat(11, 1),
+ []node.ID{"coordinator", "capture-1"},
+ )
+ challenge := requireWriteLeaseResponse(t, challengeMessages[0]).GetWitnessChallenge()
+ now = now.Add(witnessChallengeTimeout)
+
+ ackHeartbeat := newWriteLeaseHeartbeat(21, 4)
+ ackHeartbeat.WriteLeaseWitnessAck = &heartbeatpb.WriteLeaseWitnessAck{
+ CoordinatorVersion: challenge.CoordinatorVersion,
+ CoordinatorNodeEpoch: challenge.CoordinatorNodeEpoch,
+ SelfRequestSeq: challenge.SelfRequestSeq,
+ WitnessNodeEpoch: challenge.WitnessNodeEpoch,
+ Nonce: challenge.Nonce,
+ }
+ messages := controller.handleHeartbeat(
+ node.ID("capture-1"),
+ ackHeartbeat,
+ []node.ID{"coordinator", "capture-1"},
+ )
+ require.Len(t, messages, 1)
+ require.Equal(t, node.ID("capture-1"), messages[0].To)
+}
+
+func newWriteLeaseHeartbeat(nodeEpoch, requestSeq uint64) *heartbeatpb.NodeHeartbeat {
+ return &heartbeatpb.NodeHeartbeat{
+ Liveness: heartbeatpb.NodeLiveness_ALIVE,
+ NodeEpoch: nodeEpoch,
+ WriteLeaseRequestSeq: requestSeq,
+ WriteLeaseProtocolVersion: heartbeatpb.CurrentWriteLeaseProtocolVersion,
+ }
+}
+
+func enableP2PForNodes(controller *captureWriteLeaseController, ids ...node.ID) {
+ for _, id := range ids {
+ controller.observeNodeCapability(id, heartbeatpb.CurrentWriteLeaseProtocolVersion)
+ }
+ controller.updateClusterMode(ids)
+}
+
+func requireWriteLeaseResponse(t *testing.T, message *messaging.TargetMessage) *heartbeatpb.NodeHeartbeatResponse {
+ t.Helper()
+ require.Equal(t, messaging.TypeNodeHeartbeatResponse, message.Type)
+ return message.Message[0].(*heartbeatpb.NodeHeartbeatResponse)
+}
diff --git a/coordinator/controller.go b/coordinator/controller.go
index 138fc4820d..96d749af0a 100644
--- a/coordinator/controller.go
+++ b/coordinator/controller.go
@@ -19,6 +19,7 @@ import (
"sync"
"time"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/log"
"github.com/pingcap/ticdc/coordinator/changefeed"
"github.com/pingcap/ticdc/coordinator/drain"
@@ -90,6 +91,7 @@ type Controller struct {
apiLock sync.RWMutex
drainController *drain.Controller
+ writeLease *captureWriteLeaseController
// drainSession is the in-memory drain state machine for v1 drain API.
// Only one drain session is allowed at a time.
@@ -185,6 +187,7 @@ func NewController(
pdClient: pdClient,
pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock),
drainController: drainController,
+ writeLease: newCaptureWriteLeaseController(version, selfNode.ID),
}
c.nodeChanged.changed = false
@@ -207,6 +210,7 @@ func NewController(
added, _, requests, _ := c.bootstrapper.HandleNodesChange(nodes)
log.Info("coordinator bootstrap initial nodes",
zap.Int("addedCount", len(added)), zap.Any("addedNodes", nodes))
+ c.writeLease.updateClusterMode(c.bootstrapper.GetAllNodeIDs())
for _, req := range requests {
err := c.messageCenter.SendCommand(req)
@@ -421,6 +425,7 @@ func (c *Controller) onMessage(ctx context.Context, msg *messaging.TargetMessage
c.maybeBroadcastDispatcherDrainTarget(true)
}
c.syncDrainSchedulingPolicy()
+ c.handleCaptureWriteLeaseHeartbeat(msg.From, req)
case messaging.TypeSetNodeLivenessResponse:
req := msg.Message[0].(*heartbeatpb.SetNodeLivenessResponse)
c.drainController.ObserveSetNodeLivenessResponse(msg.From, req)
@@ -511,11 +516,13 @@ func (c *Controller) onNodeChanged(ctx context.Context) {
zap.Any("removedNodes", removedNodes))
for _, n := range removedNodes {
+ c.writeLease.removeNode(n)
c.RemoveNode(n)
}
for _, n := range addedNodes {
c.clearCompletedDrainTarget(n)
}
+ c.writeLease.updateClusterMode(c.bootstrapper.GetAllNodeIDs())
for _, req := range requests {
err := c.messageCenter.SendCommand(req)
if err != nil {
@@ -527,6 +534,74 @@ func (c *Controller) onNodeChanged(ctx context.Context) {
c.handleBootstrapResponses(ctx, responses)
}
+func (c *Controller) handleCaptureWriteLeaseHeartbeat(from node.ID, heartbeat *heartbeatpb.NodeHeartbeat) {
+ if c.bootstrapper == nil || !c.bootstrapper.NodeInitialized(from) {
+ metrics.CaptureLeaseHeartbeatCounter.WithLabelValues("uninitialized").Inc()
+ return
+ }
+ metrics.CaptureLeaseHeartbeatCounter.WithLabelValues("received").Inc()
+ var initializedNodes []node.ID
+ if from == c.writeLease.selfNodeID {
+ // Only the coordinator capture needs remote membership to select a witness.
+ // Remote captures can be granted directly after sender validation.
+ initializedNodes = c.bootstrapper.GetInitializedNodeIDs()
+ }
+ messages := c.writeLease.handleHeartbeat(from, heartbeat, initializedNodes)
+ if len(messages) == 0 {
+ metrics.CaptureLeaseHeartbeatCounter.WithLabelValues("no_response").Inc()
+ } else {
+ metrics.CaptureLeaseHeartbeatCounter.WithLabelValues("response").Add(float64(len(messages)))
+ }
+ hasGrant := false
+ for _, message := range messages {
+ response, ok := message.Message[0].(*heartbeatpb.NodeHeartbeatResponse)
+ if ok && response.GetRequestSeq() != 0 {
+ hasGrant = true
+ break
+ }
+ }
+ delayed := false
+ failpoint.Inject("DelayCaptureWriteLeaseResponse", func(value failpoint.Value) {
+ delayMillis, ok := value.(int)
+ if ok && delayMillis > 0 && hasGrant {
+ delay := time.Duration(delayMillis) * time.Millisecond
+ delayed = true
+ deferredMessages := append([]*messaging.TargetMessage(nil), messages...)
+ go func() {
+ time.Sleep(delay)
+ for _, message := range deferredMessages {
+ _ = c.messageCenter.SendCommand(message)
+ }
+ }()
+ }
+ })
+ if delayed {
+ return
+ }
+ dropped := false
+ failpoint.Inject("DropCaptureWriteLeaseResponse", func(value failpoint.Value) {
+ if value.(bool) && hasGrant {
+ dropped = true
+ }
+ })
+ if dropped {
+ return
+ }
+ failpoint.Inject("DuplicateCaptureWriteLeaseResponse", func(value failpoint.Value) {
+ if value.(bool) && hasGrant {
+ for _, message := range messages {
+ duplicate := *message
+ _ = c.messageCenter.SendCommand(&duplicate)
+ }
+ }
+ })
+ for _, message := range messages {
+ if err := c.messageCenter.SendCommand(message); err != nil {
+ metrics.CaptureLeaseHeartbeatCounter.WithLabelValues("send_failed").Inc()
+ }
+ }
+}
+
func (c *Controller) onMaintainerBootstrapResponse(ctx context.Context, req *messaging.TargetMessage) {
response := req.Message[0].(*heartbeatpb.CoordinatorBootstrapResponse)
c.drainController.ObserveBootstrapResponse(req.From, response)
@@ -535,6 +610,8 @@ func (c *Controller) onMaintainerBootstrapResponse(ctx context.Context, req *mes
zap.Int("maintainerCount", len(response.Statuses)))
responses := c.bootstrapper.HandleBootstrapResponse(req.From, response)
if c.bootstrapper.HasNode(req.From) {
+ c.writeLease.observeNodeCapability(req.From, response.GetWriteLeaseProtocolVersion())
+ c.writeLease.updateClusterMode(c.bootstrapper.GetAllNodeIDs())
if c.maybeAddDispatcherDrainSyncNode(req.From, response.GetDrainProtocolVersion()) {
c.maybeBroadcastDispatcherDrainTarget(true)
} else if c.observeStaleDispatcherDrainTargetSnapshot(req.From, drainTargetSnapshotFromBootstrap(response)) {
@@ -894,6 +971,7 @@ func (c *Controller) stopStaleBootstrapMaintainers(
}
func (c *Controller) Stop() {
+ metrics.CaptureP2PWitnessAvailable.Set(0)
c.taskHandlerMutex.Lock()
for _, h := range c.taskHandlers {
h.Cancel()
@@ -1195,10 +1273,15 @@ func (c *Controller) submitPeriodTask() {
func (c *Controller) newBootstrapMessage(id node.ID, addr string) *messaging.TargetMessage {
log.Info("send coordinator bootstrap request", zap.Any("nodeID", id), zap.String("nodeAddr", addr))
+ // Bootstrap every node in legacy mode while its capability is unknown. The
+ // periodic lease response enables P2P after every active node reports support.
return messaging.NewSingleTargetMessage(
id,
messaging.MaintainerManagerTopic,
- &heartbeatpb.CoordinatorBootstrapRequest{Version: c.version})
+ &heartbeatpb.CoordinatorBootstrapRequest{
+ Version: c.version,
+ WriteLeaseProtocolVersion: heartbeatpb.LegacyWriteLeaseProtocolVersion,
+ })
}
// updateChangefeedEpoch bumps the persisted owner epoch before a state change
diff --git a/coordinator/controller_drain_test.go b/coordinator/controller_drain_test.go
index cd72ad7ca6..8e93c3bfde 100644
--- a/coordinator/controller_drain_test.go
+++ b/coordinator/controller_drain_test.go
@@ -104,6 +104,7 @@ func newDrainTestController(t *testing.T) (*Controller, *drain.Controller, node.
)
},
),
+ writeLease: newCaptureWriteLeaseController(1, selfNode.ID),
}
return c, drainController, target
}
diff --git a/docs/design/capture-write-lease-design.md b/docs/design/capture-write-lease-design.md
new file mode 100644
index 0000000000..ca5d0e37ae
--- /dev/null
+++ b/docs/design/capture-write-lease-design.md
@@ -0,0 +1,790 @@
+# TiCDC Capture Write Lease Detailed Design
+
+> This document describes the complete capture write-lease design, its
+> implementation boundaries, safety proof, failure behavior, observability,
+> and test coverage. It is intended for TiCDC developers, reviewers, and test
+> engineers and does not require another design document as prerequisite
+> reading.
+
+## 1. Goals and boundaries
+
+### 1.1 Problem statement
+
+A TiCDC capture may lose connectivity to PD, the coordinator, or other
+captures while retaining access to a downstream system. If the scheduler has
+already created a replacement and the disconnected capture continues writing,
+two writers can create side effects against the same downstream state.
+
+This design establishes a capture-wide write-admission condition with the
+following goal:
+
+> After an old capture loses proof of identity or proof that it is managed by
+> the coordinator, it stops starting new downstream side effects within a
+> bounded interval. A replacement can take over only after a later boundary.
+
+### 1.2 Guarantees
+
+- All sinks stop starting new downstream side effects when the P2P lease or
+ etcd write proof required by the current mode expires.
+- P2P faults and temporary etcd TTL query failures cause recoverable write
+ blocking rather than immediate capture termination.
+- A confirmed loss of the etcd session irreversibly fences the capture and
+ terminates the process.
+- After capture-key deletion, `captureRemoveTTL` delays scheduler-visible node
+ removal so a replacement cannot take over too early.
+- During a rolling upgrade, P2P is required only after every active capture
+ has reported support for the protocol.
+- Normal writes perform only a local atomic state read and do not add a
+ per-write coordinator, PD, or downstream network round trip.
+
+### 1.3 Non-goals
+
+- The gate does not cancel SQL, producer sends, or object uploads that passed
+ the final admission check before the gate closed.
+- The design does not add a downstream fencing token and does not claim strict
+ exactly-once delivery.
+- It does not change TiCDC's existing retry and duplicate-delivery semantics
+ for an already committed event.
+- A single-capture cluster has no remote witness. In that topology P2P cannot
+ prove external connectivity, so safety primarily relies on the etcd proof.
+
+## 2. System model and core invariant
+
+Each capture creates one local `writelease.Gate`. The gate combines the
+following state:
+
+```text
+p2pValidUntil // local deadline of the P2P grant
+etcdProofValidUntil // local deadline derived from the existing etcd session
+p2pRequired // whether the current cluster mode requires P2P
+fenced // whether this process irreversibly lost its identity
+```
+
+There is one write-admission predicate:
+
+```text
+writeAllowed(now) =
+ !fenced
+ AND now < etcdProofValidUntil
+ AND (!p2pRequired OR now < p2pValidUntil)
+```
+
+The etcd proof is therefore always required. Whether the P2P proof is required
+is negotiated from the capabilities of all active captures. Unknown state is
+fail-closed, and a newly created gate starts closed.
+
+
+
+The proofs have separate responsibilities:
+
+| State or proof | What it proves | Effect of expiry | Recovery |
+| --- | --- | --- | --- |
+| P2P lease | The capture is managed by the current coordinator generation. | Stop new writes while the process remains alive. | Accept a valid new grant. |
+| etcd write proof | The capture's existing etcd session identity can still be confirmed. | Stop new writes while TTL queries continue. | Receive a successful positive TTL response. |
+| Local fence | The session is confirmed lost. | Permanently close the gate, stop local write paths, and terminate. | Start a new process. |
+
+## 3. Safety ordering and proof
+
+`captureRemoveTTL` is the scheduling barrier for a replacement. Define:
+
+```text
+Le = maximum etcd proof lifetime = 5s
+R = captureRemoveTTL = max(captureSessionTTL / 2, 10s)
+td = time the old capture key is deleted in a linearizable etcd view
+tobs = time another CDC node observes that deletion, tobs >= td
+```
+
+For any TTL query that returns a positive TTL, its linearization point precedes
+`td`, and its `requestSentAt` is no later than that linearization point. The
+proof begins at `requestSentAt` and lasts at most `Le`, so:
+
+```text
+oldLastAdmission < td + Le = td + 5s
+```
+
+Another node does not publish node removal immediately after observing the key
+deletion. It first waits `R`:
+
+```text
+newFirstAdmission >= tobs + R >= td + 10s
+```
+
+With default values:
+
+```text
+oldLastAdmission < td + 5s < td + 10s <= tobs + R <= newFirstAdmission
+```
+
+
+
+This proves that the new-operation admission windows do not overlap. P2P
+usually stops the old writer sooner, but the proof does not depend on P2P.
+Mixed-version and single-capture modes therefore retain the same time-separation
+lower bound.
+
+The proof depends on four implementation conditions:
+
+1. Every real downstream side effect passes through a transport-owned final
+ gate.
+2. Every replacement is produced through the same scheduling path that
+ observes capture removal.
+3. Capture-key deletion and observation follow etcd linearizability.
+4. In-process deadline comparison uses Go's monotonic clock, which advances
+ normally for the process.
+
+## 4. P2P lease design
+
+### 4.1 Heartbeats and grants
+
+Every bootstrapped capture sends a node heartbeat every 500 ms. A request
+contains:
+
+- A non-zero `nodeEpoch` for the current process lifetime.
+- A monotonically increasing `writeLeaseRequestSeq`.
+- The current write-lease protocol version.
+- An optional witness ACK.
+
+The coordinator processes heartbeats only from initialized nodes. For each
+capture, it records the process epoch and largest observed sequence. It does
+not issue a grant for an epoch change, a repeated or decreasing sequence, a
+stopping node, or a protocol mismatch.
+
+The capture validates a response before accepting it:
+
+```text
+sender == currentCoordinator
+coordinatorVersion == currentCoordinatorVersion
+targetNodeEpoch == localNodeEpoch
+requestSeq exists in local outstanding requests
+requestSeq > lastAppliedLeaseSeq
+duration <= 5s
+```
+
+The local deadline is derived from the request send time:
+
+```text
+p2pValidUntil = requestSentAt + grantDuration
+```
+
+A five-second grant arriving after seven seconds is already expired and cannot
+reopen the gate. A duplicate response is rejected because its sequence has
+already been applied.
+
+
+
+### 4.2 Coordinator and capture on different nodes
+
+A remote capture heartbeat already represents cross-node communication. After
+validating the request, the coordinator directly returns a grant lasting at
+most five seconds. If either the request or response direction is broken, the
+existing grant expires within five seconds.
+
+A remote heartbeat does not require scanning the complete initialized-node
+list. The coordinator performs one `NodeInitialized` check for the sender, so
+the ordinary heartbeat path is O(1).
+
+### 4.3 Coordinator and capture on the same node
+
+Local messaging cannot prove that the coordinator host is still connected to
+the rest of the cluster. Whenever a remote capture exists, the coordinator's
+own capture must first complete a remote witness challenge:
+
+```text
+self heartbeat
+ -> coordinator selects a remote initialized witness
+ -> challenge(coordinatorVersion, selfEpoch, selfSeq, witnessEpoch, nonce)
+ -> witness echoes an ACK in its heartbeat
+ -> coordinator validates every field and grants its local capture
+```
+
+One challenge attempt times out after one second, while a P2P lease lasts up to
+five seconds. If a witness becomes unreachable, the coordinator rotates to
+another witness in stable order without allowing one failed attempt to consume
+the complete lease interval.
+
+Only a heartbeat from the coordinator's own capture needs cluster membership
+for witness selection. In that path, the bootstrapper acquires its lock once
+and creates one initialized-node snapshot. This retains the required
+membership information without turning every high-frequency heartbeat into an
+O(N) scan and creating O(N^2) control-plane work.
+
+### 4.4 Single-capture cluster
+
+When no remote capture exists, the coordinator's local capture receives a
+direct grant. This keeps single-node deployments available, but P2P cannot
+prove external connectivity in that topology. The etcd proof remains
+mandatory, and etcd proof plus `captureRemoveTTL` still establishes the
+replacement ordering.
+
+### 4.5 Rolling upgrades and capability negotiation
+
+Every bootstrap response declares its supported write-lease protocol version.
+The coordinator recomputes the cluster mode when:
+
+- A node joins.
+- A node leaves.
+- A bootstrap response arrives with capability information.
+
+The rule is:
+
+```text
+if any active capture is legacy or capability is unknown:
+ p2pLeaseEnabled = false
+ return a validated zero-duration grant
+else:
+ p2pLeaseEnabled = true
+ return a normal grant up to 5s
+```
+
+After accepting a zero-duration grant, a capture sets `p2pRequired=false`; the
+etcd proof remains required. Once every active capture reports compatible
+capability, later grants carry a positive duration and captures set
+`p2pRequired=true`. No coordinator restart is needed.
+
+When the coordinator generation changes, a capture immediately invalidates its
+existing P2P proof and clears outstanding requests. A delayed response from the
+previous coordinator cannot renew the new generation.
+
+## 5. etcd write proof and process termination
+
+### 5.1 Why the design does not create a second etcd lease
+
+The capture registration key is already attached to the real etcd session
+lease. Creating another `LeaseID` would introduce two keepalive streams, two
+failure semantics, and ambiguity about whether the identity lease or write
+lease represents a live capture.
+
+The design reuses the existing session and maintains only one in-process
+deadline:
+
+```text
+etcdProofValidUntil
+```
+
+This is not another etcd lease and does not create a new key. It represents how
+long the process may still trust the latest verified positive session TTL.
+
+### 5.2 TTL query and proof calculation
+
+The server queries the existing session `LeaseID` once per second. A request
+has a maximum timeout of three seconds. If the current proof has less time
+remaining, the request timeout is shortened to that remaining time so a
+blocked query cannot cross the local safety deadline.
+
+After a successful TTL query:
+
+```text
+proofDuration = min(5s, reportedTTL - 1s)
+etcdProofValidUntil = requestSentAt + proofDuration
+```
+
+Three details matter:
+
+- Starting at `requestSentAt` prevents a slow response from adding validity.
+- Subtracting a one-second safety margin absorbs whole-second TTL rounding and
+ transport delay.
+- Capping the proof at five seconds prevents a larger session TTL from
+ extending the old writer's stop-write bound.
+
+A failed query, timeout, nil response, or `TTL == 0` does not renew the proof
+and does not independently terminate the process. Once the proof expires, the
+gate becomes non-writable. A later positive TTL response can recover it.
+
+### 5.3 Write blocking versus process exit
+
+```text
+P2P expired -> block new writes; keep process alive
+etcd proof expired -> block new writes; keep querying TTL
+TTL query error -> do not renew; keep process alive
+Session.Done() -> irreversible local fence; exit
+confirmed TTL < 0 -> irreversible local fence; exit
+```
+
+After local fencing, the server:
+
+1. Marks the gate `fenced`. This state is irreversible for the process.
+2. Advances node liveness to draining and then stopping.
+3. Tells `DispatcherOrchestrator` and all dispatcher managers to stop local
+ write paths.
+4. Returns a capture-suicide error, cancels module contexts, and exits.
+
+`TTL == 0` is not confirmation that the lease was deleted, so it only allows
+the proof to expire naturally. Only `TTL < 0` or `Session.Done()` is exit
+evidence.
+
+## 6. Gate concurrency and performance
+
+The gate publishes an immutable `leaseState` through an `atomic.Pointer`. A
+write path only performs:
+
+```text
+state = atomicLoad()
+compare monotonic deadlines
+```
+
+Renewals, mode changes, and fencing use one short mutex to serialize state
+publication. They perform no network access on the write path.
+
+### 6.1 Waiting and notification
+
+A blocked writer waits on the `changed` channel. The channel is closed and
+replaced only when the combined state changes from non-writable to writable:
+
+```text
+becameWritable = !writable(oldState, now) AND writable(newState, now)
+if becameWritable:
+ close(changed)
+ changed = new channel
+```
+
+For example, renewing the etcd proof while the required P2P lease is still
+expired does not wake every writer for a futile retry. During an outage this
+reduces wake-ups from "every proof update times the waiter count" to one
+broadcast when admission actually recovers.
+
+### 6.2 State reasons
+
+The gate exposes five states for metrics and transition logs:
+
+```text
+writable
+p2p_expired
+etcd_proof_expired
+both_expired
+fenced
+```
+
+`fenced` has the highest priority. No later renewal can reopen the gate.
+
+## 7. Gate injection and final write boundaries
+
+The server creates one capture-wide gate and publishes it through app context
+to dispatcher managers, ordinary sinks, and redo sinks. `Sink.SetWriteGate` is
+a mandatory interface method for every sink.
+
+Admission is checked in two stages:
+
+1. The outer `writeGatedSink` waits before an event enters the sink. This
+ provides backpressure and reduces unnecessary queue growth.
+2. Each transport checks again immediately before the operation that creates
+ the real downstream side effect. This closes the asynchronous-queue gap.
+
+
+
+### 7.1 Common outer gate
+
+- **DML:** waits while non-writable instead of sending more events into the
+ sink.
+- **DDL, sync point, and block events:** waits for admission and verifies it
+ again before invoking the underlying sink.
+- **Checkpoint:** skips the current checkpoint while blocked because a later,
+ larger checkpoint supersedes it. This avoids blocking a periodic message
+ stream.
+
+The outer gate is not the final safety boundary because work may already be in
+a producer, encoder, or file-writer queue. The transport checks below provide
+the final guarantee.
+
+### 7.2 MySQL and TiDB
+
+DML uses a two-phase check:
+
+```text
+loop:
+ wait for Gate outside withConn
+ acquire session mutex and sql.Conn
+ if Gate is closed immediately before SQL:
+ close/release connection
+ release mutex
+ continue
+ execute transaction
+```
+
+Writers do not occupy dedicated connections while the lease is unavailable,
+and a non-blocking final check remains immediately before SQL. DDL, sync-point
+operations, DDL-ts updates, and `RemoveDDLTsItem` cleanup use the same gate, so
+changefeed deletion cannot bypass admission and mutate TiCDC downstream
+metadata.
+
+### 7.3 Kafka
+
+The final check covers:
+
+- Topic and partition side effects.
+- Every DML `AsyncSend`.
+- DDL and checkpoint sends.
+- Claim-check object publication for large messages.
+
+An event that has already been encoded or queued cannot start a new producer
+send or claim-check object write after the gate closes.
+
+### 7.4 Pulsar
+
+The final check covers topic and partition operations and DML, DDL, and
+checkpoint producer sends. Pulsar's asynchronous queue cannot bypass the
+capture-wide gate.
+
+### 7.5 Cloud Storage
+
+The final check covers schema, data, index, and metadata publication as well
+as cleanup and delete operations. Encoded or spooled messages may remain local
+while blocked, but cannot start a new object-store mutation. Processing resumes
+after the gate reopens.
+
+### 7.6 Redo
+
+The final check covers file and memory writers for DML, DDL, rotate, flush,
+upload, metadata updates, and GC/delete. If the gate is closed during writer
+shutdown, an unpublished temporary file is not converted into a consumable
+final file.
+
+Redo is not the business downstream, but it determines which events are
+considered durable during disaster recovery. Concurrent publication of redo
+files or metadata by two captures can corrupt that recovery view, so redo must
+use the same gate.
+
+### 7.7 Blackhole
+
+Blackhole has no real external side effect, so `SetWriteGate` is a no-op. It
+still satisfies the common interface without waiting.
+
+## 8. Role of `captureRemoveTTL`
+
+`captureRemoveTTL` is neither an etcd lease nor the mechanism that terminates
+the old process. It delays scheduler-visible node removal after `NodeManager`
+observes deletion of a capture key:
+
+```text
+captureRemoveTTL = max(captureSessionTTL / 2, 10s)
+```
+
+The default `captureSessionTTL` is ten seconds, so the default
+`captureRemoveTTL` is ten seconds.
+
+The sequence is:
+
+```text
+observe capture key deletion
+ -> record pending removal time
+ -> keep capture in the node view
+ -> wait captureRemoveTTL
+ -> publish node removal
+ -> scheduler may create a replacement
+```
+
+If the same capture ID re-registers during the delay, the pending removal is
+canceled. A transient session disturbance therefore does not unnecessarily
+trigger failover.
+
+The separation of responsibilities is:
+
+- The local etcd proof establishes the latest time the old writer may admit a
+ new operation.
+- `captureRemoveTTL` establishes an earliest time when replacement scheduling
+ may begin.
+
+P2P improves stop-write latency during isolation, but it is not the replacement
+scheduling barrier.
+
+## 9. Failure behavior
+
+| Failure | Gate behavior | Capture exits? | Replacement behavior |
+| --- | --- | --- | --- |
+| Coordinator response is lost | P2P expires within five seconds and new writes stop. | No. | Capture key remains; no replacement is triggered. |
+| Coordinator capture is isolated from other nodes | Witness cannot ACK; local P2P expires. | Only if the etcd session is also confirmed lost. | Depends on capture-key deletion. |
+| One witness is unreachable | Rotate after one second; no interruption if another witness succeeds in time. | No. | Not triggered. |
+| PD/etcd TTL query temporarily fails | Do not renew etcd proof; stop writes after proof expiry. | No. | Not triggered while the key remains. |
+| etcd session is confirmed lost | Irreversible local fence. | Yes. | Wait `captureRemoveTTL` after observing key deletion. |
+| Control plane is unreachable but downstream remains reachable | At least one required proof expires and transports stop new writes. | Only after confirmed session loss. | Constrained by `captureRemoveTTL`. |
+| A legacy node participates in a rolling upgrade | P2P is not required; etcd proof remains required. | Follows etcd session semantics. | Constrained by `captureRemoveTTL`. |
+
+### 9.1 Complete example
+
+Assume an old writer on CDC-1 has just obtained its final five-second P2P grant
+and five-second etcd proof at `t0`. CDC-1 then loses connectivity to both the
+coordinator/witness and PD while retaining access to MySQL:
+
+1. From `t0` through `t0+5s`, the final proofs may remain valid, so the old
+ writer may still admit transactions.
+2. No later than `t0+5s`, the gate closes. No transport starts a new SQL
+ operation, send, object publication, or redo flush.
+3. Around `t0+10s`, the default session TTL expires and the capture key is
+ deleted from etcd.
+4. Other nodes observe the deletion and wait the default ten-second
+ `captureRemoveTTL`.
+5. Only then can the scheduler publish node removal and create a replacement;
+ the replacement's first actual write is later still.
+
+The old writer therefore stops new admission by about `t0+5s`, while a
+replacement normally cannot write until after `t0+20s`. The boundaries create
+an explicit separation interval.
+
+### 9.2 Late completion remains possible
+
+Consider this MySQL sequence:
+
+```text
+t1: old writer passes the final Gate check
+t2: old writer sends COMMIT
+t3: Gate closes
+t4: replacement starts after the removal barrier
+t5: the delayed old COMMIT finally reaches MySQL
+```
+
+The lease cannot cancel the `COMMIT` sent at `t2`. A Kafka/Pulsar send or object
+upload that already started can also finish after the gate closes. The design
+prevents new operations from starting after closure; it does not guarantee
+that every old operation completes before replacement activity.
+
+Eliminating this tail risk requires a downstream writer epoch/fencing token or
+an abort/drain protocol with a proven hard completion bound.
+
+## 10. Observability
+
+The design exposes these primary metrics:
+
+| Metric | Meaning |
+| --- | --- |
+| `ticdc_server_capture_write_gate_state{state}` | One-hot gauge for the five gate states. |
+| `ticdc_server_capture_p2p_lease_remaining_seconds` | Remaining P2P lease lifetime. |
+| `ticdc_server_capture_etcd_proof_remaining_seconds` | Remaining etcd proof lifetime. |
+| `ticdc_server_capture_write_block_total{reason}` | Number of writable-to-blocked transitions by reason. |
+| `ticdc_server_capture_last_write_admission_timestamp_seconds` | Time of the most recent admitted downstream operation. |
+| `ticdc_server_capture_lease_response_rejected_total{reason}` | Rejections by sender, epoch, sequence, duration, and other validation reasons. |
+| `ticdc_coordinator_capture_lease_heartbeat_total{result}` | Coordinator heartbeat-processing results. |
+| `ticdc_server_capture_lease_response_total{result}` | Capture response-processing results. |
+| `ticdc_coordinator_capture_p2p_witness_available` | Whether a remote witness is available for the coordinator capture. |
+| `ticdc_server_capture_safe_to_reschedule_delay_seconds` | Effective `captureRemoveTTL`. |
+
+A writable-to-blocked transition logs its reason, and a blocked-to-writable
+transition logs recovery. Local fencing has a separate explicit log, allowing
+operators to distinguish recoverable write blocking from process termination
+after confirmed identity loss.
+
+## 11. Test design and coverage
+
+Testing is layered. Each layer validates a different proof obligation rather
+than relying on one end-to-end result to infer every concurrency boundary.
+
+
+
+### 11.1 Deterministic gate, protocol, and removal tests
+
+Tests in `pkg/writelease` cover:
+
+- The two-proof truth table and fail-closed initial state.
+- Rejection of late renewals and irreversible fencing.
+- Context cancellation.
+- Notification only on a non-writable-to-writable transition.
+- Coordinator negotiation that enables or disables P2P.
+- Compatibility behavior when no gate is installed.
+
+Coordinator and maintainer tests cover:
+
+- Direct grants to remote captures.
+- The remote-witness requirement for the coordinator capture.
+- Single-capture fallback.
+- Witness rotation after one second and recovery within the five-second lease.
+- Mode transitions caused by legacy, unknown, or fully compatible membership
+ and by node joins and removals.
+- Rejection of invalid heartbeats, epoch mismatches, late witness ACKs, and
+ replayed or unknown sequences.
+- Bootstrap capability, zero-duration and positive-duration grants, and the
+ witness challenge/ACK path.
+
+Server and orchestrator tests cover:
+
+- Fencing on `Session.Done()` and `TTL < 0`.
+- No false process termination on `TTL == 0` or TTL query error.
+- Positive-TTL renewal and request timeout bounded by the current proof
+ deadline.
+- Gate state and block-transition metrics.
+- `captureRemoveTTL` calculation, delayed removal, and cancellation of pending
+ removal when the same capture re-registers.
+
+Messaging tests include in-process and remote serialization round trips for
+the response and witness fields, ensuring that the protocol works through the
+real message path rather than only through direct function calls.
+
+### 11.2 Transport boundary tests
+
+Tests close the gate at the actual side-effect API and verify that the lower
+level mock or file operation does not occur:
+
+| Transport | Primary coverage |
+| --- | --- |
+| Common sink wrapper | DML block/recovery, context cancellation, and all DDL/checkpoint write entries. |
+| MySQL | Waiting before SQL, connection release after a failed final check, shutdown rejection, and gated DDL-ts cleanup. |
+| Kafka | Blocked DML send; the same gate on DDL/checkpoint/topic paths; no claim-check object publication. |
+| Pulsar | Blocked DDL send and common-gate use by DML and checkpoint producer paths. |
+| Cloud Storage | Waiting before index publication and gate injection into schema, data, metadata, and cleanup paths. |
+| Redo | Gated file flush, close publication, memory DDL, metadata flush, and cleanup while preserving local-file semantics. |
+
+Because `Sink.SetWriteGate` is mandatory, a new sink that omits gate injection
+fails at compile time. Component tests then verify that the gate reaches the
+transport's actual mutation point.
+
+### 11.3 Repository integration case
+
+[`tests/integration_tests/capture_write_lease`](../../tests/integration_tests/capture_write_lease)
+runs three captures with continuous MySQL INSERT and UPDATE traffic. Failpoints
+inject three coordinator-to-capture grant failures:
+
+1. Delay a response beyond five seconds and verify `p2p_expired`.
+2. Drop grants in one direction while heartbeats still reach the coordinator,
+ proving that one-way control-plane loss also stops writes.
+3. Replay a response and verify that `unknown_sequence` or
+ `replayed_sequence` increases without reopening the gate.
+
+During a failure the case verifies that:
+
+- All three CDC processes remain alive, because P2P expiry must not terminate
+ a capture.
+- At least one probe table stops replicating, demonstrating that write
+ admission actually closed.
+- After fault removal the same capture returns to `writable` and drains the
+ backlog.
+- Final probe row counts are complete, YCSB produced both INSERT and UPDATE
+ traffic, and Sync Diff reports consistency.
+
+The same integration case also runs baseline synchronization and Sync Diff for
+Kafka, Pulsar, and Storage sinks. Lease-response fault assertions are in the
+MySQL branch; deterministic component tests own the final-boundary proof for
+the asynchronous transports.
+
+### 11.4 Testinfra `cdc_network_chaos_synthetic`
+
+The testinfra case runs three captures, a MySQL sink, and continuous mixed DML.
+Its standard long-running manifest uses 100 tables with 100,000 rows each, for
+ten million initial rows. Preparation uses 32 workers and committed batches of
+1,000 rows. The run phase uses 256 closed-loop workers. Each workload event
+issues two UPDATEs, one DELETE, and one INSERT, with each statement committing
+independently.
+
+The case has four phases:
+
+1. **Preparation and pressure recovery.** Wait for TiKV memory, scheduler
+ throttle, and memtable limiter metrics to recover. Preparation is resumable,
+ with a five-minute no-progress deadline and a 45-minute overall limit per
+ attempt, for at most three attempts.
+2. **Capture lifecycle failures.** Scale TiCDC from three captures to one and
+ back to three, hang one capture for ten seconds, and kill one capture
+ container. Wait for topology recovery and record the RTO of each operation.
+3. **Two-hour network chaos.** Inject ten-second faults with at least two
+ minutes after one completed round before the next. Capture ordinals and
+ failure modes rotate in stable order:
+
+ - **full ingress:** drop all traffic entering the target capture;
+ - **full egress:** drop all traffic leaving the target capture;
+ - **full bidirectional:** completely isolate the target capture;
+ - **PD-only:** isolate the target capture from PD while keeping MySQL
+ reachable;
+ - **PD+CDC:** isolate the target capture from PD and the other captures while
+ keeping MySQL reachable.
+
+ Full-node modes include the downstream path. Control-plane-only modes run a
+ MySQL TCP probe from the isolated capture, directly proving the risky state
+ in which the control plane is unavailable but the downstream remains
+ reachable. Every round must reach injected, cleanup, and inactive-rule
+ recovery. The ordinals and capture IDs of all three captures must then
+ remain unchanged for 15 seconds before another fault can start. A
+ completion-based timer prevents delayed ticker events from causing
+ back-to-back faults.
+4. **Final correctness.** Record a TSO after the workload, wait for the CDC
+ checkpoint to pass it, compare source and target CRCs for all 100 tables,
+ run row-level Sync Diff only for failed tables, and scan all CDC logs for
+ panics.
+
+EKS qualification plan `8181081`, case execution `21126582`, completed
+successfully in 2 hours 27 minutes 48 seconds. The two-hour network phase
+completed 39 rounds: full ingress, full egress, full bidirectional, and PD-only
+ran eight times each; PD+CDC ran seven times. Every round completed injection,
+cleanup, recovery, and the 15-second identity-stability barrier. All 100 final
+table CRCs matched, no failed table required repair, Sync Diff inspection
+completed, and the CDC panic scan was clean.
+
+That long-running execution used TiCDC build `c979ed56` and validates the
+dual-lease system failure model, capture lifecycle, recovery, and end-to-end
+data consistency. Deterministic unit and component tests on implementation
+baseline `e4b59cdaa` cover the transport final gates, mixed-version mode, short
+witness timeout, and MySQL connection release. These evidence types have
+different responsibilities: long-running chaos validates real system
+composition and fault shapes, while deterministic tests validate final write
+boundaries and concurrency interleavings that are difficult to reproduce
+reliably in a cluster.
+
+### 11.5 Why the coverage is sufficient
+
+For the guarantee stated by this design, coverage is sufficient because each
+assumption in the proof has direct evidence:
+
+| Proof obligation | Direct evidence |
+| --- | --- |
+| The gate opens only when every required proof is fresh. | Gate truth-table, deadline, fence, and mode tests. |
+| A delayed or replayed message cannot extend a lease. | Epoch, sequence, request-age tests and integration failpoints. |
+| Every real side effect checks final admission. | Component tests for five sink types, claim-check, and redo, plus the mandatory sink interface. |
+| Replacement does not start immediately after key deletion. | `captureRemoveTTL` state tests and lifecycle chaos. |
+| A capture stops and recovers when the control plane is isolated but downstream remains reachable. | PD-only and PD+CDC reachability probes and the two-hour rotation. |
+| Recovery leaves no residual data error. | Checkpoint catch-up, 100-table CRC, Sync Diff, and panic scan. |
+
+This evidence supports the claim that no new downstream side effect starts
+after the gate closes and that replacement admission follows the old writer's
+bounded admission interval. It is not used to claim cancellation of already
+admitted work or broker-level exactly-once, neither of which is a design
+guarantee.
+
+Before merge or release, the target build should still run the repository
+integration case and the long-running testinfra plan. That is a regression
+check of binary packaging, deployment, and external dependencies, not a
+substitute for a missing safety-proof obligation.
+
+## 12. Key parameters
+
+| Parameter | Value | Purpose |
+| --- | --- | --- |
+| Node heartbeat interval | 500 ms | Fast renewal and failure detection. |
+| P2P lease duration | 5 s | Bound new writes after coordinator loss. |
+| Witness attempt timeout | 1 s | Try another witness before the five-second lease expires. |
+| etcd TTL watch interval | 1 s | Continuously refresh session-identity proof. |
+| etcd TTL request timeout | At most 3 s | Prevent a query from blocking across the proof deadline. |
+| etcd TTL safety margin | 1 s | Absorb TTL rounding and transport time. |
+| etcd proof duration | At most 5 s | Bound new writes after PD/etcd loss. |
+| Capture session TTL | 10 s by default | Server-side lifetime of the capture key. |
+| `captureRemoveTTL` | `max(sessionTTL / 2, 10s)` | Establish a later replacement-takeover boundary. |
+| Gate monitor interval | 100 ms | Record gate metrics and transition logs. |
+
+## 13. Implementation map
+
+| Responsibility | Code |
+| --- | --- |
+| Gate state, waiting, renewal, and fencing | [`pkg/writelease/write_gate.go`](../../pkg/writelease/write_gate.go) |
+| P2P grants, capabilities, and witnesses | [`coordinator/capture_write_lease.go`](../../coordinator/capture_write_lease.go) |
+| Heartbeat admission and initialized-node snapshot | [`coordinator/controller.go`](../../coordinator/controller.go), [`pkg/bootstrap/bootstrap.go`](../../pkg/bootstrap/bootstrap.go) |
+| Capture heartbeat, response validation, and witness ACK | [`maintainer/maintainer_manager_node.go`](../../maintainer/maintainer_manager_node.go) |
+| Coordinator-generation changes | [`maintainer/maintainer_manager.go`](../../maintainer/maintainer_manager.go) |
+| etcd TTL watchdog and local fence | [`server/server.go`](../../server/server.go) |
+| Capture-wide gate injection | [`server/server.go`](../../server/server.go), [`pkg/common/context/app_context.go`](../../pkg/common/context/app_context.go) |
+| Replacement delay | [`pkg/orchestrator/reactor_state.go`](../../pkg/orchestrator/reactor_state.go) |
+| Common outer sink gate | [`downstreamadapter/sink/write_gate.go`](../../downstreamadapter/sink/write_gate.go) |
+| Transport final gates | `downstreamadapter/sink/{mysql,kafka,pulsar,cloudstorage,redo}` |
+| MySQL final SQL check | [`pkg/sink/mysql`](../../pkg/sink/mysql) |
+| Kafka claim-check | [`pkg/sink/kafka/claimcheck`](../../pkg/sink/kafka/claimcheck) |
+| Storage schema path | [`pkg/cloudstorage`](../../pkg/cloudstorage) |
+| Redo file and memory writers | [`pkg/redo/writer`](../../pkg/redo/writer) |
+| Repository integration case | [`tests/integration_tests/capture_write_lease`](../../tests/integration_tests/capture_write_lease) |
+| Testinfra case | `caselib/ticdc/testcase/cdc_network_chaos_synthetic.go` in `pingcap/test-infra` |
+| Testinfra chaos step | `caselib/pkg/steps/cdc_network_chaos.go` in `pingcap/test-infra` |
+
+## 14. Conclusion
+
+The capture write lease reduces "may this capture still write?" to one local,
+low-overhead, fail-closed gate. The P2P lease proves the current coordinator
+relationship, the etcd proof confirms the capture session identity, and
+`captureRemoveTTL` delays the replacement. Final checks at every real sink
+mutation boundary make asynchronous queues obey the same safety condition.
+
+The design proves that the new-write admission windows of the old and
+replacement writers do not overlap, and it automatically blocks and recovers
+during control-plane faults. Its boundary for already admitted in-flight work
+is explicit, so admission fencing is not overstated as downstream exactly-once
+delivery.
diff --git a/docs/design/capture-write-lease-overview.md b/docs/design/capture-write-lease-overview.md
new file mode 100644
index 0000000000..2f5a060286
--- /dev/null
+++ b/docs/design/capture-write-lease-overview.md
@@ -0,0 +1,273 @@
+# TiCDC Capture Write Lease: Design Overview and Safety Boundary
+
+> This document is intended for engineering leadership and TiCDC developers.
+> It focuses on the overall mechanism, impact, and safety proof while omitting
+> protocol fields and function-level details.
+
+## 1. Executive summary
+
+The capture write lease addresses this failure mode: after an old capture loses
+contact with the cluster, it must not continue creating new downstream side
+effects merely because it can still reach the downstream system. Before a
+replacement starts, new-write admission on the old writer must already be
+closed.
+
+The design uses one capture-wide local gate and combines two independent
+proofs:
+
+- **etcd write proof** proves that the capture's etcd session identity can
+ still be confirmed. It is always required.
+- **P2P lease** proves that the capture is still managed by the current
+ coordinator. It is required only when every active capture supports the
+ protocol.
+
+The core decision is:
+
+```text
+p2pRequired = allActiveCapturesSupportCurrentProtocol
+
+writeAllowed = !localFenced
+ AND etcdProofFresh
+ AND (!p2pRequired OR p2pLeaseFresh)
+```
+
+If any required proof expires, the gate blocks new downstream side effects,
+but the capture remains alive and continues renewing. Only `Session.Done()` or
+a successful TTL query that explicitly returns `TTL < 0` triggers an
+irreversible local fence and terminates the capture.
+
+During a rolling upgrade, any legacy capture or unknown capability switches the
+cluster to etcd-only admission. P2P admission is enabled automatically once all
+active captures have reported support. The gate is also enforced at every
+sink's actual mutation boundary, so events already present in an asynchronous
+queue cannot proceed to SQL execution, producer send, object publication, or
+redo persistence after lease expiry.
+
+
+
+## 2. Responsibilities of the two proofs
+
+### P2P lease: prove management by the current coordinator
+
+Every capture sends a heartbeat to the coordinator every 500 ms. Each request
+contains a process epoch and a monotonically increasing sequence number. A
+valid grant lasts at most five seconds and is measured from the request send
+time. A delayed or replayed response cannot extend the lease.
+
+The grant path depends on where the coordinator runs:
+
+| Deployment | P2P renewal | Purpose |
+| --- | --- | --- |
+| Coordinator and capture are on different nodes | The coordinator validates the heartbeat and directly returns a grant. | A broken request or response direction stops new writes within five seconds. |
+| Coordinator and capture are on the same node, with a remote capture available | The coordinator must complete a challenge/ACK with a remote witness before granting its local capture. | Prevents local in-process messaging from incorrectly renewing a node that is externally isolated. |
+| Single-capture cluster | No remote witness exists, so the local capture receives a direct grant. | P2P does not prove external connectivity; safety relies primarily on the etcd proof. |
+
+One witness attempt times out after one second, while the P2P lease lasts five
+seconds. If a witness becomes unreachable, the coordinator still has time to
+try another witness before the local lease expires.
+
+P2P expiry only **blocks new writes**. The capture, dispatchers, and control
+plane continue running. A fresh valid grant reopens the gate. P2P expiry by
+itself neither removes the capture key nor schedules a replacement.
+
+### etcd write proof: prove that the capture identity is still valid
+
+At startup, each capture uses one real etcd session lease to register its
+capture key. The default session TTL is ten seconds. The design does not create
+a second etcd `LeaseID`; it maintains only a local
+`etcdProofValidUntil` deadline.
+
+The server queries the existing session lease TTL once per second. A successful
+positive TTL response extends the local proof as follows:
+
+```text
+proofDuration = min(5s, reportedTTL - 1s)
+etcdProofValidUntil = requestSentAt + proofDuration
+```
+
+The proof starts at the request send time rather than the response arrival time,
+so a slow response cannot create extra validity. A failed query, timeout, or
+empty response does not renew the proof. When the proof expires, the gate stops
+writes but the capture keeps querying; a later positive TTL can recover the
+gate.
+
+Write blocking and process exit are intentionally different:
+
+```text
+TTL query failed / proof expired -> block new writes, keep running
+Session.Done or confirmed TTL < 0 -> local fence, stop write paths, exit
+```
+
+This makes write admission conservative without terminating a capture on a
+single transient etcd query failure.
+
+## 3. Downstream effects controlled by the gate
+
+`Sink.SetWriteGate` is a mandatory sink contract. An outer gate provides
+backpressure before an event enters a sink, and every transport performs a
+second check immediately before its real downstream side effect:
+
+| Write path | Final gate location | Behavior while blocked |
+| --- | --- | --- |
+| MySQL / TiDB | DML waits before acquiring the session mutex and `sql.Conn`, then rechecks immediately before SQL. DDL, sync points, and DDL-ts cleanup use the same gate. | Waiting does not occupy a connection. A failed final check releases the connection and mutex before retrying. |
+| Kafka | Before topic/partition effects, every `AsyncSend`, DDL/checkpoint send, and claim-check object publication. | Encoded or queued events stop at the producer boundary. A blocked checkpoint is skipped because a later value supersedes it. |
+| Pulsar | Before topic/partition effects and DML, DDL, or checkpoint producer sends. | Asynchronously queued events cannot bypass the gate. |
+| Cloud Storage | Before publishing schema, data, index, or metadata files and before cleanup/delete operations. | Buffered events remain local, but no new object write or deletion starts. |
+| Redo | Before file/memory DML and DDL persistence, rotate/flush/upload, metadata updates, and GC/delete. | No new redo persistence or cleanup operation starts. |
+
+Blackhole has no external side effect, so its gate injection is intentionally a
+no-op.
+
+The transport-level check matters because an entry gate only decides whether an
+event may enter an asynchronous queue. The transport gate decides whether work
+already in that queue may still touch the downstream system. The final check is
+a local atomic state load, so it does not add a coordinator, PD, or downstream
+network round trip to each DML operation.
+
+Waiters are notified only when the combined state changes from non-writable to
+writable. Renewing one proof while another required proof remains expired does
+not wake every writer for a futile retry.
+
+## 4. Rolling-upgrade compatibility
+
+Each capture reports its write-lease capability in the bootstrap response. The
+coordinator recomputes the cluster mode whenever a node joins, a node leaves,
+or a bootstrap response arrives:
+
+```text
+Any legacy or capability-unknown active capture:
+ P2P disabled for the whole cluster
+ etcd proof remains mandatory
+
+All active captures support the protocol:
+ P2P enabled automatically
+ writes require both P2P and etcd proof
+```
+
+When P2P is disabled, the coordinator returns an authenticated,
+epoch-and-sequence-validated zero-duration grant. The capture interprets it as
+`p2pRequired = false`. A coordinator upgraded before the other captures
+therefore does not permanently block its local changefeeds, and the cluster
+switches to dual-proof admission automatically after all captures are ready.
+
+The safety implication is explicit: while legacy captures are present, the
+cluster does not have P2P isolation protection. Admission safety relies on the
+etcd proof and `captureRemoveTTL` until all active captures support P2P.
+
+## 5. `captureRemoveTTL` and replacement admission
+
+`captureRemoveTTL` is not an etcd lease and does not control when the old
+process exits. It is the delay between another CDC node observing deletion of a
+capture key and publishing that capture's removal to schedulers:
+
+```text
+captureRemoveTTL = max(captureSessionTTL / 2, 10s)
+```
+
+With the default `captureSessionTTL = 10s`, `captureRemoveTTL = 10s`. During
+this delay:
+
+- The old capture remains in the node view and is not immediately replaced.
+- A re-registration of the same capture cancels the pending removal.
+- Only after the delay expires is node removal published, after which a
+ replacement may be scheduled.
+
+The local proof creates an upper bound for when the old writer stops admitting
+new work. `captureRemoveTTL` creates a later lower bound for when a replacement
+can begin.
+
+## 6. Why new-write admission does not overlap
+
+Define:
+
+```text
+Le = maximum local etcd proof lifetime = 5s
+R = captureRemoveTTL >= 10s
+td = linearizable deletion time of the old capture key
+tobs = time another CDC node observes the deletion, tobs >= td
+```
+
+The request send time for the last positive TTL proof precedes `td`, and the
+proof lasts at most five seconds:
+
+```text
+oldEtcdProofValidUntil < td + 5s
+oldLastAdmission < td + 5s
+```
+
+A replacement must wait `R` after observing the deletion:
+
+```text
+newFirstAdmission >= tobs + R >= td + 10s
+```
+
+Therefore:
+
+```text
+oldLastAdmission < td + 5s < td + 10s <= tobs + R <= newFirstAdmission
+```
+
+
+
+The proof depends on four conditions: every real downstream side effect passes
+through a transport gate; all replacements pass through `captureRemoveTTL`;
+capture-key deletion is observed with etcd linearizability; and the local
+monotonic clock advances normally. MySQL, Kafka, Pulsar, Cloud Storage, and Redo
+all implement the same gate contract, so asynchronous queues are not an
+unbounded gap in the proof.
+
+### Example
+
+Assume an old writer has just obtained its final P2P and etcd proofs at `t0`,
+then loses both coordinator/witness and PD/etcd connectivity while retaining
+access to the downstream system:
+
+1. Between `t0` and `t0+5s`, the final proofs may remain valid and the old
+ writer may still admit new operations.
+2. No later than `t0+5s`, the gate closes and no transport starts a new SQL
+ operation, send, file publication, or metadata mutation.
+3. With defaults, the session lease expires around `t0+10s` and the capture key
+ is deleted.
+4. Other nodes observe the deletion and wait another ten seconds before
+ publishing node removal and scheduling a replacement.
+5. The replacement's first actual write is normally later than `t0+20s`, about
+ fifteen seconds after the old writer stopped admitting new operations.
+
+Detection and scheduling may add wall-clock delay, but they cannot reverse the
+ordering established by the inequalities above.
+
+## 7. Residual risk
+
+The lease prevents an operation from **starting after the gate closes**. It
+cannot cancel an operation that passed the final check before closure. For
+example:
+
+1. The old writer passes the final check and sends a MySQL `COMMIT`.
+2. The gate closes, preventing new transactions.
+3. The `COMMIT` remains delayed in a proxy or the network.
+4. A replacement begins writing after the removal barrier.
+5. The delayed old `COMMIT` finally reaches MySQL.
+
+The same boundary applies to a Kafka/Pulsar send or object upload that already
+started. The design proves non-overlapping **new-admission windows**; it does
+not prove strict exactly-once or cancellation of admitted in-flight work.
+
+Eliminating that tail risk requires a downstream fencing token/epoch,
+idempotent transactional protocol, or a drain/abort mechanism with a hard
+completion bound. Those approaches are outside the design goal of avoiding
+downstream protocol changes and per-write network RTTs.
+
+## 8. Implementation entry points
+
+- Gate and proof state: [`pkg/writelease/write_gate.go`](../../pkg/writelease/write_gate.go)
+- etcd TTL watchdog and local fence: [`server/server.go`](../../server/server.go)
+- P2P, mixed-version mode, and witness: [`coordinator/capture_write_lease.go`](../../coordinator/capture_write_lease.go)
+- Capture heartbeat and capability handling: [`maintainer/maintainer_manager_node.go`](../../maintainer/maintainer_manager_node.go)
+- Replacement barrier: [`pkg/orchestrator/reactor_state.go`](../../pkg/orchestrator/reactor_state.go)
+- Common sink gate: [`downstreamadapter/sink/write_gate.go`](../../downstreamadapter/sink/write_gate.go)
+- Transport-owned final checks: `downstreamadapter/sink/{mysql,kafka,pulsar,cloudstorage,redo}`,
+ `pkg/sink/mysql`, and `pkg/redo/writer`
+
+Protocol fields, state transitions, transport boundaries, and the complete
+test strategy are described in the
+[detailed design](./capture-write-lease-design.md).
diff --git a/docs/media/capture-write-lease-architecture.svg b/docs/media/capture-write-lease-architecture.svg
new file mode 100644
index 0000000000..0dceba0581
--- /dev/null
+++ b/docs/media/capture-write-lease-architecture.svg
@@ -0,0 +1,123 @@
+
diff --git a/docs/media/capture-write-lease-p2p-sequence.svg b/docs/media/capture-write-lease-p2p-sequence.svg
new file mode 100644
index 0000000000..9ca8e5761c
--- /dev/null
+++ b/docs/media/capture-write-lease-p2p-sequence.svg
@@ -0,0 +1,86 @@
+
diff --git a/docs/media/capture-write-lease-safety-proof.svg b/docs/media/capture-write-lease-safety-proof.svg
new file mode 100644
index 0000000000..5bfff1f105
--- /dev/null
+++ b/docs/media/capture-write-lease-safety-proof.svg
@@ -0,0 +1,64 @@
+
diff --git a/docs/media/capture-write-lease-test-coverage.svg b/docs/media/capture-write-lease-test-coverage.svg
new file mode 100644
index 0000000000..0f26457444
--- /dev/null
+++ b/docs/media/capture-write-lease-test-coverage.svg
@@ -0,0 +1,87 @@
+
diff --git a/docs/media/capture-write-lease-write-path.svg b/docs/media/capture-write-lease-write-path.svg
new file mode 100644
index 0000000000..064fa89212
--- /dev/null
+++ b/docs/media/capture-write-lease-write-path.svg
@@ -0,0 +1,117 @@
+
diff --git a/downstreamadapter/dispatcher/redo_dispatcher.go b/downstreamadapter/dispatcher/redo_dispatcher.go
index 252b38df0b..2fce956bf0 100644
--- a/downstreamadapter/dispatcher/redo_dispatcher.go
+++ b/downstreamadapter/dispatcher/redo_dispatcher.go
@@ -21,8 +21,10 @@ import (
"github.com/pingcap/ticdc/downstreamadapter/sink/redo"
"github.com/pingcap/ticdc/heartbeatpb"
"github.com/pingcap/ticdc/pkg/common"
+ appcontext "github.com/pingcap/ticdc/pkg/common/context"
"github.com/pingcap/ticdc/pkg/config"
misc "github.com/pingcap/ticdc/pkg/redo/common"
+ "github.com/pingcap/ticdc/pkg/writelease"
"go.uber.org/zap"
)
@@ -98,6 +100,9 @@ func (rd *RedoDispatcher) SetRedoMeta(ctx context.Context, cfg *config.Consisten
}
ctx, rd.cancel = context.WithCancel(ctx)
rd.redoMeta = redo.NewRedoMeta(rd.sharedInfo.changefeedID, rd.startTs, cfg)
+ if gate, ok := appcontext.TryGetService[*writelease.Gate](appcontext.CaptureWriteGate); ok {
+ rd.redoMeta.SetWriteGate(gate)
+ }
go func() {
err := rd.redoMeta.PreStart(ctx)
if err != nil {
diff --git a/downstreamadapter/dispatchermanager/dispatcher_manager.go b/downstreamadapter/dispatchermanager/dispatcher_manager.go
index 18b0808cb8..42fffd1f79 100644
--- a/downstreamadapter/dispatchermanager/dispatcher_manager.go
+++ b/downstreamadapter/dispatchermanager/dispatcher_manager.go
@@ -26,7 +26,6 @@ import (
"github.com/pingcap/ticdc/downstreamadapter/eventcollector"
"github.com/pingcap/ticdc/downstreamadapter/sink"
"github.com/pingcap/ticdc/downstreamadapter/sink/mysql"
- "github.com/pingcap/ticdc/downstreamadapter/sink/redo"
"github.com/pingcap/ticdc/downstreamadapter/syncpoint"
"github.com/pingcap/ticdc/eventpb"
"github.com/pingcap/ticdc/heartbeatpb"
@@ -40,6 +39,7 @@ import (
"github.com/pingcap/ticdc/pkg/pdutil"
"github.com/pingcap/ticdc/pkg/routing"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/utils/threadpool"
"github.com/prometheus/client_golang/prometheus"
"github.com/tikv/client-go/v2/oracle"
@@ -145,13 +145,19 @@ type DispatcherManager struct {
// sink is used to send all the events to the downstream.
sink sink.Sink
+ // writeSink is the capture-write-gated view passed to dispatchers.
+ // sink remains the concrete implementation used for lifecycle and
+ // sink-specific recovery operations.
+ writeSink sink.Sink
// redo related
// redoEnabled is immutable and set to true if enabled.
redoEnabled bool
// redoReady set to true after the redo components are fully initialized and safe for concurrent access.
redoReady atomic.Bool
- redoSink *redo.Sink
+ // redoSink is the capture-write-gated sink used by redo dispatchers and for
+ // lifecycle management.
+ redoSink sink.Sink
// redoGlobalTs stores the resolved-ts of the redo metadata and blocks events in the common dispatcher where the commit-ts is greater than the resolved-ts.
redoGlobalTs atomic.Uint64
@@ -297,6 +303,7 @@ func NewDispatcherManager(
return nil, newWritePathClosedError()
}
manager.sink = createdSink
+ manager.writeSink = withCaptureWriteGate(ctx, createdSink)
manager.writePathMu.Unlock()
sinkType := manager.sink.SinkType()
@@ -424,6 +431,21 @@ func NewDispatcherManager(
return manager, nil
}
+func withCaptureWriteGate(ctx context.Context, inner sink.Sink) sink.Sink {
+ gate, ok := appcontext.TryGetService[*writelease.Gate](appcontext.CaptureWriteGate)
+ if !ok {
+ return inner
+ }
+ return sink.WithWriteGate(ctx, inner, gate)
+}
+
+func (e *DispatcherManager) getWriteSink() sink.Sink {
+ if e.writeSink != nil {
+ return e.writeSink
+ }
+ return e.sink
+}
+
func countIgnoreUpdateOnlyColumnsRules(filter *config.FilterConfig) int {
if filter == nil {
return 0
@@ -620,7 +642,7 @@ func (e *DispatcherManager) newEventDispatchers(infos map[common.DispatcherID]di
skipSyncpointAtStartTsList[idx],
skipDMLAsStartTs,
currentPdTs,
- e.sink,
+ e.getWriteSink(),
e.sharedInfo,
e.IsRedoEnabled(),
&e.redoGlobalTs,
@@ -1008,7 +1030,7 @@ func (e *DispatcherManager) mergeEventDispatcher(dispatcherIDs []common.Dispatch
false, // skipSyncpointAtStartTs
false, // skipDMLAsStartTs will be set later after calculating real startTs
0, // currentPDTs will be calculated later.
- e.sink,
+ e.getWriteSink(),
e.sharedInfo,
e.IsRedoEnabled(),
&e.redoGlobalTs,
@@ -1161,7 +1183,7 @@ func (e *DispatcherManager) addCheckpointTs(checkpointTs uint64) {
if e.writePathClosed.Load() {
return
}
- e.sink.AddCheckpointTs(checkpointTs)
+ e.getWriteSink().AddCheckpointTs(checkpointTs)
}
func (e *DispatcherManager) finishClose() {
diff --git a/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go b/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go
index 7ae077afb2..c69c94c160 100644
--- a/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go
+++ b/downstreamadapter/dispatchermanager/dispatcher_manager_redo.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/log"
"github.com/pingcap/ticdc/downstreamadapter/dispatcher"
"github.com/pingcap/ticdc/downstreamadapter/eventcollector"
+ "github.com/pingcap/ticdc/downstreamadapter/sink"
"github.com/pingcap/ticdc/downstreamadapter/sink/redo"
"github.com/pingcap/ticdc/heartbeatpb"
"github.com/pingcap/ticdc/pkg/common"
@@ -77,7 +78,7 @@ func initRedoComponet(
return newWritePathClosedError()
}
manager.redoDispatcherMap = redoDispatcherMap
- manager.redoSink = redoSink
+ manager.redoSink = withCaptureWriteGate(ctx, redoSink)
manager.redoSchemaIDToDispatchers = redoSchemaIDToDispatchers
manager.redoQuota = redoQuota
manager.sinkQuota = totalQuota - redoQuota
@@ -159,7 +160,7 @@ func (e *DispatcherManager) NewTableTriggerRedoDispatcher(id *heartbeatpb.Dispat
return nil
}
-func (e *DispatcherManager) getRedoEventCollectorBatchCountAndBytes(redoSink *redo.Sink) (int, int) {
+func (e *DispatcherManager) getRedoEventCollectorBatchCountAndBytes(redoSink sink.Sink) (int, int) {
var (
batchCount = redoSink.BatchCount()
batchBytes = redoSink.BatchBytes()
diff --git a/downstreamadapter/sink/blackhole/sink.go b/downstreamadapter/sink/blackhole/sink.go
index 6124219ad4..10c838b1fc 100644
--- a/downstreamadapter/sink/blackhole/sink.go
+++ b/downstreamadapter/sink/blackhole/sink.go
@@ -20,6 +20,7 @@ import (
"github.com/pingcap/ticdc/pkg/common"
commonEvent "github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/metrics"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/utils/chann"
"go.uber.org/zap"
)
@@ -49,6 +50,9 @@ func (s *Sink) SinkType() common.SinkType {
func (s *Sink) SetTableSchemaStore(_ *commonEvent.TableSchemaStore) {
}
+func (s *Sink) SetWriteGate(_ *writelease.Gate) {
+}
+
func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) {
// NOTE: don't change the log, integration test `lossy_ddl` depends on it.
// ref: https://github.com/pingcap/ticdc/blob/da834db76e0662ff15ef12645d1f37bfa6506d83/tests/integration_tests/lossy_ddl/run.sh#L23
diff --git a/downstreamadapter/sink/cloudstorage/dml_writers.go b/downstreamadapter/sink/cloudstorage/dml_writers.go
index 6f4d8e9eb2..2cb8332505 100644
--- a/downstreamadapter/sink/cloudstorage/dml_writers.go
+++ b/downstreamadapter/sink/cloudstorage/dml_writers.go
@@ -25,6 +25,7 @@ import (
commonEvent "github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/sink/codec/common"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/utils/chann"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"go.uber.org/atomic"
@@ -49,6 +50,12 @@ type dmlWriters struct {
closed atomic.Bool
}
+func (d *dmlWriters) setWriteGate(gate *writelease.Gate) {
+ for _, writer := range d.writers {
+ writer.setWriteGate(gate)
+ }
+}
+
func newDMLWriters(
changefeedID commonType.ChangeFeedID,
storage storeapi.Storage,
diff --git a/downstreamadapter/sink/cloudstorage/sink.go b/downstreamadapter/sink/cloudstorage/sink.go
index 297c5f1e06..fce150c04a 100644
--- a/downstreamadapter/sink/cloudstorage/sink.go
+++ b/downstreamadapter/sink/cloudstorage/sink.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/meta/model"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/robfig/cron"
@@ -74,7 +75,8 @@ type sink struct {
// we have to use the context from the struct to perceive the context done from the upper layer
// To perceive the context done from the upper layer
// it's the same as the context passed into the Run method.
- ctx context.Context
+ ctx context.Context
+ writeGate *writelease.Gate
}
func Verify(ctx context.Context, changefeedID common.ChangeFeedID, sinkURI *url.URL, sinkConfig *config.SinkConfig, enableTableAcrossNodes bool) error {
@@ -305,6 +307,9 @@ func (s *sink) writeFile(v *commonEvent.DDLEvent, schemaFile cloudstorage.Schema
}
encodedSchemaFile := schemaFile.Marshal()
path := schemaFile.Path(s.cfg.UseTableIDAsPath, v.GetTableID())
+ if err := writelease.WaitForWrite(s.ctx, s.writeGate); err != nil {
+ return err
+ }
return s.statistics.RecordDDLExecution(func() (string, error) {
err := s.storage.WriteFile(s.ctx, path, encodedSchemaFile)
if err != nil {
@@ -314,6 +319,13 @@ func (s *sink) writeFile(v *commonEvent.DDLEvent, schemaFile cloudstorage.Schema
})
}
+func (s *sink) SetWriteGate(gate *writelease.Gate) {
+ s.writeGate = gate
+ if s.dmlWriters != nil {
+ s.dmlWriters.setWriteGate(gate)
+ }
+}
+
func (s *sink) AddCheckpointTs(ts uint64) {
if !s.IsNormal() {
return
@@ -365,6 +377,9 @@ func (s *sink) sendCheckpointTs(ctx context.Context) error {
zap.Duration("duration", time.Since(start)),
zap.Error(err))
}
+ if !writelease.CanWrite(s.writeGate) {
+ continue
+ }
err = s.storage.WriteFile(ctx, "metadata", message)
if err != nil {
log.Error("cloud storage sink write file failed",
@@ -435,6 +450,9 @@ func (s *sink) genCleanupJob(ctx context.Context, uri *url.URL) []func() {
var isRemoveEmptyDirsRunning atomic.Bool
if isLocal {
ret = append(ret, func() {
+ if !writelease.CanWrite(s.writeGate) {
+ return
+ }
if !isRemoveEmptyDirsRunning.CompareAndSwap(false, true) {
log.Warn("remove empty dirs is already running, skip this round",
zap.String("keyspace", s.changefeedID.Keyspace()),
@@ -466,6 +484,9 @@ func (s *sink) genCleanupJob(ctx context.Context, uri *url.URL) []func() {
var isCleanupRunning atomic.Bool
ret = append(ret, func() {
+ if !writelease.CanWrite(s.writeGate) {
+ return
+ }
if !isCleanupRunning.CompareAndSwap(false, true) {
log.Warn("cleanup expired files is already running, skip this round",
zap.String("keyspace", s.changefeedID.Keyspace()),
diff --git a/downstreamadapter/sink/cloudstorage/writer.go b/downstreamadapter/sink/cloudstorage/writer.go
index e31a1387c6..8bc27462c0 100644
--- a/downstreamadapter/sink/cloudstorage/writer.go
+++ b/downstreamadapter/sink/cloudstorage/writer.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/ticdc/pkg/common"
"github.com/pingcap/ticdc/pkg/errors"
pmetrics "github.com/pingcap/ticdc/pkg/metrics"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
@@ -50,6 +51,12 @@ type writer struct {
metricFlushBytes prometheus.Observer
metricFlushDuration prometheus.Observer
+ writeGate *writelease.Gate
+}
+
+func (d *writer) setWriteGate(gate *writelease.Gate) {
+ d.writeGate = gate
+ d.filePathGenerator.SetWriteGate(gate)
}
// flushTask is internal and never crosses component boundary.
@@ -221,6 +228,9 @@ func (d *writer) discardEntries(entries []*spool.Entry) {
func (d *writer) writeDataFile(ctx context.Context, dataFilePath, indexFilePath string, payload *payload) error {
keyspace := d.changeFeedID.Keyspace()
changefeed := d.changeFeedID.Name()
+ if err := writelease.WaitForWrite(ctx, d.writeGate); err != nil {
+ return err
+ }
start := time.Now()
err := d.statistics.RecordBatchExecution(func() (int, int64, error) {
@@ -262,6 +272,9 @@ func (d *writer) writeDataFile(ctx context.Context, dataFilePath, indexFilePath
return err
}
+ if err := writelease.WaitForWrite(ctx, d.writeGate); err != nil {
+ return err
+ }
err = d.storage.WriteFile(ctx, indexFilePath, []byte(path.Base(dataFilePath)+"\n"))
if err != nil {
log.Error("failed to write index file to external storage",
diff --git a/downstreamadapter/sink/cloudstorage/writer_test.go b/downstreamadapter/sink/cloudstorage/writer_test.go
index 7aaf02dd82..4d34c842dc 100644
--- a/downstreamadapter/sink/cloudstorage/writer_test.go
+++ b/downstreamadapter/sink/cloudstorage/writer_test.go
@@ -35,6 +35,7 @@ import (
"github.com/pingcap/ticdc/pkg/pdutil"
"github.com/pingcap/ticdc/pkg/sink/codec/common"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/meta/model"
"github.com/pingcap/tidb/pkg/objstore/objectio"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
@@ -597,6 +598,12 @@ type failOnIndexStorage struct {
storeapi.Storage
}
+type fenceAfterDataStorage struct {
+ storeapi.Storage
+ dataFile string
+ gate *writelease.Gate
+}
+
type failOnCloseStorage struct {
storeapi.Storage
}
@@ -612,6 +619,53 @@ func (s *failOnIndexStorage) WriteFile(ctx context.Context, name string, data []
return s.Storage.WriteFile(ctx, name, data)
}
+func (s *fenceAfterDataStorage) WriteFile(ctx context.Context, name string, data []byte) error {
+ if err := s.Storage.WriteFile(ctx, name, data); err != nil {
+ return err
+ }
+ if name == s.dataFile {
+ s.gate.Fence()
+ }
+ return nil
+}
+
+func TestWriterChecksWriteGateBeforePublishingIndex(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ parentDir := t.TempDir()
+ d := testWriter(ctx, t, parentDir)
+ gate := writelease.NewGate()
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+
+ dataFile := "data.json"
+ indexFile := "meta/data.index"
+ d.storage = &fenceAfterDataStorage{
+ Storage: d.storage,
+ dataFile: dataFile,
+ gate: gate,
+ }
+ d.setWriteGate(gate)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- d.writeDataFile(ctx, dataFile, indexFile, &payload{
+ data: []byte(`{"id":1}`),
+ rowsCount: 1,
+ nBytes: 8,
+ })
+ }()
+
+ require.Eventually(t, func() bool {
+ _, err := os.Stat(path.Join(parentDir, dataFile))
+ return err == nil
+ }, time.Second, 10*time.Millisecond)
+ _, err := os.Stat(path.Join(parentDir, indexFile))
+ require.ErrorIs(t, err, os.ErrNotExist)
+
+ cancel()
+ require.ErrorIs(t, <-done, context.Canceled)
+}
+
func (s *failOnCloseStorage) Create(
ctx context.Context, name string, option *storeapi.WriterOption,
) (objectio.Writer, error) {
diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go
index c960f0cca1..7b28c9e232 100644
--- a/downstreamadapter/sink/kafka/sink.go
+++ b/downstreamadapter/sink/kafka/sink.go
@@ -33,6 +33,7 @@ import (
"github.com/pingcap/ticdc/pkg/sink/kafka"
"github.com/pingcap/ticdc/pkg/sink/kafka/claimcheck"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/utils/chann"
"go.uber.org/atomic"
"go.uber.org/zap"
@@ -64,8 +65,9 @@ type sink struct {
rowChan *chann.UnlimitedChannel[*commonEvent.MQRowEvent, any]
// isNormal indicate whether the sink is in the normal state.
- isNormal *atomic.Bool
- ctx context.Context
+ isNormal *atomic.Bool
+ ctx context.Context
+ writeGate *writelease.Gate
}
func (s *sink) SinkType() common.SinkType {
@@ -238,6 +240,13 @@ func (s *sink) AddDMLEvent(event *commonEvent.DMLEvent) {
s.eventChan.Push(event)
}
+func (s *sink) SetWriteGate(gate *writelease.Gate) {
+ s.writeGate = gate
+ if s.comp.claimCheck != nil {
+ s.comp.claimCheck.SetWriteGate(gate)
+ }
+}
+
func (s *sink) FlushDMLBeforeBlock(_ commonEvent.BlockEvent) error {
return nil
}
@@ -308,6 +317,9 @@ func (s *sink) calculateKeyPartitions(ctx context.Context) error {
schema := event.TableInfo.GetSchemaName()
table := event.TableInfo.GetTableName()
topic := s.comp.eventRouter.GetTopicForRowChange(schema, table)
+ if err := writelease.WaitForWrite(ctx, s.writeGate); err != nil {
+ return err
+ }
partitionNum, err := s.comp.topicManager.GetPartitionNum(ctx, topic)
if err != nil {
return err
@@ -425,6 +437,9 @@ func (s *sink) sendMessages(ctx context.Context) error {
return err
}
for _, message := range future.Messages {
+ if err = writelease.WaitForWrite(ctx, s.writeGate); err != nil {
+ return err
+ }
start := time.Now()
if err = s.statistics.RecordBatchExecution(func() (int, int64, error) {
message.SetPartitionKey(future.Key.PartitionKey)
@@ -460,6 +475,9 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error {
}
codecCommon.SetDDLMessageLogInfo(message, e)
topic := s.comp.eventRouter.GetTopicForDDL(e)
+ if err := writelease.WaitForWrite(s.ctx, s.writeGate); err != nil {
+ return err
+ }
// Notice: We must call GetPartitionNum here,
// which will be responsible for automatically creating topics when they don't exist.
// If it is not called here and kafka has `auto.create.topics.enable` turned on,
@@ -468,6 +486,9 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error {
if err != nil {
return err
}
+ if err := writelease.WaitForWrite(s.ctx, s.writeGate); err != nil {
+ return err
+ }
ddlType := e.GetDDLType().String()
if s.partitionRule == helper.PartitionAll {
err = s.statistics.RecordDDLExecution(func() (string, error) {
@@ -530,6 +551,9 @@ func (s *sink) sendCheckpoint(ctx context.Context) error {
continue
}
codecCommon.SetCheckpointMessageLogInfo(msg, ts)
+ if !writelease.CanWrite(s.writeGate) {
+ continue
+ }
tableNames := s.getAllTableNames(ts)
// NOTICE: When there are no tables to replicate,
@@ -541,6 +565,9 @@ func (s *sink) sendCheckpoint(ctx context.Context) error {
if err != nil {
return err
}
+ if !writelease.CanWrite(s.writeGate) {
+ continue
+ }
err = s.ddlProducer.SendMessages(topic, partitionNum, msg)
if err != nil {
return err
@@ -552,6 +579,9 @@ func (s *sink) sendCheckpoint(ctx context.Context) error {
if err != nil {
return err
}
+ if !writelease.CanWrite(s.writeGate) {
+ break
+ }
err = s.ddlProducer.SendMessages(topic, partitionNum, msg)
if err != nil {
return err
diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go
index 47133a2723..c193784d2e 100644
--- a/downstreamadapter/sink/kafka/sink_test.go
+++ b/downstreamadapter/sink/kafka/sink_test.go
@@ -33,6 +33,7 @@ import (
"github.com/pingcap/ticdc/pkg/sink/codec"
codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common"
"github.com/pingcap/ticdc/pkg/sink/kafka"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/meta/model"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
@@ -230,6 +231,59 @@ func TestKafkaSinkBasicFunctionality(t *testing.T) {
kafkaSink.AddCheckpointTs(12345)
}
+func TestKafkaSinkWriteGateBlocksDMLSend(t *testing.T) {
+ eventHelper := commonEvent.NewEventTestHelper(t)
+ defer eventHelper.Close()
+ eventHelper.Tk().MustExec("use test")
+ require.NotNil(t, eventHelper.DDL2Job("create table t (id int primary key)"))
+ dmlEvent := eventHelper.DML2Event("test", "t", "insert into t values (1)")
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ kafkaSink, topicManager, asyncProducer, _ := newKafkaSinkForTest(
+ t, ctx, config.ProtocolOpen, &config.SinkConfig{})
+ topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(1), nil)
+ sent := make(chan struct{}, 1)
+ asyncProducer.EXPECT().AsyncRunCallback(gomock.Any()).Return(nil).AnyTimes()
+ asyncProducer.EXPECT().AsyncSend(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+ DoAndReturn(func(
+ _ context.Context,
+ _ string,
+ _ int32,
+ message *codecCommon.Message,
+ ) error {
+ if message.Callback != nil {
+ message.Callback()
+ }
+ sent <- struct{}{}
+ return nil
+ }).Times(1)
+ gate := writelease.NewGate()
+ kafkaSink.SetWriteGate(gate)
+
+ runDone := make(chan error, 1)
+ go func() {
+ runDone <- kafkaSink.Run(ctx)
+ }()
+ kafkaSink.AddDMLEvent(dmlEvent)
+
+ select {
+ case <-sent:
+ t.Fatal("Kafka DML was sent while the capture write gate was closed")
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ select {
+ case <-sent:
+ case <-time.After(5 * time.Second):
+ t.Fatal("Kafka DML was not sent after the capture write gate reopened")
+ }
+
+ cancel()
+ require.ErrorIs(t, <-runDone, context.Canceled)
+}
+
func TestKafkaSinkBatchConfig(t *testing.T) {
sink := &sink{}
require.Equal(t, 4096, sink.BatchCount())
diff --git a/downstreamadapter/sink/mock/sink_mock.go b/downstreamadapter/sink/mock/sink_mock.go
index 40e2ea2a67..67ca7f6478 100644
--- a/downstreamadapter/sink/mock/sink_mock.go
+++ b/downstreamadapter/sink/mock/sink_mock.go
@@ -11,6 +11,7 @@ import (
gomock "github.com/golang/mock/gomock"
common "github.com/pingcap/ticdc/pkg/common"
event "github.com/pingcap/ticdc/pkg/common/event"
+ writelease "github.com/pingcap/ticdc/pkg/writelease"
)
// MockSink is a mock of Sink interface.
@@ -154,6 +155,18 @@ func (mr *MockSinkMockRecorder) SetTableSchemaStore(tableSchemaStore interface{}
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTableSchemaStore", reflect.TypeOf((*MockSink)(nil).SetTableSchemaStore), tableSchemaStore)
}
+// SetWriteGate mocks base method.
+func (m *MockSink) SetWriteGate(gate *writelease.Gate) {
+ m.ctrl.T.Helper()
+ m.ctrl.Call(m, "SetWriteGate", gate)
+}
+
+// SetWriteGate indicates an expected call of SetWriteGate.
+func (mr *MockSinkMockRecorder) SetWriteGate(gate interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWriteGate", reflect.TypeOf((*MockSink)(nil).SetWriteGate), gate)
+}
+
// SinkType mocks base method.
func (m *MockSink) SinkType() common.SinkType {
m.ctrl.T.Helper()
diff --git a/downstreamadapter/sink/mysql/sink.go b/downstreamadapter/sink/mysql/sink.go
index 642e664837..d9fbec1fe3 100644
--- a/downstreamadapter/sink/mysql/sink.go
+++ b/downstreamadapter/sink/mysql/sink.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/sink/mysql"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/parser/ast"
"go.uber.org/atomic"
"go.uber.org/zap"
@@ -62,6 +63,7 @@ type Sink struct {
// isNormal indicate whether the sink is in the normal state.
isNormal *atomic.Bool
cfg *mysql.Config
+ writeGate *writelease.Gate
maxTxnRows int
bdrMode bool
// enableActiveActive enables active-active replication behaviors in the MySQL-class sink.
@@ -356,6 +358,16 @@ func (s *Sink) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaStor
}
}
+// SetWriteGate delegates admission to the transport writers, where each
+// downstream write is checked immediately before it is executed.
+func (s *Sink) SetWriteGate(gate *writelease.Gate) {
+ s.writeGate = gate
+ for _, writer := range s.dmlWriter {
+ writer.SetWriteGate(gate)
+ }
+ s.ddlWriter.SetWriteGate(gate)
+}
+
func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) {
s.conflictDetector.Add(event)
}
@@ -538,6 +550,7 @@ func (s *Sink) CleanupRemovedChangefeed() error {
cleanupWriter := mysql.NewWriter(context.Background(), -1, db, s.cfg, s.changefeedID, nil, nil)
defer cleanupWriter.Close()
+ cleanupWriter.SetWriteGate(s.writeGate)
return cleanupWriter.RemoveDDLTsItem()
}
diff --git a/downstreamadapter/sink/mysql/sink_test.go b/downstreamadapter/sink/mysql/sink_test.go
index b80e25fc7a..f10f0f3920 100644
--- a/downstreamadapter/sink/mysql/sink_test.go
+++ b/downstreamadapter/sink/mysql/sink_test.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/ticdc/pkg/common"
commonEvent "github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/sink/mysql"
+ "github.com/pingcap/ticdc/pkg/writelease"
timodel "github.com/pingcap/tidb/pkg/meta/model"
"github.com/pingcap/tidb/pkg/sessionctx/vardef"
"github.com/stretchr/testify/require"
@@ -600,8 +601,35 @@ func TestGetTableRecoveryInfo_RemoveDDLTs(t *testing.T) {
mock.ExpectCommit()
mock.ExpectClose() // Expect database close when sink.Close() is called
- // Call GetTableRecoveryInfo with removeDDLTs=true
- resultStartTsList, skipSyncpointList, skipDMLList, err := sink.GetTableRecoveryInfo(tableIDs, inputStartTsList, true)
+ gate := writelease.NewGate()
+ gate.SetP2PRequired(true)
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ sink.SetWriteGate(gate)
+
+ type recoveryResult struct {
+ startTsList []int64
+ skipSyncpoint []bool
+ skipDML []bool
+ err error
+ }
+ resultCh := make(chan recoveryResult, 1)
+ go func() {
+ startTsList, skipSyncpoint, skipDML, err := sink.GetTableRecoveryInfo(tableIDs, inputStartTsList, true)
+ resultCh <- recoveryResult{startTsList, skipSyncpoint, skipDML, err}
+ }()
+
+ select {
+ case result := <-resultCh:
+ t.Fatalf("DDL-ts cleanup passed through a closed capture write gate: %v", result.err)
+ case <-time.After(50 * time.Millisecond):
+ }
+
+ require.True(t, gate.RenewP2P(time.Now(), writelease.P2PLeaseDuration))
+ result := <-resultCh
+ resultStartTsList := result.startTsList
+ skipSyncpointList := result.skipSyncpoint
+ skipDMLList := result.skipDML
+ err := result.err
require.NoError(t, err)
require.Len(t, resultStartTsList, 3)
diff --git a/downstreamadapter/sink/pulsar/sink.go b/downstreamadapter/sink/pulsar/sink.go
index 9895541233..ad31bee9ed 100644
--- a/downstreamadapter/sink/pulsar/sink.go
+++ b/downstreamadapter/sink/pulsar/sink.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/sink/codec/common"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/utils/chann"
"go.uber.org/atomic"
"go.uber.org/zap"
@@ -64,6 +65,7 @@ type sink struct {
checkpointTsChan chan uint64
eventChan *chann.UnlimitedChannel[*commonEvent.DMLEvent, any]
rowChan *chann.UnlimitedChannel[*commonEvent.MQRowEvent, any]
+ writeGate *writelease.Gate
}
func (s *sink) SinkType() commonType.SinkType {
@@ -200,6 +202,10 @@ func (s *sink) AddDMLEvent(event *commonEvent.DMLEvent) {
s.eventChan.Push(event)
}
+func (s *sink) SetWriteGate(gate *writelease.Gate) {
+ s.writeGate = gate
+}
+
func (s *sink) FlushDMLBeforeBlock(_ commonEvent.BlockEvent) error {
return nil
}
@@ -240,6 +246,9 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error {
}
common.SetDDLMessageLogInfo(message, e)
topic := s.comp.eventRouter.GetTopicForDDL(e)
+ if err := writelease.WaitForWrite(s.ctx, s.writeGate); err != nil {
+ return err
+ }
// Notice: We must call GetPartitionNum here,
// which will be responsible for automatically creating topics when they don't exist.
// If it is not called here and kafka has `auto.create.topics.enable` turned on,
@@ -248,6 +257,9 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error {
if err != nil {
return err
}
+ if err := writelease.WaitForWrite(s.ctx, s.writeGate); err != nil {
+ return err
+ }
ddlType := e.GetDDLType().String()
if s.partitionRule == helper.PartitionAll {
err = s.statistics.RecordDDLExecution(func() (string, error) {
@@ -316,6 +328,9 @@ func (s *sink) sendCheckpoint(ctx context.Context) error {
continue
}
common.SetCheckpointMessageLogInfo(msg, ts)
+ if !writelease.CanWrite(s.writeGate) {
+ continue
+ }
tableNames := s.getAllTableNames(ts)
// NOTICE: When there are no tables to replicate,
@@ -327,6 +342,9 @@ func (s *sink) sendCheckpoint(ctx context.Context) error {
if err != nil {
return errors.Trace(err)
}
+ if !writelease.CanWrite(s.writeGate) {
+ continue
+ }
err = s.ddlProducer.syncBroadcastMessage(ctx, topic, msg, common.MessageTypeResolved)
if err != nil {
return errors.Trace(err)
@@ -338,6 +356,9 @@ func (s *sink) sendCheckpoint(ctx context.Context) error {
if err != nil {
return errors.Trace(err)
}
+ if !writelease.CanWrite(s.writeGate) {
+ break
+ }
err = s.ddlProducer.syncBroadcastMessage(ctx, topic, msg, common.MessageTypeResolved)
if err != nil {
return errors.Trace(err)
@@ -398,6 +419,9 @@ func (s *sink) calculateKeyPartitions(ctx context.Context) error {
schema := event.TableInfo.GetSchemaName()
table := event.TableInfo.GetTableName()
topic := s.comp.eventRouter.GetTopicForRowChange(schema, table)
+ if err := writelease.WaitForWrite(ctx, s.writeGate); err != nil {
+ return errors.Trace(err)
+ }
partitionNum, err := s.comp.topicManager.GetPartitionNum(ctx, topic)
if err != nil {
return errors.Trace(err)
@@ -532,6 +556,9 @@ func (s *sink) sendMessages(ctx context.Context) error {
return errors.Trace(err)
}
for _, message := range future.Messages {
+ if err = writelease.WaitForWrite(ctx, s.writeGate); err != nil {
+ return errors.Trace(err)
+ }
start := time.Now()
if err = s.statistics.RecordBatchExecution(func() (int, int64, error) {
message.SetPartitionKey(future.Key.PartitionKey)
diff --git a/downstreamadapter/sink/pulsar/sink_test.go b/downstreamadapter/sink/pulsar/sink_test.go
index 973e4f15e6..7371dc3fec 100644
--- a/downstreamadapter/sink/pulsar/sink_test.go
+++ b/downstreamadapter/sink/pulsar/sink_test.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/ticdc/pkg/config"
cerror "github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/metrics"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/utils/chann"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
@@ -66,13 +67,17 @@ func newPulsarSinkForTest(t *testing.T) (*sink, error) {
statistics: statistics,
ctx: ctx,
}
- go pulsarSink.Run(ctx)
return pulsarSink, nil
}
func TestPulsarSinkBasicFunctionality(t *testing.T) {
pulsarSink, err := newPulsarSinkForTest(t)
require.NoError(t, err)
+ ctx, cancel := context.WithCancel(context.Background())
+ runDone := make(chan error, 1)
+ go func() {
+ runDone <- pulsarSink.Run(ctx)
+ }()
var count atomic.Int64
@@ -132,6 +137,41 @@ func TestPulsarSinkBasicFunctionality(t *testing.T) {
require.Len(t, pulsarSink.ddlProducer.(*mockProducer).GetAllEvents(), 1)
require.Equal(t, count.Load(), int64(3))
+ cancel()
+ require.ErrorIs(t, <-runDone, context.Canceled)
+ pulsarSink.Close()
+}
+
+func TestPulsarSinkWriteGateBlocksDDLSend(t *testing.T) {
+ pulsarSink, err := newPulsarSinkForTest(t)
+ require.NoError(t, err)
+ defer pulsarSink.Close()
+ gate := writelease.NewGate()
+ pulsarSink.SetWriteGate(gate)
+
+ ddlEvent := &commonEvent.DDLEvent{
+ Query: "create table t (id int primary key)",
+ FinishedTs: 1,
+ BlockedTables: &commonEvent.InfluencedTables{
+ InfluenceType: commonEvent.InfluenceTypeNormal,
+ TableIDs: []int64{0},
+ },
+ }
+ done := make(chan error, 1)
+ go func() {
+ done <- pulsarSink.WriteBlockEvent(ddlEvent)
+ }()
+
+ select {
+ case err := <-done:
+ t.Fatalf("Pulsar DDL returned while the capture write gate was closed: %v", err)
+ case <-time.After(100 * time.Millisecond):
+ }
+ require.Empty(t, pulsarSink.ddlProducer.(*mockProducer).GetAllEvents())
+
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ require.NoError(t, <-done)
+ require.Len(t, pulsarSink.ddlProducer.(*mockProducer).GetAllEvents(), 1)
}
func TestPulsarSinkBatchConfig(t *testing.T) {
diff --git a/downstreamadapter/sink/redo/meta.go b/downstreamadapter/sink/redo/meta.go
index ab75d5d887..ddd8355053 100644
--- a/downstreamadapter/sink/redo/meta.go
+++ b/downstreamadapter/sink/redo/meta.go
@@ -29,6 +29,7 @@ import (
misc "github.com/pingcap/ticdc/pkg/redo/common"
"github.com/pingcap/ticdc/pkg/util"
"github.com/pingcap/ticdc/pkg/uuid"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/prometheus/client_golang/prometheus"
@@ -63,6 +64,7 @@ type RedoMeta struct {
metricResolvedTs prometheus.Gauge
flushIntervalInMs int64
+ writeGate *writelease.Gate
}
// NewRedoMeta creates a new redo meta.
@@ -101,6 +103,10 @@ func (m *RedoMeta) Running() bool {
return m.running.Load()
}
+func (m *RedoMeta) SetWriteGate(gate *writelease.Gate) {
+ m.writeGate = gate
+}
+
func (m *RedoMeta) PreStart(ctx context.Context) (err error) {
uri, err := objstore.ParseRawURL(util.GetOrZero(m.cfg.Storage))
if err != nil {
@@ -263,6 +269,11 @@ func (m *RedoMeta) initMeta(ctx context.Context) error {
zap.Uint64("checkpointTs", flushedMeta.CheckpointTs),
zap.Uint64("resolvedTs", flushedMeta.ResolvedTs))
+ if len(toRemoveMetaFiles) != 0 {
+ if err := writelease.WaitForWrite(ctx, m.writeGate); err != nil {
+ return err
+ }
+ }
return util.DeleteFilesInExtStorage(ctx, m.extStorage, toRemoveMetaFiles)
}
@@ -275,6 +286,9 @@ func (m *RedoMeta) preCleanupExtStorage(ctx context.Context) error {
if !ret {
return nil
}
+ if err := writelease.WaitForWrite(ctx, m.writeGate); err != nil {
+ return err
+ }
changefeedMatcher := getChangefeedMatcher(m.changeFeedID)
err = util.RemoveFilesIf(ctx, m.extStorage, func(path string) bool {
@@ -287,6 +301,9 @@ func (m *RedoMeta) preCleanupExtStorage(ctx context.Context) error {
return err
}
+ if err := writelease.WaitForWrite(ctx, m.writeGate); err != nil {
+ return err
+ }
err = m.extStorage.DeleteFile(ctx, deleteMarker)
if err != nil && !util.IsNotExistInExtStorage(err) {
return errors.WrapError(errors.ErrExternalStorageAPI, err)
@@ -345,6 +362,9 @@ func (m *RedoMeta) deleteAllLogs(ctx context.Context) error {
}
// Write deleted mark before clean any files.
deleteMarker := getDeletedChangefeedMarker(m.changeFeedID)
+ if err := writelease.WaitForWrite(ctx, m.writeGate); err != nil {
+ return err
+ }
if err := m.extStorage.WriteFile(ctx, deleteMarker, []byte("D")); err != nil {
return errors.WrapError(errors.ErrExternalStorageAPI, err)
}
@@ -352,6 +372,9 @@ func (m *RedoMeta) deleteAllLogs(ctx context.Context) error {
zap.String("keyspace", m.changeFeedID.Keyspace()),
zap.String("changefeed", m.changeFeedID.Name()))
+ if err := writelease.WaitForWrite(ctx, m.writeGate); err != nil {
+ return err
+ }
changefeedMatcher := getChangefeedMatcher(m.changeFeedID)
return util.RemoveFilesIf(ctx, m.extStorage, func(path string) bool {
if path == deleteMarker || !strings.Contains(path, changefeedMatcher) {
@@ -418,6 +441,9 @@ func (m *RedoMeta) flush(ctx context.Context, meta misc.LogMeta) error {
return errors.WrapError(errors.ErrMarshalFailed, err)
}
metaFile := getMetafileName(m.captureID, m.changeFeedID, m.uuidGenerator)
+ if err := writelease.WaitForWrite(ctx, m.writeGate); err != nil {
+ return err
+ }
if err := m.extStorage.WriteFile(ctx, metaFile, data); err != nil {
log.Error("redo: meta manager flush meta write file failed",
zap.String("keyspace", m.changeFeedID.Keyspace()),
@@ -426,7 +452,7 @@ func (m *RedoMeta) flush(ctx context.Context, meta misc.LogMeta) error {
return errors.WrapError(errors.ErrExternalStorageAPI, err)
}
- if m.preMetaFile != "" {
+ if m.preMetaFile != "" && writelease.CanWrite(m.writeGate) {
if m.preMetaFile == metaFile {
// This should only happen when use a constant uuid generator in test.
return nil
@@ -517,6 +543,9 @@ func (m *RedoMeta) bgGC(egCtx context.Context) error {
if ckpt == preCkpt {
continue
}
+ if !writelease.CanWrite(m.writeGate) {
+ continue
+ }
preCkpt = ckpt
log.Debug("redo meta GC is triggered",
zap.Uint64("checkpointTs", ckpt),
diff --git a/downstreamadapter/sink/redo/meta_test.go b/downstreamadapter/sink/redo/meta_test.go
index 6d6378c39f..6a0efd4404 100644
--- a/downstreamadapter/sink/redo/meta_test.go
+++ b/downstreamadapter/sink/redo/meta_test.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/ticdc/pkg/redo/testutil"
"github.com/pingcap/ticdc/pkg/util"
"github.com/pingcap/ticdc/pkg/uuid"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/mockobjstore"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
promtestutil "github.com/prometheus/client_golang/prometheus/testutil"
@@ -113,6 +114,45 @@ func TestInitAndWriteMeta(t *testing.T) {
require.ErrorIs(t, eg.Wait(), context.Canceled)
}
+func TestRedoMetaFlushWaitsForWriteGate(t *testing.T) {
+ ctx := t.Context()
+ changefeedID := common.NewChangeFeedIDWithName(t.Name(), common.DefaultKeyspaceName)
+ _, uri, err := util.GetTestExtStorage(ctx, t.TempDir())
+ require.NoError(t, err)
+ m := NewRedoMeta(changefeedID, 1, testutil.NewConsistentConfig(uri.String()))
+ gate := writelease.NewGate()
+ m.SetWriteGate(gate)
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ require.NoError(t, m.PreStart(ctx))
+ t.Cleanup(func() {
+ m.closeExtStorage()
+ m.CleanupMetrics()
+ })
+ initialMetaFile := m.preMetaFile
+ require.NotEmpty(t, initialMetaFile)
+ gate.SetP2PRequired(true)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- m.flush(ctx, misc.NewMeta(2, 3))
+ }()
+
+ time.Sleep(100 * time.Millisecond)
+ require.Equal(t, initialMetaFile, m.preMetaFile)
+
+ require.True(t, gate.RenewP2P(time.Now(), writelease.P2PLeaseDuration))
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(5 * time.Second):
+ t.Fatal("redo metadata flush did not resume after the capture write gate reopened")
+ }
+ require.NotEqual(t, initialMetaFile, m.preMetaFile)
+ exists, err := m.extStorage.FileExists(ctx, initialMetaFile)
+ require.NoError(t, err)
+ require.False(t, exists)
+}
+
func TestPreCleanupAndWriteMeta(t *testing.T) {
t.Parallel()
diff --git a/downstreamadapter/sink/redo/sink.go b/downstreamadapter/sink/redo/sink.go
index d19fbf9c1e..20ffbed811 100644
--- a/downstreamadapter/sink/redo/sink.go
+++ b/downstreamadapter/sink/redo/sink.go
@@ -27,6 +27,7 @@ import (
"github.com/pingcap/ticdc/pkg/redo/writer"
"github.com/pingcap/ticdc/pkg/redo/writer/factory"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/utils/chann"
"go.uber.org/atomic"
"go.uber.org/zap"
@@ -193,6 +194,11 @@ func (s *Sink) AddDMLEvent(event *commonEvent.DMLEvent) {
s.logBuffer.Push(events...)
}
+func (s *Sink) SetWriteGate(gate *writelease.Gate) {
+ s.dmlWriter.SetWriteGate(gate)
+ s.ddlWriter.SetWriteGate(gate)
+}
+
func (s *Sink) IsNormal() bool {
return s.isNormal.Load()
}
diff --git a/downstreamadapter/sink/sink.go b/downstreamadapter/sink/sink.go
index 7231a144b7..c7b3ca88c2 100644
--- a/downstreamadapter/sink/sink.go
+++ b/downstreamadapter/sink/sink.go
@@ -27,6 +27,7 @@ import (
"github.com/pingcap/ticdc/pkg/config"
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
)
type Sink interface {
@@ -42,6 +43,9 @@ type Sink interface {
// implementations are expected to call event.PostFlush().
WriteBlockEvent(event commonEvent.BlockEvent) error
AddCheckpointTs(ts uint64)
+ // SetWriteGate installs the capture-wide write admission gate. Every sink
+ // must enforce it again at its actual downstream mutation boundary.
+ SetWriteGate(gate *writelease.Gate)
SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaStore)
Close()
diff --git a/downstreamadapter/sink/write_gate.go b/downstreamadapter/sink/write_gate.go
new file mode 100644
index 0000000000..662f90d9dc
--- /dev/null
+++ b/downstreamadapter/sink/write_gate.go
@@ -0,0 +1,97 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sink
+
+import (
+ "context"
+
+ commonEvent "github.com/pingcap/ticdc/pkg/common/event"
+ "github.com/pingcap/ticdc/pkg/writelease"
+)
+
+type writeGatedSink struct {
+ Sink
+ ctx context.Context
+ gate *writelease.Gate
+}
+
+// WithWriteGate prevents new downstream side effects from entering a sink
+// while the capture cannot prove that it still owns the write lease.
+func WithWriteGate(ctx context.Context, inner Sink, gate *writelease.Gate) Sink {
+ if gate == nil {
+ return inner
+ }
+ inner.SetWriteGate(gate)
+ return &writeGatedSink{
+ Sink: inner,
+ ctx: ctx,
+ gate: gate,
+ }
+}
+
+func (s *writeGatedSink) AddDMLEvent(event *commonEvent.DMLEvent) {
+ if s.waitUntilWritable() {
+ s.Sink.AddDMLEvent(event)
+ }
+}
+
+func (s *writeGatedSink) FlushDMLBeforeBlock(event commonEvent.BlockEvent) error {
+ if err := s.gate.WaitUntilWritable(s.ctx); err != nil {
+ return err
+ }
+ if err := s.ensureWritable(); err != nil {
+ return err
+ }
+ return s.Sink.FlushDMLBeforeBlock(event)
+}
+
+func (s *writeGatedSink) WriteBlockEvent(event commonEvent.BlockEvent) error {
+ if err := s.gate.WaitUntilWritable(s.ctx); err != nil {
+ return err
+ }
+ if err := s.ensureWritable(); err != nil {
+ return err
+ }
+ return s.Sink.WriteBlockEvent(event)
+}
+
+func (s *writeGatedSink) AddCheckpointTs(ts uint64) {
+ // Checkpoints are periodic and superseded by newer values, so dropping one
+ // while blocked avoids stalling the checkpoint message stream.
+ if s.gate.IsWritable() {
+ s.Sink.AddCheckpointTs(ts)
+ }
+}
+
+func (s *writeGatedSink) waitUntilWritable() bool {
+ for {
+ if err := s.gate.WaitUntilWritable(s.ctx); err != nil {
+ return false
+ }
+ if s.gate.IsWritable() {
+ return true
+ }
+ }
+}
+
+func (s *writeGatedSink) ensureWritable() error {
+ for {
+ if s.gate.IsWritable() {
+ return nil
+ }
+ if err := s.gate.WaitUntilWritable(s.ctx); err != nil {
+ return err
+ }
+ }
+}
diff --git a/downstreamadapter/sink/write_gate_test.go b/downstreamadapter/sink/write_gate_test.go
new file mode 100644
index 0000000000..d7af4d1caf
--- /dev/null
+++ b/downstreamadapter/sink/write_gate_test.go
@@ -0,0 +1,100 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sink_test
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/golang/mock/gomock"
+ "github.com/pingcap/ticdc/downstreamadapter/sink"
+ "github.com/pingcap/ticdc/downstreamadapter/sink/mock"
+ "github.com/pingcap/ticdc/pkg/writelease"
+ "github.com/stretchr/testify/require"
+)
+
+func TestWriteGatedSinkBlocksAndResumesDML(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ inner := mock.NewMockSink(ctrl)
+ gate := writelease.NewGate()
+ inner.EXPECT().SetWriteGate(gate)
+ gated := sink.WithWriteGate(t.Context(), inner, gate)
+
+ written := make(chan struct{})
+ inner.EXPECT().AddDMLEvent(nil).Do(func(_ any) { close(written) })
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ gated.AddDMLEvent(nil)
+ }()
+
+ select {
+ case <-written:
+ t.Fatal("DML passed through a closed capture write gate")
+ case <-time.After(50 * time.Millisecond):
+ }
+
+ now := time.Now()
+ require.True(t, gate.RenewP2P(now, writelease.P2PLeaseDuration))
+ require.True(t, gate.RenewEtcd(now, writelease.EtcdProofDuration))
+ require.Eventually(t, func() bool {
+ select {
+ case <-written:
+ return true
+ default:
+ return false
+ }
+ }, time.Second, 10*time.Millisecond)
+ <-done
+}
+
+func TestWriteGatedSinkStopsWaitingWhenContextIsCanceled(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ inner := mock.NewMockSink(ctrl)
+ gate := writelease.NewGate()
+ ctx, cancel := context.WithCancel(context.Background())
+ inner.EXPECT().SetWriteGate(gate)
+ gated := sink.WithWriteGate(ctx, inner, gate)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- gated.WriteBlockEvent(nil)
+ }()
+ cancel()
+ require.ErrorIs(t, <-done, context.Canceled)
+}
+
+func TestWriteGatedSinkCoversEveryWriteEntry(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ inner := mock.NewMockSink(ctrl)
+ gate := writelease.NewGate()
+ inner.EXPECT().SetWriteGate(gate)
+ gated := sink.WithWriteGate(t.Context(), inner, gate)
+
+ // A checkpoint is safe to drop while closed because later checkpoints
+ // supersede it.
+ gated.AddCheckpointTs(41)
+
+ now := time.Now()
+ require.True(t, gate.RenewP2P(now, writelease.P2PLeaseDuration))
+ require.True(t, gate.RenewEtcd(now, writelease.EtcdProofDuration))
+
+ inner.EXPECT().FlushDMLBeforeBlock(nil).Return(nil)
+ inner.EXPECT().WriteBlockEvent(nil).Return(nil)
+ inner.EXPECT().AddCheckpointTs(uint64(42))
+ require.NoError(t, gated.FlushDMLBeforeBlock(nil))
+ require.NoError(t, gated.WriteBlockEvent(nil))
+ gated.AddCheckpointTs(42)
+}
diff --git a/heartbeatpb/heartbeat.pb.go b/heartbeatpb/heartbeat.pb.go
index 94ff2f969d..f2d2be03db 100644
--- a/heartbeatpb/heartbeat.pb.go
+++ b/heartbeatpb/heartbeat.pb.go
@@ -1623,8 +1623,11 @@ type NodeHeartbeat struct {
NodeEpoch uint64 `protobuf:"varint,2,opt,name=node_epoch,json=nodeEpoch,proto3" json:"node_epoch,omitempty"`
// dispatcher_drain_target_* reports the manager-level dispatcher drain target
// currently applied on this node. Empty target means the drain target is clear.
- DispatcherDrainTargetNodeId string `protobuf:"bytes,3,opt,name=dispatcher_drain_target_node_id,json=dispatcherDrainTargetNodeId,proto3" json:"dispatcher_drain_target_node_id,omitempty"`
- DispatcherDrainTargetEpoch uint64 `protobuf:"varint,4,opt,name=dispatcher_drain_target_epoch,json=dispatcherDrainTargetEpoch,proto3" json:"dispatcher_drain_target_epoch,omitempty"`
+ DispatcherDrainTargetNodeId string `protobuf:"bytes,3,opt,name=dispatcher_drain_target_node_id,json=dispatcherDrainTargetNodeId,proto3" json:"dispatcher_drain_target_node_id,omitempty"`
+ DispatcherDrainTargetEpoch uint64 `protobuf:"varint,4,opt,name=dispatcher_drain_target_epoch,json=dispatcherDrainTargetEpoch,proto3" json:"dispatcher_drain_target_epoch,omitempty"`
+ WriteLeaseRequestSeq uint64 `protobuf:"varint,5,opt,name=write_lease_request_seq,json=writeLeaseRequestSeq,proto3" json:"write_lease_request_seq,omitempty"`
+ WriteLeaseProtocolVersion uint32 `protobuf:"varint,6,opt,name=write_lease_protocol_version,json=writeLeaseProtocolVersion,proto3" json:"write_lease_protocol_version,omitempty"`
+ WriteLeaseWitnessAck *WriteLeaseWitnessAck `protobuf:"bytes,7,opt,name=write_lease_witness_ack,json=writeLeaseWitnessAck,proto3" json:"write_lease_witness_ack,omitempty"`
}
func (m *NodeHeartbeat) Reset() { *m = NodeHeartbeat{} }
@@ -1688,6 +1691,255 @@ func (m *NodeHeartbeat) GetDispatcherDrainTargetEpoch() uint64 {
return 0
}
+func (m *NodeHeartbeat) GetWriteLeaseRequestSeq() uint64 {
+ if m != nil {
+ return m.WriteLeaseRequestSeq
+ }
+ return 0
+}
+
+func (m *NodeHeartbeat) GetWriteLeaseProtocolVersion() uint32 {
+ if m != nil {
+ return m.WriteLeaseProtocolVersion
+ }
+ return 0
+}
+
+func (m *NodeHeartbeat) GetWriteLeaseWitnessAck() *WriteLeaseWitnessAck {
+ if m != nil {
+ return m.WriteLeaseWitnessAck
+ }
+ return nil
+}
+
+type WriteLeaseWitnessChallenge struct {
+ CoordinatorVersion int64 `protobuf:"varint,1,opt,name=coordinator_version,json=coordinatorVersion,proto3" json:"coordinator_version,omitempty"`
+ CoordinatorNodeEpoch uint64 `protobuf:"varint,2,opt,name=coordinator_node_epoch,json=coordinatorNodeEpoch,proto3" json:"coordinator_node_epoch,omitempty"`
+ SelfRequestSeq uint64 `protobuf:"varint,3,opt,name=self_request_seq,json=selfRequestSeq,proto3" json:"self_request_seq,omitempty"`
+ WitnessNodeEpoch uint64 `protobuf:"varint,4,opt,name=witness_node_epoch,json=witnessNodeEpoch,proto3" json:"witness_node_epoch,omitempty"`
+ Nonce []byte `protobuf:"bytes,5,opt,name=nonce,proto3" json:"nonce,omitempty"`
+}
+
+func (m *WriteLeaseWitnessChallenge) Reset() { *m = WriteLeaseWitnessChallenge{} }
+func (m *WriteLeaseWitnessChallenge) String() string { return proto.CompactTextString(m) }
+func (*WriteLeaseWitnessChallenge) ProtoMessage() {}
+func (*WriteLeaseWitnessChallenge) Descriptor() ([]byte, []int) {
+ return fileDescriptor_6d584080fdadb670, []int{20}
+}
+func (m *WriteLeaseWitnessChallenge) XXX_Unmarshal(b []byte) error {
+ return m.Unmarshal(b)
+}
+func (m *WriteLeaseWitnessChallenge) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ if deterministic {
+ return xxx_messageInfo_WriteLeaseWitnessChallenge.Marshal(b, m, deterministic)
+ } else {
+ b = b[:cap(b)]
+ n, err := m.MarshalToSizedBuffer(b)
+ if err != nil {
+ return nil, err
+ }
+ return b[:n], nil
+ }
+}
+func (m *WriteLeaseWitnessChallenge) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_WriteLeaseWitnessChallenge.Merge(m, src)
+}
+func (m *WriteLeaseWitnessChallenge) XXX_Size() int {
+ return m.Size()
+}
+func (m *WriteLeaseWitnessChallenge) XXX_DiscardUnknown() {
+ xxx_messageInfo_WriteLeaseWitnessChallenge.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_WriteLeaseWitnessChallenge proto.InternalMessageInfo
+
+func (m *WriteLeaseWitnessChallenge) GetCoordinatorVersion() int64 {
+ if m != nil {
+ return m.CoordinatorVersion
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessChallenge) GetCoordinatorNodeEpoch() uint64 {
+ if m != nil {
+ return m.CoordinatorNodeEpoch
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessChallenge) GetSelfRequestSeq() uint64 {
+ if m != nil {
+ return m.SelfRequestSeq
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessChallenge) GetWitnessNodeEpoch() uint64 {
+ if m != nil {
+ return m.WitnessNodeEpoch
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessChallenge) GetNonce() []byte {
+ if m != nil {
+ return m.Nonce
+ }
+ return nil
+}
+
+type WriteLeaseWitnessAck struct {
+ CoordinatorVersion int64 `protobuf:"varint,1,opt,name=coordinator_version,json=coordinatorVersion,proto3" json:"coordinator_version,omitempty"`
+ CoordinatorNodeEpoch uint64 `protobuf:"varint,2,opt,name=coordinator_node_epoch,json=coordinatorNodeEpoch,proto3" json:"coordinator_node_epoch,omitempty"`
+ SelfRequestSeq uint64 `protobuf:"varint,3,opt,name=self_request_seq,json=selfRequestSeq,proto3" json:"self_request_seq,omitempty"`
+ WitnessNodeEpoch uint64 `protobuf:"varint,4,opt,name=witness_node_epoch,json=witnessNodeEpoch,proto3" json:"witness_node_epoch,omitempty"`
+ Nonce []byte `protobuf:"bytes,5,opt,name=nonce,proto3" json:"nonce,omitempty"`
+}
+
+func (m *WriteLeaseWitnessAck) Reset() { *m = WriteLeaseWitnessAck{} }
+func (m *WriteLeaseWitnessAck) String() string { return proto.CompactTextString(m) }
+func (*WriteLeaseWitnessAck) ProtoMessage() {}
+func (*WriteLeaseWitnessAck) Descriptor() ([]byte, []int) {
+ return fileDescriptor_6d584080fdadb670, []int{21}
+}
+func (m *WriteLeaseWitnessAck) XXX_Unmarshal(b []byte) error {
+ return m.Unmarshal(b)
+}
+func (m *WriteLeaseWitnessAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ if deterministic {
+ return xxx_messageInfo_WriteLeaseWitnessAck.Marshal(b, m, deterministic)
+ } else {
+ b = b[:cap(b)]
+ n, err := m.MarshalToSizedBuffer(b)
+ if err != nil {
+ return nil, err
+ }
+ return b[:n], nil
+ }
+}
+func (m *WriteLeaseWitnessAck) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_WriteLeaseWitnessAck.Merge(m, src)
+}
+func (m *WriteLeaseWitnessAck) XXX_Size() int {
+ return m.Size()
+}
+func (m *WriteLeaseWitnessAck) XXX_DiscardUnknown() {
+ xxx_messageInfo_WriteLeaseWitnessAck.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_WriteLeaseWitnessAck proto.InternalMessageInfo
+
+func (m *WriteLeaseWitnessAck) GetCoordinatorVersion() int64 {
+ if m != nil {
+ return m.CoordinatorVersion
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessAck) GetCoordinatorNodeEpoch() uint64 {
+ if m != nil {
+ return m.CoordinatorNodeEpoch
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessAck) GetSelfRequestSeq() uint64 {
+ if m != nil {
+ return m.SelfRequestSeq
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessAck) GetWitnessNodeEpoch() uint64 {
+ if m != nil {
+ return m.WitnessNodeEpoch
+ }
+ return 0
+}
+
+func (m *WriteLeaseWitnessAck) GetNonce() []byte {
+ if m != nil {
+ return m.Nonce
+ }
+ return nil
+}
+
+type NodeHeartbeatResponse struct {
+ CoordinatorVersion int64 `protobuf:"varint,1,opt,name=coordinator_version,json=coordinatorVersion,proto3" json:"coordinator_version,omitempty"`
+ TargetNodeEpoch uint64 `protobuf:"varint,2,opt,name=target_node_epoch,json=targetNodeEpoch,proto3" json:"target_node_epoch,omitempty"`
+ RequestSeq uint64 `protobuf:"varint,3,opt,name=request_seq,json=requestSeq,proto3" json:"request_seq,omitempty"`
+ LeaseDurationMs uint64 `protobuf:"varint,4,opt,name=lease_duration_ms,json=leaseDurationMs,proto3" json:"lease_duration_ms,omitempty"`
+ WitnessChallenge *WriteLeaseWitnessChallenge `protobuf:"bytes,5,opt,name=witness_challenge,json=witnessChallenge,proto3" json:"witness_challenge,omitempty"`
+}
+
+func (m *NodeHeartbeatResponse) Reset() { *m = NodeHeartbeatResponse{} }
+func (m *NodeHeartbeatResponse) String() string { return proto.CompactTextString(m) }
+func (*NodeHeartbeatResponse) ProtoMessage() {}
+func (*NodeHeartbeatResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_6d584080fdadb670, []int{22}
+}
+func (m *NodeHeartbeatResponse) XXX_Unmarshal(b []byte) error {
+ return m.Unmarshal(b)
+}
+func (m *NodeHeartbeatResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ if deterministic {
+ return xxx_messageInfo_NodeHeartbeatResponse.Marshal(b, m, deterministic)
+ } else {
+ b = b[:cap(b)]
+ n, err := m.MarshalToSizedBuffer(b)
+ if err != nil {
+ return nil, err
+ }
+ return b[:n], nil
+ }
+}
+func (m *NodeHeartbeatResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_NodeHeartbeatResponse.Merge(m, src)
+}
+func (m *NodeHeartbeatResponse) XXX_Size() int {
+ return m.Size()
+}
+func (m *NodeHeartbeatResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_NodeHeartbeatResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_NodeHeartbeatResponse proto.InternalMessageInfo
+
+func (m *NodeHeartbeatResponse) GetCoordinatorVersion() int64 {
+ if m != nil {
+ return m.CoordinatorVersion
+ }
+ return 0
+}
+
+func (m *NodeHeartbeatResponse) GetTargetNodeEpoch() uint64 {
+ if m != nil {
+ return m.TargetNodeEpoch
+ }
+ return 0
+}
+
+func (m *NodeHeartbeatResponse) GetRequestSeq() uint64 {
+ if m != nil {
+ return m.RequestSeq
+ }
+ return 0
+}
+
+func (m *NodeHeartbeatResponse) GetLeaseDurationMs() uint64 {
+ if m != nil {
+ return m.LeaseDurationMs
+ }
+ return 0
+}
+
+func (m *NodeHeartbeatResponse) GetWitnessChallenge() *WriteLeaseWitnessChallenge {
+ if m != nil {
+ return m.WitnessChallenge
+ }
+ return nil
+}
+
// SetNodeLivenessRequest asks a node to transition its local liveness.
type SetNodeLivenessRequest struct {
Target NodeLiveness `protobuf:"varint,1,opt,name=target,proto3,enum=heartbeatpb.NodeLiveness" json:"target,omitempty"`
@@ -1698,7 +1950,7 @@ func (m *SetNodeLivenessRequest) Reset() { *m = SetNodeLivenessRequest{}
func (m *SetNodeLivenessRequest) String() string { return proto.CompactTextString(m) }
func (*SetNodeLivenessRequest) ProtoMessage() {}
func (*SetNodeLivenessRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{20}
+ return fileDescriptor_6d584080fdadb670, []int{23}
}
func (m *SetNodeLivenessRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -1751,7 +2003,7 @@ func (m *SetNodeLivenessResponse) Reset() { *m = SetNodeLivenessResponse
func (m *SetNodeLivenessResponse) String() string { return proto.CompactTextString(m) }
func (*SetNodeLivenessResponse) ProtoMessage() {}
func (*SetNodeLivenessResponse) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{21}
+ return fileDescriptor_6d584080fdadb670, []int{24}
}
func (m *SetNodeLivenessResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -1805,7 +2057,7 @@ func (m *SetDispatcherDrainTargetRequest) Reset() { *m = SetDispatcherDr
func (m *SetDispatcherDrainTargetRequest) String() string { return proto.CompactTextString(m) }
func (*SetDispatcherDrainTargetRequest) ProtoMessage() {}
func (*SetDispatcherDrainTargetRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{22}
+ return fileDescriptor_6d584080fdadb670, []int{25}
}
func (m *SetDispatcherDrainTargetRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -1850,13 +2102,16 @@ func (m *SetDispatcherDrainTargetRequest) GetTargetEpoch() uint64 {
type CoordinatorBootstrapRequest struct {
Version int64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"`
+ // Zero keeps the peer in legacy mode. A non-zero version activates the
+ // capture P2P write lease only on peers that understand this protocol.
+ WriteLeaseProtocolVersion uint32 `protobuf:"varint,2,opt,name=write_lease_protocol_version,json=writeLeaseProtocolVersion,proto3" json:"write_lease_protocol_version,omitempty"`
}
func (m *CoordinatorBootstrapRequest) Reset() { *m = CoordinatorBootstrapRequest{} }
func (m *CoordinatorBootstrapRequest) String() string { return proto.CompactTextString(m) }
func (*CoordinatorBootstrapRequest) ProtoMessage() {}
func (*CoordinatorBootstrapRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{23}
+ return fileDescriptor_6d584080fdadb670, []int{26}
}
func (m *CoordinatorBootstrapRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -1892,6 +2147,13 @@ func (m *CoordinatorBootstrapRequest) GetVersion() int64 {
return 0
}
+func (m *CoordinatorBootstrapRequest) GetWriteLeaseProtocolVersion() uint32 {
+ if m != nil {
+ return m.WriteLeaseProtocolVersion
+ }
+ return 0
+}
+
type CoordinatorBootstrapResponse struct {
Statuses []*MaintainerStatus `protobuf:"bytes,1,rep,name=statuses,proto3" json:"statuses,omitempty"`
// drain_protocol_version is the node-scoped drain capability supported by
@@ -1905,13 +2167,14 @@ type CoordinatorBootstrapResponse struct {
// scheduling resumes.
DispatcherDrainTargetNodeId string `protobuf:"bytes,3,opt,name=dispatcher_drain_target_node_id,json=dispatcherDrainTargetNodeId,proto3" json:"dispatcher_drain_target_node_id,omitempty"`
DispatcherDrainTargetEpoch uint64 `protobuf:"varint,4,opt,name=dispatcher_drain_target_epoch,json=dispatcherDrainTargetEpoch,proto3" json:"dispatcher_drain_target_epoch,omitempty"`
+ WriteLeaseProtocolVersion uint32 `protobuf:"varint,5,opt,name=write_lease_protocol_version,json=writeLeaseProtocolVersion,proto3" json:"write_lease_protocol_version,omitempty"`
}
func (m *CoordinatorBootstrapResponse) Reset() { *m = CoordinatorBootstrapResponse{} }
func (m *CoordinatorBootstrapResponse) String() string { return proto.CompactTextString(m) }
func (*CoordinatorBootstrapResponse) ProtoMessage() {}
func (*CoordinatorBootstrapResponse) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{24}
+ return fileDescriptor_6d584080fdadb670, []int{27}
}
func (m *CoordinatorBootstrapResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -1968,6 +2231,13 @@ func (m *CoordinatorBootstrapResponse) GetDispatcherDrainTargetEpoch() uint64 {
return 0
}
+func (m *CoordinatorBootstrapResponse) GetWriteLeaseProtocolVersion() uint32 {
+ if m != nil {
+ return m.WriteLeaseProtocolVersion
+ }
+ return 0
+}
+
type AddMaintainerRequest struct {
Id *ChangefeedID `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
Config []byte `protobuf:"bytes,2,opt,name=config,proto3" json:"config,omitempty"`
@@ -1981,7 +2251,7 @@ func (m *AddMaintainerRequest) Reset() { *m = AddMaintainerRequest{} }
func (m *AddMaintainerRequest) String() string { return proto.CompactTextString(m) }
func (*AddMaintainerRequest) ProtoMessage() {}
func (*AddMaintainerRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{25}
+ return fileDescriptor_6d584080fdadb670, []int{28}
}
func (m *AddMaintainerRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2064,7 +2334,7 @@ func (m *RemoveMaintainerRequest) Reset() { *m = RemoveMaintainerRequest
func (m *RemoveMaintainerRequest) String() string { return proto.CompactTextString(m) }
func (*RemoveMaintainerRequest) ProtoMessage() {}
func (*RemoveMaintainerRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{26}
+ return fileDescriptor_6d584080fdadb670, []int{29}
}
func (m *RemoveMaintainerRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2143,7 +2413,7 @@ func (m *MaintainerBootstrapRequest) Reset() { *m = MaintainerBootstrapR
func (m *MaintainerBootstrapRequest) String() string { return proto.CompactTextString(m) }
func (*MaintainerBootstrapRequest) ProtoMessage() {}
func (*MaintainerBootstrapRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{27}
+ return fileDescriptor_6d584080fdadb670, []int{30}
}
func (m *MaintainerBootstrapRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2254,7 +2524,7 @@ func (m *MaintainerBootstrapResponse) Reset() { *m = MaintainerBootstrap
func (m *MaintainerBootstrapResponse) String() string { return proto.CompactTextString(m) }
func (*MaintainerBootstrapResponse) ProtoMessage() {}
func (*MaintainerBootstrapResponse) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{28}
+ return fileDescriptor_6d584080fdadb670, []int{31}
}
func (m *MaintainerBootstrapResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2351,7 +2621,7 @@ func (m *MaintainerPostBootstrapRequest) Reset() { *m = MaintainerPostBo
func (m *MaintainerPostBootstrapRequest) String() string { return proto.CompactTextString(m) }
func (*MaintainerPostBootstrapRequest) ProtoMessage() {}
func (*MaintainerPostBootstrapRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{29}
+ return fileDescriptor_6d584080fdadb670, []int{32}
}
func (m *MaintainerPostBootstrapRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2426,7 +2696,7 @@ func (m *MaintainerPostBootstrapResponse) Reset() { *m = MaintainerPostB
func (m *MaintainerPostBootstrapResponse) String() string { return proto.CompactTextString(m) }
func (*MaintainerPostBootstrapResponse) ProtoMessage() {}
func (*MaintainerPostBootstrapResponse) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{30}
+ return fileDescriptor_6d584080fdadb670, []int{33}
}
func (m *MaintainerPostBootstrapResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2493,7 +2763,7 @@ func (m *SchemaInfo) Reset() { *m = SchemaInfo{} }
func (m *SchemaInfo) String() string { return proto.CompactTextString(m) }
func (*SchemaInfo) ProtoMessage() {}
func (*SchemaInfo) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{31}
+ return fileDescriptor_6d584080fdadb670, []int{34}
}
func (m *SchemaInfo) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2552,7 +2822,7 @@ func (m *TableInfo) Reset() { *m = TableInfo{} }
func (m *TableInfo) String() string { return proto.CompactTextString(m) }
func (*TableInfo) ProtoMessage() {}
func (*TableInfo) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{32}
+ return fileDescriptor_6d584080fdadb670, []int{35}
}
func (m *TableInfo) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2609,7 +2879,7 @@ func (m *BootstrapTableSpan) Reset() { *m = BootstrapTableSpan{} }
func (m *BootstrapTableSpan) String() string { return proto.CompactTextString(m) }
func (*BootstrapTableSpan) ProtoMessage() {}
func (*BootstrapTableSpan) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{33}
+ return fileDescriptor_6d584080fdadb670, []int{36}
}
func (m *BootstrapTableSpan) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2698,7 +2968,7 @@ func (m *MaintainerCloseRequest) Reset() { *m = MaintainerCloseRequest{}
func (m *MaintainerCloseRequest) String() string { return proto.CompactTextString(m) }
func (*MaintainerCloseRequest) ProtoMessage() {}
func (*MaintainerCloseRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{34}
+ return fileDescriptor_6d584080fdadb670, []int{37}
}
func (m *MaintainerCloseRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2758,7 +3028,7 @@ func (m *MaintainerCloseResponse) Reset() { *m = MaintainerCloseResponse
func (m *MaintainerCloseResponse) String() string { return proto.CompactTextString(m) }
func (*MaintainerCloseResponse) ProtoMessage() {}
func (*MaintainerCloseResponse) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{35}
+ return fileDescriptor_6d584080fdadb670, []int{38}
}
func (m *MaintainerCloseResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2820,7 +3090,7 @@ func (m *InfluencedTables) Reset() { *m = InfluencedTables{} }
func (m *InfluencedTables) String() string { return proto.CompactTextString(m) }
func (*InfluencedTables) ProtoMessage() {}
func (*InfluencedTables) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{36}
+ return fileDescriptor_6d584080fdadb670, []int{39}
}
func (m *InfluencedTables) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2880,7 +3150,7 @@ func (m *Table) Reset() { *m = Table{} }
func (m *Table) String() string { return proto.CompactTextString(m) }
func (*Table) ProtoMessage() {}
func (*Table) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{37}
+ return fileDescriptor_6d584080fdadb670, []int{40}
}
func (m *Table) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -2945,7 +3215,7 @@ func (m *RouteTableAdmission) Reset() { *m = RouteTableAdmission{} }
func (m *RouteTableAdmission) String() string { return proto.CompactTextString(m) }
func (*RouteTableAdmission) ProtoMessage() {}
func (*RouteTableAdmission) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{38}
+ return fileDescriptor_6d584080fdadb670, []int{41}
}
func (m *RouteTableAdmission) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3019,7 +3289,7 @@ func (m *SchemaIDChange) Reset() { *m = SchemaIDChange{} }
func (m *SchemaIDChange) String() string { return proto.CompactTextString(m) }
func (*SchemaIDChange) ProtoMessage() {}
func (*SchemaIDChange) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{39}
+ return fileDescriptor_6d584080fdadb670, []int{42}
}
func (m *SchemaIDChange) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3088,7 +3358,7 @@ func (m *State) Reset() { *m = State{} }
func (m *State) String() string { return proto.CompactTextString(m) }
func (*State) ProtoMessage() {}
func (*State) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{40}
+ return fileDescriptor_6d584080fdadb670, []int{43}
}
func (m *State) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3190,7 +3460,7 @@ func (m *TableSpanBlockStatus) Reset() { *m = TableSpanBlockStatus{} }
func (m *TableSpanBlockStatus) String() string { return proto.CompactTextString(m) }
func (*TableSpanBlockStatus) ProtoMessage() {}
func (*TableSpanBlockStatus) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{41}
+ return fileDescriptor_6d584080fdadb670, []int{44}
}
func (m *TableSpanBlockStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3252,7 +3522,7 @@ func (m *TableSpanStatus) Reset() { *m = TableSpanStatus{} }
func (m *TableSpanStatus) String() string { return proto.CompactTextString(m) }
func (*TableSpanStatus) ProtoMessage() {}
func (*TableSpanStatus) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{42}
+ return fileDescriptor_6d584080fdadb670, []int{45}
}
func (m *TableSpanStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3326,7 +3596,7 @@ func (m *BlockStatusRequest) Reset() { *m = BlockStatusRequest{} }
func (m *BlockStatusRequest) String() string { return proto.CompactTextString(m) }
func (*BlockStatusRequest) ProtoMessage() {}
func (*BlockStatusRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{43}
+ return fileDescriptor_6d584080fdadb670, []int{46}
}
func (m *BlockStatusRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3387,7 +3657,7 @@ func (m *RunningError) Reset() { *m = RunningError{} }
func (m *RunningError) String() string { return proto.CompactTextString(m) }
func (*RunningError) ProtoMessage() {}
func (*RunningError) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{44}
+ return fileDescriptor_6d584080fdadb670, []int{47}
}
func (m *RunningError) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3453,7 +3723,7 @@ func (m *DispatcherID) Reset() { *m = DispatcherID{} }
func (m *DispatcherID) String() string { return proto.CompactTextString(m) }
func (*DispatcherID) ProtoMessage() {}
func (*DispatcherID) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{45}
+ return fileDescriptor_6d584080fdadb670, []int{48}
}
func (m *DispatcherID) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3507,7 +3777,7 @@ func (m *ChangefeedID) Reset() { *m = ChangefeedID{} }
func (m *ChangefeedID) String() string { return proto.CompactTextString(m) }
func (*ChangefeedID) ProtoMessage() {}
func (*ChangefeedID) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{46}
+ return fileDescriptor_6d584080fdadb670, []int{49}
}
func (m *ChangefeedID) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3572,7 +3842,7 @@ func (m *LogCoordinatorResolvedTsRequest) Reset() { *m = LogCoordinatorR
func (m *LogCoordinatorResolvedTsRequest) String() string { return proto.CompactTextString(m) }
func (*LogCoordinatorResolvedTsRequest) ProtoMessage() {}
func (*LogCoordinatorResolvedTsRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{47}
+ return fileDescriptor_6d584080fdadb670, []int{50}
}
func (m *LogCoordinatorResolvedTsRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3617,7 +3887,7 @@ func (m *LogCoordinatorResolvedTsResponse) Reset() { *m = LogCoordinator
func (m *LogCoordinatorResolvedTsResponse) String() string { return proto.CompactTextString(m) }
func (*LogCoordinatorResolvedTsResponse) ProtoMessage() {}
func (*LogCoordinatorResolvedTsResponse) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{48}
+ return fileDescriptor_6d584080fdadb670, []int{51}
}
func (m *LogCoordinatorResolvedTsResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3671,7 +3941,7 @@ func (m *ChecksumMeta) Reset() { *m = ChecksumMeta{} }
func (m *ChecksumMeta) String() string { return proto.CompactTextString(m) }
func (*ChecksumMeta) ProtoMessage() {}
func (*ChecksumMeta) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{49}
+ return fileDescriptor_6d584080fdadb670, []int{52}
}
func (m *ChecksumMeta) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3726,7 +3996,7 @@ func (m *DispatcherSetChecksum) Reset() { *m = DispatcherSetChecksum{} }
func (m *DispatcherSetChecksum) String() string { return proto.CompactTextString(m) }
func (*DispatcherSetChecksum) ProtoMessage() {}
func (*DispatcherSetChecksum) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{50}
+ return fileDescriptor_6d584080fdadb670, []int{53}
}
func (m *DispatcherSetChecksum) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3803,7 +4073,7 @@ func (m *DispatcherSetChecksumAckResponse) Reset() { *m = DispatcherSetC
func (m *DispatcherSetChecksumAckResponse) String() string { return proto.CompactTextString(m) }
func (*DispatcherSetChecksumAckResponse) ProtoMessage() {}
func (*DispatcherSetChecksumAckResponse) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{51}
+ return fileDescriptor_6d584080fdadb670, []int{54}
}
func (m *DispatcherSetChecksumAckResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3874,7 +4144,7 @@ func (m *DispatcherSetChecksumUpdateRequest) Reset() { *m = DispatcherSe
func (m *DispatcherSetChecksumUpdateRequest) String() string { return proto.CompactTextString(m) }
func (*DispatcherSetChecksumUpdateRequest) ProtoMessage() {}
func (*DispatcherSetChecksumUpdateRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_6d584080fdadb670, []int{52}
+ return fileDescriptor_6d584080fdadb670, []int{55}
}
func (m *DispatcherSetChecksumUpdateRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@@ -3968,6 +4238,9 @@ func init() {
proto.RegisterType((*DrainProgress)(nil), "heartbeatpb.DrainProgress")
proto.RegisterType((*MaintainerStatus)(nil), "heartbeatpb.MaintainerStatus")
proto.RegisterType((*NodeHeartbeat)(nil), "heartbeatpb.NodeHeartbeat")
+ proto.RegisterType((*WriteLeaseWitnessChallenge)(nil), "heartbeatpb.WriteLeaseWitnessChallenge")
+ proto.RegisterType((*WriteLeaseWitnessAck)(nil), "heartbeatpb.WriteLeaseWitnessAck")
+ proto.RegisterType((*NodeHeartbeatResponse)(nil), "heartbeatpb.NodeHeartbeatResponse")
proto.RegisterType((*SetNodeLivenessRequest)(nil), "heartbeatpb.SetNodeLivenessRequest")
proto.RegisterType((*SetNodeLivenessResponse)(nil), "heartbeatpb.SetNodeLivenessResponse")
proto.RegisterType((*SetDispatcherDrainTargetRequest)(nil), "heartbeatpb.SetDispatcherDrainTargetRequest")
@@ -4006,196 +4279,213 @@ func init() {
func init() { proto.RegisterFile("heartbeatpb/heartbeat.proto", fileDescriptor_6d584080fdadb670) }
var fileDescriptor_6d584080fdadb670 = []byte{
- // 3020 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x1a, 0x4d, 0x8f, 0x1c, 0x47,
- 0xd5, 0xdd, 0xf3, 0xfd, 0x76, 0x67, 0xb7, 0x5d, 0xb6, 0xd7, 0x6b, 0x7b, 0xbd, 0xde, 0x74, 0x02,
- 0xda, 0x4c, 0x82, 0x8d, 0x9d, 0x98, 0x8f, 0x10, 0x12, 0xc6, 0x33, 0x9b, 0x78, 0xe4, 0x9d, 0xdd,
- 0x55, 0xcd, 0x26, 0x46, 0xe1, 0x30, 0xf4, 0x76, 0x97, 0x67, 0x3b, 0x3b, 0xd3, 0x35, 0xe9, 0xee,
- 0xf1, 0xda, 0x96, 0x00, 0x45, 0x88, 0x1b, 0x07, 0xb8, 0x71, 0x20, 0x17, 0x4e, 0x9c, 0x10, 0x7f,
- 0x00, 0xc1, 0x81, 0x03, 0x27, 0x14, 0x71, 0x40, 0x39, 0x41, 0x94, 0xdc, 0x11, 0x12, 0x12, 0x5c,
- 0x51, 0x7d, 0x74, 0x77, 0xf5, 0x4c, 0xcf, 0x7e, 0xb0, 0xa3, 0x88, 0xd3, 0xf4, 0x7b, 0xf5, 0xde,
- 0xab, 0x57, 0xaf, 0x5e, 0xbd, 0x7a, 0xef, 0xd5, 0xc0, 0xb5, 0x7d, 0x62, 0xf9, 0xe1, 0x1e, 0xb1,
- 0xc2, 0xe1, 0xde, 0xad, 0xf8, 0xfb, 0xe6, 0xd0, 0xa7, 0x21, 0x45, 0x73, 0xca, 0xa0, 0xf9, 0x14,
- 0x2a, 0xbb, 0xd6, 0x5e, 0x9f, 0x74, 0x86, 0x96, 0x87, 0x96, 0xa1, 0xc4, 0x81, 0x56, 0x73, 0x59,
- 0x5b, 0xd3, 0xd6, 0x73, 0x38, 0x02, 0xd1, 0x55, 0x28, 0x77, 0x42, 0xcb, 0x0f, 0x1f, 0x90, 0xa7,
- 0xcb, 0xfa, 0x9a, 0xb6, 0x3e, 0x8f, 0x63, 0x18, 0x2d, 0x41, 0x71, 0xc3, 0x73, 0xd8, 0x48, 0x8e,
- 0x8f, 0x48, 0x08, 0xad, 0x02, 0x3c, 0x20, 0x4f, 0x83, 0xa1, 0x65, 0x33, 0x81, 0xf9, 0x35, 0x6d,
- 0xbd, 0x8a, 0x15, 0x8c, 0xf9, 0x57, 0x1d, 0x8c, 0xfb, 0x4c, 0x95, 0x7b, 0xc4, 0x0a, 0x31, 0xf9,
- 0x60, 0x44, 0x82, 0x10, 0x7d, 0x1b, 0xe6, 0xed, 0x7d, 0xcb, 0xeb, 0x91, 0x47, 0x84, 0x38, 0x52,
- 0x8f, 0xb9, 0x3b, 0x57, 0x6e, 0x2a, 0x3a, 0xdf, 0x6c, 0x28, 0x04, 0x38, 0x45, 0x8e, 0x5e, 0x85,
- 0xca, 0xa1, 0x15, 0x12, 0x7f, 0x60, 0xf9, 0x07, 0x5c, 0xd1, 0xb9, 0x3b, 0x4b, 0x29, 0xde, 0x87,
- 0xd1, 0x28, 0x4e, 0x08, 0xd1, 0xeb, 0x50, 0xf5, 0x89, 0x43, 0xe3, 0x31, 0xbe, 0x90, 0xe9, 0x9c,
- 0x69, 0x62, 0xf4, 0x0d, 0x28, 0x07, 0xa1, 0x15, 0x8e, 0x02, 0x12, 0x2c, 0xe7, 0xd7, 0x72, 0xeb,
- 0x73, 0x77, 0x56, 0x52, 0x8c, 0xb1, 0x7d, 0x3b, 0x9c, 0x0a, 0xc7, 0xd4, 0x68, 0x1d, 0x16, 0x6d,
- 0x3a, 0x18, 0x92, 0x3e, 0x09, 0x89, 0x18, 0x5c, 0x2e, 0xac, 0x69, 0xeb, 0x65, 0x3c, 0x8e, 0x46,
- 0x2f, 0x41, 0x8e, 0xf8, 0xfe, 0x72, 0x31, 0xc3, 0x1a, 0x78, 0xe4, 0x79, 0xae, 0xd7, 0xdb, 0xf0,
- 0x7d, 0xea, 0x63, 0x46, 0x65, 0xfe, 0x44, 0x83, 0x4a, 0xa2, 0x9e, 0xc9, 0x2c, 0x4a, 0xec, 0x83,
- 0x21, 0x75, 0xbd, 0x70, 0x37, 0xe0, 0x16, 0xcd, 0xe3, 0x14, 0x8e, 0x6d, 0x95, 0x4f, 0x02, 0xda,
- 0x7f, 0x4c, 0x9c, 0xdd, 0x80, 0xdb, 0x2d, 0x8f, 0x15, 0x0c, 0x32, 0x20, 0x17, 0x90, 0x0f, 0xb8,
- 0x59, 0xf2, 0x98, 0x7d, 0x32, 0xa9, 0x7d, 0x2b, 0x08, 0x3b, 0x4f, 0x3d, 0x9b, 0xf3, 0xe4, 0x85,
- 0x54, 0x15, 0x67, 0xfe, 0x00, 0x8c, 0xa6, 0x1b, 0x0c, 0xad, 0xd0, 0xde, 0x27, 0x7e, 0xdd, 0x0e,
- 0x5d, 0xea, 0xa1, 0x97, 0xa0, 0x68, 0xf1, 0x2f, 0xae, 0xc7, 0xc2, 0x9d, 0x0b, 0xa9, 0xb5, 0x08,
- 0x22, 0x2c, 0x49, 0x98, 0xd7, 0x35, 0xe8, 0x60, 0xe0, 0x86, 0xb1, 0x52, 0x31, 0x8c, 0xd6, 0x60,
- 0xae, 0x15, 0xb0, 0xa9, 0x76, 0xd8, 0x1a, 0xb8, 0x6a, 0x65, 0xac, 0xa2, 0xcc, 0x06, 0xe4, 0xea,
- 0x8d, 0x07, 0x29, 0x21, 0xda, 0xd1, 0x42, 0xf4, 0x49, 0x21, 0x18, 0x50, 0xab, 0xe7, 0x51, 0x9f,
- 0x38, 0xf7, 0xfa, 0xd4, 0x3e, 0x90, 0xdb, 0x71, 0x36, 0x99, 0x3f, 0xd6, 0xe1, 0x52, 0xcb, 0x7b,
- 0xd4, 0x1f, 0x11, 0x66, 0xa8, 0xc4, 0x44, 0x01, 0xfa, 0x0e, 0x54, 0xe3, 0x81, 0xdd, 0xa7, 0x43,
- 0x22, 0x8d, 0x74, 0x35, 0x65, 0xa4, 0x14, 0x05, 0x4e, 0x33, 0xa0, 0x37, 0xa1, 0x9a, 0x08, 0x6c,
- 0x35, 0x99, 0xdd, 0x72, 0x13, 0x2e, 0xa3, 0x52, 0xe0, 0x34, 0x3d, 0x3f, 0xe9, 0xf6, 0x3e, 0x19,
- 0x58, 0xad, 0x26, 0x37, 0x6a, 0x0e, 0xc7, 0x30, 0x7a, 0x00, 0x17, 0xc8, 0x13, 0xbb, 0x3f, 0x72,
- 0x88, 0xc2, 0xe3, 0xf0, 0xbd, 0x3f, 0x72, 0x8a, 0x2c, 0x2e, 0xf3, 0x17, 0xba, 0xea, 0x1e, 0xd2,
- 0xb0, 0xdf, 0x85, 0x4b, 0x6e, 0x96, 0x65, 0x64, 0x1c, 0x30, 0xb3, 0x0d, 0xa1, 0x52, 0xe2, 0x6c,
- 0x01, 0xe8, 0x6e, 0xec, 0x78, 0x22, 0x2c, 0x5c, 0x9f, 0xa2, 0xee, 0x98, 0x0b, 0x9a, 0x90, 0xb3,
- 0xec, 0x28, 0x20, 0x18, 0x69, 0x67, 0x6d, 0x3c, 0xc0, 0x6c, 0x10, 0x6d, 0x03, 0x72, 0x27, 0x7c,
- 0x44, 0x5a, 0xe5, 0x46, 0x5a, 0xe3, 0x09, 0x32, 0x9c, 0xc1, 0x6a, 0x7e, 0xaa, 0xc1, 0x79, 0x25,
- 0x32, 0x06, 0x43, 0xea, 0x05, 0xe4, 0xac, 0xa1, 0xb1, 0x0d, 0xc8, 0x19, 0x33, 0x37, 0x89, 0xdc,
- 0x63, 0x9a, 0x31, 0x22, 0x1d, 0x27, 0x19, 0x11, 0x82, 0xfc, 0x80, 0x3a, 0x44, 0xfa, 0x08, 0xff,
- 0x46, 0x2f, 0x82, 0x31, 0xb0, 0x5c, 0x2f, 0xb4, 0x5c, 0x8f, 0xf8, 0x5d, 0x32, 0xa4, 0xf6, 0xbe,
- 0x0c, 0x0c, 0x8b, 0x09, 0x7e, 0x83, 0xa1, 0xcd, 0x27, 0x70, 0xa1, 0xa1, 0x44, 0xa0, 0x36, 0x09,
- 0x02, 0xab, 0x77, 0xe6, 0x35, 0x8e, 0xc7, 0x3a, 0x7d, 0x32, 0xd6, 0x99, 0xbf, 0xd7, 0x60, 0x11,
- 0x13, 0x87, 0xb6, 0x49, 0x68, 0xcd, 0x68, 0xda, 0xe3, 0xc2, 0xe7, 0xb8, 0x5a, 0xb9, 0x8c, 0x10,
- 0x7c, 0x0a, 0xdb, 0xfd, 0x10, 0xae, 0xb3, 0x05, 0xe0, 0x78, 0x82, 0x1d, 0x9f, 0xf6, 0x7c, 0x12,
- 0x04, 0x5f, 0xcc, 0x72, 0xcc, 0x5f, 0x6b, 0xb0, 0x92, 0x56, 0xe0, 0x2d, 0xea, 0x1f, 0x5a, 0xbe,
- 0xf3, 0x05, 0x99, 0x33, 0xcb, 0x54, 0xb9, 0x6c, 0x53, 0xfd, 0x4b, 0x53, 0x83, 0x4c, 0x83, 0x7a,
- 0x8f, 0xdc, 0x1e, 0xaa, 0x41, 0x3e, 0x18, 0x5a, 0x9e, 0x54, 0x6b, 0x29, 0xfb, 0xb2, 0xc6, 0x9c,
- 0x86, 0xa5, 0x44, 0x01, 0x4b, 0x74, 0x62, 0x45, 0x22, 0x90, 0x2d, 0xd2, 0x51, 0x82, 0x9c, 0x0c,
- 0x11, 0x47, 0x44, 0xc1, 0x14, 0x39, 0x8b, 0xb3, 0x41, 0x14, 0x67, 0xf3, 0x22, 0xce, 0x46, 0x70,
- 0x7c, 0xb6, 0x0a, 0xca, 0xd9, 0xaa, 0x81, 0x11, 0x1c, 0xb8, 0xc3, 0x66, 0x7b, 0xb3, 0x1e, 0x74,
- 0xa4, 0x46, 0x45, 0x7e, 0xb7, 0x4c, 0xe0, 0xcd, 0x3f, 0xe8, 0x70, 0x85, 0x05, 0x6d, 0x67, 0xd4,
- 0x57, 0x62, 0xee, 0x8c, 0x52, 0xac, 0xbb, 0x50, 0xb4, 0xb9, 0x1d, 0x8f, 0x09, 0xa4, 0xc2, 0xd8,
- 0x58, 0x12, 0xa3, 0x06, 0x2c, 0x04, 0x52, 0x25, 0x11, 0x62, 0xb9, 0xc1, 0x16, 0xee, 0x5c, 0x4b,
- 0xb1, 0x77, 0x52, 0x24, 0x78, 0x8c, 0x85, 0xa9, 0x4e, 0x87, 0xc4, 0xb7, 0x42, 0xea, 0xf3, 0xeb,
- 0x31, 0xcf, 0x45, 0xa4, 0x55, 0xdf, 0x56, 0x08, 0x70, 0x8a, 0x3c, 0xd3, 0x71, 0x0a, 0xd9, 0x8e,
- 0xf3, 0x2b, 0x1d, 0x96, 0xda, 0xc4, 0xef, 0xcd, 0xde, 0x7e, 0x6f, 0x42, 0xd5, 0x39, 0xe5, 0x0d,
- 0x9d, 0xa2, 0x47, 0x2d, 0x40, 0x03, 0xa6, 0x99, 0xd3, 0x3c, 0x95, 0xfb, 0x65, 0x30, 0xc5, 0x8e,
- 0x96, 0x3f, 0x26, 0x88, 0x4f, 0x31, 0xd2, 0x0e, 0x5c, 0x68, 0xc7, 0xa8, 0xfb, 0xd1, 0xc4, 0xe8,
- 0x9b, 0x4a, 0x42, 0xac, 0x65, 0xdc, 0x2f, 0x09, 0xcf, 0x78, 0x46, 0x6c, 0x7e, 0xa2, 0x41, 0xb5,
- 0xe9, 0x5b, 0xae, 0x17, 0x85, 0x34, 0xf4, 0x02, 0x2c, 0x84, 0x96, 0xdf, 0x23, 0x61, 0xd7, 0xa3,
- 0x0e, 0xe9, 0xba, 0x0e, 0xb7, 0x77, 0x05, 0xcf, 0x0b, 0xec, 0x16, 0x75, 0x48, 0xcb, 0x41, 0xcf,
- 0x81, 0x84, 0xa5, 0xc2, 0xe2, 0xac, 0xce, 0x09, 0x1c, 0x57, 0x16, 0x7d, 0x0d, 0x2e, 0x4b, 0x92,
- 0xc4, 0x9c, 0x5d, 0x9b, 0x8e, 0x64, 0xf2, 0x58, 0xc5, 0x97, 0xc4, 0xb0, 0xea, 0xc1, 0x23, 0x2f,
- 0x44, 0x6f, 0xc1, 0x9a, 0xe4, 0x63, 0x89, 0x85, 0xdb, 0xdb, 0x0f, 0xbb, 0x0e, 0xd3, 0xb0, 0x3b,
- 0xa0, 0x8f, 0x89, 0x14, 0x20, 0x8a, 0x9b, 0x15, 0x41, 0xd7, 0x92, 0x64, 0x7c, 0x1d, 0x6d, 0xfa,
- 0x98, 0x70, 0x39, 0xe6, 0x6f, 0x72, 0x60, 0x8c, 0xaf, 0xfc, 0xac, 0xbe, 0x74, 0x1d, 0x80, 0x7d,
- 0x75, 0x99, 0xfd, 0x08, 0x5f, 0x74, 0x05, 0x57, 0x18, 0x86, 0x89, 0x27, 0xe8, 0x36, 0x14, 0xc4,
- 0x48, 0xd6, 0x51, 0x6b, 0xd0, 0xc1, 0x90, 0x7a, 0xc4, 0x0b, 0x39, 0x2d, 0x16, 0x94, 0xe8, 0x79,
- 0xa8, 0x26, 0xd7, 0x52, 0x37, 0x8c, 0x13, 0xfb, 0xd4, 0x5d, 0x25, 0xab, 0x91, 0x42, 0x86, 0xe3,
- 0x4e, 0x54, 0x23, 0xe8, 0x4b, 0xb0, 0xb0, 0x47, 0x69, 0x18, 0x84, 0xbe, 0x35, 0xec, 0x3a, 0xd4,
- 0x23, 0x32, 0x6c, 0x55, 0x63, 0x6c, 0x93, 0x7a, 0x64, 0xa2, 0xa0, 0x28, 0x4d, 0x16, 0x14, 0xa8,
- 0x0e, 0x0b, 0xc2, 0xf4, 0x43, 0xe9, 0x1d, 0xcb, 0x65, 0x6e, 0xaf, 0x74, 0x7e, 0x9c, 0xf2, 0x1f,
- 0x5c, 0x75, 0x52, 0xee, 0x94, 0xe5, 0xdd, 0x95, 0x6c, 0xef, 0xfe, 0x87, 0x06, 0x55, 0xe6, 0x5e,
- 0x89, 0x63, 0xdf, 0x85, 0x72, 0xdf, 0x7d, 0x4c, 0x3c, 0x36, 0xb3, 0x96, 0x11, 0x7a, 0x18, 0xf5,
- 0xa6, 0x24, 0xc0, 0x31, 0x29, 0xdb, 0x25, 0xee, 0xbb, 0xaa, 0x6b, 0x56, 0x18, 0x46, 0x38, 0x66,
- 0x13, 0x6e, 0x28, 0x1e, 0x29, 0x16, 0x38, 0xe6, 0xf2, 0x39, 0xbe, 0xb3, 0xd7, 0x12, 0x32, 0xbe,
- 0xc6, 0x5d, 0xf5, 0x04, 0xd4, 0xe1, 0xfa, 0x34, 0x29, 0x6a, 0x32, 0x71, 0x35, 0x53, 0x86, 0x58,
- 0xf0, 0xfb, 0xb0, 0xd4, 0x11, 0xf2, 0xe2, 0x45, 0xc8, 0x90, 0x77, 0x1b, 0x8a, 0x42, 0xd6, 0xf1,
- 0xcb, 0x96, 0x84, 0xc7, 0x2c, 0xda, 0x1c, 0xc0, 0xe5, 0x89, 0xb9, 0x64, 0x9e, 0xfb, 0x0a, 0x94,
- 0xac, 0xe1, 0xb0, 0xef, 0x12, 0xe7, 0xf8, 0xd9, 0x22, 0xca, 0xe3, 0xa6, 0x7b, 0x1f, 0x6e, 0x74,
- 0xd4, 0xa3, 0xad, 0xac, 0x3d, 0x5a, 0xe3, 0xac, 0x02, 0x8d, 0xf9, 0x75, 0xb8, 0xd6, 0xa0, 0xd4,
- 0x77, 0x5c, 0x8f, 0x5d, 0x3c, 0xf7, 0x22, 0x2f, 0x8f, 0xe6, 0x59, 0x86, 0xd2, 0x63, 0xe2, 0x07,
- 0x51, 0x09, 0x9c, 0xc3, 0x11, 0xc8, 0x2a, 0xa2, 0x95, 0x6c, 0x4e, 0x69, 0x99, 0xff, 0x3d, 0xb0,
- 0xa2, 0x57, 0x61, 0x29, 0x3e, 0x3a, 0x21, 0xb5, 0x69, 0xbf, 0x1b, 0x29, 0xa1, 0xf3, 0xd8, 0x75,
- 0x31, 0x3a, 0x26, 0x7c, 0xf0, 0x5d, 0x31, 0xf6, 0xff, 0xe3, 0x9a, 0xff, 0xd6, 0xe0, 0x62, 0xdd,
- 0x71, 0x92, 0x05, 0x46, 0xd6, 0x7c, 0x11, 0x74, 0xb9, 0x53, 0x47, 0x86, 0x4d, 0xdd, 0x75, 0xd0,
- 0x52, 0x2a, 0x71, 0x99, 0x8f, 0x33, 0x93, 0x89, 0x90, 0x97, 0x95, 0x9e, 0xd7, 0xe0, 0xbc, 0x1b,
- 0x74, 0x3d, 0x72, 0xd8, 0x4d, 0x02, 0x30, 0xd7, 0xbb, 0x8c, 0x17, 0xdd, 0x60, 0x8b, 0x1c, 0x26,
- 0xd3, 0xa1, 0x1b, 0x30, 0x77, 0x20, 0xdb, 0x5c, 0xcc, 0x42, 0x05, 0xd1, 0xf9, 0x8a, 0x50, 0x2d,
- 0x27, 0x33, 0x08, 0x15, 0xb3, 0x83, 0xd0, 0x1f, 0x35, 0xb8, 0x8c, 0x09, 0xbb, 0x6a, 0xce, 0xb4,
- 0xf6, 0x65, 0x28, 0xd9, 0x56, 0x60, 0x5b, 0x0e, 0x91, 0x0d, 0x89, 0x08, 0x64, 0x23, 0x3e, 0x97,
- 0xef, 0xc8, 0x1e, 0x4a, 0x04, 0x8e, 0x2f, 0x23, 0x7f, 0xa2, 0x65, 0x4c, 0xc9, 0x14, 0xfe, 0x9c,
- 0x83, 0xab, 0xc9, 0x02, 0x26, 0xce, 0xc4, 0x19, 0xaf, 0xc1, 0x69, 0x3b, 0x7b, 0x85, 0x9f, 0x17,
- 0x5f, 0xd9, 0xd4, 0x38, 0x7b, 0xb7, 0xe1, 0xb9, 0x90, 0xa5, 0xfa, 0xdd, 0xd0, 0x77, 0x7b, 0x3d,
- 0xa6, 0xfe, 0x63, 0xe2, 0xa5, 0x52, 0x03, 0xf7, 0x04, 0x8d, 0x8d, 0xeb, 0x5c, 0xc6, 0xae, 0x10,
- 0xb1, 0xc1, 0x24, 0xa8, 0x2d, 0x8e, 0x6c, 0xa7, 0x29, 0x64, 0x3b, 0x8d, 0xc5, 0xd2, 0x0c, 0x55,
- 0x21, 0x9f, 0x38, 0x74, 0x4c, 0x9f, 0xe2, 0x71, 0xfa, 0xac, 0xa8, 0xfa, 0xb0, 0x12, 0x2d, 0xa5,
- 0xce, 0xd8, 0x86, 0x96, 0x4e, 0xb4, 0xa1, 0xe5, 0xec, 0x0d, 0xfd, 0x4b, 0x0e, 0xae, 0x65, 0x6e,
- 0xe8, 0x6c, 0x9a, 0x15, 0x77, 0xa1, 0xc0, 0xca, 0xaf, 0x28, 0x39, 0x4e, 0x77, 0x51, 0xe2, 0xd9,
- 0x92, 0x62, 0x4d, 0x50, 0x47, 0x89, 0x49, 0xee, 0x24, 0x6d, 0xd2, 0x93, 0xa5, 0x3a, 0x2f, 0x03,
- 0xe2, 0x1b, 0x91, 0xa6, 0x14, 0x5e, 0x6e, 0xb0, 0x11, 0xb5, 0x8b, 0x81, 0x9a, 0x50, 0x89, 0x0a,
- 0x0e, 0x56, 0x9d, 0x31, 0xd5, 0xbf, 0x9c, 0x59, 0xdf, 0x4c, 0x54, 0x15, 0x38, 0x61, 0xcc, 0xdc,
- 0x86, 0x52, 0xe6, 0x36, 0xa0, 0x4d, 0x58, 0xe4, 0x69, 0x7d, 0x37, 0x99, 0xb6, 0xcc, 0xa7, 0x7d,
- 0x3e, 0x7d, 0x31, 0x64, 0x56, 0x32, 0x78, 0x81, 0xf3, 0x46, 0x05, 0x53, 0x60, 0xfe, 0x4d, 0x87,
- 0xd5, 0x64, 0x53, 0x77, 0x68, 0x10, 0xce, 0xfa, 0xa4, 0x9e, 0xe8, 0xd8, 0xe9, 0x67, 0x3c, 0x76,
- 0xb7, 0xa1, 0x24, 0x4a, 0x69, 0x76, 0xea, 0x99, 0x31, 0x2e, 0x4f, 0xec, 0xc1, 0xc0, 0x6a, 0x79,
- 0x8f, 0x28, 0x8e, 0xe8, 0xd0, 0x6b, 0x30, 0xcf, 0xb7, 0x39, 0xe2, 0xcb, 0x1f, 0xcd, 0x37, 0xc7,
- 0x88, 0x3b, 0x92, 0xf7, 0x14, 0x61, 0xf0, 0x23, 0x1d, 0x6e, 0x4c, 0x35, 0xf0, 0x6c, 0x4e, 0xce,
- 0x17, 0x62, 0xe1, 0x53, 0x9d, 0xb3, 0x53, 0x75, 0x05, 0x21, 0xb1, 0x72, 0xaa, 0x15, 0xad, 0x8d,
- 0xb5, 0xa2, 0x57, 0x23, 0xca, 0x2d, 0x6b, 0x10, 0x55, 0x3e, 0x0a, 0x06, 0xdd, 0x84, 0x22, 0x8f,
- 0x0e, 0x91, 0x0b, 0x64, 0x74, 0x79, 0xf8, 0x4e, 0x4a, 0x2a, 0xb3, 0x21, 0xdf, 0xc1, 0xf8, 0xc4,
- 0xd3, 0xdf, 0xc1, 0x56, 0x24, 0x99, 0x32, 0x6b, 0x82, 0x30, 0x7f, 0xa7, 0x03, 0x9a, 0x0c, 0x4e,
- 0xec, 0x9e, 0x9e, 0xb2, 0x8f, 0x29, 0x9b, 0xeb, 0xf2, 0x9d, 0x2d, 0x5a, 0xb2, 0x3e, 0xb6, 0xe4,
- 0xa8, 0x6d, 0x95, 0x3b, 0x41, 0xdb, 0xea, 0x2d, 0x30, 0xec, 0xa8, 0xbe, 0xeb, 0x06, 0x49, 0x43,
- 0xfa, 0x98, 0x22, 0x70, 0xd1, 0x56, 0xe1, 0x51, 0x30, 0x19, 0x23, 0x0b, 0x19, 0x31, 0xf2, 0x15,
- 0x98, 0xdb, 0xeb, 0x53, 0xfb, 0x40, 0x96, 0xa1, 0xe2, 0x96, 0x42, 0xe9, 0xb3, 0xc3, 0xc5, 0xc3,
- 0x5e, 0xd4, 0xe4, 0x26, 0x71, 0xeb, 0xa1, 0x94, 0xb4, 0x1e, 0xcc, 0x5f, 0x6a, 0xb0, 0x94, 0x1c,
- 0x8f, 0x46, 0x9f, 0x06, 0x64, 0x46, 0x71, 0x47, 0xc9, 0x72, 0xf4, 0x74, 0x96, 0x73, 0x8a, 0x66,
- 0xe2, 0x47, 0x1a, 0x5c, 0x9e, 0x50, 0x6f, 0x36, 0xa7, 0x76, 0x19, 0x4a, 0xc1, 0xc8, 0xb6, 0x59,
- 0x61, 0x29, 0xf5, 0x93, 0xe0, 0x69, 0xf4, 0xfb, 0xa9, 0x06, 0x46, 0xf2, 0x26, 0x22, 0x1c, 0x7b,
- 0x06, 0x4f, 0x4a, 0x57, 0xa1, 0x2c, 0xdd, 0x5f, 0x5c, 0xc7, 0x39, 0x1c, 0xc3, 0x47, 0xbd, 0x16,
- 0x99, 0xdf, 0x83, 0x02, 0xa7, 0x3b, 0xe6, 0x59, 0x79, 0x9a, 0xbb, 0xaf, 0x40, 0xa5, 0x33, 0xec,
- 0xbb, 0x3c, 0x10, 0xc9, 0xd4, 0x34, 0x41, 0x98, 0x1f, 0xea, 0x70, 0x01, 0xd3, 0x51, 0x48, 0xb8,
- 0xa8, 0xba, 0x33, 0x70, 0x03, 0x5e, 0xb1, 0xd4, 0xc0, 0xe8, 0xd0, 0x91, 0x6f, 0x13, 0x25, 0x3a,
- 0x88, 0x3a, 0x6e, 0x02, 0x8f, 0xd6, 0x61, 0x51, 0xe0, 0xc6, 0x8f, 0xf4, 0x38, 0x9a, 0x49, 0x15,
- 0xd5, 0x88, 0x22, 0x55, 0x14, 0x3e, 0x13, 0x78, 0x26, 0x55, 0xe0, 0x12, 0xa9, 0x79, 0x21, 0x75,
- 0x0c, 0x8d, 0xde, 0x80, 0xa2, 0x6c, 0x85, 0x16, 0xf8, 0x9e, 0xa4, 0x53, 0x85, 0x8c, 0xd5, 0x45,
- 0x6f, 0x53, 0xe2, 0xd7, 0xf4, 0x60, 0x21, 0xb2, 0x96, 0x70, 0xad, 0x23, 0x2c, 0xbd, 0x06, 0x73,
- 0xdb, 0x7d, 0x67, 0xcc, 0xd8, 0x2a, 0x8a, 0x51, 0x6c, 0x91, 0xc3, 0xb1, 0xdd, 0x54, 0x51, 0xe6,
- 0x7f, 0x72, 0x50, 0x10, 0x87, 0x77, 0x05, 0x2a, 0xad, 0x80, 0xbf, 0x58, 0xc9, 0x22, 0xbd, 0x8c,
- 0x13, 0x04, 0xd3, 0x82, 0x7f, 0x26, 0x3d, 0x73, 0x09, 0xa2, 0x37, 0x61, 0x4e, 0x7c, 0x46, 0xa1,
- 0x79, 0xb2, 0x81, 0x3c, 0xee, 0xc0, 0x58, 0xe5, 0x40, 0x0f, 0xe0, 0xfc, 0x16, 0x21, 0x4e, 0xd3,
- 0xa7, 0xc3, 0x61, 0x44, 0x21, 0xd3, 0xf4, 0x63, 0xc4, 0x4c, 0xf2, 0xa1, 0xd7, 0x61, 0x91, 0x21,
- 0xeb, 0x8e, 0x13, 0x8b, 0x12, 0x2d, 0x2d, 0x34, 0x19, 0x5b, 0xf1, 0x38, 0x29, 0x6a, 0xc0, 0xc2,
- 0x3b, 0x43, 0xc7, 0x0a, 0x89, 0x34, 0x61, 0x94, 0xf0, 0x5d, 0xcb, 0x4a, 0x1a, 0xe4, 0x06, 0xe1,
- 0x31, 0x96, 0xf1, 0xc7, 0xe2, 0xd2, 0xc4, 0x63, 0x31, 0xfa, 0x0a, 0xef, 0xe1, 0xf5, 0x08, 0x4f,
- 0xc4, 0x17, 0xc6, 0x52, 0x92, 0xe8, 0xd1, 0xb0, 0x27, 0xfa, 0x77, 0x3d, 0x82, 0x76, 0xe1, 0x62,
- 0x86, 0xe3, 0x04, 0xcb, 0x15, 0xae, 0xdb, 0xda, 0x71, 0x1e, 0x86, 0x33, 0xb9, 0xcd, 0x1f, 0xc1,
- 0xc5, 0xf8, 0x86, 0x51, 0xdf, 0xc1, 0x4f, 0x71, 0xb3, 0xad, 0x47, 0xbd, 0x48, 0x7d, 0xea, 0xf5,
- 0x20, 0x5b, 0x90, 0x19, 0x2f, 0x8b, 0xe6, 0x3f, 0x35, 0x76, 0xaa, 0x52, 0xff, 0xa3, 0x38, 0xcd,
- 0xe4, 0x59, 0xd7, 0xa1, 0x3e, 0x8b, 0xeb, 0x30, 0xab, 0x55, 0x70, 0x1b, 0x2e, 0x89, 0x9c, 0x2b,
- 0x70, 0x9f, 0x91, 0xee, 0x90, 0xf8, 0xdd, 0x80, 0xd8, 0xd4, 0x13, 0xe5, 0xa4, 0x8e, 0x11, 0x1f,
- 0xec, 0xb8, 0xcf, 0xc8, 0x0e, 0xf1, 0x3b, 0x7c, 0x24, 0xeb, 0xc1, 0xc7, 0xfc, 0xad, 0x06, 0x48,
- 0x7d, 0x28, 0x9e, 0xcd, 0x45, 0xf8, 0x36, 0x54, 0xf7, 0x12, 0xa1, 0xf1, 0x03, 0xf0, 0x73, 0xd9,
- 0xd9, 0x84, 0x3a, 0x7f, 0x9a, 0x2f, 0x73, 0x97, 0x1c, 0x98, 0x57, 0xd3, 0x3f, 0x46, 0x13, 0xba,
- 0x71, 0x00, 0xe6, 0xdf, 0x0c, 0xe7, 0x51, 0x27, 0x8a, 0xb4, 0xfc, 0x9b, 0xe1, 0xec, 0x48, 0x56,
- 0x05, 0xf3, 0x6f, 0x16, 0x44, 0x06, 0xe2, 0x39, 0x51, 0x86, 0xcf, 0x08, 0x34, 0x5f, 0x85, 0xf9,
- 0xf1, 0x47, 0x8c, 0x7d, 0xb7, 0xb7, 0x2f, 0xff, 0x88, 0xc1, 0xbf, 0x91, 0x01, 0xb9, 0x3e, 0x3d,
- 0x94, 0xe1, 0x87, 0x7d, 0x32, 0xdd, 0x54, 0xb3, 0x9c, 0x8c, 0x8b, 0x6b, 0x9b, 0x04, 0x7b, 0xfe,
- 0xcd, 0x2e, 0xad, 0xa8, 0x66, 0x96, 0xaa, 0xc5, 0xb0, 0xf9, 0x7d, 0xb8, 0xb1, 0x49, 0x7b, 0x4a,
- 0x13, 0x2f, 0x79, 0x23, 0x9d, 0xcd, 0x06, 0x9a, 0x1f, 0x6a, 0xb0, 0x36, 0x7d, 0x8a, 0xd9, 0x64,
- 0x23, 0xc7, 0x3d, 0x00, 0xf7, 0x99, 0x2d, 0x89, 0x7d, 0x10, 0x8c, 0x06, 0x6d, 0x12, 0x5a, 0xe8,
- 0xab, 0xd1, 0xd9, 0xce, 0xca, 0x2d, 0x22, 0xca, 0xd4, 0x19, 0xaf, 0x81, 0x61, 0xab, 0xf8, 0x0e,
- 0xf9, 0x40, 0xce, 0x33, 0x81, 0x37, 0x7f, 0xae, 0xc1, 0x25, 0xe5, 0x2f, 0x09, 0x24, 0x8c, 0x24,
- 0xa2, 0x8b, 0x50, 0x10, 0xef, 0x2f, 0x62, 0x13, 0x05, 0xc0, 0x3c, 0xe7, 0x09, 0xf5, 0xef, 0xb3,
- 0xcd, 0x95, 0xd7, 0x8f, 0x04, 0xd1, 0x12, 0x14, 0x9f, 0x50, 0x7f, 0x93, 0x1e, 0xca, 0x73, 0x2b,
- 0x21, 0x91, 0x7d, 0x0d, 0x38, 0x47, 0x5e, 0xb6, 0x89, 0x04, 0xc8, 0x38, 0x82, 0xd1, 0x80, 0x71,
- 0x88, 0xc4, 0x57, 0x42, 0x2c, 0x15, 0x5c, 0xcb, 0xd4, 0xa9, 0x6e, 0x1f, 0xcc, 0x6a, 0x17, 0x2e,
- 0x42, 0x41, 0xed, 0x31, 0x0b, 0x20, 0xf3, 0x7f, 0x17, 0xf2, 0xef, 0x59, 0xf9, 0xf8, 0xef, 0x59,
- 0xe6, 0xdf, 0x35, 0x30, 0x33, 0xf5, 0x13, 0xf7, 0xcf, 0x8c, 0x82, 0xc9, 0x19, 0x34, 0x44, 0x6f,
- 0x40, 0x39, 0xda, 0x69, 0x6e, 0xdb, 0xf1, 0x3f, 0xf7, 0x64, 0x6a, 0x8f, 0x63, 0x9e, 0xda, 0xf5,
- 0x28, 0x79, 0x42, 0x15, 0x28, 0x3c, 0xf4, 0xdd, 0x90, 0x18, 0xe7, 0x50, 0x19, 0xf2, 0x3b, 0x56,
- 0x10, 0x18, 0x5a, 0x6d, 0x5d, 0xe4, 0x46, 0xca, 0xdb, 0x31, 0x40, 0xb1, 0xe1, 0x13, 0x8b, 0xd3,
- 0x01, 0x14, 0x45, 0x53, 0xd5, 0xd0, 0x6a, 0x6d, 0x98, 0x57, 0x9f, 0x8c, 0x99, 0xb8, 0xed, 0x6e,
- 0xdd, 0x71, 0x8c, 0x73, 0x68, 0x1e, 0xca, 0xdb, 0xdd, 0x88, 0x90, 0x31, 0x6d, 0x77, 0xdb, 0xec,
- 0x5b, 0x47, 0x73, 0x50, 0xda, 0xee, 0xf2, 0x6c, 0xd4, 0xc8, 0x09, 0x80, 0xb7, 0x58, 0x8c, 0x7c,
- 0xed, 0x2e, 0xcc, 0xab, 0x2f, 0x14, 0x4c, 0x5c, 0x7d, 0xb3, 0xf5, 0xee, 0x86, 0x10, 0xd7, 0xc4,
- 0xf5, 0xd6, 0x56, 0x6b, 0xeb, 0x6d, 0x43, 0x63, 0x50, 0x67, 0x77, 0x7b, 0x67, 0x87, 0x41, 0x7a,
- 0xed, 0x35, 0x80, 0xe4, 0x32, 0x67, 0xeb, 0xd8, 0xda, 0xde, 0x62, 0x3c, 0x73, 0x50, 0x7a, 0x58,
- 0x6f, 0xed, 0x0a, 0x16, 0x06, 0x60, 0x01, 0xe8, 0x8c, 0xa6, 0xc9, 0x68, 0x72, 0xb5, 0x97, 0xc7,
- 0x52, 0x7c, 0x54, 0x82, 0x5c, 0xbd, 0xdf, 0x37, 0xce, 0xa1, 0x22, 0xe8, 0xcd, 0x7b, 0x42, 0xf5,
- 0x2d, 0xea, 0x0f, 0xac, 0xbe, 0xa1, 0xd7, 0xde, 0x86, 0x2b, 0x53, 0x53, 0x4b, 0xae, 0x6d, 0xb3,
- 0xdd, 0xda, 0x15, 0x33, 0xe3, 0x8d, 0xcd, 0x8d, 0x7a, 0x67, 0xc3, 0xd0, 0x10, 0x82, 0x05, 0x09,
- 0x74, 0x3b, 0x8d, 0xfb, 0x1b, 0xed, 0xba, 0xa1, 0xd7, 0x9e, 0xc1, 0x42, 0xfa, 0xbe, 0xe4, 0xfa,
- 0x51, 0xff, 0xc0, 0xf5, 0x7a, 0x82, 0xbf, 0x13, 0xf2, 0x74, 0x4b, 0x68, 0x2e, 0xec, 0xe8, 0x18,
- 0x3a, 0x32, 0x60, 0xbe, 0xe5, 0xb9, 0xa1, 0x6b, 0xf5, 0xdd, 0x67, 0x8c, 0x36, 0x87, 0xaa, 0x50,
- 0xd9, 0xf1, 0xc9, 0xd0, 0xf2, 0x19, 0x98, 0x47, 0x0b, 0x00, 0xdc, 0x9c, 0x98, 0x58, 0xce, 0x53,
- 0xa3, 0xc0, 0x18, 0x1e, 0x5a, 0x6e, 0xe8, 0x7a, 0x3d, 0x61, 0xe5, 0x62, 0xed, 0x5b, 0x50, 0x4d,
- 0xc5, 0x15, 0x74, 0x1e, 0xaa, 0xef, 0x6c, 0xb5, 0xb6, 0x5a, 0xbb, 0xad, 0xfa, 0x66, 0xeb, 0xbd,
- 0x8d, 0xa6, 0x30, 0x77, 0xbb, 0xd5, 0x69, 0xd7, 0x77, 0x1b, 0xf7, 0x0d, 0x8d, 0xad, 0x4c, 0x7c,
- 0xea, 0xf7, 0xde, 0xf8, 0xd3, 0x67, 0xab, 0xda, 0xc7, 0x9f, 0xad, 0x6a, 0x9f, 0x7e, 0xb6, 0xaa,
- 0xfd, 0xec, 0xf3, 0xd5, 0x73, 0x1f, 0x7f, 0xbe, 0x7a, 0xee, 0x93, 0xcf, 0x57, 0xcf, 0xbd, 0xf7,
- 0x42, 0xcf, 0x0d, 0xf7, 0x47, 0x7b, 0x37, 0x6d, 0x3a, 0xb8, 0x35, 0x74, 0xbd, 0x9e, 0x6d, 0x0d,
- 0x6f, 0x85, 0xae, 0xed, 0xd8, 0xb7, 0x14, 0xd7, 0xdc, 0x2b, 0xf2, 0x37, 0x94, 0x57, 0xfe, 0x1b,
- 0x00, 0x00, 0xff, 0xff, 0x50, 0xf3, 0xe4, 0x7c, 0x66, 0x2b, 0x00, 0x00,
+ // 3285 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x1a, 0x4d, 0x73, 0x1c, 0x47,
+ 0xd5, 0x33, 0xfb, 0xa5, 0x7d, 0xd2, 0x4a, 0xe3, 0xb6, 0x2c, 0xcb, 0xb6, 0x2c, 0xcb, 0x93, 0x00,
+ 0xca, 0x26, 0xd8, 0xd8, 0x89, 0x29, 0x08, 0x21, 0x66, 0xbd, 0xab, 0xc4, 0x5b, 0xd6, 0x4a, 0xaa,
+ 0x59, 0x25, 0x4e, 0x85, 0xc3, 0x32, 0x9a, 0x69, 0xaf, 0x26, 0xda, 0x9d, 0x59, 0xcf, 0xcc, 0x5a,
+ 0xb6, 0xab, 0x80, 0x4a, 0x51, 0xdc, 0x38, 0xc0, 0x09, 0x0e, 0xe4, 0xc2, 0x89, 0x13, 0xc5, 0x1f,
+ 0xa0, 0xc8, 0x81, 0x03, 0x27, 0x2a, 0xc5, 0x81, 0x0a, 0x17, 0x48, 0x25, 0x47, 0x2e, 0x50, 0x54,
+ 0xc1, 0x95, 0xea, 0xaf, 0x99, 0x9e, 0x9d, 0x59, 0xad, 0x84, 0xb6, 0x52, 0x14, 0xa7, 0x99, 0x7e,
+ 0xfd, 0xde, 0xeb, 0xd7, 0xef, 0xbd, 0x7e, 0xfd, 0xfa, 0x75, 0xc3, 0xe5, 0x7d, 0x6c, 0xfa, 0xe1,
+ 0x1e, 0x36, 0xc3, 0xc1, 0xde, 0x8d, 0xe8, 0xff, 0xfa, 0xc0, 0xf7, 0x42, 0x0f, 0xcd, 0x4a, 0x9d,
+ 0xfa, 0x53, 0x28, 0xef, 0x9a, 0x7b, 0x3d, 0xdc, 0x1e, 0x98, 0x2e, 0x5a, 0x86, 0x12, 0x6d, 0x34,
+ 0x1b, 0xcb, 0xca, 0x9a, 0xb2, 0x9e, 0x33, 0x44, 0x13, 0x5d, 0x82, 0x99, 0x76, 0x68, 0xfa, 0xe1,
+ 0x7d, 0xfc, 0x74, 0x59, 0x5d, 0x53, 0xd6, 0xe7, 0x8c, 0xa8, 0x8d, 0x96, 0xa0, 0xb8, 0xe1, 0xda,
+ 0xa4, 0x27, 0x47, 0x7b, 0x78, 0x0b, 0xad, 0x02, 0xdc, 0xc7, 0x4f, 0x83, 0x81, 0x69, 0x11, 0x86,
+ 0xf9, 0x35, 0x65, 0xbd, 0x62, 0x48, 0x10, 0xfd, 0x4f, 0x2a, 0x68, 0xf7, 0x88, 0x28, 0x77, 0xb1,
+ 0x19, 0x1a, 0xf8, 0xd1, 0x10, 0x07, 0x21, 0xfa, 0x26, 0xcc, 0x59, 0xfb, 0xa6, 0xdb, 0xc5, 0x0f,
+ 0x31, 0xb6, 0xb9, 0x1c, 0xb3, 0xb7, 0x2e, 0x5e, 0x97, 0x64, 0xbe, 0x5e, 0x97, 0x10, 0x8c, 0x04,
+ 0x3a, 0x7a, 0x05, 0xca, 0x87, 0x66, 0x88, 0xfd, 0xbe, 0xe9, 0x1f, 0x50, 0x41, 0x67, 0x6f, 0x2d,
+ 0x25, 0x68, 0x1f, 0x88, 0x5e, 0x23, 0x46, 0x44, 0xaf, 0x41, 0xc5, 0xc7, 0xb6, 0x17, 0xf5, 0xd1,
+ 0x89, 0x8c, 0xa7, 0x4c, 0x22, 0xa3, 0xaf, 0xc1, 0x4c, 0x10, 0x9a, 0xe1, 0x30, 0xc0, 0xc1, 0x72,
+ 0x7e, 0x2d, 0xb7, 0x3e, 0x7b, 0x6b, 0x25, 0x41, 0x18, 0xe9, 0xb7, 0x4d, 0xb1, 0x8c, 0x08, 0x1b,
+ 0xad, 0xc3, 0x82, 0xe5, 0xf5, 0x07, 0xb8, 0x87, 0x43, 0xcc, 0x3a, 0x97, 0x0b, 0x6b, 0xca, 0xfa,
+ 0x8c, 0x31, 0x0a, 0x46, 0x2f, 0x42, 0x0e, 0xfb, 0xfe, 0x72, 0x31, 0x43, 0x1b, 0xc6, 0xd0, 0x75,
+ 0x1d, 0xb7, 0xbb, 0xe1, 0xfb, 0x9e, 0x6f, 0x10, 0x2c, 0xfd, 0x87, 0x0a, 0x94, 0x63, 0xf1, 0x74,
+ 0xa2, 0x51, 0x6c, 0x1d, 0x0c, 0x3c, 0xc7, 0x0d, 0x77, 0x03, 0xaa, 0xd1, 0xbc, 0x91, 0x80, 0x11,
+ 0x53, 0xf9, 0x38, 0xf0, 0x7a, 0x8f, 0xb1, 0xbd, 0x1b, 0x50, 0xbd, 0xe5, 0x0d, 0x09, 0x82, 0x34,
+ 0xc8, 0x05, 0xf8, 0x11, 0x55, 0x4b, 0xde, 0x20, 0xbf, 0x84, 0x6b, 0xcf, 0x0c, 0xc2, 0xf6, 0x53,
+ 0xd7, 0xa2, 0x34, 0x79, 0xc6, 0x55, 0x86, 0xe9, 0xdf, 0x05, 0xad, 0xe1, 0x04, 0x03, 0x33, 0xb4,
+ 0xf6, 0xb1, 0x5f, 0xb3, 0x42, 0xc7, 0x73, 0xd1, 0x8b, 0x50, 0x34, 0xe9, 0x1f, 0x95, 0x63, 0xfe,
+ 0xd6, 0xb9, 0xc4, 0x5c, 0x18, 0x92, 0xc1, 0x51, 0x88, 0xd7, 0xd5, 0xbd, 0x7e, 0xdf, 0x09, 0x23,
+ 0xa1, 0xa2, 0x36, 0x5a, 0x83, 0xd9, 0x66, 0x40, 0x86, 0xda, 0x21, 0x73, 0xa0, 0xa2, 0xcd, 0x18,
+ 0x32, 0x48, 0xaf, 0x43, 0xae, 0x56, 0xbf, 0x9f, 0x60, 0xa2, 0x1c, 0xcd, 0x44, 0x4d, 0x33, 0x31,
+ 0x00, 0x35, 0xbb, 0xae, 0xe7, 0x63, 0xfb, 0x6e, 0xcf, 0xb3, 0x0e, 0xb8, 0x39, 0x4e, 0xc7, 0xf3,
+ 0x07, 0x2a, 0x9c, 0x6f, 0xba, 0x0f, 0x7b, 0x43, 0x4c, 0x14, 0x15, 0xab, 0x28, 0x40, 0xdf, 0x82,
+ 0x4a, 0xd4, 0xb1, 0xfb, 0x74, 0x80, 0xb9, 0x92, 0x2e, 0x25, 0x94, 0x94, 0xc0, 0x30, 0x92, 0x04,
+ 0xe8, 0x0e, 0x54, 0x62, 0x86, 0xcd, 0x06, 0xd1, 0x5b, 0x2e, 0xe5, 0x32, 0x32, 0x86, 0x91, 0xc4,
+ 0xa7, 0x2b, 0xdd, 0xda, 0xc7, 0x7d, 0xb3, 0xd9, 0xa0, 0x4a, 0xcd, 0x19, 0x51, 0x1b, 0xdd, 0x87,
+ 0x73, 0xf8, 0x89, 0xd5, 0x1b, 0xda, 0x58, 0xa2, 0xb1, 0xa9, 0xed, 0x8f, 0x1c, 0x22, 0x8b, 0x4a,
+ 0xff, 0x99, 0x2a, 0xbb, 0x07, 0x57, 0xec, 0x3b, 0x70, 0xde, 0xc9, 0xd2, 0x0c, 0x8f, 0x03, 0x7a,
+ 0xb6, 0x22, 0x64, 0x4c, 0x23, 0x9b, 0x01, 0xba, 0x1d, 0x39, 0x1e, 0x0b, 0x0b, 0x57, 0xc6, 0x88,
+ 0x3b, 0xe2, 0x82, 0x3a, 0xe4, 0x4c, 0x4b, 0x04, 0x04, 0x2d, 0xe9, 0xac, 0xf5, 0xfb, 0x06, 0xe9,
+ 0x44, 0xdb, 0x80, 0x9c, 0x94, 0x8f, 0x70, 0xad, 0x5c, 0x4d, 0x4a, 0x9c, 0x42, 0x33, 0x32, 0x48,
+ 0xf5, 0x4f, 0x14, 0x38, 0x2b, 0x45, 0xc6, 0x60, 0xe0, 0xb9, 0x01, 0x3e, 0x6d, 0x68, 0x6c, 0x01,
+ 0xb2, 0x47, 0xd4, 0x8d, 0x85, 0x7b, 0x8c, 0x53, 0x86, 0x90, 0x31, 0x4d, 0x88, 0x10, 0xe4, 0xfb,
+ 0x9e, 0x8d, 0xb9, 0x8f, 0xd0, 0x7f, 0xf4, 0x02, 0x68, 0x7d, 0xd3, 0x71, 0x43, 0xd3, 0x71, 0xb1,
+ 0xdf, 0xc1, 0x03, 0xcf, 0xda, 0xe7, 0x81, 0x61, 0x21, 0x86, 0x6f, 0x10, 0xb0, 0xfe, 0x04, 0xce,
+ 0xd5, 0xa5, 0x08, 0xd4, 0xc2, 0x41, 0x60, 0x76, 0x4f, 0x3d, 0xc7, 0xd1, 0x58, 0xa7, 0xa6, 0x63,
+ 0x9d, 0xfe, 0x5b, 0x05, 0x16, 0x0c, 0x6c, 0x7b, 0x2d, 0x1c, 0x9a, 0x53, 0x1a, 0x76, 0x52, 0xf8,
+ 0x1c, 0x15, 0x2b, 0x97, 0x11, 0x82, 0x4f, 0xa0, 0xbb, 0xef, 0xc1, 0x15, 0x32, 0x01, 0x23, 0x1a,
+ 0x60, 0xc7, 0xf7, 0xba, 0x3e, 0x0e, 0x82, 0xcf, 0x67, 0x3a, 0xfa, 0x2f, 0x15, 0x58, 0x49, 0x0a,
+ 0xf0, 0x86, 0xe7, 0x1f, 0x9a, 0xbe, 0xfd, 0x39, 0xa9, 0x33, 0x4b, 0x55, 0xb9, 0x6c, 0x55, 0xfd,
+ 0x53, 0x91, 0x83, 0x4c, 0xdd, 0x73, 0x1f, 0x3a, 0x5d, 0x54, 0x85, 0x7c, 0x30, 0x30, 0x5d, 0x2e,
+ 0xd6, 0x52, 0xf6, 0x66, 0x6d, 0x50, 0x1c, 0x92, 0x12, 0x05, 0x24, 0xd1, 0x89, 0x04, 0x11, 0x4d,
+ 0x32, 0x49, 0x5b, 0x0a, 0x72, 0x3c, 0x44, 0x1c, 0x11, 0x05, 0x13, 0xe8, 0x24, 0xce, 0x06, 0x22,
+ 0xce, 0xe6, 0x59, 0x9c, 0x15, 0xed, 0x68, 0x6d, 0x15, 0xa4, 0xb5, 0x55, 0x05, 0x2d, 0x38, 0x70,
+ 0x06, 0x8d, 0xd6, 0x66, 0x2d, 0x68, 0x73, 0x89, 0x8a, 0x74, 0x6f, 0x49, 0xc1, 0xf5, 0x0f, 0x55,
+ 0xb8, 0x48, 0x82, 0xb6, 0x3d, 0xec, 0x49, 0x31, 0x77, 0x4a, 0x29, 0xd6, 0x6d, 0x28, 0x5a, 0x54,
+ 0x8f, 0x13, 0x02, 0x29, 0x53, 0xb6, 0xc1, 0x91, 0x51, 0x1d, 0xe6, 0x03, 0x2e, 0x12, 0x0b, 0xb1,
+ 0x54, 0x61, 0xf3, 0xb7, 0x2e, 0x27, 0xc8, 0xdb, 0x09, 0x14, 0x63, 0x84, 0x84, 0x88, 0xee, 0x0d,
+ 0xb0, 0x6f, 0x86, 0x9e, 0x4f, 0xb7, 0xc7, 0x3c, 0x65, 0x91, 0x14, 0x7d, 0x5b, 0x42, 0x30, 0x12,
+ 0xe8, 0x99, 0x8e, 0x53, 0xc8, 0x76, 0x9c, 0x5f, 0xa8, 0xb0, 0xd4, 0xc2, 0x7e, 0x77, 0xfa, 0xfa,
+ 0xbb, 0x03, 0x15, 0xfb, 0x84, 0x3b, 0x74, 0x02, 0x1f, 0x35, 0x01, 0xf5, 0x89, 0x64, 0x76, 0xe3,
+ 0x44, 0xee, 0x97, 0x41, 0x14, 0x39, 0x5a, 0x7e, 0x42, 0x10, 0x1f, 0xa3, 0xa4, 0x1d, 0x38, 0xd7,
+ 0x8a, 0x40, 0xf7, 0xc4, 0xc0, 0xe8, 0xeb, 0x52, 0x42, 0xac, 0x64, 0xec, 0x2f, 0x31, 0xcd, 0x68,
+ 0x46, 0xac, 0x7f, 0xac, 0x40, 0xa5, 0xe1, 0x9b, 0x8e, 0x2b, 0x42, 0x1a, 0x7a, 0x1e, 0xe6, 0x43,
+ 0xd3, 0xef, 0xe2, 0xb0, 0xe3, 0x7a, 0x36, 0xee, 0x38, 0x36, 0xd5, 0x77, 0xd9, 0x98, 0x63, 0xd0,
+ 0x2d, 0xcf, 0xc6, 0x4d, 0x1b, 0x5d, 0x03, 0xde, 0xe6, 0x02, 0xb3, 0xb5, 0x3a, 0xcb, 0x60, 0x54,
+ 0x58, 0xf4, 0x55, 0xb8, 0xc0, 0x51, 0x62, 0x75, 0x76, 0x2c, 0x6f, 0xc8, 0x93, 0xc7, 0x8a, 0x71,
+ 0x9e, 0x75, 0xcb, 0x1e, 0x3c, 0x74, 0x43, 0xf4, 0x06, 0xac, 0x71, 0x3a, 0x92, 0x58, 0x38, 0xdd,
+ 0xfd, 0xb0, 0x63, 0x13, 0x09, 0x3b, 0x7d, 0xef, 0x31, 0xe6, 0x0c, 0xd8, 0xe1, 0x66, 0x85, 0xe1,
+ 0x35, 0x39, 0x1a, 0x9d, 0x47, 0xcb, 0x7b, 0x8c, 0x29, 0x1f, 0xfd, 0x57, 0x39, 0xd0, 0x46, 0x67,
+ 0x7e, 0x5a, 0x5f, 0xba, 0x02, 0x40, 0xfe, 0x3a, 0x44, 0x7f, 0x98, 0x4e, 0xba, 0x6c, 0x94, 0x09,
+ 0x84, 0xb0, 0xc7, 0xe8, 0x26, 0x14, 0x58, 0x4f, 0xd6, 0x52, 0xab, 0x7b, 0xfd, 0x81, 0xe7, 0x62,
+ 0x37, 0xa4, 0xb8, 0x06, 0xc3, 0x44, 0xcf, 0x41, 0x25, 0xde, 0x96, 0x3a, 0x61, 0x94, 0xd8, 0x27,
+ 0xf6, 0x2a, 0x7e, 0x1a, 0x29, 0x64, 0x38, 0x6e, 0xea, 0x34, 0x82, 0xbe, 0x00, 0xf3, 0x7b, 0x9e,
+ 0x17, 0x06, 0xa1, 0x6f, 0x0e, 0x3a, 0xb6, 0xe7, 0x62, 0x1e, 0xb6, 0x2a, 0x11, 0xb4, 0xe1, 0xb9,
+ 0x38, 0x75, 0xa0, 0x28, 0xa5, 0x0f, 0x14, 0xa8, 0x06, 0xf3, 0x4c, 0xf5, 0x03, 0xee, 0x1d, 0xcb,
+ 0x33, 0x54, 0x5f, 0xc9, 0xfc, 0x38, 0xe1, 0x3f, 0x46, 0xc5, 0x4e, 0xb8, 0x53, 0x96, 0x77, 0x97,
+ 0xb3, 0xbd, 0xfb, 0xc3, 0x1c, 0x54, 0x88, 0x7b, 0xc5, 0x8e, 0x7d, 0x1b, 0x66, 0x7a, 0xce, 0x63,
+ 0xec, 0x92, 0x91, 0x95, 0x8c, 0xd0, 0x43, 0xb0, 0x37, 0x39, 0x82, 0x11, 0xa1, 0x12, 0x2b, 0x51,
+ 0xdf, 0x95, 0x5d, 0xb3, 0x4c, 0x20, 0xcc, 0x31, 0x1b, 0x70, 0x55, 0xf2, 0x48, 0x36, 0xc1, 0x11,
+ 0x97, 0xcf, 0x51, 0xcb, 0x5e, 0x8e, 0xd1, 0xe8, 0x1c, 0x77, 0xe5, 0x15, 0x50, 0x83, 0x2b, 0xe3,
+ 0xb8, 0xc8, 0xc9, 0xc4, 0xa5, 0x4c, 0x1e, 0x4c, 0x90, 0xdb, 0x70, 0xe1, 0xd0, 0x77, 0x42, 0xdc,
+ 0xe9, 0x61, 0x33, 0xc0, 0x1d, 0x9f, 0xc5, 0xbb, 0x0e, 0x39, 0xf9, 0xb1, 0x00, 0xb0, 0x48, 0xbb,
+ 0x37, 0x49, 0x2f, 0x0f, 0x86, 0x6d, 0xfc, 0x08, 0xdd, 0x81, 0x15, 0x99, 0x8c, 0x16, 0x19, 0x2c,
+ 0xaf, 0xd7, 0x79, 0x8c, 0xfd, 0x80, 0xc4, 0xf9, 0x22, 0x5d, 0x1c, 0x17, 0x63, 0xda, 0x1d, 0x8e,
+ 0xf1, 0x36, 0x43, 0x40, 0xef, 0x24, 0xc7, 0x3d, 0x74, 0x42, 0xa2, 0xb6, 0x0e, 0xc9, 0xbb, 0x4b,
+ 0xd4, 0xbe, 0xd7, 0x92, 0x07, 0xf1, 0x88, 0xd1, 0x03, 0x86, 0x59, 0xb3, 0x0e, 0x64, 0xd1, 0x62,
+ 0xa8, 0xfe, 0x0f, 0x05, 0x2e, 0xa5, 0xd0, 0xeb, 0xfb, 0x66, 0xaf, 0x87, 0xdd, 0x2e, 0x46, 0x37,
+ 0xe0, 0x9c, 0xe5, 0x79, 0xbe, 0xed, 0xb8, 0x64, 0x8b, 0x88, 0x04, 0x66, 0xb5, 0x0f, 0x24, 0x75,
+ 0x09, 0x49, 0x5f, 0x81, 0x25, 0x99, 0x20, 0x65, 0xd5, 0x45, 0xa9, 0x77, 0x2b, 0x32, 0xf0, 0x3a,
+ 0x68, 0x01, 0xee, 0x3d, 0x4c, 0x28, 0x94, 0xe5, 0x2b, 0xf3, 0x04, 0x2e, 0xa9, 0xf2, 0x25, 0x40,
+ 0x62, 0xf6, 0x12, 0x6f, 0x66, 0x39, 0x8d, 0xf7, 0xc4, 0x7c, 0x17, 0xa1, 0xe0, 0x7a, 0xae, 0xc5,
+ 0xf2, 0x84, 0x39, 0x83, 0x35, 0xf4, 0xbf, 0x29, 0xb0, 0x98, 0xa5, 0xa2, 0xff, 0xcf, 0xd9, 0xfe,
+ 0x54, 0x85, 0xf3, 0x89, 0x45, 0x1a, 0x1d, 0x97, 0x4e, 0x3c, 0xdd, 0x2a, 0x9c, 0x95, 0x97, 0x9d,
+ 0x3c, 0xd3, 0x85, 0x78, 0xb3, 0x61, 0xc2, 0x5c, 0x85, 0xd9, 0xf4, 0xfc, 0xc0, 0x8f, 0xe7, 0x56,
+ 0x85, 0xb3, 0xcc, 0x9b, 0xed, 0xa1, 0x6f, 0x92, 0xdc, 0xa5, 0xd3, 0x17, 0xb1, 0x74, 0x81, 0x76,
+ 0x34, 0x38, 0xbc, 0x15, 0xa0, 0x5d, 0x38, 0x2b, 0xf4, 0x60, 0x09, 0xdf, 0xa4, 0xb3, 0x9c, 0xbd,
+ 0xf5, 0xa5, 0xa3, 0x3d, 0x3f, 0x72, 0xe5, 0x48, 0x5f, 0x11, 0x44, 0x7f, 0x0f, 0x96, 0xda, 0x4c,
+ 0xe4, 0x28, 0x24, 0xf1, 0x04, 0xe6, 0x26, 0x14, 0xd9, 0x7c, 0x26, 0x07, 0x31, 0x8e, 0x38, 0x21,
+ 0x84, 0xe9, 0x7d, 0xb8, 0x90, 0x1a, 0x8b, 0x9b, 0xe1, 0x65, 0x28, 0x99, 0x83, 0x41, 0xcf, 0xc1,
+ 0xf6, 0xe4, 0xd1, 0x04, 0xe6, 0xa4, 0xe1, 0xde, 0x83, 0xab, 0x6d, 0x79, 0xa3, 0x96, 0x22, 0x99,
+ 0x98, 0xe3, 0xb4, 0xd2, 0x06, 0xfd, 0x09, 0x5c, 0xae, 0xc7, 0xbe, 0x72, 0x57, 0xec, 0x59, 0x62,
+ 0x9c, 0x65, 0x28, 0x25, 0x3d, 0x4b, 0x34, 0x27, 0x86, 0x45, 0x75, 0x42, 0x58, 0xd4, 0xff, 0xac,
+ 0xc2, 0x4a, 0xf6, 0xd0, 0x5c, 0xb5, 0xff, 0x7d, 0x9e, 0x45, 0x96, 0x76, 0xb4, 0x93, 0x66, 0x89,
+ 0xb5, 0x28, 0x76, 0xcd, 0x44, 0xa0, 0xfe, 0x9f, 0xd9, 0xa9, 0x26, 0xe9, 0xb6, 0x30, 0x49, 0xb7,
+ 0xff, 0x52, 0x60, 0xb1, 0x66, 0xdb, 0xb1, 0x86, 0x84, 0x3d, 0x5f, 0x00, 0x95, 0xfb, 0xca, 0x91,
+ 0x69, 0x98, 0xea, 0xd8, 0x68, 0x29, 0x71, 0x10, 0x9a, 0x8b, 0x4e, 0x3a, 0xa9, 0x14, 0x2a, 0xeb,
+ 0xb8, 0x5f, 0x85, 0xb3, 0x4e, 0xd0, 0x71, 0xf1, 0x61, 0x27, 0x4e, 0xe8, 0xe8, 0xc4, 0x67, 0x8c,
+ 0x05, 0x27, 0xd8, 0xc2, 0x87, 0xf1, 0x70, 0x24, 0xd8, 0x1c, 0xf0, 0xb2, 0x39, 0x51, 0x31, 0x9b,
+ 0x1c, 0x08, 0x50, 0xd3, 0xce, 0x4c, 0x6a, 0x8a, 0xd9, 0x49, 0xcd, 0xef, 0x14, 0xb8, 0x60, 0x60,
+ 0x92, 0xba, 0x9e, 0x6a, 0xee, 0xcb, 0x50, 0xb2, 0xcc, 0xc0, 0x32, 0x6d, 0xcc, 0x0b, 0x9c, 0xa2,
+ 0x49, 0x7a, 0x7c, 0xca, 0xdf, 0xe6, 0x35, 0x59, 0xd1, 0x1c, 0x9d, 0x46, 0xfe, 0x58, 0xd3, 0x18,
+ 0x73, 0xf2, 0xf8, 0x43, 0x0e, 0x2e, 0xc5, 0x13, 0x48, 0xad, 0xca, 0x53, 0xa6, 0xd5, 0xe3, 0x2c,
+ 0x7b, 0x91, 0x2e, 0x38, 0x5f, 0x32, 0x6a, 0x54, 0x0d, 0xb0, 0xe0, 0x5a, 0x68, 0xee, 0xf5, 0x70,
+ 0x27, 0xf4, 0x9d, 0x6e, 0x97, 0x88, 0xff, 0x18, 0xbb, 0x89, 0xa3, 0x86, 0x73, 0x8c, 0x42, 0xe9,
+ 0x15, 0xca, 0x63, 0x97, 0xb1, 0xd8, 0x20, 0x1c, 0xe4, 0x92, 0x69, 0xb6, 0xd3, 0x14, 0xb2, 0x9d,
+ 0xc6, 0x24, 0xc7, 0x16, 0x59, 0x20, 0x1f, 0xdb, 0xde, 0x88, 0x3c, 0xc5, 0x49, 0xf2, 0xac, 0xc8,
+ 0xf2, 0x18, 0xd8, 0xf6, 0x12, 0xe2, 0x8c, 0x18, 0xb4, 0x74, 0x2c, 0x83, 0xce, 0x64, 0x1b, 0xf4,
+ 0x8f, 0x39, 0xb8, 0x9c, 0x69, 0xd0, 0xe9, 0x14, 0x3f, 0x6f, 0x43, 0x21, 0x18, 0x98, 0xae, 0x38,
+ 0x6c, 0x27, 0xab, 0xb2, 0xd1, 0x68, 0x71, 0xf1, 0x87, 0x61, 0x8b, 0x83, 0x4e, 0xee, 0x38, 0xd7,
+ 0x2e, 0xc7, 0x3b, 0x3a, 0xbd, 0x04, 0x88, 0x1a, 0x22, 0x89, 0xc9, 0xbc, 0x5c, 0x23, 0x3d, 0x72,
+ 0x55, 0x14, 0x35, 0xa0, 0x2c, 0x0a, 0x18, 0xc1, 0x72, 0x91, 0x8a, 0xfe, 0xc5, 0xcc, 0x7a, 0x49,
+ 0xaa, 0x4a, 0x61, 0xc4, 0x84, 0x99, 0x66, 0x28, 0x65, 0x9a, 0x01, 0x6d, 0xc2, 0x02, 0x2d, 0x13,
+ 0x74, 0xe2, 0x61, 0x67, 0xe8, 0xb0, 0xcf, 0x25, 0x77, 0x96, 0xcc, 0xca, 0x88, 0x31, 0x4f, 0x69,
+ 0x45, 0x01, 0x26, 0xd0, 0xff, 0xa2, 0xc2, 0x6a, 0x6c, 0xd4, 0x1d, 0x2f, 0x08, 0xa7, 0xbd, 0x52,
+ 0x8f, 0xb5, 0xec, 0xd4, 0x53, 0x2e, 0xbb, 0x9b, 0x50, 0x62, 0xa5, 0x39, 0xb2, 0xea, 0x89, 0x32,
+ 0x2e, 0xa4, 0x6c, 0xd0, 0x37, 0x9b, 0xee, 0x43, 0xcf, 0x10, 0x78, 0xe8, 0x55, 0x98, 0xa3, 0x66,
+ 0x16, 0x74, 0xf9, 0xa3, 0xe9, 0x66, 0x09, 0x72, 0x9b, 0xd3, 0x9e, 0x20, 0x0c, 0x7e, 0xa0, 0xc2,
+ 0xd5, 0xb1, 0x0a, 0x9e, 0xce, 0xca, 0xf9, 0x5c, 0x34, 0x7c, 0xa2, 0x75, 0x76, 0xa2, 0x5b, 0x06,
+ 0x88, 0xb5, 0x9c, 0xb8, 0xda, 0x52, 0x46, 0xae, 0xb6, 0x56, 0x05, 0xe6, 0x96, 0xd9, 0x17, 0x95,
+ 0x14, 0x09, 0x82, 0xae, 0x43, 0x91, 0x46, 0x07, 0xe1, 0x02, 0x19, 0x55, 0x63, 0x6a, 0x49, 0x8e,
+ 0xa5, 0xd7, 0xf9, 0xbd, 0x3a, 0x1d, 0x78, 0xfc, 0xbd, 0xfa, 0x0a, 0x47, 0x93, 0x46, 0x8d, 0x01,
+ 0xfa, 0x6f, 0x54, 0x40, 0xe9, 0xe0, 0x44, 0xf6, 0xe9, 0x31, 0x76, 0x4c, 0xe8, 0x5c, 0xe5, 0xf7,
+ 0xf6, 0x62, 0xca, 0xea, 0xc8, 0x94, 0x45, 0x19, 0x3c, 0x77, 0x8c, 0x32, 0xf8, 0x1b, 0xa0, 0x59,
+ 0xa2, 0x5e, 0xd4, 0x09, 0xe2, 0x0b, 0xae, 0x09, 0x45, 0xa5, 0x05, 0x4b, 0x6e, 0x0f, 0x83, 0x74,
+ 0x8c, 0x2c, 0x64, 0xc4, 0xc8, 0x97, 0x61, 0x76, 0xaf, 0xe7, 0x59, 0x07, 0xbc, 0xac, 0xc5, 0x76,
+ 0x29, 0x94, 0x5c, 0x3b, 0x94, 0x3d, 0xec, 0x89, 0x4b, 0x33, 0x1c, 0x95, 0x32, 0x4b, 0x71, 0x29,
+ 0x53, 0xff, 0xb9, 0x02, 0x4b, 0xf1, 0xf2, 0xa8, 0xf7, 0xbc, 0xa8, 0x6e, 0x71, 0xda, 0x55, 0x21,
+ 0x65, 0x39, 0x6a, 0x32, 0xcb, 0x39, 0xc1, 0xe5, 0xc4, 0x07, 0x0a, 0x5c, 0x48, 0x89, 0x37, 0x9d,
+ 0x55, 0xbb, 0x0c, 0xa5, 0x60, 0x68, 0x59, 0x38, 0x08, 0x84, 0x7c, 0xbc, 0x79, 0x12, 0xf9, 0x7e,
+ 0xa4, 0x80, 0x16, 0xdf, 0xb1, 0x32, 0xc7, 0x9e, 0xc2, 0x15, 0xf5, 0x25, 0x98, 0xe1, 0xee, 0xcf,
+ 0xb6, 0xe3, 0x9c, 0x11, 0xb5, 0x8f, 0xba, 0x7d, 0xd6, 0xbf, 0x0d, 0x05, 0x8a, 0x37, 0xe1, 0x99,
+ 0xca, 0x38, 0x77, 0x5f, 0x81, 0x72, 0x7b, 0xd0, 0x73, 0x68, 0x20, 0xe2, 0xa9, 0x69, 0x0c, 0xd0,
+ 0xdf, 0x57, 0xe1, 0x9c, 0xe1, 0x0d, 0x43, 0x4c, 0x59, 0xd5, 0xec, 0xbe, 0x13, 0xf0, 0xa2, 0x80,
+ 0xd6, 0xf6, 0x86, 0xbe, 0x85, 0xa5, 0xe8, 0xc0, 0x4e, 0x92, 0x29, 0x38, 0x5a, 0x87, 0x05, 0x06,
+ 0x1b, 0x5d, 0xd2, 0xa3, 0x60, 0xc2, 0x95, 0x1d, 0x67, 0x24, 0xae, 0xec, 0xe4, 0x94, 0x82, 0x13,
+ 0xae, 0x0c, 0x16, 0x73, 0xcd, 0x33, 0xae, 0x23, 0x60, 0xf4, 0x3a, 0x14, 0xf9, 0xd5, 0x4a, 0x81,
+ 0xda, 0x24, 0x99, 0x2a, 0x64, 0xcc, 0x4e, 0xdc, 0x75, 0xb3, 0xaf, 0xee, 0xc2, 0xbc, 0xd0, 0x16,
+ 0x73, 0xad, 0x23, 0x34, 0xbd, 0x06, 0xb3, 0xdb, 0x3d, 0x7b, 0x44, 0xd9, 0x32, 0x88, 0x60, 0x6c,
+ 0xe1, 0xc3, 0x11, 0x6b, 0xca, 0x20, 0xfd, 0xdf, 0x39, 0x28, 0xb0, 0xc5, 0xbb, 0x02, 0xe5, 0x66,
+ 0x40, 0x6f, 0xc0, 0x79, 0x99, 0x60, 0xc6, 0x88, 0x01, 0x44, 0x0a, 0xfa, 0x1b, 0xdf, 0xc1, 0xf1,
+ 0x26, 0xba, 0x03, 0xb3, 0xec, 0x57, 0x84, 0xe6, 0xf4, 0x85, 0xd4, 0xa8, 0x03, 0x1b, 0x32, 0x05,
+ 0xba, 0x0f, 0x67, 0xb7, 0x30, 0xb6, 0x1b, 0xbe, 0x37, 0x18, 0x08, 0x0c, 0x9e, 0xa6, 0x4f, 0x60,
+ 0x93, 0xa6, 0x43, 0xaf, 0xc1, 0x02, 0x01, 0xd6, 0x6c, 0x3b, 0x62, 0xc5, 0x4a, 0xe4, 0x28, 0x1d,
+ 0x5b, 0x8d, 0x51, 0x54, 0x54, 0x87, 0xf9, 0xb7, 0x06, 0xb6, 0x19, 0x62, 0xae, 0x42, 0x91, 0xf0,
+ 0x5d, 0xce, 0x4a, 0x1a, 0xb8, 0x81, 0x8c, 0x11, 0x92, 0xd1, 0xc7, 0x27, 0xa5, 0xd4, 0xe3, 0x13,
+ 0xf4, 0x65, 0x7a, 0x27, 0xd0, 0xc5, 0x34, 0x11, 0x9f, 0x1f, 0x49, 0x49, 0xc4, 0x23, 0x84, 0x2e,
+ 0xbb, 0x0f, 0xe8, 0x62, 0xb4, 0x0b, 0x8b, 0x19, 0x8e, 0x13, 0x2c, 0x97, 0xa9, 0x6c, 0x6b, 0x93,
+ 0x3c, 0xcc, 0xc8, 0xa4, 0xd6, 0xbf, 0x0f, 0x8b, 0xd1, 0x0e, 0x23, 0xbf, 0xab, 0x39, 0xc1, 0xce,
+ 0xb6, 0x2e, 0xee, 0x36, 0xd4, 0xb1, 0xdb, 0x03, 0xbf, 0xd2, 0xc8, 0x78, 0xa9, 0xa0, 0xff, 0x5d,
+ 0x21, 0xab, 0x2a, 0xf1, 0x2e, 0xeb, 0x24, 0x83, 0x67, 0x6d, 0x87, 0xea, 0x34, 0xb6, 0xc3, 0xac,
+ 0x52, 0xc1, 0x4d, 0x38, 0xcf, 0x72, 0xae, 0xc0, 0x79, 0x86, 0x3b, 0x03, 0xec, 0x77, 0x02, 0x6c,
+ 0x79, 0x2e, 0x3b, 0x4e, 0xaa, 0x06, 0xa2, 0x9d, 0x6d, 0xe7, 0x19, 0xde, 0xc1, 0x7e, 0x9b, 0xf6,
+ 0x64, 0x5d, 0x20, 0xeb, 0xbf, 0x56, 0x00, 0xc9, 0x0f, 0x4f, 0xa6, 0xb3, 0x11, 0xbe, 0x09, 0x95,
+ 0xbd, 0x98, 0x69, 0xf4, 0xa0, 0xe4, 0x5a, 0x76, 0x36, 0x21, 0x8f, 0x9f, 0xa4, 0xcb, 0xb4, 0x92,
+ 0x0d, 0x73, 0x72, 0xfa, 0x47, 0x70, 0x42, 0x27, 0x0a, 0xc0, 0xf4, 0x9f, 0xc0, 0x5c, 0xcf, 0x16,
+ 0x91, 0x96, 0xfe, 0x13, 0x98, 0x25, 0x78, 0x95, 0x0d, 0xfa, 0x4f, 0x82, 0x48, 0x9f, 0x3d, 0x4f,
+ 0xe0, 0xe1, 0x53, 0x34, 0xf5, 0x57, 0x60, 0x6e, 0xf4, 0x52, 0x74, 0xdf, 0xe9, 0xee, 0xf3, 0x87,
+ 0x5d, 0xf4, 0x1f, 0x69, 0x90, 0xeb, 0x79, 0x87, 0x3c, 0xfc, 0x90, 0x5f, 0x22, 0x9b, 0xac, 0x96,
+ 0xe3, 0x51, 0x51, 0x69, 0xe3, 0x60, 0x4f, 0xff, 0xc9, 0xa6, 0x25, 0xce, 0xcc, 0x5c, 0xb4, 0xa8,
+ 0xad, 0x7f, 0x07, 0xae, 0x6e, 0x7a, 0x5d, 0xa9, 0x0a, 0x18, 0xbf, 0xb9, 0x98, 0x8e, 0x01, 0xf5,
+ 0xf7, 0x15, 0x58, 0x1b, 0x3f, 0xc4, 0x74, 0xb2, 0x91, 0x49, 0x0f, 0x4a, 0x7a, 0x44, 0x97, 0xd8,
+ 0x3a, 0x08, 0x86, 0xfd, 0x16, 0x0e, 0x4d, 0xf4, 0x15, 0xb1, 0xb6, 0xb3, 0x72, 0x0b, 0x81, 0x99,
+ 0x58, 0xe3, 0x55, 0xd0, 0x2c, 0x19, 0xde, 0xc6, 0x8f, 0xf8, 0x38, 0x29, 0xb8, 0xfe, 0x13, 0x05,
+ 0xce, 0x4b, 0x4f, 0x9c, 0x70, 0x28, 0x38, 0xa2, 0x45, 0x28, 0xb0, 0xfb, 0x5c, 0x66, 0x44, 0xd6,
+ 0x20, 0x9e, 0xf3, 0xc4, 0xf3, 0xef, 0x11, 0xe3, 0xf2, 0xed, 0x87, 0x37, 0xd1, 0x12, 0x14, 0x9f,
+ 0x78, 0xfe, 0xa6, 0x77, 0xc8, 0xd7, 0x2d, 0x6f, 0xb1, 0xec, 0xab, 0x4f, 0x29, 0xf2, 0xbc, 0x4c,
+ 0xc4, 0x9a, 0x84, 0x22, 0x18, 0xf6, 0x09, 0x05, 0x4b, 0x7c, 0x79, 0x8b, 0xa4, 0x82, 0x6b, 0x99,
+ 0x32, 0xd5, 0xac, 0x83, 0x69, 0x59, 0x61, 0x11, 0x0a, 0x72, 0x95, 0x9b, 0x35, 0x32, 0xdf, 0x71,
+ 0xf1, 0xe7, 0x9e, 0xf9, 0xe8, 0xb9, 0xa7, 0xfe, 0x57, 0x05, 0xf4, 0x4c, 0xf9, 0xd8, 0xfe, 0x33,
+ 0xa5, 0x60, 0x72, 0x0a, 0x09, 0xd1, 0xeb, 0x30, 0x23, 0x2c, 0xcd, 0xef, 0x4e, 0xf4, 0x71, 0x8f,
+ 0xda, 0x62, 0xe9, 0x8d, 0x88, 0xa6, 0x7a, 0x45, 0x24, 0x4f, 0xa8, 0x0c, 0x05, 0x7a, 0xd1, 0xa2,
+ 0x9d, 0x41, 0x33, 0x90, 0xdf, 0x31, 0x83, 0x40, 0x53, 0xaa, 0xeb, 0x2c, 0x37, 0x92, 0xde, 0xa2,
+ 0x00, 0x14, 0xeb, 0x3e, 0x36, 0x29, 0x1e, 0x40, 0x91, 0x15, 0x55, 0x35, 0xa5, 0xda, 0x82, 0x39,
+ 0xf9, 0x09, 0x0a, 0x61, 0xb7, 0xdd, 0xa9, 0xd9, 0xb6, 0x76, 0x06, 0xcd, 0xc1, 0xcc, 0x76, 0x47,
+ 0x20, 0x12, 0xa2, 0xed, 0x4e, 0x8b, 0xfc, 0xab, 0x68, 0x16, 0x4a, 0xdb, 0x1d, 0x9a, 0x8d, 0x6a,
+ 0x39, 0xd6, 0xa0, 0x25, 0x16, 0x2d, 0x5f, 0xbd, 0x0d, 0x73, 0xf2, 0x1d, 0x09, 0x61, 0x57, 0xdb,
+ 0x6c, 0xbe, 0xbd, 0xc1, 0xd8, 0x35, 0x8c, 0x5a, 0x73, 0xab, 0xb9, 0xf5, 0xa6, 0xa6, 0x90, 0x56,
+ 0x7b, 0x77, 0x7b, 0x67, 0x87, 0xb4, 0xd4, 0xea, 0xab, 0x00, 0xf1, 0x66, 0x4e, 0xe6, 0xb1, 0xb5,
+ 0xbd, 0x45, 0x68, 0x66, 0xa1, 0xf4, 0xa0, 0xd6, 0xdc, 0x65, 0x24, 0xa4, 0x61, 0xb0, 0x86, 0x4a,
+ 0x70, 0x1a, 0x04, 0x27, 0x57, 0x7d, 0x69, 0x24, 0xc5, 0x47, 0x25, 0xc8, 0xd5, 0x7a, 0x3d, 0xed,
+ 0x0c, 0x2a, 0x82, 0xda, 0xb8, 0xcb, 0x44, 0xdf, 0xf2, 0xfc, 0xbe, 0xd9, 0xd3, 0xd4, 0xea, 0x9b,
+ 0x70, 0x71, 0x6c, 0x6a, 0x49, 0xa5, 0x6d, 0xb4, 0x9a, 0xbb, 0x6c, 0x64, 0x63, 0x63, 0x73, 0xa3,
+ 0xd6, 0xde, 0xd0, 0x14, 0x84, 0x60, 0x9e, 0x37, 0x3a, 0xed, 0xfa, 0xbd, 0x8d, 0x56, 0x4d, 0x53,
+ 0xab, 0xcf, 0x60, 0x3e, 0xb9, 0x5f, 0x52, 0xf9, 0x3c, 0xff, 0xc0, 0x71, 0xbb, 0x8c, 0xbe, 0x1d,
+ 0xd2, 0x74, 0x8b, 0x49, 0xce, 0xf4, 0x68, 0x6b, 0x2a, 0xd2, 0x60, 0xae, 0xe9, 0x3a, 0xa1, 0x63,
+ 0xf6, 0x9c, 0x67, 0x04, 0x37, 0x87, 0x2a, 0x50, 0xde, 0xf1, 0xf1, 0xc0, 0xf4, 0x49, 0x33, 0x8f,
+ 0xe6, 0x01, 0xa8, 0x3a, 0x0d, 0x6c, 0xda, 0x4f, 0xb5, 0x02, 0x21, 0x78, 0x60, 0x3a, 0xa1, 0xe3,
+ 0x76, 0x99, 0x96, 0x8b, 0xd5, 0x6f, 0x40, 0x25, 0x11, 0x57, 0xd0, 0x59, 0xa8, 0xbc, 0xb5, 0xd5,
+ 0xdc, 0x6a, 0xee, 0x36, 0x6b, 0x9b, 0xcd, 0x77, 0x37, 0x1a, 0x4c, 0xdd, 0xad, 0x66, 0xbb, 0x55,
+ 0xdb, 0xad, 0xdf, 0xd3, 0x14, 0x32, 0x33, 0xf6, 0xab, 0xde, 0x7d, 0xfd, 0xf7, 0x9f, 0xae, 0x2a,
+ 0x1f, 0x7d, 0xba, 0xaa, 0x7c, 0xf2, 0xe9, 0xaa, 0xf2, 0xe3, 0xcf, 0x56, 0xcf, 0x7c, 0xf4, 0xd9,
+ 0xea, 0x99, 0x8f, 0x3f, 0x5b, 0x3d, 0xf3, 0xee, 0xf3, 0x5d, 0x27, 0xdc, 0x1f, 0xee, 0x5d, 0xb7,
+ 0xbc, 0xfe, 0x8d, 0x81, 0xe3, 0x76, 0x2d, 0x73, 0x70, 0x23, 0x74, 0x2c, 0xdb, 0xba, 0x21, 0xb9,
+ 0xe6, 0x5e, 0x91, 0xde, 0x5f, 0xbc, 0xfc, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x40, 0x67, 0x9a,
+ 0x8d, 0xb6, 0x2f, 0x00, 0x00,
}
func (m *TableSpan) Marshal() (dAtA []byte, err error) {
@@ -5271,6 +5561,28 @@ func (m *NodeHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
+ if m.WriteLeaseWitnessAck != nil {
+ {
+ size, err := m.WriteLeaseWitnessAck.MarshalToSizedBuffer(dAtA[:i])
+ if err != nil {
+ return 0, err
+ }
+ i -= size
+ i = encodeVarintHeartbeat(dAtA, i, uint64(size))
+ }
+ i--
+ dAtA[i] = 0x3a
+ }
+ if m.WriteLeaseProtocolVersion != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.WriteLeaseProtocolVersion))
+ i--
+ dAtA[i] = 0x30
+ }
+ if m.WriteLeaseRequestSeq != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.WriteLeaseRequestSeq))
+ i--
+ dAtA[i] = 0x28
+ }
if m.DispatcherDrainTargetEpoch != 0 {
i = encodeVarintHeartbeat(dAtA, i, uint64(m.DispatcherDrainTargetEpoch))
i--
@@ -5296,7 +5608,7 @@ func (m *NodeHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
-func (m *SetNodeLivenessRequest) Marshal() (dAtA []byte, err error) {
+func (m *WriteLeaseWitnessChallenge) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
@@ -5306,25 +5618,180 @@ func (m *SetNodeLivenessRequest) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
-func (m *SetNodeLivenessRequest) MarshalTo(dAtA []byte) (int, error) {
+func (m *WriteLeaseWitnessChallenge) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
-func (m *SetNodeLivenessRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+func (m *WriteLeaseWitnessChallenge) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
- if m.NodeEpoch != 0 {
- i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch))
+ if len(m.Nonce) > 0 {
+ i -= len(m.Nonce)
+ copy(dAtA[i:], m.Nonce)
+ i = encodeVarintHeartbeat(dAtA, i, uint64(len(m.Nonce)))
i--
- dAtA[i] = 0x10
+ dAtA[i] = 0x2a
}
- if m.Target != 0 {
- i = encodeVarintHeartbeat(dAtA, i, uint64(m.Target))
+ if m.WitnessNodeEpoch != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.WitnessNodeEpoch))
i--
- dAtA[i] = 0x8
+ dAtA[i] = 0x20
+ }
+ if m.SelfRequestSeq != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.SelfRequestSeq))
+ i--
+ dAtA[i] = 0x18
+ }
+ if m.CoordinatorNodeEpoch != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.CoordinatorNodeEpoch))
+ i--
+ dAtA[i] = 0x10
+ }
+ if m.CoordinatorVersion != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.CoordinatorVersion))
+ i--
+ dAtA[i] = 0x8
+ }
+ return len(dAtA) - i, nil
+}
+
+func (m *WriteLeaseWitnessAck) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBuffer(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *WriteLeaseWitnessAck) MarshalTo(dAtA []byte) (int, error) {
+ size := m.Size()
+ return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *WriteLeaseWitnessAck) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if len(m.Nonce) > 0 {
+ i -= len(m.Nonce)
+ copy(dAtA[i:], m.Nonce)
+ i = encodeVarintHeartbeat(dAtA, i, uint64(len(m.Nonce)))
+ i--
+ dAtA[i] = 0x2a
+ }
+ if m.WitnessNodeEpoch != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.WitnessNodeEpoch))
+ i--
+ dAtA[i] = 0x20
+ }
+ if m.SelfRequestSeq != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.SelfRequestSeq))
+ i--
+ dAtA[i] = 0x18
+ }
+ if m.CoordinatorNodeEpoch != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.CoordinatorNodeEpoch))
+ i--
+ dAtA[i] = 0x10
+ }
+ if m.CoordinatorVersion != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.CoordinatorVersion))
+ i--
+ dAtA[i] = 0x8
+ }
+ return len(dAtA) - i, nil
+}
+
+func (m *NodeHeartbeatResponse) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBuffer(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *NodeHeartbeatResponse) MarshalTo(dAtA []byte) (int, error) {
+ size := m.Size()
+ return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *NodeHeartbeatResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if m.WitnessChallenge != nil {
+ {
+ size, err := m.WitnessChallenge.MarshalToSizedBuffer(dAtA[:i])
+ if err != nil {
+ return 0, err
+ }
+ i -= size
+ i = encodeVarintHeartbeat(dAtA, i, uint64(size))
+ }
+ i--
+ dAtA[i] = 0x2a
+ }
+ if m.LeaseDurationMs != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.LeaseDurationMs))
+ i--
+ dAtA[i] = 0x20
+ }
+ if m.RequestSeq != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.RequestSeq))
+ i--
+ dAtA[i] = 0x18
+ }
+ if m.TargetNodeEpoch != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.TargetNodeEpoch))
+ i--
+ dAtA[i] = 0x10
+ }
+ if m.CoordinatorVersion != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.CoordinatorVersion))
+ i--
+ dAtA[i] = 0x8
+ }
+ return len(dAtA) - i, nil
+}
+
+func (m *SetNodeLivenessRequest) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBuffer(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *SetNodeLivenessRequest) MarshalTo(dAtA []byte) (int, error) {
+ size := m.Size()
+ return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *SetNodeLivenessRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if m.NodeEpoch != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.NodeEpoch))
+ i--
+ dAtA[i] = 0x10
+ }
+ if m.Target != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.Target))
+ i--
+ dAtA[i] = 0x8
}
return len(dAtA) - i, nil
}
@@ -5417,6 +5884,11 @@ func (m *CoordinatorBootstrapRequest) MarshalToSizedBuffer(dAtA []byte) (int, er
_ = i
var l int
_ = l
+ if m.WriteLeaseProtocolVersion != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.WriteLeaseProtocolVersion))
+ i--
+ dAtA[i] = 0x10
+ }
if m.Version != 0 {
i = encodeVarintHeartbeat(dAtA, i, uint64(m.Version))
i--
@@ -5445,6 +5917,11 @@ func (m *CoordinatorBootstrapResponse) MarshalToSizedBuffer(dAtA []byte) (int, e
_ = i
var l int
_ = l
+ if m.WriteLeaseProtocolVersion != 0 {
+ i = encodeVarintHeartbeat(dAtA, i, uint64(m.WriteLeaseProtocolVersion))
+ i--
+ dAtA[i] = 0x28
+ }
if m.DispatcherDrainTargetEpoch != 0 {
i = encodeVarintHeartbeat(dAtA, i, uint64(m.DispatcherDrainTargetEpoch))
i--
@@ -6239,21 +6716,21 @@ func (m *InfluencedTables) MarshalToSizedBuffer(dAtA []byte) (int, error) {
dAtA[i] = 0x18
}
if len(m.TableIDs) > 0 {
- dAtA41 := make([]byte, len(m.TableIDs)*10)
- var j40 int
+ dAtA43 := make([]byte, len(m.TableIDs)*10)
+ var j42 int
for _, num1 := range m.TableIDs {
num := uint64(num1)
for num >= 1<<7 {
- dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80)
+ dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
- j40++
+ j42++
}
- dAtA41[j40] = uint8(num)
- j40++
+ dAtA43[j42] = uint8(num)
+ j42++
}
- i -= j40
- copy(dAtA[i:], dAtA41[:j40])
- i = encodeVarintHeartbeat(dAtA, i, uint64(j40))
+ i -= j42
+ copy(dAtA[i:], dAtA43[:j42])
+ i = encodeVarintHeartbeat(dAtA, i, uint64(j42))
i--
dAtA[i] = 0x12
}
@@ -7549,6 +8026,91 @@ func (m *NodeHeartbeat) Size() (n int) {
if m.DispatcherDrainTargetEpoch != 0 {
n += 1 + sovHeartbeat(uint64(m.DispatcherDrainTargetEpoch))
}
+ if m.WriteLeaseRequestSeq != 0 {
+ n += 1 + sovHeartbeat(uint64(m.WriteLeaseRequestSeq))
+ }
+ if m.WriteLeaseProtocolVersion != 0 {
+ n += 1 + sovHeartbeat(uint64(m.WriteLeaseProtocolVersion))
+ }
+ if m.WriteLeaseWitnessAck != nil {
+ l = m.WriteLeaseWitnessAck.Size()
+ n += 1 + l + sovHeartbeat(uint64(l))
+ }
+ return n
+}
+
+func (m *WriteLeaseWitnessChallenge) Size() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if m.CoordinatorVersion != 0 {
+ n += 1 + sovHeartbeat(uint64(m.CoordinatorVersion))
+ }
+ if m.CoordinatorNodeEpoch != 0 {
+ n += 1 + sovHeartbeat(uint64(m.CoordinatorNodeEpoch))
+ }
+ if m.SelfRequestSeq != 0 {
+ n += 1 + sovHeartbeat(uint64(m.SelfRequestSeq))
+ }
+ if m.WitnessNodeEpoch != 0 {
+ n += 1 + sovHeartbeat(uint64(m.WitnessNodeEpoch))
+ }
+ l = len(m.Nonce)
+ if l > 0 {
+ n += 1 + l + sovHeartbeat(uint64(l))
+ }
+ return n
+}
+
+func (m *WriteLeaseWitnessAck) Size() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if m.CoordinatorVersion != 0 {
+ n += 1 + sovHeartbeat(uint64(m.CoordinatorVersion))
+ }
+ if m.CoordinatorNodeEpoch != 0 {
+ n += 1 + sovHeartbeat(uint64(m.CoordinatorNodeEpoch))
+ }
+ if m.SelfRequestSeq != 0 {
+ n += 1 + sovHeartbeat(uint64(m.SelfRequestSeq))
+ }
+ if m.WitnessNodeEpoch != 0 {
+ n += 1 + sovHeartbeat(uint64(m.WitnessNodeEpoch))
+ }
+ l = len(m.Nonce)
+ if l > 0 {
+ n += 1 + l + sovHeartbeat(uint64(l))
+ }
+ return n
+}
+
+func (m *NodeHeartbeatResponse) Size() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if m.CoordinatorVersion != 0 {
+ n += 1 + sovHeartbeat(uint64(m.CoordinatorVersion))
+ }
+ if m.TargetNodeEpoch != 0 {
+ n += 1 + sovHeartbeat(uint64(m.TargetNodeEpoch))
+ }
+ if m.RequestSeq != 0 {
+ n += 1 + sovHeartbeat(uint64(m.RequestSeq))
+ }
+ if m.LeaseDurationMs != 0 {
+ n += 1 + sovHeartbeat(uint64(m.LeaseDurationMs))
+ }
+ if m.WitnessChallenge != nil {
+ l = m.WitnessChallenge.Size()
+ n += 1 + l + sovHeartbeat(uint64(l))
+ }
return n
}
@@ -7607,6 +8169,9 @@ func (m *CoordinatorBootstrapRequest) Size() (n int) {
if m.Version != 0 {
n += 1 + sovHeartbeat(uint64(m.Version))
}
+ if m.WriteLeaseProtocolVersion != 0 {
+ n += 1 + sovHeartbeat(uint64(m.WriteLeaseProtocolVersion))
+ }
return n
}
@@ -7632,6 +8197,9 @@ func (m *CoordinatorBootstrapResponse) Size() (n int) {
if m.DispatcherDrainTargetEpoch != 0 {
n += 1 + sovHeartbeat(uint64(m.DispatcherDrainTargetEpoch))
}
+ if m.WriteLeaseProtocolVersion != 0 {
+ n += 1 + sovHeartbeat(uint64(m.WriteLeaseProtocolVersion))
+ }
return n
}
@@ -11320,6 +11888,80 @@ func (m *NodeHeartbeat) Unmarshal(dAtA []byte) error {
break
}
}
+ case 5:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WriteLeaseRequestSeq", wireType)
+ }
+ m.WriteLeaseRequestSeq = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.WriteLeaseRequestSeq |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 6:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WriteLeaseProtocolVersion", wireType)
+ }
+ m.WriteLeaseProtocolVersion = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.WriteLeaseProtocolVersion |= uint32(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 7:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WriteLeaseWitnessAck", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ postIndex := iNdEx + msglen
+ if postIndex < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ if m.WriteLeaseWitnessAck == nil {
+ m.WriteLeaseWitnessAck = &WriteLeaseWitnessAck{}
+ }
+ if err := m.WriteLeaseWitnessAck.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipHeartbeat(dAtA[iNdEx:])
@@ -11341,7 +11983,7 @@ func (m *NodeHeartbeat) Unmarshal(dAtA []byte) error {
}
return nil
}
-func (m *SetNodeLivenessRequest) Unmarshal(dAtA []byte) error {
+func (m *WriteLeaseWitnessChallenge) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@@ -11364,17 +12006,17 @@ func (m *SetNodeLivenessRequest) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
- return fmt.Errorf("proto: SetNodeLivenessRequest: wiretype end group for non-group")
+ return fmt.Errorf("proto: WriteLeaseWitnessChallenge: wiretype end group for non-group")
}
if fieldNum <= 0 {
- return fmt.Errorf("proto: SetNodeLivenessRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+ return fmt.Errorf("proto: WriteLeaseWitnessChallenge: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Target", wireType)
+ return fmt.Errorf("proto: wrong wireType = %d for field CoordinatorVersion", wireType)
}
- m.Target = 0
+ m.CoordinatorVersion = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowHeartbeat
@@ -11384,16 +12026,16 @@ func (m *SetNodeLivenessRequest) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
- m.Target |= NodeLiveness(b&0x7F) << shift
+ m.CoordinatorVersion |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType)
+ return fmt.Errorf("proto: wrong wireType = %d for field CoordinatorNodeEpoch", wireType)
}
- m.NodeEpoch = 0
+ m.CoordinatorNodeEpoch = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowHeartbeat
@@ -11403,16 +12045,498 @@ func (m *SetNodeLivenessRequest) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
- m.NodeEpoch |= uint64(b&0x7F) << shift
+ m.CoordinatorNodeEpoch |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
- default:
- iNdEx = preIndex
- skippy, err := skipHeartbeat(dAtA[iNdEx:])
- if err != nil {
- return err
+ case 3:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field SelfRequestSeq", wireType)
+ }
+ m.SelfRequestSeq = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.SelfRequestSeq |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 4:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WitnessNodeEpoch", wireType)
+ }
+ m.WitnessNodeEpoch = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.WitnessNodeEpoch |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 5:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Nonce", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Nonce = append(m.Nonce[:0], dAtA[iNdEx:postIndex]...)
+ if m.Nonce == nil {
+ m.Nonce = []byte{}
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipHeartbeat(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if (skippy < 0) || (iNdEx+skippy) < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *WriteLeaseWitnessAck) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: WriteLeaseWitnessAck: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: WriteLeaseWitnessAck: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field CoordinatorVersion", wireType)
+ }
+ m.CoordinatorVersion = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.CoordinatorVersion |= int64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 2:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field CoordinatorNodeEpoch", wireType)
+ }
+ m.CoordinatorNodeEpoch = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.CoordinatorNodeEpoch |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 3:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field SelfRequestSeq", wireType)
+ }
+ m.SelfRequestSeq = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.SelfRequestSeq |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 4:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WitnessNodeEpoch", wireType)
+ }
+ m.WitnessNodeEpoch = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.WitnessNodeEpoch |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 5:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Nonce", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Nonce = append(m.Nonce[:0], dAtA[iNdEx:postIndex]...)
+ if m.Nonce == nil {
+ m.Nonce = []byte{}
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipHeartbeat(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if (skippy < 0) || (iNdEx+skippy) < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *NodeHeartbeatResponse) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: NodeHeartbeatResponse: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: NodeHeartbeatResponse: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field CoordinatorVersion", wireType)
+ }
+ m.CoordinatorVersion = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.CoordinatorVersion |= int64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 2:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field TargetNodeEpoch", wireType)
+ }
+ m.TargetNodeEpoch = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.TargetNodeEpoch |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 3:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field RequestSeq", wireType)
+ }
+ m.RequestSeq = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.RequestSeq |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 4:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field LeaseDurationMs", wireType)
+ }
+ m.LeaseDurationMs = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.LeaseDurationMs |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 5:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WitnessChallenge", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ postIndex := iNdEx + msglen
+ if postIndex < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ if m.WitnessChallenge == nil {
+ m.WitnessChallenge = &WriteLeaseWitnessChallenge{}
+ }
+ if err := m.WitnessChallenge.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipHeartbeat(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if (skippy < 0) || (iNdEx+skippy) < 0 {
+ return ErrInvalidLengthHeartbeat
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *SetNodeLivenessRequest) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: SetNodeLivenessRequest: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: SetNodeLivenessRequest: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Target", wireType)
+ }
+ m.Target = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.Target |= NodeLiveness(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ case 2:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field NodeEpoch", wireType)
+ }
+ m.NodeEpoch = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.NodeEpoch |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ default:
+ iNdEx = preIndex
+ skippy, err := skipHeartbeat(dAtA[iNdEx:])
+ if err != nil {
+ return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthHeartbeat
@@ -11666,6 +12790,25 @@ func (m *CoordinatorBootstrapRequest) Unmarshal(dAtA []byte) error {
break
}
}
+ case 2:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WriteLeaseProtocolVersion", wireType)
+ }
+ m.WriteLeaseProtocolVersion = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.WriteLeaseProtocolVersion |= uint32(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
default:
iNdEx = preIndex
skippy, err := skipHeartbeat(dAtA[iNdEx:])
@@ -11820,6 +12963,25 @@ func (m *CoordinatorBootstrapResponse) Unmarshal(dAtA []byte) error {
break
}
}
+ case 5:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field WriteLeaseProtocolVersion", wireType)
+ }
+ m.WriteLeaseProtocolVersion = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowHeartbeat
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.WriteLeaseProtocolVersion |= uint32(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
default:
iNdEx = preIndex
skippy, err := skipHeartbeat(dAtA[iNdEx:])
diff --git a/heartbeatpb/heartbeat.proto b/heartbeatpb/heartbeat.proto
index b55a9117af..4f16a90539 100644
--- a/heartbeatpb/heartbeat.proto
+++ b/heartbeatpb/heartbeat.proto
@@ -206,6 +206,33 @@ message NodeHeartbeat {
// currently applied on this node. Empty target means the drain target is clear.
string dispatcher_drain_target_node_id = 3;
uint64 dispatcher_drain_target_epoch = 4;
+ uint64 write_lease_request_seq = 5;
+ uint32 write_lease_protocol_version = 6;
+ WriteLeaseWitnessAck write_lease_witness_ack = 7;
+}
+
+message WriteLeaseWitnessChallenge {
+ int64 coordinator_version = 1;
+ uint64 coordinator_node_epoch = 2;
+ uint64 self_request_seq = 3;
+ uint64 witness_node_epoch = 4;
+ bytes nonce = 5;
+}
+
+message WriteLeaseWitnessAck {
+ int64 coordinator_version = 1;
+ uint64 coordinator_node_epoch = 2;
+ uint64 self_request_seq = 3;
+ uint64 witness_node_epoch = 4;
+ bytes nonce = 5;
+}
+
+message NodeHeartbeatResponse {
+ int64 coordinator_version = 1;
+ uint64 target_node_epoch = 2;
+ uint64 request_seq = 3;
+ uint64 lease_duration_ms = 4;
+ WriteLeaseWitnessChallenge witness_challenge = 5;
}
// SetNodeLivenessRequest asks a node to transition its local liveness.
@@ -229,6 +256,9 @@ message SetDispatcherDrainTargetRequest {
message CoordinatorBootstrapRequest {
int64 version = 1;
+ // Zero keeps the peer in legacy mode. A non-zero version activates the
+ // capture P2P write lease only on peers that understand this protocol.
+ uint32 write_lease_protocol_version = 2;
}
message CoordinatorBootstrapResponse {
@@ -244,6 +274,7 @@ message CoordinatorBootstrapResponse {
// scheduling resumes.
string dispatcher_drain_target_node_id = 3;
uint64 dispatcher_drain_target_epoch = 4;
+ uint32 write_lease_protocol_version = 5;
}
message AddMaintainerRequest {
diff --git a/heartbeatpb/write_lease_protocol.go b/heartbeatpb/write_lease_protocol.go
new file mode 100644
index 0000000000..d82c870bdf
--- /dev/null
+++ b/heartbeatpb/write_lease_protocol.go
@@ -0,0 +1,19 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package heartbeatpb
+
+const (
+ LegacyWriteLeaseProtocolVersion uint32 = 0
+ CurrentWriteLeaseProtocolVersion uint32 = 1
+)
diff --git a/maintainer/maintainer_manager.go b/maintainer/maintainer_manager.go
index 04aa842137..acc7c523ff 100644
--- a/maintainer/maintainer_manager.go
+++ b/maintainer/maintainer_manager.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/ticdc/pkg/liveness"
"github.com/pingcap/ticdc/pkg/messaging"
"github.com/pingcap/ticdc/pkg/node"
+ "github.com/pingcap/ticdc/pkg/writelease"
"go.uber.org/zap"
)
@@ -53,6 +54,7 @@ type Manager struct {
node *managerNodeState
// maintainers holds changefeed-scoped state and lifecycle operations.
maintainers *managerMaintainerSet
+ writeGate *writelease.Gate
}
// NewMaintainerManager create a changefeed maintainer manager instance
@@ -67,6 +69,10 @@ func NewMaintainerManager(
) *Manager {
mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter)
heartbeatCh := make(chan struct{}, 1)
+ writeGate, ok := appcontext.TryGetService[*writelease.Gate](appcontext.CaptureWriteGate)
+ if !ok {
+ writeGate = writelease.NewGate()
+ }
m := &Manager{
mc: mc,
nodeInfo: nodeInfo,
@@ -74,6 +80,7 @@ func NewMaintainerManager(
heartbeatCh: heartbeatCh,
node: newManagerNodeState(nodeLiveness),
maintainers: newManagerMaintainerSet(conf, nodeInfo, heartbeatCh),
+ writeGate: writeGate,
}
mc.RegisterHandler(messaging.MaintainerManagerTopic, m.recvMessages)
@@ -93,7 +100,8 @@ func (m *Manager) recvMessages(ctx context.Context, msg *messaging.TargetMessage
messaging.TypeRemoveMaintainerRequest,
messaging.TypeCoordinatorBootstrapRequest,
messaging.TypeSetNodeLivenessRequest,
- messaging.TypeSetDispatcherDrainTargetRequest:
+ messaging.TypeSetDispatcherDrainTargetRequest,
+ messaging.TypeNodeHeartbeatResponse:
select {
case <-ctx.Done():
return ctx.Err()
@@ -135,6 +143,8 @@ func (m *Manager) Name() string {
func (m *Manager) Run(ctx context.Context) error {
ticker := time.NewTicker(defaultManagerHeartbeatInterval)
defer ticker.Stop()
+ nodeHeartbeatTicker := time.NewTicker(writelease.NodeHeartbeatInterval)
+ defer nodeHeartbeatTicker.Stop()
for {
select {
case <-ctx.Done():
@@ -144,9 +154,10 @@ func (m *Manager) Run(ctx context.Context) error {
case <-m.heartbeatCh:
m.sendHeartbeat()
case <-ticker.C:
- m.sendNodeHeartbeat(false)
m.sendHeartbeat()
m.cleanupRemovedMaintainers()
+ case <-nodeHeartbeatTicker.C:
+ m.sendNodeHeartbeat(false)
}
}
}
@@ -184,6 +195,13 @@ func (m *Manager) onCoordinatorBootstrapRequest(msg *messaging.TargetMessage) {
zap.Int64("version", req.Version))
return
}
+ if m.coordinatorID != msg.From || m.coordinatorVersion != req.Version {
+ m.writeGate.InvalidateP2P()
+ m.node.resetWriteLeaseRequests()
+ }
+ m.writeGate.SetP2PRequired(
+ req.GetWriteLeaseProtocolVersion() == heartbeatpb.CurrentWriteLeaseProtocolVersion,
+ )
m.coordinatorID = msg.From
m.coordinatorVersion = req.Version
@@ -227,6 +245,8 @@ func (m *Manager) handleMessage(msg *messaging.TargetMessage) {
m.onSetNodeLivenessRequest(msg)
case messaging.TypeSetDispatcherDrainTargetRequest:
m.onSetDispatcherDrainTargetRequest(msg)
+ case messaging.TypeNodeHeartbeatResponse:
+ m.onNodeHeartbeatResponse(msg)
default:
}
}
diff --git a/maintainer/maintainer_manager_maintainers.go b/maintainer/maintainer_manager_maintainers.go
index c94fa77527..0f1916e280 100644
--- a/maintainer/maintainer_manager_maintainers.go
+++ b/maintainer/maintainer_manager_maintainers.go
@@ -170,7 +170,8 @@ func (p *managerMaintainerSet) closeAll() {
// buildBootstrapResponse snapshots all local maintainer states for coordinator bootstrap.
func (p *managerMaintainerSet) buildBootstrapResponse() *heartbeatpb.CoordinatorBootstrapResponse {
response := &heartbeatpb.CoordinatorBootstrapResponse{
- DrainProtocolVersion: heartbeatpb.CurrentDrainProtocolVersion,
+ DrainProtocolVersion: heartbeatpb.CurrentDrainProtocolVersion,
+ WriteLeaseProtocolVersion: heartbeatpb.CurrentWriteLeaseProtocolVersion,
}
p.registry.Range(func(_, value interface{}) bool {
maintainer := value.(*Maintainer)
diff --git a/maintainer/maintainer_manager_node.go b/maintainer/maintainer_manager_node.go
index 3b67b3d5f1..2b53f96a70 100644
--- a/maintainer/maintainer_manager_node.go
+++ b/maintainer/maintainer_manager_node.go
@@ -21,14 +21,12 @@ import (
"github.com/pingcap/ticdc/heartbeatpb"
"github.com/pingcap/ticdc/pkg/liveness"
"github.com/pingcap/ticdc/pkg/messaging"
+ "github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/node"
+ "github.com/pingcap/ticdc/pkg/writelease"
"go.uber.org/zap"
)
-// nodeHeartbeatInterval bounds background node heartbeat frequency.
-// Forced heartbeats bypass this throttle to acknowledge state changes immediately.
-const nodeHeartbeatInterval = 5 * time.Second
-
// managerNodeState owns node-scoped state shared by all local maintainers.
type managerNodeState struct {
// liveness points to the server-wide node liveness state shared with other
@@ -54,13 +52,19 @@ type managerNodeState struct {
// lastNodeHeartbeatSentAt records the last successful periodic node heartbeat
// send so background heartbeats can be throttled.
lastNodeHeartbeatSentAt time.Time
+
+ writeLeaseRequestSeq uint64
+ writeLeaseRequestSentAt map[uint64]time.Time
+ lastAppliedLeaseSeq uint64
+ pendingWitnessAck *heartbeatpb.WriteLeaseWitnessAck
}
// newManagerNodeState initializes the node-scoped state owned by a manager.
func newManagerNodeState(nodeLiveness *liveness.Liveness) *managerNodeState {
return &managerNodeState{
- liveness: nodeLiveness,
- nodeEpoch: newNodeEpoch(),
+ liveness: nodeLiveness,
+ nodeEpoch: newNodeEpoch(),
+ writeLeaseRequestSentAt: make(map[uint64]time.Time),
}
}
@@ -83,7 +87,7 @@ func (m *Manager) sendNodeHeartbeat(force bool) {
}
now := time.Now()
- if !force && now.Sub(m.node.lastNodeHeartbeatSentAt) < nodeHeartbeatInterval {
+ if !force && now.Sub(m.node.lastNodeHeartbeatSentAt) < writelease.NodeHeartbeatInterval {
return
}
// Update before sending so a transient send failure will not cause
@@ -95,6 +99,10 @@ func (m *Manager) sendNodeHeartbeat(force bool) {
currentLiveness = m.node.liveness.Load()
}
drainTarget, drainEpoch := m.getDispatcherDrainTarget()
+ m.node.writeLeaseRequestSeq++
+ requestSeq := m.node.writeLeaseRequestSeq
+ m.node.writeLeaseRequestSentAt[requestSeq] = now
+ m.node.pruneWriteLeaseRequests(now)
hb := &heartbeatpb.NodeHeartbeat{
Liveness: m.toNodeLivenessPB(currentLiveness),
NodeEpoch: m.node.nodeEpoch,
@@ -102,15 +110,107 @@ func (m *Manager) sendNodeHeartbeat(force bool) {
// confirm both activation and clearing even when no maintainers exist.
DispatcherDrainTargetNodeId: drainTarget.String(),
DispatcherDrainTargetEpoch: drainEpoch,
+ WriteLeaseRequestSeq: requestSeq,
+ WriteLeaseProtocolVersion: heartbeatpb.CurrentWriteLeaseProtocolVersion,
+ WriteLeaseWitnessAck: m.node.pendingWitnessAck,
}
target := m.newCoordinatorTopicMessage(hb)
if err := m.mc.SendCommand(target); err != nil {
+ delete(m.node.writeLeaseRequestSentAt, requestSeq)
log.Warn("send node heartbeat failed",
zap.Stringer("from", m.nodeInfo.ID),
zap.Stringer("target", target.To),
zap.Error(err))
return
}
+ m.node.pendingWitnessAck = nil
+}
+
+func (m *Manager) onNodeHeartbeatResponse(msg *messaging.TargetMessage) {
+ metrics.CaptureLeaseResponseCounter.WithLabelValues("received").Inc()
+ if msg.From != m.coordinatorID {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("sender").Inc()
+ return
+ }
+ response := msg.Message[0].(*heartbeatpb.NodeHeartbeatResponse)
+ if response.GetCoordinatorVersion() != m.coordinatorVersion {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("coordinator_version").Inc()
+ return
+ }
+ if response.GetTargetNodeEpoch() != m.node.nodeEpoch {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("node_epoch").Inc()
+ return
+ }
+
+ challenge := response.GetWitnessChallenge()
+ if challenge != nil &&
+ challenge.GetCoordinatorVersion() == m.coordinatorVersion &&
+ challenge.GetWitnessNodeEpoch() == m.node.nodeEpoch &&
+ len(challenge.GetNonce()) > 0 {
+ m.node.pendingWitnessAck = &heartbeatpb.WriteLeaseWitnessAck{
+ CoordinatorVersion: challenge.GetCoordinatorVersion(),
+ CoordinatorNodeEpoch: challenge.GetCoordinatorNodeEpoch(),
+ SelfRequestSeq: challenge.GetSelfRequestSeq(),
+ WitnessNodeEpoch: challenge.GetWitnessNodeEpoch(),
+ Nonce: append([]byte(nil), challenge.GetNonce()...),
+ }
+ m.sendNodeHeartbeat(true)
+ }
+
+ requestSeq := response.GetRequestSeq()
+ if requestSeq == 0 {
+ if challenge == nil {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("request_sequence").Inc()
+ }
+ return
+ }
+ if requestSeq <= m.node.lastAppliedLeaseSeq {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("replayed_sequence").Inc()
+ return
+ }
+ requestSentAt, ok := m.node.writeLeaseRequestSentAt[requestSeq]
+ if !ok {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("unknown_sequence").Inc()
+ return
+ }
+ leaseDurationMs := response.GetLeaseDurationMs()
+ if leaseDurationMs == 0 {
+ // The coordinator observed an unknown or legacy capture, so the whole
+ // cluster temporarily falls back to etcd-only write admission.
+ m.writeGate.SetP2PRequired(false)
+ } else {
+ duration := time.Duration(leaseDurationMs) * time.Millisecond
+ if duration <= 0 || duration > writelease.P2PLeaseDuration {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("lease_duration").Inc()
+ return
+ }
+ if !m.writeGate.RenewP2P(requestSentAt, duration) {
+ metrics.CaptureLeaseResponseRejectedCounter.WithLabelValues("expired_or_fenced").Inc()
+ return
+ }
+ m.writeGate.SetP2PRequired(true)
+ }
+ metrics.CaptureLeaseResponseCounter.WithLabelValues("accepted").Inc()
+ m.node.lastAppliedLeaseSeq = requestSeq
+ for seq := range m.node.writeLeaseRequestSentAt {
+ if seq <= requestSeq {
+ delete(m.node.writeLeaseRequestSentAt, seq)
+ }
+ }
+}
+
+func (n *managerNodeState) pruneWriteLeaseRequests(now time.Time) {
+ for seq, sentAt := range n.writeLeaseRequestSentAt {
+ if !sentAt.Add(writelease.P2PLeaseDuration).After(now) {
+ delete(n.writeLeaseRequestSentAt, seq)
+ }
+ }
+}
+
+func (n *managerNodeState) resetWriteLeaseRequests() {
+ n.writeLeaseRequestSentAt = make(map[uint64]time.Time)
+ n.lastAppliedLeaseSeq = 0
+ n.pendingWitnessAck = nil
}
// onSetNodeLivenessRequest applies a coordinator-driven liveness transition if
diff --git a/maintainer/node_liveness_test.go b/maintainer/node_liveness_test.go
index b35e056dec..89f1c92a9b 100644
--- a/maintainer/node_liveness_test.go
+++ b/maintainer/node_liveness_test.go
@@ -15,6 +15,7 @@ package maintainer
import (
"encoding/json"
"testing"
+ "time"
"github.com/pingcap/ticdc/heartbeatpb"
"github.com/pingcap/ticdc/pkg/common"
@@ -23,6 +24,7 @@ import (
"github.com/pingcap/ticdc/pkg/liveness"
"github.com/pingcap/ticdc/pkg/messaging"
"github.com/pingcap/ticdc/pkg/node"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/stretchr/testify/require"
)
@@ -218,6 +220,182 @@ func TestCoordinatorBootstrapResponseIncludesDispatcherDrainTarget(t *testing.T)
resp := out.Message[0].(*heartbeatpb.CoordinatorBootstrapResponse)
require.Equal(t, "n2", resp.DispatcherDrainTargetNodeId)
require.Equal(t, uint64(7), resp.DispatcherDrainTargetEpoch)
+ require.Equal(t, heartbeatpb.CurrentWriteLeaseProtocolVersion, resp.WriteLeaseProtocolVersion)
+}
+
+func TestCoordinatorBootstrapNegotiatesP2PWriteLease(t *testing.T) {
+ mc := messaging.NewMockMessageCenter()
+ appcontext.SetService(appcontext.MessageCenter, mc)
+ gate := writelease.NewGate()
+ appcontext.SetService(appcontext.CaptureWriteGate, gate)
+
+ var nodeLiveness liveness.Liveness
+ m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness)
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ require.True(t, gate.IsWritable())
+
+ bootstrap := func(coordinator node.ID, protocolVersion uint32) {
+ message := messaging.NewSingleTargetMessage(
+ m.nodeInfo.ID,
+ messaging.MaintainerManagerTopic,
+ &heartbeatpb.CoordinatorBootstrapRequest{
+ Version: 1,
+ WriteLeaseProtocolVersion: protocolVersion,
+ },
+ )
+ message.From = coordinator
+ m.onCoordinatorBootstrapRequest(message)
+ // Bootstrap response and the forced node heartbeat.
+ <-mc.GetMessageChannel()
+ <-mc.GetMessageChannel()
+ }
+
+ bootstrap(node.ID("new-coordinator"), heartbeatpb.CurrentWriteLeaseProtocolVersion)
+ require.True(t, gate.Status().P2PRequired)
+ require.False(t, gate.IsWritable())
+
+ bootstrap(node.ID("legacy-coordinator"), heartbeatpb.LegacyWriteLeaseProtocolVersion)
+ require.False(t, gate.Status().P2PRequired)
+ require.True(t, gate.IsWritable())
+
+ bootstrap(node.ID("future-coordinator"), heartbeatpb.CurrentWriteLeaseProtocolVersion+1)
+ require.False(t, gate.Status().P2PRequired)
+ require.True(t, gate.IsWritable())
+}
+
+func TestNodeHeartbeatResponseRenewsP2PWriteLease(t *testing.T) {
+ mc := messaging.NewMockMessageCenter()
+ appcontext.SetService(appcontext.MessageCenter, mc)
+ gate := writelease.NewGate()
+ appcontext.SetService(appcontext.CaptureWriteGate, gate)
+
+ var nodeLiveness liveness.Liveness
+ m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness)
+ m.coordinatorID = node.ID("coordinator")
+ m.coordinatorVersion = 10
+ gate.SetP2PRequired(true)
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+
+ m.sendNodeHeartbeat(true)
+ heartbeatMessage := <-mc.GetMessageChannel()
+ heartbeat := heartbeatMessage.Message[0].(*heartbeatpb.NodeHeartbeat)
+ require.Equal(t, heartbeatpb.CurrentWriteLeaseProtocolVersion, heartbeat.WriteLeaseProtocolVersion)
+ require.NotZero(t, heartbeat.WriteLeaseRequestSeq)
+
+ responseMessage := messaging.NewSingleTargetMessage(
+ m.nodeInfo.ID,
+ messaging.MaintainerManagerTopic,
+ &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: 10,
+ TargetNodeEpoch: m.node.nodeEpoch,
+ RequestSeq: heartbeat.WriteLeaseRequestSeq,
+ LeaseDurationMs: uint64(writelease.P2PLeaseDuration.Milliseconds()),
+ },
+ )
+ responseMessage.From = m.coordinatorID
+ m.onNodeHeartbeatResponse(responseMessage)
+
+ require.True(t, gate.IsWritable())
+
+ // An old coordinator response cannot renew a new request.
+ m.writeGate.InvalidateP2P()
+ responseMessage.Message[0].(*heartbeatpb.NodeHeartbeatResponse).CoordinatorVersion = 9
+ m.onNodeHeartbeatResponse(responseMessage)
+ require.False(t, gate.IsWritable())
+
+ // Replaying an already applied sequence from the current coordinator is also rejected.
+ responseMessage.Message[0].(*heartbeatpb.NodeHeartbeatResponse).CoordinatorVersion = 10
+ m.onNodeHeartbeatResponse(responseMessage)
+ require.False(t, gate.IsWritable())
+}
+
+func TestNodeHeartbeatResponseUpdatesClusterP2PMode(t *testing.T) {
+ mc := messaging.NewMockMessageCenter()
+ appcontext.SetService(appcontext.MessageCenter, mc)
+ gate := writelease.NewGate()
+ appcontext.SetService(appcontext.CaptureWriteGate, gate)
+
+ var nodeLiveness liveness.Liveness
+ m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness)
+ m.coordinatorID = node.ID("coordinator")
+ m.coordinatorVersion = 10
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+
+ m.sendNodeHeartbeat(true)
+ heartbeatMessage := <-mc.GetMessageChannel()
+ heartbeat := heartbeatMessage.Message[0].(*heartbeatpb.NodeHeartbeat)
+ grant := messaging.NewSingleTargetMessage(
+ m.nodeInfo.ID,
+ messaging.MaintainerManagerTopic,
+ &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: 10,
+ TargetNodeEpoch: m.node.nodeEpoch,
+ RequestSeq: heartbeat.WriteLeaseRequestSeq,
+ LeaseDurationMs: uint64(writelease.P2PLeaseDuration.Milliseconds()),
+ },
+ )
+ grant.From = m.coordinatorID
+ m.onNodeHeartbeatResponse(grant)
+ require.True(t, gate.Status().P2PRequired)
+ require.True(t, gate.IsWritable())
+
+ m.sendNodeHeartbeat(true)
+ heartbeatMessage = <-mc.GetMessageChannel()
+ heartbeat = heartbeatMessage.Message[0].(*heartbeatpb.NodeHeartbeat)
+ disable := messaging.NewSingleTargetMessage(
+ m.nodeInfo.ID,
+ messaging.MaintainerManagerTopic,
+ &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: 10,
+ TargetNodeEpoch: m.node.nodeEpoch,
+ RequestSeq: heartbeat.WriteLeaseRequestSeq,
+ LeaseDurationMs: 0,
+ },
+ )
+ disable.From = m.coordinatorID
+ m.onNodeHeartbeatResponse(disable)
+ require.False(t, gate.Status().P2PRequired)
+ require.True(t, gate.IsWritable())
+}
+
+func TestNodeHeartbeatResponseEchoesWitnessChallenge(t *testing.T) {
+ mc := messaging.NewMockMessageCenter()
+ appcontext.SetService(appcontext.MessageCenter, mc)
+ appcontext.SetService(appcontext.CaptureWriteGate, writelease.NewGate())
+
+ var nodeLiveness liveness.Liveness
+ m := NewMaintainerManager(&node.Info{ID: node.ID("n1")}, &config.SchedulerConfig{}, &nodeLiveness)
+ m.coordinatorID = node.ID("coordinator")
+ m.coordinatorVersion = 10
+
+ challenge := &heartbeatpb.WriteLeaseWitnessChallenge{
+ CoordinatorVersion: 10,
+ CoordinatorNodeEpoch: 11,
+ SelfRequestSeq: 7,
+ WitnessNodeEpoch: m.node.nodeEpoch,
+ Nonce: []byte("nonce"),
+ }
+ message := messaging.NewSingleTargetMessage(
+ m.nodeInfo.ID,
+ messaging.MaintainerManagerTopic,
+ &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: 10,
+ TargetNodeEpoch: m.node.nodeEpoch,
+ WitnessChallenge: challenge,
+ },
+ )
+ message.From = m.coordinatorID
+ m.onNodeHeartbeatResponse(message)
+
+ heartbeatMessage := <-mc.GetMessageChannel()
+ heartbeat := heartbeatMessage.Message[0].(*heartbeatpb.NodeHeartbeat)
+ ack := heartbeat.GetWriteLeaseWitnessAck()
+ require.NotNil(t, ack)
+ require.Equal(t, challenge.CoordinatorVersion, ack.CoordinatorVersion)
+ require.Equal(t, challenge.CoordinatorNodeEpoch, ack.CoordinatorNodeEpoch)
+ require.Equal(t, challenge.SelfRequestSeq, ack.SelfRequestSeq)
+ require.Equal(t, challenge.WitnessNodeEpoch, ack.WitnessNodeEpoch)
+ require.Equal(t, challenge.Nonce, ack.Nonce)
}
func TestAddMaintainerIgnoreInvalidConfig(t *testing.T) {
diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go
index d3e8c989a2..e2ff489c16 100644
--- a/pkg/bootstrap/bootstrap.go
+++ b/pkg/bootstrap/bootstrap.go
@@ -152,6 +152,21 @@ func (b *Bootstrapper[T]) GetAllNodeIDs() []node.ID {
return result
}
+// GetInitializedNodeIDs returns a snapshot of nodes that have reported a
+// bootstrap response.
+func (b *Bootstrapper[T]) GetInitializedNodeIDs() []node.ID {
+ b.mutex.Lock()
+ defer b.mutex.Unlock()
+
+ result := make([]node.ID, 0, len(b.nodes))
+ for id, status := range b.nodes {
+ if status.Initialized() {
+ result = append(result, id)
+ }
+ }
+ return result
+}
+
// HasNode returns whether the bootstrapper is still tracking the given node.
func (b *Bootstrapper[T]) HasNode(id node.ID) bool {
b.mutex.Lock()
diff --git a/pkg/bootstrap/bootstrap_test.go b/pkg/bootstrap/bootstrap_test.go
index badccae2cd..a69f7bfda0 100644
--- a/pkg/bootstrap/bootstrap_test.go
+++ b/pkg/bootstrap/bootstrap_test.go
@@ -48,6 +48,7 @@ func TestHandleNewNodes(t *testing.T) {
require.Len(t, removed, 0)
require.Len(t, requests, 2)
require.Len(t, b.GetAllNodeIDs(), 2)
+ require.Empty(t, b.GetInitializedNodeIDs())
require.Nil(t, responses)
require.False(t, b.AllNodesReady())
@@ -72,6 +73,7 @@ func TestHandleNewNodes(t *testing.T) {
Spans: []*heartbeatpb.BootstrapTableSpan{{}},
})
require.False(t, b.AllNodesReady())
+ require.Equal(t, []node.ID{node1.ID}, b.GetInitializedNodeIDs())
require.Nil(t, responses)
// all nodes responses received, bootstrapped
@@ -83,6 +85,7 @@ func TestHandleNewNodes(t *testing.T) {
})
require.True(t, b.AllNodesReady())
require.Len(t, responses, 2)
+ require.ElementsMatch(t, []node.ID{node1.ID, node2.ID}, b.GetInitializedNodeIDs())
require.Equal(t, 1, len(responses[node1.ID].Spans))
require.Equal(t, 2, len(responses[node2.ID].Spans))
b.ClearBootstrapResponses()
diff --git a/pkg/cloudstorage/generator.go b/pkg/cloudstorage/generator.go
index f17d2bd145..28a651f6a1 100644
--- a/pkg/cloudstorage/generator.go
+++ b/pkg/cloudstorage/generator.go
@@ -33,6 +33,7 @@ import (
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/pdutil"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/tikv/client-go/v2/oracle"
"go.uber.org/zap"
@@ -187,6 +188,7 @@ type FilePathGenerator struct {
// This can differ from TableInfoVersion when reusing an existing schema file
// with the same checksum.
versionMap map[VersionedTableName]uint64
+ writeGate *writelease.Gate
}
// NewFilePathGenerator creates a FilePathGenerator for one changefeed storage
@@ -209,6 +211,11 @@ func NewFilePathGenerator(
}
}
+// SetWriteGate installs capture-wide admission for schema-file publication.
+func (f *FilePathGenerator) SetWriteGate(gate *writelease.Gate) {
+ f.writeGate = gate
+}
+
// CheckOrWriteSchema ensures the schema file for table/tableInfo exists.
// The first return value is the schema version that must be used in later
// data/index paths. The second return value is true when storage already has a
@@ -306,8 +313,14 @@ func (f *FilePathGenerator) CheckOrWriteSchema(
zap.Uint32("checksum", checksum))
}
encodedSchemaFile := schemaFile.Marshal()
+ if err := writelease.WaitForWrite(ctx, f.writeGate); err != nil {
+ return 0, false, err
+ }
+ if err := f.storage.WriteFile(ctx, schemaFilePath, encodedSchemaFile); err != nil {
+ return 0, false, err
+ }
f.versionMap[table] = table.TableInfoVersion
- return table.TableInfoVersion, false, f.storage.WriteFile(ctx, schemaFilePath, encodedSchemaFile)
+ return table.TableInfoVersion, false, nil
}
// SetClock sets the clock used by GenerateDateStr. It is used by tests.
diff --git a/pkg/common/context/app_context.go b/pkg/common/context/app_context.go
index 84bca4d842..26c7b5c142 100644
--- a/pkg/common/context/app_context.go
+++ b/pkg/common/context/app_context.go
@@ -39,6 +39,7 @@ const (
PDAPIClient = "PDAPIClient"
RegionCache = "RegionCache"
KeyspaceManager = "keyspaceManager"
+ CaptureWriteGate = "CaptureWriteGate"
)
// Put all the global instances here.
diff --git a/pkg/common/table_info_shared_schema_guard_test.go b/pkg/common/table_info_shared_schema_guard_test.go
index 87bbb342be..d18bc6cdc4 100644
--- a/pkg/common/table_info_shared_schema_guard_test.go
+++ b/pkg/common/table_info_shared_schema_guard_test.go
@@ -84,6 +84,8 @@ func TestLatestTiDBTableInfoSharedSchemaGuard(t *testing.T) {
"TempTableType", "TableCacheStatusType", "PlacementPolicyRef", "StatsOptions",
"ExchangePartitionInfo", "TTLInfo", "IsActiveActive", "SoftdeleteInfo", "Affinity",
"Revision", "DBID",
+ // Materialized-view metadata is table-level and does not affect the shared column schema.
+ "MaterializedViewBase", "MaterializedView", "MaterializedViewLog",
// These table-level storage settings do not affect the shared column schema.
"EngineAttribute", "StorageClassTier", "StorageClassTransitions", "Mode",
},
diff --git a/pkg/messaging/message.go b/pkg/messaging/message.go
index 5eaf5b71a1..91118c076a 100644
--- a/pkg/messaging/message.go
+++ b/pkg/messaging/message.go
@@ -109,6 +109,7 @@ const (
TypeSetNodeLivenessRequest IOType = 43
TypeSetNodeLivenessResponse IOType = 44
TypeSetDispatcherDrainTargetRequest IOType = 45
+ TypeNodeHeartbeatResponse IOType = 46
)
func (t IOType) String() string {
@@ -203,6 +204,8 @@ func (t IOType) String() string {
return "SetNodeLivenessResponse"
case TypeSetDispatcherDrainTargetRequest:
return "SetDispatcherDrainTargetRequest"
+ case TypeNodeHeartbeatResponse:
+ return "NodeHeartbeatResponse"
default:
}
return "Unknown"
@@ -403,6 +406,8 @@ func decodeIOType(ioType IOType, value []byte) (IOTypeT, error) {
m = &heartbeatpb.SetNodeLivenessResponse{}
case TypeSetDispatcherDrainTargetRequest:
m = &heartbeatpb.SetDispatcherDrainTargetRequest{}
+ case TypeNodeHeartbeatResponse:
+ m = &heartbeatpb.NodeHeartbeatResponse{}
default:
log.Debug("Unimplemented IOType, ignore the message", zap.Stringer("Type", ioType))
return nil, errors.ErrUnimplementedIOType.GenWithStackByArgs(int(ioType))
@@ -523,6 +528,8 @@ func NewSingleTargetMessage(To node.ID, Topic string, Message IOTypeT, Group ...
ioType = TypeSetNodeLivenessResponse
case *heartbeatpb.SetDispatcherDrainTargetRequest:
ioType = TypeSetDispatcherDrainTargetRequest
+ case *heartbeatpb.NodeHeartbeatResponse:
+ ioType = TypeNodeHeartbeatResponse
default:
panic("unknown io type")
}
diff --git a/pkg/messaging/message_write_lease_test.go b/pkg/messaging/message_write_lease_test.go
new file mode 100644
index 0000000000..c7946f7f29
--- /dev/null
+++ b/pkg/messaging/message_write_lease_test.go
@@ -0,0 +1,87 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package messaging
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/pingcap/ticdc/heartbeatpb"
+ "github.com/pingcap/ticdc/pkg/node"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNodeHeartbeatResponseRemoteRoundTrip(t *testing.T) {
+ sender, senderAddr, stopSender := NewMessageCenterForTest(t)
+ receiver, receiverAddr, stopReceiver := NewMessageCenterForTest(t)
+ t.Cleanup(stopSender)
+ t.Cleanup(stopReceiver)
+
+ sender.addTarget(receiver.id, receiverAddr)
+ receiver.addTarget(sender.id, senderAddr)
+ require.Eventually(t, func() bool {
+ return sender.IsReadyToSend(receiver.id) && receiver.IsReadyToSend(sender.id)
+ }, 10*time.Second, 100*time.Millisecond)
+
+ received := make(chan *TargetMessage, 1)
+ receiver.RegisterHandler(MaintainerManagerTopic, func(_ context.Context, message *TargetMessage) error {
+ received <- message
+ return nil
+ })
+
+ response := &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: 10,
+ TargetNodeEpoch: 11,
+ RequestSeq: 12,
+ LeaseDurationMs: 5000,
+ }
+ require.NoError(t, sender.SendCommand(
+ NewSingleTargetMessage(receiver.id, MaintainerManagerTopic, response),
+ ))
+
+ select {
+ case message := <-received:
+ require.Equal(t, sender.id, message.From)
+ require.Equal(t, receiver.id, message.To)
+ require.Equal(t, TypeNodeHeartbeatResponse, message.Type)
+ require.Equal(t, response, message.Message[0])
+ case <-time.After(10 * time.Second):
+ t.Fatal("timed out waiting for node heartbeat response")
+ }
+}
+
+func TestNodeHeartbeatResponseIOTypeRoundTrip(t *testing.T) {
+ response := &heartbeatpb.NodeHeartbeatResponse{
+ CoordinatorVersion: 10,
+ TargetNodeEpoch: 11,
+ RequestSeq: 12,
+ LeaseDurationMs: 5000,
+ WitnessChallenge: &heartbeatpb.WriteLeaseWitnessChallenge{
+ CoordinatorVersion: 10,
+ CoordinatorNodeEpoch: 11,
+ SelfRequestSeq: 12,
+ WitnessNodeEpoch: 13,
+ Nonce: []byte("nonce"),
+ },
+ }
+ message := NewSingleTargetMessage(node.ID("capture"), MaintainerManagerTopic, response)
+ require.Equal(t, TypeNodeHeartbeatResponse, message.Type)
+
+ data, err := response.Marshal()
+ require.NoError(t, err)
+ decoded, err := decodeIOType(TypeNodeHeartbeatResponse, data)
+ require.NoError(t, err)
+ require.Equal(t, response, decoded)
+}
diff --git a/pkg/metrics/init.go b/pkg/metrics/init.go
index 67d73c86bc..79e5cedc68 100644
--- a/pkg/metrics/init.go
+++ b/pkg/metrics/init.go
@@ -39,6 +39,7 @@ func InitMetrics(registry *prometheus.Registry) {
initLogPullerMetrics(registry)
common.InitCommonMetrics(registry)
initDynamicStreamMetrics(registry)
+ initCaptureWriteLeaseMetrics(registry)
kafka.InitMetrics(registry)
gc.InitMetrics(registry)
diff --git a/pkg/metrics/write_lease.go b/pkg/metrics/write_lease.go
new file mode 100644
index 0000000000..27187de944
--- /dev/null
+++ b/pkg/metrics/write_lease.go
@@ -0,0 +1,102 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+ CaptureWriteGateState = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_write_gate_state",
+ Help: "Whether the capture write gate is in the labeled state.",
+ }, []string{"state"})
+ CaptureP2PLeaseRemainingSeconds = prometheus.NewGauge(
+ prometheus.GaugeOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_p2p_lease_remaining_seconds",
+ Help: "Remaining lifetime of the capture P2P write lease.",
+ })
+ CaptureEtcdProofRemainingSeconds = prometheus.NewGauge(
+ prometheus.GaugeOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_etcd_proof_remaining_seconds",
+ Help: "Remaining lifetime of the capture etcd write proof.",
+ })
+ CaptureWriteBlockCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_write_block_total",
+ Help: "Number of capture write gate transitions from writable to blocked.",
+ }, []string{"reason"})
+ CaptureLastWriteAdmissionTimestamp = prometheus.NewGauge(
+ prometheus.GaugeOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_last_write_admission_timestamp_seconds",
+ Help: "Unix timestamp of the most recent downstream write admitted by this capture.",
+ })
+ CaptureLeaseResponseRejectedCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_lease_response_rejected_total",
+ Help: "Number of rejected P2P write lease responses.",
+ }, []string{"reason"})
+ CaptureLeaseHeartbeatCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "ticdc",
+ Subsystem: "coordinator",
+ Name: "capture_lease_heartbeat_total",
+ Help: "Number of capture write lease heartbeats by handling result.",
+ }, []string{"result"})
+ CaptureLeaseResponseCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_lease_response_total",
+ Help: "Number of capture write lease responses by handling result.",
+ }, []string{"result"})
+ CaptureSafeToRescheduleDelaySeconds = prometheus.NewGauge(
+ prometheus.GaugeOpts{
+ Namespace: "ticdc",
+ Subsystem: "server",
+ Name: "capture_safe_to_reschedule_delay_seconds",
+ Help: "Delay after capture lease-key deletion before removal is published.",
+ })
+ CaptureP2PWitnessAvailable = prometheus.NewGauge(
+ prometheus.GaugeOpts{
+ Namespace: "ticdc",
+ Subsystem: "coordinator",
+ Name: "capture_p2p_witness_available",
+ Help: "Whether a remote capture is available to witness the coordinator capture.",
+ })
+)
+
+func initCaptureWriteLeaseMetrics(registry *prometheus.Registry) {
+ registry.MustRegister(CaptureWriteGateState)
+ registry.MustRegister(CaptureP2PLeaseRemainingSeconds)
+ registry.MustRegister(CaptureEtcdProofRemainingSeconds)
+ registry.MustRegister(CaptureWriteBlockCounter)
+ registry.MustRegister(CaptureLastWriteAdmissionTimestamp)
+ registry.MustRegister(CaptureLeaseResponseRejectedCounter)
+ registry.MustRegister(CaptureLeaseHeartbeatCounter)
+ registry.MustRegister(CaptureLeaseResponseCounter)
+ registry.MustRegister(CaptureSafeToRescheduleDelaySeconds)
+ registry.MustRegister(CaptureP2PWitnessAvailable)
+}
diff --git a/pkg/orchestrator/reactor_state.go b/pkg/orchestrator/reactor_state.go
index b2ec901dbf..33b00b9c16 100644
--- a/pkg/orchestrator/reactor_state.go
+++ b/pkg/orchestrator/reactor_state.go
@@ -28,7 +28,7 @@ import (
"go.uber.org/zap"
)
-const defaultCaptureRemoveTTL = 5
+const defaultCaptureRemoveTTL = 10
// GlobalReactorState represents a global state which stores all key-value pairs in ETCD
type GlobalReactorState struct {
@@ -66,6 +66,12 @@ func NewGlobalState(clusterID string, captureSessionTTL int) *GlobalReactorState
}
}
+// CaptureRemoveTTLSeconds returns the delay between observing capture-key
+// deletion and publishing the capture removal to schedulers.
+func (s *GlobalReactorState) CaptureRemoveTTLSeconds() int {
+ return s.captureRemoveTTL
+}
+
// NewGlobalStateForTest creates a new global state for test.
func NewGlobalStateForTest(clusterID string) *GlobalReactorState {
return NewGlobalState(clusterID, 0)
diff --git a/pkg/orchestrator/reactor_state_capture_test.go b/pkg/orchestrator/reactor_state_capture_test.go
index 032368e7f7..931c174df0 100644
--- a/pkg/orchestrator/reactor_state_capture_test.go
+++ b/pkg/orchestrator/reactor_state_capture_test.go
@@ -23,6 +23,14 @@ import (
"github.com/stretchr/testify/require"
)
+func TestGlobalReactorStateCaptureRemoveTTL(t *testing.T) {
+ t.Parallel()
+
+ require.Equal(t, 10, NewGlobalState(etcd.DefaultCDCClusterID, 0).captureRemoveTTL)
+ require.Equal(t, 10, NewGlobalState(etcd.DefaultCDCClusterID, 10).captureRemoveTTL)
+ require.Equal(t, 15, NewGlobalState(etcd.DefaultCDCClusterID, 30).captureRemoveTTL)
+}
+
func TestGlobalReactorStateKeepsCaptureAfterReRegister(t *testing.T) {
t.Parallel()
diff --git a/pkg/redo/writer/blackhole/writer.go b/pkg/redo/writer/blackhole/writer.go
index ec276ac9f2..625f72bc0a 100644
--- a/pkg/redo/writer/blackhole/writer.go
+++ b/pkg/redo/writer/blackhole/writer.go
@@ -20,6 +20,7 @@ import (
"github.com/pingcap/log"
"github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/redo/writer"
+ "github.com/pingcap/ticdc/pkg/writelease"
"go.uber.org/zap"
)
@@ -82,6 +83,9 @@ func (bs *blackHoleDMLWriter) AddDMLEvents(_ context.Context, events ...*event.R
return
}
+func (bs *blackHoleDMLWriter) SetWriteGate(_ *writelease.Gate) {
+}
+
func (bs *blackHoleDMLWriter) Close() error {
return nil
}
@@ -100,6 +104,9 @@ func (bs *blackHoleDDLWriter) WriteDDLEvent(_ context.Context, event *event.DDLE
func (bs *blackHoleDDLWriter) SetTableSchemaStore(_ *event.TableSchemaStore) {
}
+func (bs *blackHoleDDLWriter) SetWriteGate(_ *writelease.Gate) {
+}
+
func (bs *blackHoleDDLWriter) Close() error {
return nil
}
diff --git a/pkg/redo/writer/file/file.go b/pkg/redo/writer/file/file.go
index 759d429337..86ea4f7cb1 100644
--- a/pkg/redo/writer/file/file.go
+++ b/pkg/redo/writer/file/file.go
@@ -34,6 +34,7 @@ import (
"github.com/pingcap/ticdc/pkg/redo/codec"
"github.com/pingcap/ticdc/pkg/redo/writer"
"github.com/pingcap/ticdc/pkg/uuid"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/prometheus/client_golang/prometheus"
"github.com/uber-go/atomic"
@@ -45,11 +46,12 @@ import (
type fileWriter interface {
Run(ctx context.Context) error
IsRunning() bool
- SyncWrite(event writer.RedoEvent) error
+ SyncWrite(ctx context.Context, event writer.RedoEvent) error
GetInputCh() chan writer.RedoEvent
Flush() error
Close() error
SetTableSchemaStore(*commonEvent.TableSchemaStore)
+ SetWriteGate(*writelease.Gate)
}
type fileWriterConfig interface {
@@ -125,6 +127,7 @@ type Writer struct {
metricFlushAllDuration prometheus.Observer
metricWriteBytes prometheus.Gauge
tableSchemaStore *commonEvent.TableSchemaStore
+ writeGate *writelease.Gate
}
func newWriter(
@@ -211,6 +214,10 @@ func (w *Writer) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaSt
w.tableSchemaStore = tableSchemaStore
}
+func (w *Writer) SetWriteGate(gate *writelease.Gate) {
+ w.writeGate = gate
+}
+
func (w *Writer) Run(ctx context.Context) error {
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
@@ -222,6 +229,14 @@ func (w *Writer) Run(ctx context.Context) error {
// Write implement write interface
// TODO: more general api with fileName generated by caller
func (w *Writer) Write(rawData []byte) (int, error) {
+ ctx := context.Background()
+ if err := writelease.WaitForWrite(ctx, w.writeGate); err != nil {
+ return 0, err
+ }
+ return w.writeRaw(ctx, rawData)
+}
+
+func (w *Writer) writeRaw(ctx context.Context, rawData []byte) (int, error) {
w.Lock()
defer w.Unlock()
@@ -237,7 +252,7 @@ func (w *Writer) Write(rawData []byte) (int, error) {
}
if w.size+writeLen > w.cfg.MaxLogSizeInBytes() {
- if err := w.rotate(); err != nil {
+ if err := w.rotate(ctx); err != nil {
return 0, err
}
}
@@ -306,6 +321,11 @@ func (w *Writer) Close() error {
DeleteLabelValues(w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), w.logType)
metrics.RedoWriteBytesGauge.
DeleteLabelValues(w.cfg.ChangeFeedID().Keyspace(), w.cfg.ChangeFeedID().Name(), w.logType)
+ if w.file != nil && !writelease.CanWrite(w.writeGate) {
+ err := w.file.Close()
+ w.file = nil
+ return errors.WrapError(errors.ErrRedoFileOp, err)
+ }
ctx, cancel := context.WithTimeout(context.Background(), redo.CloseTimeout)
defer cancel()
@@ -321,7 +341,7 @@ func (w *Writer) GetInputCh() chan writer.RedoEvent {
return w.inputCh
}
-func (w *Writer) write(event writer.RedoEvent) error {
+func (w *Writer) writeEvent(ctx context.Context, event writer.RedoEvent) error {
rl := event.ToRedoLog()
if rl.Type == commonEvent.RedoLogTypeDDL {
rl.RedoDDL.SetTableSchemaStore(w.tableSchemaStore)
@@ -331,19 +351,19 @@ func (w *Writer) write(event writer.RedoEvent) error {
return errors.WrapError(errors.ErrMarshalFailed, err)
}
w.AdvanceTs(rl.GetCommitTs())
- _, err = w.Write(data)
+ _, err = w.writeRaw(ctx, data)
if err != nil {
return err
}
return nil
}
-func (w *Writer) SyncWrite(event writer.RedoEvent) error {
- err := w.write(event)
+func (w *Writer) SyncWrite(ctx context.Context, event writer.RedoEvent) error {
+ err := w.writeEvent(ctx, event)
if err != nil {
return err
}
- err = w.Flush()
+ err = w.flushWithContext(ctx)
if err != nil {
return errors.Trace(err)
}
@@ -358,7 +378,7 @@ func (w *Writer) encode(ctx context.Context) error {
num := 0
cacheEventPostFlush := make([]func(), 0, redo.DefaultFlushBatchSize)
flush := func() error {
- err := w.Flush()
+ err := w.flushWithContext(ctx)
if err != nil {
return err
}
@@ -379,7 +399,10 @@ func (w *Writer) encode(ctx context.Context) error {
return errors.Trace(err)
}
case e := <-w.inputCh:
- err := w.write(e)
+ if err := writelease.WaitForWrite(ctx, w.writeGate); err != nil {
+ return err
+ }
+ err := w.writeEvent(ctx, e)
if err != nil {
return err
}
@@ -405,6 +428,9 @@ func (w *Writer) close(ctx context.Context) error {
if err := w.flush(); err != nil {
return err
}
+ if err := writelease.WaitForWrite(ctx, w.writeGate); err != nil {
+ return err
+ }
if w.cfg.UseExternalStorage() {
off, err := w.file.Seek(0, io.SeekCurrent)
@@ -445,6 +471,9 @@ func (w *Writer) close(ctx context.Context) error {
// We only write content to S3 before closing the local file.
// By this way, we no longer need renaming object in S3.
if w.cfg.UseExternalStorage() {
+ if err := writelease.WaitForWrite(ctx, w.writeGate); err != nil {
+ return err
+ }
err = w.writeToS3(ctx, w.ongoingFilePath)
if err != nil {
w.file.Close()
@@ -531,9 +560,7 @@ func (w *Writer) newPageWriter() error {
return nil
}
-func (w *Writer) rotate() error {
- ctx, cancel := context.WithTimeout(context.Background(), redo.DefaultTimeout)
- defer cancel()
+func (w *Writer) rotate(ctx context.Context) error {
if err := w.close(ctx); err != nil {
return err
}
@@ -541,10 +568,15 @@ func (w *Writer) rotate() error {
}
// flushAndRotateFile flushes the file to disk and rotate it if S3 storage is used.
-func (w *Writer) flushAndRotateFile() error {
+func (w *Writer) flushAndRotateFile(ctx context.Context) error {
if w.file == nil {
return nil
}
+ if !w.cfg.UseExternalStorage() && w.size != 0 {
+ if err := writelease.WaitForWrite(ctx, w.writeGate); err != nil {
+ return err
+ }
+ }
start := time.Now()
err := w.flush()
@@ -563,7 +595,7 @@ func (w *Writer) flushAndRotateFile() error {
// for s3 storage, when the file is flushed to disk, we need an immediate
// file rotate. Otherwise, the existing file content would be repeatedly written to S3,
// which could cause considerable network bandwidth waste.
- err = w.rotate()
+ err = w.rotate(ctx)
if err != nil {
return err
}
@@ -574,10 +606,14 @@ func (w *Writer) flushAndRotateFile() error {
// Flush implement Flush interface
func (w *Writer) Flush() error {
+ return w.flushWithContext(context.Background())
+}
+
+func (w *Writer) flushWithContext(ctx context.Context) error {
w.Lock()
defer w.Unlock()
- return w.flushAndRotateFile()
+ return w.flushAndRotateFile(ctx)
}
func (w *Writer) flush() error {
diff --git a/pkg/redo/writer/file/file_log_writer.go b/pkg/redo/writer/file/file_log_writer.go
index 1fedb9b0c8..5f59f4789e 100644
--- a/pkg/redo/writer/file/file_log_writer.go
+++ b/pkg/redo/writer/file/file_log_writer.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/redo"
"github.com/pingcap/ticdc/pkg/redo/writer"
+ "github.com/pingcap/ticdc/pkg/writelease"
"go.uber.org/zap"
)
@@ -32,6 +33,7 @@ var (
type logWriter struct {
cfg *writer.Config
backendWriter fileWriter
+ writeGate *writelease.Gate
}
type dmlWriter struct {
@@ -78,6 +80,11 @@ func (l *logWriter) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchem
l.backendWriter.SetTableSchemaStore(tableSchemaStore)
}
+func (l *logWriter) SetWriteGate(gate *writelease.Gate) {
+ l.writeGate = gate
+ l.backendWriter.SetWriteGate(gate)
+}
+
func (l *logWriter) Run(ctx context.Context) error {
return l.backendWriter.Run(ctx)
}
@@ -117,7 +124,10 @@ func (l *ddlWriter) WriteDDLEvent(ctx context.Context, event *commonEvent.DDLEve
zap.String("capture", l.cfg.CaptureID()))
return nil
}
- if err := l.backendWriter.SyncWrite(event); err != nil {
+ if err := writelease.WaitForWrite(ctx, l.writeGate); err != nil {
+ return err
+ }
+ if err := l.backendWriter.SyncWrite(ctx, event); err != nil {
return errors.Trace(err)
}
return nil
diff --git a/pkg/redo/writer/file/file_log_writer_test.go b/pkg/redo/writer/file/file_log_writer_test.go
index a83dc5a2dc..74402585ee 100644
--- a/pkg/redo/writer/file/file_log_writer_test.go
+++ b/pkg/redo/writer/file/file_log_writer_test.go
@@ -78,7 +78,7 @@ func TestLogWriterWriteDDL(t *testing.T) {
for _, tt := range tests {
mockWriter := &mockFileWriter{}
mockWriter.On("IsRunning").Return(tt.isRunning)
- mockWriter.On("SyncWrite", mock.Anything).Return(tt.writerErr)
+ mockWriter.On("SyncWrite", mock.Anything, mock.Anything).Return(tt.writerErr)
w := ddlWriter{logWriter: &logWriter{
cfg: newTestWriterConfig(t, common.ChangeFeedID{}, nil),
backendWriter: mockWriter,
diff --git a/pkg/redo/writer/file/file_mock.go b/pkg/redo/writer/file/file_mock.go
index 40679247ce..ec7856f14e 100644
--- a/pkg/redo/writer/file/file_mock.go
+++ b/pkg/redo/writer/file/file_mock.go
@@ -8,6 +8,8 @@ import (
event "github.com/pingcap/ticdc/pkg/common/event"
mock "github.com/stretchr/testify/mock"
+ writelease "github.com/pingcap/ticdc/pkg/writelease"
+
writer "github.com/pingcap/ticdc/pkg/redo/writer"
)
@@ -113,17 +115,22 @@ func (_m *mockFileWriter) SetTableSchemaStore(_a0 *event.TableSchemaStore) {
_m.Called(_a0)
}
-// SyncWrite provides a mock function with given fields: _a0
-func (_m *mockFileWriter) SyncWrite(_a0 writer.RedoEvent) error {
- ret := _m.Called(_a0)
+// SetWriteGate provides a mock function with given fields: _a0
+func (_m *mockFileWriter) SetWriteGate(_a0 *writelease.Gate) {
+ _m.Called(_a0)
+}
+
+// SyncWrite provides a mock function with given fields: ctx, _a1
+func (_m *mockFileWriter) SyncWrite(ctx context.Context, _a1 writer.RedoEvent) error {
+ ret := _m.Called(ctx, _a1)
if len(ret) == 0 {
panic("no return value specified for SyncWrite")
}
var r0 error
- if rf, ok := ret.Get(0).(func(writer.RedoEvent) error); ok {
- r0 = rf(_a0)
+ if rf, ok := ret.Get(0).(func(context.Context, writer.RedoEvent) error); ok {
+ r0 = rf(ctx, _a1)
} else {
r0 = ret.Error(0)
}
diff --git a/pkg/redo/writer/file/file_test.go b/pkg/redo/writer/file/file_test.go
index 02f890ec47..6e569c5686 100644
--- a/pkg/redo/writer/file/file_test.go
+++ b/pkg/redo/writer/file/file_test.go
@@ -31,6 +31,7 @@ import (
"github.com/pingcap/ticdc/pkg/redo/writer"
"github.com/pingcap/ticdc/pkg/util"
"github.com/pingcap/ticdc/pkg/uuid"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/mockobjstore"
"github.com/stretchr/testify/require"
"github.com/uber-go/atomic"
@@ -283,6 +284,68 @@ func TestNewWriter(t *testing.T) {
require.Equal(t, w.running.Load(), false)
}
+func TestWriterFlushWaitsForWriteGate(t *testing.T) {
+ dir := t.TempDir()
+ writerCfg := newTestWriterConfig(
+ t,
+ common.NewChangeFeedIDWithName(t.Name(), common.DefaultKeyspaceName),
+ &config.ConsistentConfig{Storage: util.AddressOf("file://" + dir)},
+ )
+ w, err := NewFileWriter(t.Context(), writerCfg, redo.RedoRowLogFileType)
+ require.NoError(t, err)
+ w.AdvanceTs(1)
+ _, err = w.Write([]byte("redo-event"))
+ require.NoError(t, err)
+ gate := writelease.NewGate()
+ w.SetWriteGate(gate)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- w.flushWithContext(t.Context())
+ }()
+
+ select {
+ case err := <-done:
+ t.Fatalf("redo file flush returned while the capture write gate was closed: %v", err)
+ case <-time.After(100 * time.Millisecond):
+ }
+ files, err := filepath.Glob(filepath.Join(dir, "*"+redo.LogEXT))
+ require.NoError(t, err)
+ require.Empty(t, files)
+
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(5 * time.Second):
+ t.Fatal("redo file flush did not resume after the capture write gate reopened")
+ }
+ files, err = filepath.Glob(filepath.Join(dir, "*"+redo.LogEXT))
+ require.NoError(t, err)
+ require.NotEmpty(t, files)
+ require.NoError(t, w.Close())
+}
+
+func TestWriterCloseDoesNotPublishWithClosedWriteGate(t *testing.T) {
+ dir := t.TempDir()
+ writerCfg := newTestWriterConfig(
+ t,
+ common.NewChangeFeedIDWithName(t.Name(), common.DefaultKeyspaceName),
+ &config.ConsistentConfig{Storage: util.AddressOf("file://" + dir)},
+ )
+ w, err := NewFileWriter(t.Context(), writerCfg, redo.RedoRowLogFileType)
+ require.NoError(t, err)
+ w.AdvanceTs(1)
+ _, err = w.Write([]byte("redo-event"))
+ require.NoError(t, err)
+ w.SetWriteGate(writelease.NewGate())
+
+ require.NoError(t, w.Close())
+ files, err := filepath.Glob(filepath.Join(dir, "*"+redo.LogEXT))
+ require.NoError(t, err)
+ require.Empty(t, files)
+}
+
func TestNewLocalFileWriterKeepsLocalOnlySemantics(t *testing.T) {
t.Parallel()
@@ -359,13 +422,13 @@ func TestRotateFileWithFileAllocator(t *testing.T) {
_, err := w.Write([]byte("test"))
require.Nil(t, err)
- err = w.rotate()
+ err = w.rotate(context.Background())
require.Nil(t, err)
w.AdvanceTs(100)
_, err = w.Write([]byte("test"))
require.Nil(t, err)
- err = w.rotate()
+ err = w.rotate(context.Background())
require.Nil(t, err)
w.Close()
@@ -424,13 +487,13 @@ func TestRotateFileWithoutFileAllocator(t *testing.T) {
_, err := w.Write([]byte("test"))
require.Nil(t, err)
- err = w.rotate()
+ err = w.rotate(context.Background())
require.Nil(t, err)
w.AdvanceTs(100)
_, err = w.Write([]byte("test"))
require.Nil(t, err)
- err = w.rotate()
+ err = w.rotate(context.Background())
require.Nil(t, err)
w.Close()
diff --git a/pkg/redo/writer/memory/ddl_writer.go b/pkg/redo/writer/memory/ddl_writer.go
index e21c0ebbfe..b27dc979d8 100644
--- a/pkg/redo/writer/memory/ddl_writer.go
+++ b/pkg/redo/writer/memory/ddl_writer.go
@@ -32,6 +32,7 @@ import (
"github.com/pingcap/ticdc/pkg/redo/codec"
"github.com/pingcap/ticdc/pkg/redo/writer"
"github.com/pingcap/ticdc/pkg/uuid"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
@@ -51,6 +52,7 @@ type ddlWriter struct {
closed bool
writeMetric prometheus.Gauge
flushMetric prometheus.Observer
+ writeGate *writelease.Gate
}
// NewDDLWriter creates a new memory DDL writer.
@@ -90,6 +92,10 @@ func (l *ddlWriter) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchem
l.tableSchema = tableSchemaStore
}
+func (l *ddlWriter) SetWriteGate(gate *writelease.Gate) {
+ l.writeGate = gate
+}
+
func (l *ddlWriter) WriteDDLEvent(ctx context.Context, event *commonEvent.DDLEvent) error {
if event == nil {
log.Warn("writing nil event to redo log, ignore this",
@@ -134,6 +140,9 @@ func (l *ddlWriter) write(ctx context.Context, event *polymorphicRedoEvent) erro
return errors.ErrRedoFileSizeExceed.GenWithStackByArgs(writeLen, l.cfg.MaxLogSizeInBytes())
}
defer l.writeMetric.Add(float64(writeLen))
+ if err := writelease.WaitForWrite(ctx, l.writeGate); err != nil {
+ return err
+ }
start := time.Now()
data, err := l.prepareWriteData(event.data)
@@ -141,6 +150,9 @@ func (l *ddlWriter) write(ctx context.Context, event *polymorphicRedoEvent) erro
return err
}
fileName := l.getLogFileName(event.commitTs)
+ if err := writelease.WaitForWrite(ctx, l.writeGate); err != nil {
+ return err
+ }
if l.cfg.FlushConcurrency() <= 1 {
err = l.extStorage.WriteFile(ctx, fileName, data)
} else {
diff --git a/pkg/redo/writer/memory/ddl_writer_test.go b/pkg/redo/writer/memory/ddl_writer_test.go
index f80308c97c..6fd38f0b24 100644
--- a/pkg/redo/writer/memory/ddl_writer_test.go
+++ b/pkg/redo/writer/memory/ddl_writer_test.go
@@ -16,12 +16,14 @@ package memory
import (
"context"
"testing"
+ "time"
"github.com/pingcap/ticdc/pkg/common"
pevent "github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/redo/testutil"
"github.com/pingcap/ticdc/pkg/redo/writer"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/stretchr/testify/require"
)
@@ -64,3 +66,40 @@ func TestWriteDDL(t *testing.T) {
require.NoError(t, lw.Close())
require.NoError(t, lw.Close())
}
+
+func TestWriteDDLWaitsForWriteGate(t *testing.T) {
+ ctx := t.Context()
+
+ extStorage, uri, err := util.GetTestExtStorage(ctx, t.TempDir())
+ require.NoError(t, err)
+ cfg, err := writer.NewConfig(
+ common.NewChangeFeedIDWithName("test-changefeed", common.DefaultKeyspaceName),
+ testutil.NewConsistentConfig(uri.String()),
+ )
+ require.NoError(t, err)
+
+ const filename = "gated-ddl.log"
+ lw, err := NewDDLWriter(ctx, cfg, writer.WithLogFileName(func() string {
+ return filename
+ }))
+ require.NoError(t, err)
+ defer func() { require.NoError(t, lw.Close()) }()
+ gate := writelease.NewGate()
+ lw.SetWriteGate(gate)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- lw.WriteDDLEvent(ctx, &pevent.DDLEvent{FinishedTs: 1})
+ }()
+
+ time.Sleep(100 * time.Millisecond)
+ exists, err := extStorage.FileExists(ctx, filename)
+ require.NoError(t, err)
+ require.False(t, exists)
+
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ require.NoError(t, <-done)
+ exists, err = extStorage.FileExists(ctx, filename)
+ require.NoError(t, err)
+ require.True(t, exists)
+}
diff --git a/pkg/redo/writer/memory/dml_writer.go b/pkg/redo/writer/memory/dml_writer.go
index 649126f7b5..c1aa20e37b 100644
--- a/pkg/redo/writer/memory/dml_writer.go
+++ b/pkg/redo/writer/memory/dml_writer.go
@@ -20,6 +20,7 @@ import (
commonEvent "github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/redo"
"github.com/pingcap/ticdc/pkg/redo/writer"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
@@ -85,6 +86,10 @@ func (l *dmlWriter) AddDMLEvents(ctx context.Context, events ...*commonEvent.Red
return nil
}
+func (l *dmlWriter) SetWriteGate(gate *writelease.Gate) {
+ l.fileWorkers.setWriteGate(gate)
+}
+
func (l *dmlWriter) Close() error {
if l.cancel != nil {
l.cancel()
diff --git a/pkg/redo/writer/memory/file_worker.go b/pkg/redo/writer/memory/file_worker.go
index ba67651544..4241068710 100644
--- a/pkg/redo/writer/memory/file_worker.go
+++ b/pkg/redo/writer/memory/file_worker.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/ticdc/pkg/redo"
"github.com/pingcap/ticdc/pkg/redo/writer"
"github.com/pingcap/ticdc/pkg/uuid"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
@@ -97,6 +98,11 @@ type fileWorkerGroup struct {
metricWriteBytes prometheus.Gauge
metricFlushAllDuration prometheus.Observer
+ writeGate *writelease.Gate
+}
+
+func (f *fileWorkerGroup) setWriteGate(gate *writelease.Gate) {
+ f.writeGate = gate
}
// newFileWorkerGroup creates a DML fileWorkerGroup.
@@ -260,11 +266,17 @@ func (f *fileWorkerGroup) bgWriteLogs(
func (f *fileWorkerGroup) syncWriteFile(egCtx context.Context, file *fileCache) error {
var err error
+ if err = writelease.WaitForWrite(egCtx, f.writeGate); err != nil {
+ return err
+ }
start := time.Now()
file.filename = f.getLogFileName(file.maxCommitTs)
if err = file.writer.Close(); err != nil {
return err
}
+ if err = writelease.WaitForWrite(egCtx, f.writeGate); err != nil {
+ return err
+ }
if f.cfg.FlushConcurrency() <= 1 {
err = f.extStorage.WriteFile(egCtx, file.filename, file.writer.buf.Bytes())
} else {
diff --git a/pkg/redo/writer/writer.go b/pkg/redo/writer/writer.go
index c349531565..63414031bc 100644
--- a/pkg/redo/writer/writer.go
+++ b/pkg/redo/writer/writer.go
@@ -18,6 +18,7 @@ import (
commonEvent "github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/uuid"
+ "github.com/pingcap/ticdc/pkg/writelease"
)
var (
@@ -34,6 +35,7 @@ type RedoEvent interface {
// RedoDMLWriter writes row redo events, all operations are thread-safe.
type RedoDMLWriter interface {
AddDMLEvents(ctx context.Context, events ...*commonEvent.RedoRowEvent) error
+ SetWriteGate(gate *writelease.Gate)
Run(ctx context.Context) error
Close() error
}
@@ -41,6 +43,7 @@ type RedoDMLWriter interface {
// RedoDDLWriter writes DDL redo events, all operations are thread-safe.
type RedoDDLWriter interface {
WriteDDLEvent(ctx context.Context, event *commonEvent.DDLEvent) error
+ SetWriteGate(gate *writelease.Gate)
Close() error
SetTableSchemaStore(*commonEvent.TableSchemaStore)
}
diff --git a/pkg/redo/writer/writer_mock.go b/pkg/redo/writer/writer_mock.go
index c05c56f3de..94180e0859 100644
--- a/pkg/redo/writer/writer_mock.go
+++ b/pkg/redo/writer/writer_mock.go
@@ -10,6 +10,7 @@ import (
gomock "github.com/golang/mock/gomock"
event "github.com/pingcap/ticdc/pkg/common/event"
+ writelease "github.com/pingcap/ticdc/pkg/writelease"
)
// MockRedoEvent is a mock of RedoEvent interface.
@@ -131,6 +132,18 @@ func (mr *MockRedoDMLWriterMockRecorder) Run(ctx interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockRedoDMLWriter)(nil).Run), ctx)
}
+// SetWriteGate mocks base method.
+func (m *MockRedoDMLWriter) SetWriteGate(gate *writelease.Gate) {
+ m.ctrl.T.Helper()
+ m.ctrl.Call(m, "SetWriteGate", gate)
+}
+
+// SetWriteGate indicates an expected call of SetWriteGate.
+func (mr *MockRedoDMLWriterMockRecorder) SetWriteGate(gate interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWriteGate", reflect.TypeOf((*MockRedoDMLWriter)(nil).SetWriteGate), gate)
+}
+
// MockRedoDDLWriter is a mock of RedoDDLWriter interface.
type MockRedoDDLWriter struct {
ctrl *gomock.Controller
@@ -180,6 +193,18 @@ func (mr *MockRedoDDLWriterMockRecorder) SetTableSchemaStore(arg0 interface{}) *
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTableSchemaStore", reflect.TypeOf((*MockRedoDDLWriter)(nil).SetTableSchemaStore), arg0)
}
+// SetWriteGate mocks base method.
+func (m *MockRedoDDLWriter) SetWriteGate(gate *writelease.Gate) {
+ m.ctrl.T.Helper()
+ m.ctrl.Call(m, "SetWriteGate", gate)
+}
+
+// SetWriteGate indicates an expected call of SetWriteGate.
+func (mr *MockRedoDDLWriterMockRecorder) SetWriteGate(gate interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWriteGate", reflect.TypeOf((*MockRedoDDLWriter)(nil).SetWriteGate), gate)
+}
+
// WriteDDLEvent mocks base method.
func (m *MockRedoDDLWriter) WriteDDLEvent(ctx context.Context, event *event.DDLEvent) error {
m.ctrl.T.Helper()
diff --git a/pkg/sink/kafka/claimcheck/claim_check.go b/pkg/sink/kafka/claimcheck/claim_check.go
index 21a53b58c3..fc18664e3f 100644
--- a/pkg/sink/kafka/claimcheck/claim_check.go
+++ b/pkg/sink/kafka/claimcheck/claim_check.go
@@ -26,6 +26,7 @@ import (
"github.com/pingcap/ticdc/pkg/errors"
codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common"
"github.com/pingcap/ticdc/pkg/util"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore/storeapi"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
@@ -41,6 +42,7 @@ type ClaimCheck struct {
// cost on send messages to the claim check external storage.
metricSendMessageDuration prometheus.Observer
metricSendMessageCount prometheus.Counter
+ writeGate *writelease.Gate
}
// New return a new ClaimCheck.
@@ -82,6 +84,9 @@ func (c *ClaimCheck) WriteMessage(ctx context.Context, key, value []byte, fileNa
return errors.WrapError(errors.ErrMarshalFailed, err)
}
}
+ if err := writelease.WaitForWrite(ctx, c.writeGate); err != nil {
+ return err
+ }
start := time.Now()
err = c.storage.WriteFile(ctx, fileName, value)
if err != nil {
@@ -92,6 +97,12 @@ func (c *ClaimCheck) WriteMessage(ctx context.Context, key, value []byte, fileNa
return nil
}
+// SetWriteGate installs capture-wide write admission for claim-check object
+// publication.
+func (c *ClaimCheck) SetWriteGate(gate *writelease.Gate) {
+ c.writeGate = gate
+}
+
// FileNameWithPrefix returns the file name with prefix, the full path.
func (c *ClaimCheck) FileNameWithPrefix(fileName string) string {
return strings.TrimSuffix(c.storage.URI(), "/") + "/" + fileName
diff --git a/pkg/sink/kafka/claimcheck/claim_check_test.go b/pkg/sink/kafka/claimcheck/claim_check_test.go
index 080e62de0f..26647bd63a 100644
--- a/pkg/sink/kafka/claimcheck/claim_check_test.go
+++ b/pkg/sink/kafka/claimcheck/claim_check_test.go
@@ -18,10 +18,12 @@ import (
"fmt"
"strings"
"testing"
+ "time"
"github.com/pingcap/ticdc/pkg/common"
"github.com/pingcap/ticdc/pkg/config"
"github.com/pingcap/ticdc/pkg/errors"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/pkg/objstore"
"github.com/pingcap/tidb/pkg/objstore/mockobjstore"
"github.com/stretchr/testify/require"
@@ -108,3 +110,35 @@ func TestClaimCheckConcurrentWrites(t *testing.T) {
require.Equal(t, fileName, string(data))
}
}
+
+func TestClaimCheckWriteGateBlocksObjectPublication(t *testing.T) {
+ ctx := t.Context()
+ storage := objstore.NewMemStorage()
+ changefeedID := common.NewChangeFeedIDWithName("test", "default")
+ claimCheck := &ClaimCheck{
+ storage: storage,
+ rawValue: true,
+ changefeedID: changefeedID,
+ metricSendMessageDuration: claimCheckSendMessageDuration.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name()),
+ metricSendMessageCount: claimCheckSendMessageCount.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name()),
+ }
+ t.Cleanup(claimCheck.Close)
+ gate := writelease.NewGate()
+ claimCheck.SetWriteGate(gate)
+
+ done := make(chan error, 1)
+ go func() {
+ done <- claimCheck.WriteMessage(ctx, nil, []byte("large-message"), "message.json")
+ }()
+
+ time.Sleep(100 * time.Millisecond)
+ exists, err := storage.FileExists(ctx, "message.json")
+ require.NoError(t, err)
+ require.False(t, exists)
+
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ require.NoError(t, <-done)
+ exists, err = storage.FileExists(ctx, "message.json")
+ require.NoError(t, err)
+ require.True(t, exists)
+}
diff --git a/pkg/sink/mysql/mysql_writer.go b/pkg/sink/mysql/mysql_writer.go
index 6f65a2dd05..eab457d6c9 100644
--- a/pkg/sink/mysql/mysql_writer.go
+++ b/pkg/sink/mysql/mysql_writer.go
@@ -26,6 +26,7 @@ import (
commonEvent "github.com/pingcap/ticdc/pkg/common/event"
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/metrics"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
)
@@ -90,6 +91,8 @@ type Writer struct {
// for dry-run mode
blockerTicker *time.Ticker
+
+ writeGate *writelease.Gate
}
func NewWriter(
@@ -138,6 +141,30 @@ func (w *Writer) SetTableSchemaStore(tableSchemaStore *commonEvent.TableSchemaSt
w.tableSchemaStore = tableSchemaStore
}
+// SetWriteGate configures capture-wide DML write admission for this transport
+// writer. A nil gate preserves the legacy behavior.
+func (w *Writer) SetWriteGate(gate *writelease.Gate) {
+ w.writeGate = gate
+}
+
+// grantWrite waits for a valid capture write lease. It returns false only when
+// the writer is shutting down, so callers must not execute the downstream write.
+func (w *Writer) grantWrite() bool {
+ if w.writeGate == nil {
+ metrics.CaptureLastWriteAdmissionTimestamp.SetToCurrentTime()
+ return true
+ }
+ for {
+ if err := w.writeGate.WaitUntilWritable(w.ctx); err != nil {
+ return false
+ }
+ if w.writeGate.IsWritable() {
+ metrics.CaptureLastWriteAdmissionTimestamp.SetToCurrentTime()
+ return true
+ }
+ }
+}
+
// SetControlAsyncDB sets the DB pool used to execute TiDB ADD INDEX DDLs.
func (w *Writer) SetControlAsyncDB(db *sql.DB) {
w.asyncDB = db
diff --git a/pkg/sink/mysql/mysql_writer_dml_exec.go b/pkg/sink/mysql/mysql_writer_dml_exec.go
index c60d43dfa9..7145391295 100644
--- a/pkg/sink/mysql/mysql_writer_dml_exec.go
+++ b/pkg/sink/mysql/mysql_writer_dml_exec.go
@@ -54,42 +54,51 @@ func (w *Writer) execDMLWithMaxRetries(dmls *preparedDMLs) error {
log.Info("Slow Query", zap.Any("sql", dmls.LogWithoutValues()), zap.Any("writerID", w.id))
}
}()
- err := w.dmlSession.withConn(w, writeTimeout, func(conn *sql.Conn) error {
- if fallbackToSeqWay || !w.cfg.MultiStmtEnable {
- // use sequence way to execute the dmls
- tx, err := conn.BeginTx(w.ctx, nil)
- if err != nil {
- return errors.Trace(err)
- }
+ for {
+ if !w.grantWrite() {
+ return 0, 0, errors.Trace(w.ctx.Err())
+ }
+ admitted, err := w.dmlSession.withConn(w, writeTimeout, func() bool {
+ return w.writeGate == nil || w.writeGate.IsWritable()
+ }, func(conn *sql.Conn) error {
+ if fallbackToSeqWay || !w.cfg.MultiStmtEnable {
+ // use sequence way to execute the dmls
+ tx, err := conn.BeginTx(w.ctx, nil)
+ if err != nil {
+ return errors.Trace(err)
+ }
- err = w.sequenceExecute(dmls, tx, writeTimeout)
- if err != nil {
- return err
+ err = w.sequenceExecute(dmls, tx, writeTimeout)
+ if err != nil {
+ return err
+ }
+
+ if err = tx.Commit(); err != nil {
+ return err
+ }
+
+ log.Debug("Exec Rows succeeded", zap.Any("rowCount", dmls.rowCount), zap.Int("writerID", w.id))
+ return nil
}
- if err = tx.Commit(); err != nil {
+ // use multi stmt way to execute the dmls
+ if err := w.multiStmtExecute(conn, dmls, writeTimeout); err != nil {
+ log.Warn("multiStmtExecute failed, fallback to sequence way",
+ zap.Error(err),
+ zap.Any("sql", dmls.LogWithoutValues()),
+ zap.Int("writerID", w.id))
+ fallbackToSeqWay = true
return err
}
-
- log.Debug("Exec Rows succeeded", zap.Any("rowCount", dmls.rowCount), zap.Int("writerID", w.id))
return nil
+ })
+ if err != nil {
+ return 0, 0, err
}
-
- // use multi stmt way to execute the dmls
- if err := w.multiStmtExecute(conn, dmls, writeTimeout); err != nil {
- log.Warn("multiStmtExecute failed, fallback to sequence way",
- zap.Error(err),
- zap.Any("sql", dmls.LogWithoutValues()),
- zap.Int("writerID", w.id))
- fallbackToSeqWay = true
- return err
+ if admitted {
+ return dmls.rowCount, dmls.approximateSize, nil
}
- return nil
- })
- if err != nil {
- return 0, 0, err
}
- return dmls.rowCount, dmls.approximateSize, nil
}
return retry.Do(w.ctx, func() error {
failpoint.Inject("MySQLSinkTxnRandomError", func() {
diff --git a/pkg/sink/mysql/mysql_writer_dml_session.go b/pkg/sink/mysql/mysql_writer_dml_session.go
index 9c00fbb240..19ef7c59f5 100644
--- a/pkg/sink/mysql/mysql_writer_dml_session.go
+++ b/pkg/sink/mysql/mysql_writer_dml_session.go
@@ -65,17 +65,25 @@ func NewDMLSession(idleTimeout time.Duration) *dmlSession {
//
// The lock is held during fn to guarantee that the underlying sql.Conn is never
// used concurrently by DML execution and background maintenance.
+// If admit rejects the write after connection acquisition, the connection is
+// released and the caller can wait for admission again without holding session
+// resources.
func (s *dmlSession) withConn(
w *Writer,
writeTimeout time.Duration,
+ admit func() bool,
fn func(conn *sql.Conn) error,
-) error {
+) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
conn, err := s.getOrCreateLocked(w, writeTimeout)
if err != nil {
- return err
+ return false, err
+ }
+ if !admit() {
+ s.closeLocked(w)
+ return false, nil
}
if err := fn(conn); err != nil {
// fn must best-effort clean up any explicit transaction state it started on conn
@@ -83,10 +91,10 @@ func (s *dmlSession) withConn(
// handle on error to avoid reusing a connection with uncertain state.
// Discard the session on error to avoid reusing a session with unknown txn state.
s.closeLocked(w)
- return err
+ return true, err
}
s.lastActive = time.Now()
- return nil
+ return true, nil
}
// CheckStats performs best-effort maintenance for the current session:
diff --git a/pkg/sink/mysql/mysql_writer_for_ddl_ts.go b/pkg/sink/mysql/mysql_writer_for_ddl_ts.go
index 423fce61fc..7d4eaad4ef 100644
--- a/pkg/sink/mysql/mysql_writer_for_ddl_ts.go
+++ b/pkg/sink/mysql/mysql_writer_for_ddl_ts.go
@@ -512,6 +512,10 @@ func selectDDLTsQuery(tableIDs []int64, ticdcClusterID string, changefeedID stri
}
func (w *Writer) RemoveDDLTsItem() error {
+ if !w.grantWrite() {
+ return w.ctx.Err()
+ }
+
tx, err := w.db.BeginTx(w.ctx, nil)
if err != nil {
return errors.WrapError(errors.ErrMySQLTxnError, errors.WithMessage(err, "select ddl ts table: begin Tx fail;"))
diff --git a/pkg/sink/mysql/mysql_writer_test.go b/pkg/sink/mysql/mysql_writer_test.go
index 3814ff373c..85bc9714b6 100644
--- a/pkg/sink/mysql/mysql_writer_test.go
+++ b/pkg/sink/mysql/mysql_writer_test.go
@@ -33,6 +33,7 @@ import (
cerror "github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/routing"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/tidb/br/pkg/version"
ticonfig "github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/dxf/framework/handle"
@@ -127,6 +128,79 @@ func TestMysqlWriter_FlushDML(t *testing.T) {
require.NoError(t, err)
}
+func TestMysqlWriterWaitsForWriteGrantBeforeExecute(t *testing.T) {
+ writer, db, mock := newTestMysqlWriter(t)
+ defer db.Close()
+
+ helper := commonEvent.NewEventTestHelper(t)
+ defer helper.Close()
+
+ helper.Tk().MustExec("use test")
+ require.NotNil(t, helper.DDL2Job("create table t (id int primary key, name varchar(32));"))
+ dmlEvent := helper.DML2Event("test", "t", "insert into t values (1, 'test')")
+ dmlEvent.CommitTs = 2
+ dmlEvent.ReplicatingTs = 1
+ dmlEvent.DispatcherID = common.NewDispatcherID()
+
+ gate := writelease.NewGate()
+ gate.SetP2PRequired(true)
+ require.True(t, gate.RenewEtcd(time.Now(), writelease.EtcdProofDuration))
+ writer.SetWriteGate(gate)
+
+ mock.ExpectExec("BEGIN;INSERT INTO `test`.`t` (`id`,`name`) VALUES (?,?);COMMIT;").
+ WithArgs(1, "test").
+ WillReturnResult(sqlmock.NewResult(1, 1))
+
+ done := make(chan error, 1)
+ go func() {
+ done <- writer.Flush([]*commonEvent.DMLEvent{dmlEvent})
+ }()
+
+ require.Never(t, func() bool {
+ return db.Stats().InUse != 0
+ }, 50*time.Millisecond, time.Millisecond, "writer held a connection while waiting for a write grant")
+
+ select {
+ case err := <-done:
+ t.Fatalf("DML execute returned before the transport received a write grant: %v", err)
+ case <-time.After(50 * time.Millisecond):
+ }
+
+ require.True(t, gate.RenewP2P(time.Now(), writelease.P2PLeaseDuration))
+ require.NoError(t, <-done)
+ require.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestMysqlWriterReleasesConnectionWhenFinalAdmissionFails(t *testing.T) {
+ writer, db, _ := newTestMysqlWriter(t)
+ defer db.Close()
+
+ callbackCalled := false
+ admitted, err := writer.dmlSession.withConn(writer, time.Second, func() bool {
+ return false
+ }, func(*sql.Conn) error {
+ callbackCalled = true
+ return nil
+ })
+ require.NoError(t, err)
+ require.False(t, admitted)
+ require.False(t, callbackCalled)
+ require.Nil(t, writer.dmlSession.conn)
+ require.Zero(t, db.Stats().InUse)
+}
+
+func TestMysqlWriterGrantWriteRejectsAfterShutdown(t *testing.T) {
+ writer, db, _ := newTestMysqlWriter(t)
+ defer db.Close()
+
+ gate := writelease.NewGate()
+ gate.SetP2PRequired(true)
+ writer.SetWriteGate(gate)
+ writer.cancel()
+
+ require.False(t, writer.grantWrite())
+}
+
func TestMysqlWriter_FlushNoopWhenActiveActiveRowsDropped(t *testing.T) {
writer, db, mock := newTestMysqlWriter(t)
defer db.Close()
diff --git a/pkg/writelease/write_gate.go b/pkg/writelease/write_gate.go
new file mode 100644
index 0000000000..af56e977dc
--- /dev/null
+++ b/pkg/writelease/write_gate.go
@@ -0,0 +1,290 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writelease
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+const (
+ // NodeHeartbeatInterval is the cadence for capture heartbeats and P2P write-lease requests.
+ NodeHeartbeatInterval = 500 * time.Millisecond
+ // P2PLeaseDuration is the maximum validity period of a coordinator-issued P2P write proof.
+ P2PLeaseDuration = 5 * time.Second
+ // EtcdProofDuration caps the validity period derived from a positive etcd session TTL check.
+ EtcdProofDuration = 5 * time.Second
+)
+
+// BlockReason reports whether the capture write gate is writable or why it is
+// blocked. The gate is writable only when its etcd proof is fresh and, after
+// the current write-lease protocol is enabled, its P2P proof is also fresh.
+//
+// writable means all required proofs are fresh. etcd_proof_expired means the
+// required etcd proof is absent or expired; p2p_expired means the required P2P
+// proof is absent or expired while the etcd proof is fresh; and both_expired
+// means both required proofs are absent or expired. Renewing the missing proof
+// moves those recoverable states toward writable; a proof expiring or a
+// coordinator change invalidating P2P proof moves the state back to blocked.
+//
+// fenced takes precedence over every proof state. It is entered only when the
+// local capture fences itself after losing its etcd session or detecting that
+// the session expired. Fencing is irreversible for the process lifetime, so
+// later proof renewals cannot make the gate writable.
+type BlockReason string
+
+const (
+ // BlockReasonWritable indicates that all currently required write proofs are fresh.
+ BlockReasonWritable BlockReason = "writable"
+ // BlockReasonP2PExpired indicates that the required P2P proof is absent or expired.
+ BlockReasonP2PExpired BlockReason = "p2p_expired"
+ // BlockReasonEtcdExpired indicates that the required etcd proof is absent or expired.
+ BlockReasonEtcdExpired BlockReason = "etcd_proof_expired"
+ // BlockReasonBothExpired indicates that both required write proofs are absent or expired.
+ BlockReasonBothExpired BlockReason = "both_expired"
+ // BlockReasonFenced indicates that this capture was locally and irreversibly fenced.
+ BlockReasonFenced BlockReason = "fenced"
+)
+
+// Status is a point-in-time view used for metrics and transition logs. Write
+// admission still reads the immutable lease snapshot directly.
+type Status struct {
+ Reason BlockReason
+ Writable bool
+ P2PRequired bool
+ P2PRemaining time.Duration
+ EtcdProofRemaining time.Duration
+}
+
+type leaseState struct {
+ p2pValidUntil time.Time
+ etcdProofValidUntil time.Time
+ p2pRequired bool
+ fenced bool
+}
+
+// Gate controls capture-wide admission of downstream side effects.
+// Renewals publish immutable snapshots so the write path only needs an atomic
+// load and local monotonic-clock comparisons.
+type Gate struct {
+ now func() time.Time
+
+ state atomic.Pointer[leaseState]
+
+ mu sync.Mutex
+ changed chan struct{}
+}
+
+// NewGate creates a fail-closed write gate. Etcd proof is always required.
+// P2P proof becomes mandatory only after a coordinator negotiates the current
+// write-lease protocol, so rolling upgrades do not stop legacy nodes.
+func NewGate() *Gate {
+ return newGate(time.Now)
+}
+
+func newGate(now func() time.Time) *Gate {
+ g := &Gate{
+ now: now,
+ changed: make(chan struct{}),
+ }
+ g.state.Store(&leaseState{})
+ return g
+}
+
+// IsWritable returns whether both write proofs are fresh and the capture has
+// not entered the irreversible fenced state.
+func (g *Gate) IsWritable() bool {
+ return g.isWritableAt(g.now())
+}
+
+// CanWrite reports whether writes are admitted. A nil gate means write-lease
+// enforcement is not installed, which preserves the legacy sink behavior.
+func CanWrite(gate *Gate) bool {
+ return gate == nil || gate.IsWritable()
+}
+
+// Status returns the current gate state and non-negative lease lifetimes.
+func (g *Gate) Status() Status {
+ now := g.now()
+ state := g.state.Load()
+ p2pRemaining := max(state.p2pValidUntil.Sub(now), 0)
+ etcdRemaining := max(state.etcdProofValidUntil.Sub(now), 0)
+
+ reason := BlockReasonWritable
+ switch {
+ case state.fenced:
+ reason = BlockReasonFenced
+ case state.p2pRequired && p2pRemaining == 0 && etcdRemaining == 0:
+ reason = BlockReasonBothExpired
+ case state.p2pRequired && p2pRemaining == 0:
+ reason = BlockReasonP2PExpired
+ case etcdRemaining == 0:
+ reason = BlockReasonEtcdExpired
+ }
+ return Status{
+ Reason: reason,
+ Writable: reason == BlockReasonWritable,
+ P2PRequired: state.p2pRequired,
+ P2PRemaining: p2pRemaining,
+ EtcdProofRemaining: etcdRemaining,
+ }
+}
+
+func (g *Gate) isWritableAt(now time.Time) bool {
+ return isStateWritableAt(g.state.Load(), now)
+}
+
+func isStateWritableAt(state *leaseState, now time.Time) bool {
+ return !state.fenced &&
+ (!state.p2pRequired || now.Before(state.p2pValidUntil)) &&
+ now.Before(state.etcdProofValidUntil)
+}
+
+// WaitUntilWritable blocks until writes are admitted again or ctx is done.
+func (g *Gate) WaitUntilWritable(ctx context.Context) error {
+ for {
+ if g.IsWritable() {
+ return nil
+ }
+
+ g.mu.Lock()
+ changed := g.changed
+ g.mu.Unlock()
+
+ // Avoid missing a renewal between the first state check and loading the
+ // notification channel.
+ if g.IsWritable() {
+ return nil
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-changed:
+ }
+ }
+}
+
+// WaitForWrite waits for write admission when a gate is installed. A nil gate
+// preserves the legacy sink behavior.
+func WaitForWrite(ctx context.Context, gate *Gate) error {
+ if gate == nil {
+ return nil
+ }
+ return gate.WaitUntilWritable(ctx)
+}
+
+// RenewP2P renews the coordinator-issued proof from the request send time.
+// It rejects grants that were already expired when they arrived.
+func (g *Gate) RenewP2P(requestSentAt time.Time, duration time.Duration) bool {
+ return g.renew(requestSentAt.Add(duration), true)
+}
+
+// RenewEtcd renews the positive etcd proof from the TTL request send time.
+// It rejects responses that were already expired when they arrived.
+func (g *Gate) RenewEtcd(requestSentAt time.Time, duration time.Duration) bool {
+ return g.renew(requestSentAt.Add(duration), false)
+}
+
+// SetP2PRequired activates or deactivates P2P enforcement for the current
+// coordinator generation. Legacy coordinators leave it disabled; a current
+// coordinator enables it before its first grant is accepted.
+func (g *Gate) SetP2PRequired(required bool) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ current := g.state.Load()
+ if current.fenced || current.p2pRequired == required {
+ return
+ }
+ next := *current
+ next.p2pRequired = required
+ g.publishLocked(&next)
+}
+
+func (g *Gate) renew(validUntil time.Time, p2p bool) bool {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ if !validUntil.After(g.now()) {
+ return false
+ }
+
+ current := g.state.Load()
+ if current.fenced {
+ return false
+ }
+ if p2p && !validUntil.After(current.p2pValidUntil) {
+ return false
+ }
+ if !p2p && !validUntil.After(current.etcdProofValidUntil) {
+ return false
+ }
+
+ next := *current
+ if p2p {
+ next.p2pValidUntil = validUntil
+ } else {
+ next.etcdProofValidUntil = validUntil
+ }
+ g.publishLocked(&next)
+ return true
+}
+
+// InvalidateP2P closes write admission until a fresh coordinator generation
+// grants another P2P lease.
+func (g *Gate) InvalidateP2P() {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ current := g.state.Load()
+ if current.p2pValidUntil.IsZero() {
+ return
+ }
+ next := *current
+ next.p2pValidUntil = time.Time{}
+ g.publishLocked(&next)
+}
+
+// Fence irreversibly closes the gate for this capture process lifetime.
+func (g *Gate) Fence() {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+
+ current := g.state.Load()
+ if current.fenced {
+ return
+ }
+ next := *current
+ next.fenced = true
+ g.publishLocked(&next)
+}
+
+// EtcdProofValidUntil returns the current positive proof deadline.
+func (g *Gate) EtcdProofValidUntil() time.Time {
+ return g.state.Load().etcdProofValidUntil
+}
+
+func (g *Gate) publishLocked(next *leaseState) {
+ now := g.now()
+ becameWritable := !isStateWritableAt(g.state.Load(), now) && isStateWritableAt(next, now)
+ g.state.Store(next)
+ if !becameWritable {
+ return
+ }
+ close(g.changed)
+ g.changed = make(chan struct{})
+}
diff --git a/pkg/writelease/write_gate_test.go b/pkg/writelease/write_gate_test.go
new file mode 100644
index 0000000000..d910c04c70
--- /dev/null
+++ b/pkg/writelease/write_gate_test.go
@@ -0,0 +1,149 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writelease
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGateRequiresBothProofs(t *testing.T) {
+ now := time.Unix(100, 0)
+ gate := newGate(func() time.Time { return now })
+ gate.SetP2PRequired(true)
+
+ require.False(t, gate.IsWritable())
+ require.Equal(t, BlockReasonBothExpired, gate.Status().Reason)
+ require.True(t, gate.RenewP2P(now, P2PLeaseDuration))
+ require.False(t, gate.IsWritable())
+ require.Equal(t, BlockReasonEtcdExpired, gate.Status().Reason)
+ require.True(t, gate.RenewEtcd(now, EtcdProofDuration))
+ require.True(t, gate.IsWritable())
+ require.Equal(t, BlockReasonWritable, gate.Status().Reason)
+
+ now = now.Add(P2PLeaseDuration)
+ require.False(t, gate.IsWritable())
+ require.Equal(t, BlockReasonBothExpired, gate.Status().Reason)
+
+ require.True(t, gate.RenewP2P(now, P2PLeaseDuration))
+ require.False(t, gate.IsWritable())
+ require.True(t, gate.RenewEtcd(now, EtcdProofDuration))
+ require.True(t, gate.IsWritable())
+}
+
+func TestGateRejectsLateRenewalAndFenceIsIrreversible(t *testing.T) {
+ now := time.Unix(100, 0)
+ gate := newGate(func() time.Time { return now })
+ gate.SetP2PRequired(true)
+
+ require.False(t, gate.RenewP2P(now.Add(-P2PLeaseDuration), P2PLeaseDuration))
+ require.False(t, gate.RenewEtcd(now.Add(-EtcdProofDuration), EtcdProofDuration))
+
+ require.True(t, gate.RenewP2P(now, P2PLeaseDuration))
+ require.True(t, gate.RenewEtcd(now, EtcdProofDuration))
+ gate.Fence()
+ require.False(t, gate.IsWritable())
+ require.Equal(t, BlockReasonFenced, gate.Status().Reason)
+ require.False(t, gate.RenewP2P(now, 2*P2PLeaseDuration))
+ require.False(t, gate.RenewEtcd(now, 2*EtcdProofDuration))
+}
+
+func TestGateWaitUntilWritable(t *testing.T) {
+ gate := NewGate()
+ gate.SetP2PRequired(true)
+ done := make(chan error, 1)
+ go func() {
+ done <- gate.WaitUntilWritable(context.Background())
+ }()
+
+ gate.RenewP2P(time.Now(), P2PLeaseDuration)
+ select {
+ case <-done:
+ t.Fatal("wait returned without an etcd proof")
+ case <-time.After(10 * time.Millisecond):
+ }
+
+ gate.RenewEtcd(time.Now(), EtcdProofDuration)
+ require.NoError(t, <-done)
+
+ gate.InvalidateP2P()
+ require.False(t, gate.IsWritable())
+}
+
+func TestGateNotifiesOnlyWhenWritable(t *testing.T) {
+ now := time.Unix(100, 0)
+ gate := newGate(func() time.Time { return now })
+ gate.SetP2PRequired(true)
+
+ changed := gate.changed
+ require.True(t, gate.RenewP2P(now, P2PLeaseDuration))
+ select {
+ case <-changed:
+ t.Fatal("P2P renewal notified waiters while etcd proof was still expired")
+ default:
+ }
+
+ require.True(t, gate.RenewEtcd(now, EtcdProofDuration))
+ select {
+ case <-changed:
+ default:
+ t.Fatal("etcd renewal did not notify waiters when the gate became writable")
+ }
+
+ changed = gate.changed
+ require.True(t, gate.RenewEtcd(now.Add(time.Second), EtcdProofDuration))
+ select {
+ case <-changed:
+ t.Fatal("etcd renewal notified waiters while the gate remained writable")
+ default:
+ }
+}
+
+func TestGateNegotiatesP2PPerCoordinator(t *testing.T) {
+ now := time.Unix(100, 0)
+ gate := newGate(func() time.Time { return now })
+
+ // Legacy mode never requires a P2P grant, but it still requires fresh etcd
+ // proof and remains protected by an irreversible local fence.
+ require.False(t, gate.IsWritable())
+ require.True(t, gate.RenewEtcd(now, EtcdProofDuration))
+ require.True(t, gate.IsWritable())
+ require.False(t, gate.Status().P2PRequired)
+
+ gate.SetP2PRequired(true)
+ require.False(t, gate.IsWritable())
+ require.Equal(t, BlockReasonP2PExpired, gate.Status().Reason)
+ require.True(t, gate.RenewP2P(now, P2PLeaseDuration))
+ require.True(t, gate.IsWritable())
+
+ gate.InvalidateP2P()
+ gate.SetP2PRequired(false)
+ require.True(t, gate.IsWritable())
+}
+
+func TestGateWaitReturnsOnContextCancellation(t *testing.T) {
+ gate := NewGate()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ require.ErrorIs(t, gate.WaitUntilWritable(ctx), context.Canceled)
+}
+
+func TestOptionalGatePreservesLegacyWrites(t *testing.T) {
+ require.True(t, CanWrite(nil))
+ require.NoError(t, WaitForWrite(t.Context(), nil))
+}
diff --git a/server/server.go b/server/server.go
index 8a604d35cc..27faa3bb8e 100644
--- a/server/server.go
+++ b/server/server.go
@@ -40,12 +40,14 @@ import (
"github.com/pingcap/ticdc/pkg/keyspace"
"github.com/pingcap/ticdc/pkg/liveness"
"github.com/pingcap/ticdc/pkg/messaging"
+ "github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/node"
"github.com/pingcap/ticdc/pkg/pdutil"
"github.com/pingcap/ticdc/pkg/security"
tiserver "github.com/pingcap/ticdc/pkg/server"
"github.com/pingcap/ticdc/pkg/tcpserver"
"github.com/pingcap/ticdc/pkg/upstream"
+ "github.com/pingcap/ticdc/pkg/writelease"
"github.com/pingcap/ticdc/server/watcher"
pd "github.com/tikv/pd/client"
clientv3 "go.etcd.io/etcd/client/v3"
@@ -55,10 +57,20 @@ import (
)
const (
- closeServiceTimeout = 15 * time.Second
- cleanMetaDuration = 10 * time.Second
+ // closeServiceTimeout bounds shutdown of all pre-services.
+ closeServiceTimeout = 15 * time.Second
+ // cleanMetaDuration bounds deletion of this capture's etcd registration during shutdown.
+ cleanMetaDuration = 10 * time.Second
+ // oldArchCheckInterval is the retry interval while waiting for the old-architecture capture to stop.
oldArchCheckInterval = 100 * time.Millisecond
+ // sessionWatchInterval is the cadence for checking the etcd session TTL.
sessionWatchInterval = time.Second
+ // etcdTTLRequestTimeout bounds one etcd session TTL request.
+ etcdTTLRequestTimeout = 3 * time.Second
+ // etcdTTLSafetyMargin is subtracted from the observed TTL before accepting it as write proof.
+ etcdTTLSafetyMargin = time.Second
+ // writeGateMonitorTick is the cadence for recording write-gate metrics and state transitions.
+ writeGateMonitorTick = 100 * time.Millisecond
// GracefulShutdownTimeout is used to prevent the CDC process from hanging for an extended period due to certain modules don't exit immediately.
GracefulShutdownTimeout = 30 * time.Second
)
@@ -145,6 +157,7 @@ type server struct {
closed atomic.Bool
localFenceOnce atomic.Bool
+ writeGate *writelease.Gate
}
// New returns a new Server instance
@@ -250,6 +263,9 @@ func (c *server) setPreServices(ctx context.Context) error {
// Set ID to Global Context
appctx.SetID(c.info.ID.String())
+ c.writeGate = writelease.NewGate()
+ appctx.SetService(appctx.CaptureWriteGate, c.writeGate)
+
// Set PDClock to Global Context
var err error
c.PDClock, err = pdutil.NewClock(ctx, c.pdClient)
@@ -378,6 +394,10 @@ func (c *server) Run(ctx context.Context) error {
if err != nil {
return errors.Trace(err)
}
+ g.Go(func() error {
+ c.monitorCaptureWriteGate(gctx, writeGateMonitorTick)
+ return nil
+ })
fatalErrCh := make(chan error, 1)
go func() {
@@ -431,7 +451,10 @@ func (c *server) watchEtcdSession(
c.localFence("etcd session done")
return errors.ErrCaptureSuicide.GenWithStackByArgs()
case <-ticker.C:
- ttl, err := c.EtcdClient.GetEtcdClient().TimeToLive(ctx, leaseID)
+ requestSentAt := time.Now()
+ ttlCtx, cancel := context.WithTimeout(ctx, c.etcdTTLRequestTimeout(requestSentAt))
+ ttl, err := c.EtcdClient.GetEtcdClient().TimeToLive(ttlCtx, leaseID)
+ cancel()
if err != nil {
if ctx.Err() != nil {
return nil
@@ -439,20 +462,90 @@ func (c *server) watchEtcdSession(
log.Warn("check etcd session ttl failed", zap.Error(err))
continue
}
- if ttl != nil && ttl.TTL == -1 {
+ if ttl == nil {
+ continue
+ }
+ if ttl.TTL < 0 {
c.localFence("etcd lease expired")
return errors.ErrCaptureSuicide.GenWithStackByArgs()
}
+ proofDuration := time.Duration(ttl.TTL)*time.Second - etcdTTLSafetyMargin
+ proofDuration = min(proofDuration, writelease.EtcdProofDuration)
+ if proofDuration > 0 && c.writeGate != nil {
+ c.writeGate.RenewEtcd(requestSentAt, proofDuration)
+ }
+ }
+ }
+}
+
+func (c *server) etcdTTLRequestTimeout(now time.Time) time.Duration {
+ if c.writeGate == nil {
+ return etcdTTLRequestTimeout
+ }
+ proofValidUntil := c.writeGate.EtcdProofValidUntil()
+ if proofValidUntil.IsZero() || !proofValidUntil.After(now) {
+ return etcdTTLRequestTimeout
+ }
+ return min(etcdTTLRequestTimeout, proofValidUntil.Sub(now))
+}
+
+func (c *server) monitorCaptureWriteGate(ctx context.Context, interval time.Duration) {
+ if c.writeGate == nil {
+ return
+ }
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+
+ previous := c.writeGate.Status()
+ c.recordCaptureWriteGateMetrics(previous)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ current := c.writeGate.Status()
+ c.recordCaptureWriteGateMetrics(current)
+ if current.Reason == previous.Reason {
+ continue
+ }
+ if previous.Writable && !current.Writable {
+ metrics.CaptureWriteBlockCounter.WithLabelValues(string(current.Reason)).Inc()
+ log.Warn("capture write gate blocked", zap.String("reason", string(current.Reason)))
+ } else if !previous.Writable && current.Writable {
+ log.Info("capture write gate recovered")
+ }
+ previous = current
}
}
}
+func (c *server) recordCaptureWriteGateMetrics(status writelease.Status) {
+ for _, reason := range []writelease.BlockReason{
+ writelease.BlockReasonWritable,
+ writelease.BlockReasonP2PExpired,
+ writelease.BlockReasonEtcdExpired,
+ writelease.BlockReasonBothExpired,
+ writelease.BlockReasonFenced,
+ } {
+ value := float64(0)
+ if status.Reason == reason {
+ value = 1
+ }
+ metrics.CaptureWriteGateState.WithLabelValues(string(reason)).Set(value)
+ }
+ metrics.CaptureP2PLeaseRemainingSeconds.Set(status.P2PRemaining.Seconds())
+ metrics.CaptureEtcdProofRemainingSeconds.Set(status.EtcdProofRemaining.Seconds())
+}
+
func (c *server) localFence(reason string) {
if !c.localFenceOnce.CompareAndSwap(false, true) {
return
}
log.Warn("local fence triggered", zap.String("reason", reason))
+ if c.writeGate != nil {
+ c.writeGate.Fence()
+ }
c.liveness.Store(liveness.CaptureDraining)
c.liveness.Store(liveness.CaptureStopping)
diff --git a/server/server_session_watchdog_test.go b/server/server_session_watchdog_test.go
index fea9788382..179fbc260f 100644
--- a/server/server_session_watchdog_test.go
+++ b/server/server_session_watchdog_test.go
@@ -24,6 +24,9 @@ import (
"github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/etcd"
"github.com/pingcap/ticdc/pkg/liveness"
+ "github.com/pingcap/ticdc/pkg/metrics"
+ "github.com/pingcap/ticdc/pkg/writelease"
+ "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
clientv3 "go.etcd.io/etcd/client/v3"
)
@@ -75,6 +78,118 @@ func TestSessionWatchdogFencesOnExpiredLease(t *testing.T) {
require.Equal(t, liveness.CaptureStopping, c.liveness.Load())
}
+func TestSessionWatchdogDoesNotFenceOnLiveLeaseWithZeroTTL(t *testing.T) {
+ fencer := &testLocalFencer{}
+ appctx.SetService(appctx.DispatcherOrchestrator, fencer)
+
+ ctrl := gomock.NewController(t)
+ cdcEtcdClient := etcd.NewMockCDCEtcdClient(ctrl)
+ rawEtcdClient := etcd.NewMockClient(ctrl)
+ cdcEtcdClient.EXPECT().GetEtcdClient().Return(rawEtcdClient).AnyTimes()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ rawEtcdClient.EXPECT().
+ TimeToLive(gomock.Any(), clientv3.LeaseID(100)).
+ DoAndReturn(func(context.Context, clientv3.LeaseID, ...clientv3.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) {
+ cancel()
+ return &clientv3.LeaseTimeToLiveResponse{TTL: 0}, nil
+ }).MinTimes(1)
+
+ gate := writelease.NewGate()
+ require.True(t, gate.RenewP2P(time.Now(), writelease.P2PLeaseDuration))
+ c := &server{EtcdClient: cdcEtcdClient, writeGate: gate}
+
+ err := c.watchEtcdSession(ctx, make(chan struct{}), 100, time.Millisecond)
+
+ require.NoError(t, err)
+ require.Equal(t, int32(0), fencer.count.Load())
+ require.False(t, gate.IsWritable())
+}
+
+func TestSessionWatchdogRenewsEtcdWriteProof(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ cdcEtcdClient := etcd.NewMockCDCEtcdClient(ctrl)
+ rawEtcdClient := etcd.NewMockClient(ctrl)
+ cdcEtcdClient.EXPECT().GetEtcdClient().Return(rawEtcdClient).AnyTimes()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ rawEtcdClient.EXPECT().
+ TimeToLive(gomock.Any(), clientv3.LeaseID(100)).
+ DoAndReturn(func(context.Context, clientv3.LeaseID, ...clientv3.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) {
+ cancel()
+ return &clientv3.LeaseTimeToLiveResponse{TTL: 10}, nil
+ }).MinTimes(1)
+
+ gate := writelease.NewGate()
+ requestSentAt := time.Now()
+ require.True(t, gate.RenewP2P(requestSentAt, writelease.P2PLeaseDuration))
+ c := &server{EtcdClient: cdcEtcdClient, writeGate: gate}
+
+ err := c.watchEtcdSession(ctx, make(chan struct{}), 100, time.Millisecond)
+
+ require.NoError(t, err)
+ require.True(t, gate.IsWritable())
+}
+
+func TestSessionWatchdogDoesNotFenceOnTTLQueryError(t *testing.T) {
+ fencer := &testLocalFencer{}
+ appctx.SetService(appctx.DispatcherOrchestrator, fencer)
+
+ ctrl := gomock.NewController(t)
+ cdcEtcdClient := etcd.NewMockCDCEtcdClient(ctrl)
+ rawEtcdClient := etcd.NewMockClient(ctrl)
+ cdcEtcdClient.EXPECT().GetEtcdClient().Return(rawEtcdClient).AnyTimes()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ callCount := 0
+ rawEtcdClient.EXPECT().
+ TimeToLive(gomock.Any(), clientv3.LeaseID(100)).
+ DoAndReturn(func(context.Context, clientv3.LeaseID, ...clientv3.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) {
+ callCount++
+ if callCount == 1 {
+ return nil, context.DeadlineExceeded
+ }
+ cancel()
+ return nil, context.Canceled
+ }).Times(2)
+
+ c := &server{EtcdClient: cdcEtcdClient, writeGate: writelease.NewGate()}
+ err := c.watchEtcdSession(ctx, make(chan struct{}), 100, time.Millisecond)
+
+ require.NoError(t, err)
+ require.Equal(t, int32(0), fencer.count.Load())
+}
+
+func TestEtcdTTLRequestTimeoutUsesCurrentProofDeadline(t *testing.T) {
+ gate := writelease.NewGate()
+ now := time.Now()
+ require.True(t, gate.RenewEtcd(now, 500*time.Millisecond))
+ c := &server{writeGate: gate}
+
+ timeout := c.etcdTTLRequestTimeout(now)
+ require.Equal(t, 500*time.Millisecond, timeout)
+ require.Equal(t, etcdTTLRequestTimeout, (&server{writeGate: writelease.NewGate()}).etcdTTLRequestTimeout(now))
+}
+
+func TestCaptureWriteGateMonitorRecordsBlockTransition(t *testing.T) {
+ gate := writelease.NewGate()
+ gate.SetP2PRequired(true)
+ now := time.Now()
+ require.True(t, gate.RenewP2P(now, 200*time.Millisecond))
+ require.True(t, gate.RenewEtcd(now, 200*time.Millisecond))
+ c := &server{writeGate: gate}
+
+ counter := metrics.CaptureWriteBlockCounter.WithLabelValues(string(writelease.BlockReasonBothExpired))
+ before := testutil.ToFloat64(counter)
+ go c.monitorCaptureWriteGate(t.Context(), 5*time.Millisecond)
+
+ require.Eventually(t, func() bool {
+ return testutil.ToFloat64(counter) == before+1
+ }, time.Second, 10*time.Millisecond)
+ require.Equal(t, float64(1), testutil.ToFloat64(
+ metrics.CaptureWriteGateState.WithLabelValues(string(writelease.BlockReasonBothExpired))))
+}
+
func TestLocalFenceIsIdempotent(t *testing.T) {
fencer := &testLocalFencer{}
appctx.SetService(appctx.DispatcherOrchestrator, fencer)
diff --git a/server/watcher/module_node_manager.go b/server/watcher/module_node_manager.go
index b54bc327a1..8125e2b92a 100644
--- a/server/watcher/module_node_manager.go
+++ b/server/watcher/module_node_manager.go
@@ -22,6 +22,7 @@ import (
"github.com/pingcap/log"
"github.com/pingcap/ticdc/pkg/config"
"github.com/pingcap/ticdc/pkg/etcd"
+ "github.com/pingcap/ticdc/pkg/metrics"
"github.com/pingcap/ticdc/pkg/node"
"github.com/pingcap/ticdc/pkg/orchestrator"
"go.etcd.io/etcd/client/v3/concurrency"
@@ -166,9 +167,9 @@ func (c *NodeManager) Run(ctx context.Context) error {
etcd.BaseKey(c.etcdClient.GetClusterID())+"/__cdc_meta__/capture",
"capture-manager")
- return watcher.RunEtcdWorker(ctx, c,
- orchestrator.NewGlobalState(c.etcdClient.GetClusterID(),
- cfg.CaptureSessionTTL), time.Millisecond*50)
+ state := orchestrator.NewGlobalState(c.etcdClient.GetClusterID(), cfg.CaptureSessionTTL)
+ metrics.CaptureSafeToRescheduleDelaySeconds.Set(float64(state.CaptureRemoveTTLSeconds()))
+ return watcher.RunEtcdWorker(ctx, c, state, time.Millisecond*50)
}
func (c *NodeManager) RegisterNodeChangeHandler(name node.ID, handler NodeChangeHandler) {
diff --git a/tests/integration_tests/capture_write_lease/conf/changefeed-main.toml b/tests/integration_tests/capture_write_lease/conf/changefeed-main.toml
new file mode 100644
index 0000000000..5e3532b4c2
--- /dev/null
+++ b/tests/integration_tests/capture_write_lease/conf/changefeed-main.toml
@@ -0,0 +1,2 @@
+[filter]
+rules = ["*.*", "!capture_write_lease_redo.*"]
diff --git a/tests/integration_tests/capture_write_lease/conf/changefeed-redo.toml b/tests/integration_tests/capture_write_lease/conf/changefeed-redo.toml
new file mode 100644
index 0000000000..6ec33640d6
--- /dev/null
+++ b/tests/integration_tests/capture_write_lease/conf/changefeed-redo.toml
@@ -0,0 +1,7 @@
+[consistent]
+level = "eventual"
+storage = "file:///tmp/tidb_cdc_test/capture_write_lease/redo"
+meta-flush-interval = 200
+
+[filter]
+rules = ["capture_write_lease_redo.*"]
diff --git a/tests/integration_tests/multi_capture/conf/diff_config.toml b/tests/integration_tests/capture_write_lease/conf/diff_config.toml
similarity index 61%
rename from tests/integration_tests/multi_capture/conf/diff_config.toml
rename to tests/integration_tests/capture_write_lease/conf/diff_config.toml
index f6cd453010..9bbbae05aa 100644
--- a/tests/integration_tests/multi_capture/conf/diff_config.toml
+++ b/tests/integration_tests/capture_write_lease/conf/diff_config.toml
@@ -7,13 +7,13 @@ export-fix-sql = true
check-struct-only = false
[task]
- output-dir = "/tmp/tidb_cdc_test/multi_capture/sync_diff/output"
+ output-dir = "/tmp/tidb_cdc_test/capture_write_lease/sync_diff/output"
source-instances = ["mysql1"]
target-instance = "tidb0"
- target-check-tables = ["multi_capture_1.usertable", "multi_capture_2.usertable", "multi_capture_3.usertable", "multi_capture_4.usertable"]
+ target-check-tables = ["capture_write_lease_1.usertable", "capture_write_lease_2.usertable", "capture_write_lease_3.usertable", "capture_write_lease_4.usertable"]
[data-sources]
[data-sources.mysql1]
diff --git a/tests/integration_tests/multi_capture/conf/workload1 b/tests/integration_tests/capture_write_lease/conf/workload1
similarity index 100%
rename from tests/integration_tests/multi_capture/conf/workload1
rename to tests/integration_tests/capture_write_lease/conf/workload1
diff --git a/tests/integration_tests/multi_capture/conf/workload2 b/tests/integration_tests/capture_write_lease/conf/workload2
similarity index 100%
rename from tests/integration_tests/multi_capture/conf/workload2
rename to tests/integration_tests/capture_write_lease/conf/workload2
diff --git a/tests/integration_tests/capture_write_lease/conf/write_lease_diff_config.toml b/tests/integration_tests/capture_write_lease/conf/write_lease_diff_config.toml
new file mode 100644
index 0000000000..ce80245e5f
--- /dev/null
+++ b/tests/integration_tests/capture_write_lease/conf/write_lease_diff_config.toml
@@ -0,0 +1,34 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+ output-dir = "/tmp/tidb_cdc_test/capture_write_lease/sync_diff/write_lease_output"
+
+ source-instances = ["mysql1"]
+
+ target-instance = "tidb0"
+
+ target-check-tables = [
+ "capture_write_lease_redo.lease_probe_1",
+ "capture_write_lease_redo.lease_probe_2",
+ "capture_write_lease_redo.lease_probe_3",
+ "capture_write_lease.usertable",
+ ]
+
+[data-sources]
+[data-sources.mysql1]
+ host = "127.0.0.1"
+ port = 4000
+ user = "root"
+ password = ""
+
+[data-sources.tidb0]
+ host = "127.0.0.1"
+ port = 3306
+ user = "root"
+ password = ""
diff --git a/tests/integration_tests/capture_write_lease/conf/write_lease_workload b/tests/integration_tests/capture_write_lease/conf/write_lease_workload
new file mode 100644
index 0000000000..e01719cbe2
--- /dev/null
+++ b/tests/integration_tests/capture_write_lease/conf/write_lease_workload
@@ -0,0 +1,13 @@
+threadcount=4
+workload=core
+
+readallfields=true
+fieldcount=1
+fieldlength=32
+
+readproportion=0
+updateproportion=0.5
+scanproportion=0
+insertproportion=0.5
+
+requestdistribution=uniform
diff --git a/tests/integration_tests/capture_write_lease/run.sh b/tests/integration_tests/capture_write_lease/run.sh
new file mode 100755
index 0000000000..f5a4f34dcd
--- /dev/null
+++ b/tests/integration_tests/capture_write_lease/run.sh
@@ -0,0 +1,554 @@
+#!/bin/bash
+
+# Capture write-lease integration test.
+#
+# Runs three captures with rate-limited INSERT/UPDATE traffic. For the MySQL
+# sink, it delays and drops coordinator-to-capture write-lease grants, then
+# verifies write admission closes, Redo publication stops, both recover, stale
+# grants are rejected, and the data remains consistent. The drop simulates
+# one-way P2P control-plane loss only.
+
+set -eu
+
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+source $CUR/../_utils/test_prepare
+WORK_DIR=$OUT_DIR/$TEST_NAME
+CDC_BINARY=cdc.test
+SINK_TYPE=$1
+
+CDC_COUNT=3
+DB_COUNT=4
+CDC_BASE_PORT=${CDC_PORT}
+LEASE_DB=capture_write_lease
+REDO_DB=capture_write_lease_redo
+LEASE_PROBE_TABLE_PREFIX=lease_probe
+LEASE_PROBE_TABLE_COUNT=3
+YCSB_TABLE=usertable
+YCSB_RECORD_COUNT=10000
+YCSB_OPERATION_COUNT=285000
+YCSB_TARGET=1000
+YCSB_THREADS=4
+LEASE_DELAY_MS=7000
+LEASE_DELAY_SECONDS=9
+LEASE_DROP_SECONDS=15
+LEASE_DELAY_FAILPOINT=github.com/pingcap/ticdc/coordinator/DelayCaptureWriteLeaseResponse
+LEASE_DROP_FAILPOINT=github.com/pingcap/ticdc/coordinator/DropCaptureWriteLeaseResponse
+LEASE_DUPLICATE_FAILPOINT=github.com/pingcap/ticdc/coordinator/DuplicateCaptureWriteLeaseResponse
+MYSQL_HANG_FAILPOINT=github.com/pingcap/ticdc/pkg/sink/mysql/MySQLSinkHangLongTime
+MAIN_CHANGEFEED_ID=capture-write-lease-main-test
+REDO_CHANGEFEED_ID=capture-write-lease-redo-test
+REDO_STORAGE_PATH="file://$WORK_DIR/redo"
+REDO_DOWNLOAD_PATH="$WORK_DIR/cdc_data/redo/$REDO_CHANGEFEED_ID"
+
+function ycsb_load() {
+ go-ycsb load mysql -P "$CUR/conf/write_lease_workload" \
+ --threads="$YCSB_THREADS" \
+ -p mysql.host=${UP_TIDB_HOST} \
+ -p mysql.port=${UP_TIDB_PORT} \
+ -p mysql.user=root \
+ -p mysql.db=${LEASE_DB} \
+ -p table=${YCSB_TABLE} \
+ -p recordcount=${YCSB_RECORD_COUNT} \
+ -p operationcount=0
+}
+
+function ycsb_run() {
+ go-ycsb run mysql -P "$CUR/conf/write_lease_workload" \
+ --target="$YCSB_TARGET" \
+ --threads="$YCSB_THREADS" \
+ -p mysql.host=${UP_TIDB_HOST} \
+ -p mysql.port=${UP_TIDB_PORT} \
+ -p mysql.user=root \
+ -p mysql.db=${LEASE_DB} \
+ -p table=${YCSB_TABLE} \
+ -p recordcount=${YCSB_RECORD_COUNT} \
+ -p operationcount=${YCSB_OPERATION_COUNT}
+}
+
+function probe_count() {
+ local expression=""
+ local index
+
+ for index in $(seq "$LEASE_PROBE_TABLE_COUNT"); do
+ if [ -n "$expression" ]; then
+ expression+=" + "
+ fi
+ expression+="(SELECT COUNT(*) FROM ${REDO_DB}.${LEASE_PROBE_TABLE_PREFIX}_${index})"
+ done
+ mysql -h${DOWN_TIDB_HOST} -P${DOWN_TIDB_PORT} -uroot -N -s \
+ -e "SELECT ${expression};"
+}
+
+function wait_for_probe_count() {
+ local expected=$1
+ local count
+
+ for ((i = 0; i < 60; i++)); do
+ count=$(probe_count 2>/dev/null || true)
+ if [ "$count" = "$expected" ]; then
+ return
+ fi
+ sleep 1
+ done
+
+ echo "downstream probe row count is not ${expected}" >&2
+ return 1
+}
+
+function capture_has_gate_state() {
+ local port=$1
+ local state=$2
+
+ curl -fsS --max-time 5 "http://127.0.0.1:${port}/metrics" |
+ grep -E "^ticdc_server_capture_write_gate_state\\{state=\\\"${state}\\\"\\}[[:space:]]+1(\\.0+)?$" >/dev/null
+}
+
+function wait_for_gate_state() {
+ local state=$1
+ local target_port=${2:-}
+ local port
+
+ for ((i = 0; i < 30; i++)); do
+ if [ -n "$target_port" ]; then
+ if capture_has_gate_state "$target_port" "$state"; then
+ return
+ fi
+ else
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ if capture_has_gate_state "$port" "$state"; then
+ lease_gate_state_port=$port
+ return
+ fi
+ done
+ fi
+ sleep 1
+ done
+
+ echo "write gates did not reach ${state}" >&2
+ return 1
+}
+
+function wait_for_all_gate_state() {
+ local state=$1
+ local port
+ local ready
+
+ for ((i = 0; i < 30; i++)); do
+ ready=true
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ if ! capture_has_gate_state "$port" "$state"; then
+ ready=false
+ break
+ fi
+ done
+ if [ "$ready" = true ]; then
+ return
+ fi
+ sleep 1
+ done
+
+ echo "not all write gates reached ${state}" >&2
+ return 1
+}
+
+function capture_has_active_p2p_lease() {
+ local port=$1
+
+ curl -fsS --max-time 5 "http://127.0.0.1:${port}/metrics" |
+ awk '$1 == "ticdc_server_capture_p2p_lease_remaining_seconds" && $2 > 0 { found = 1 } END { exit !found }'
+}
+
+function wait_for_active_p2p_leases() {
+ local port
+ local ready
+
+ for ((i = 0; i < 30; i++)); do
+ ready=true
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ if ! capture_has_active_p2p_lease "$port"; then
+ ready=false
+ break
+ fi
+ done
+ if [ "$ready" = true ]; then
+ return
+ fi
+ sleep 1
+ done
+
+ echo "captures did not obtain active P2P write leases" >&2
+ return 1
+}
+
+function redo_resolved_ts() {
+ cdc redo meta --storage="$REDO_STORAGE_PATH" --tmp-dir="$REDO_DOWNLOAD_PATH/meta" |
+ grep -oE "resolved-ts:[0-9]+" | awk -F: '{print $2}'
+}
+
+function assert_redo_resolved_before() {
+ local upper_bound=$1
+ local duration=$2
+ local resolved_ts
+
+ for ((i = 0; i < duration; i++)); do
+ resolved_ts=$(redo_resolved_ts)
+ if ! [[ "$resolved_ts" =~ ^[0-9]+$ ]]; then
+ echo "invalid redo resolved ts: ${resolved_ts}" >&2
+ return 1
+ fi
+ if [ "$resolved_ts" -ge "$upper_bound" ]; then
+ echo "redo resolved ts ${resolved_ts} advanced to ${upper_bound} while write gates were closed" >&2
+ return 1
+ fi
+ sleep 1
+ done
+}
+
+function enable_lease_failpoint() {
+ local name=$1
+ local expr=$2
+ local port
+
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ enable_failpoint --addr "127.0.0.1:${port}" --name "$name" --expr "$expr"
+ done
+}
+
+function disable_lease_failpoint() {
+ local name=$1
+ local port
+
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ disable_failpoint --addr "127.0.0.1:${port}" --name "$name"
+ done
+}
+
+function disable_lease_failpoint_best_effort() {
+ local name=$1
+ local port
+
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ disable_failpoint --addr "127.0.0.1:${port}" --name "$name" >/dev/null 2>&1 || true
+ done
+}
+
+function rejected_lease_response_count() {
+ local reason=$1
+ local port
+
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ curl -fsS --max-time 5 "http://127.0.0.1:${port}/metrics" 2>/dev/null || true
+ done | awk -v reason="$reason" '
+ $0 ~ "^ticdc_server_capture_lease_response_rejected_total\\{reason=\"" reason "\"\\}" {
+ total += $NF
+ }
+ END {
+ printf "%.0f\n", total
+ }
+ '
+}
+
+function wait_for_rejected_lease_response() {
+ local reason=$1
+ local previous_count=$2
+ local count
+
+ for ((i = 0; i < 30; i++)); do
+ count=$(rejected_lease_response_count "$reason")
+ if [ "$count" -gt "$previous_count" ]; then
+ return
+ fi
+ sleep 1
+ done
+
+ echo "no new ${reason} lease response rejection observed" >&2
+ return 1
+}
+
+function stale_lease_response_rejected_count() {
+ local unknown_count
+ local replayed_count
+
+ unknown_count=$(rejected_lease_response_count unknown_sequence)
+ replayed_count=$(rejected_lease_response_count replayed_sequence)
+ echo $((unknown_count + replayed_count))
+}
+
+function wait_for_stale_lease_response_rejection() {
+ local previous_count=$1
+ local count
+
+ for ((i = 0; i < 30; i++)); do
+ count=$(stale_lease_response_rejected_count)
+ if [ "$count" -gt "$previous_count" ]; then
+ return
+ fi
+ sleep 1
+ done
+
+ echo "no new stale lease response rejection observed" >&2
+ return 1
+}
+
+function assert_cdc_processes_alive() {
+ local port
+ local pid
+
+ for port in $(seq $((CDC_BASE_PORT + 1)) $((CDC_BASE_PORT + CDC_COUNT))); do
+ pid=$(get_cdc_pid 127.0.0.1 "$port")
+ if ! kill -0 "$pid" >/dev/null 2>&1; then
+ echo "cdc on port ${port} exited during P2P lease expiry" >&2
+ return 1
+ fi
+ done
+}
+
+function start_lease_response_delay() {
+ enable_lease_failpoint "$LEASE_DELAY_FAILPOINT" "return(${LEASE_DELAY_MS})"
+ (
+ sleep "$LEASE_DELAY_SECONDS"
+ disable_lease_failpoint "$LEASE_DELAY_FAILPOINT"
+ ) &
+ lease_fault_pid=$!
+}
+
+function start_lease_response_drop() {
+ enable_lease_failpoint "$LEASE_DROP_FAILPOINT" "return(true)"
+ (
+ sleep "$LEASE_DROP_SECONDS"
+ disable_lease_failpoint "$LEASE_DROP_FAILPOINT"
+ ) &
+ lease_fault_pid=$!
+}
+
+function insert_probe_rows() {
+ local start=$1
+ local index
+ local values
+ local id
+
+ for index in $(seq "$LEASE_PROBE_TABLE_COUNT"); do
+ values=""
+ for id in $(seq "$start" $((start + 99))); do
+ if [ -n "$values" ]; then
+ values+=","
+ fi
+ values+="(${id}, ${id})"
+ done
+ run_sql "INSERT INTO ${REDO_DB}.${LEASE_PROBE_TABLE_PREFIX}_${index} VALUES ${values};" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
+ done
+}
+
+function run_lease_expiry_round() {
+ local round=$1
+ local fault=$2
+ local expected_after=$((round * 100 * LEASE_PROBE_TABLE_COUNT))
+ local blocked_cdc_port
+ local count
+ local rejected_count
+ local redo_target_tso
+
+ case "$fault" in
+ delay)
+ rejected_count=$(stale_lease_response_rejected_count)
+ start_lease_response_delay
+ ;;
+ drop) start_lease_response_drop ;;
+ *)
+ echo "unknown lease response fault ${fault}" >&2
+ return 1
+ ;;
+ esac
+ wait_for_gate_state p2p_expired
+ blocked_cdc_port=$lease_gate_state_port
+ assert_cdc_processes_alive
+ if [ "$fault" = drop ]; then
+ # Close every capture gate so neither Redo writers nor RedoMeta can publish
+ # progress for events produced below.
+ wait_for_all_gate_state p2p_expired
+ sleep 1
+ fi
+ insert_probe_rows $(((round - 1) * 100 + 1))
+ if [ "$fault" = drop ]; then
+ redo_target_tso=$(run_cdc_cli_tso_query "$UP_PD_HOST_1" "$UP_PD_PORT_1")
+ assert_redo_resolved_before "$redo_target_tso" 3
+ fi
+ sleep 2
+ count=$(probe_count)
+ if [ "$count" -ge "$expected_after" ]; then
+ echo "all probe rows replicated while cdc on port ${blocked_cdc_port} was blocked" >&2
+ return 1
+ fi
+ wait "$lease_fault_pid"
+ if [ "$fault" = delay ]; then
+ # Depending on whether a newer grant was applied first, the delayed grant is
+ # rejected as an unknown or replayed sequence; neither may reopen admission.
+ wait_for_stale_lease_response_rejection "$rejected_count"
+ fi
+ wait_for_gate_state writable "$blocked_cdc_port"
+ wait_for_probe_count "$expected_after"
+ if [ "$fault" = drop ]; then
+ ensure 60 check_redo_resolved_ts "$REDO_CHANGEFEED_ID" "$redo_target_tso" \
+ "$REDO_STORAGE_PATH" "$REDO_DOWNLOAD_PATH/meta"
+ fi
+}
+
+function run_write_lease_test() {
+ local duplicate_rejected_count
+ local redo_start_tso
+
+ if [ "$SINK_TYPE" != mysql ]; then
+ return
+ fi
+
+ run_sql "CREATE DATABASE ${LEASE_DB};" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
+ run_sql "CREATE DATABASE ${REDO_DB};" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
+ # Dynamic table scheduling assigns each small probe table to a capture. Three
+ # tables cover the three captures while keeping probe traffic negligible.
+ for i in $(seq "$LEASE_PROBE_TABLE_COUNT"); do
+ run_sql \
+ "CREATE TABLE ${REDO_DB}.${LEASE_PROBE_TABLE_PREFIX}_${i} (id BIGINT PRIMARY KEY, v BIGINT NOT NULL);" \
+ ${UP_TIDB_HOST} ${UP_TIDB_PORT}
+ done
+ cdc_cli_changefeed create --start-ts="$start_ts" --sink-uri="$SINK_URI" \
+ --changefeed-id="$REDO_CHANGEFEED_ID" --config="$CUR/conf/changefeed-redo.toml" \
+ --server="127.0.0.1:$((CDC_BASE_PORT + 1))"
+ ycsb_load
+ for i in $(seq "$LEASE_PROBE_TABLE_COUNT"); do
+ check_table_exists "${REDO_DB}.${LEASE_PROBE_TABLE_PREFIX}_${i}" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+ done
+ check_table_exists "${LEASE_DB}.${YCSB_TABLE}" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+ check_sync_diff "$WORK_DIR" "$CUR/conf/write_lease_diff_config.toml" 120
+ wait_for_active_p2p_leases
+ redo_start_tso=$(run_cdc_cli_tso_query "$UP_PD_HOST_1" "$UP_PD_PORT_1")
+ ensure 60 check_redo_resolved_ts "$REDO_CHANGEFEED_ID" "$redo_start_tso" \
+ "$REDO_STORAGE_PATH" "$REDO_DOWNLOAD_PATH/meta"
+
+ ycsb_run >"$WORK_DIR/ycsb.log" 2>&1 &
+ ycsb_pid=$!
+ sleep 30
+ for round in $(seq 1 3); do
+ fault=delay
+ if [ "$round" = 3 ]; then
+ # Drop grants while still receiving heartbeats: a deterministic one-way
+ # P2P control-plane network-loss fault.
+ fault=drop
+ fi
+ assert_cdc_processes_alive
+ run_lease_expiry_round "$round" "$fault"
+ sleep 30
+ done
+
+ duplicate_rejected_count=$(rejected_lease_response_count replayed_sequence)
+ enable_lease_failpoint "$LEASE_DUPLICATE_FAILPOINT" "return(true)"
+ wait_for_rejected_lease_response replayed_sequence "$duplicate_rejected_count"
+ disable_lease_failpoint "$LEASE_DUPLICATE_FAILPOINT"
+ wait "$ycsb_pid"
+ grep -Eq '^INSERT - .*Count: [1-9][0-9]*,' "$WORK_DIR/ycsb.log"
+ grep -Eq '^UPDATE - .*Count: [1-9][0-9]*,' "$WORK_DIR/ycsb.log"
+ check_sync_diff "$WORK_DIR" "$CUR/conf/write_lease_diff_config.toml" 120
+}
+
+function run_redo_apply_test() {
+ local count
+ local expected_after=$((4 * 100 * LEASE_PROBE_TABLE_COUNT))
+ local redo_apply_tso
+
+ if [ "$SINK_TYPE" != mysql ]; then
+ return
+ fi
+
+ # Hold the normal MySQL sink while Redo continues, then recover the missing
+ # downstream rows from Redo after the captures stop.
+ enable_lease_failpoint "$MYSQL_HANG_FAILPOINT" "return(true)"
+ insert_probe_rows 1001
+ redo_apply_tso=$(run_cdc_cli_tso_query "$UP_PD_HOST_1" "$UP_PD_PORT_1")
+ ensure 60 check_redo_resolved_ts "$REDO_CHANGEFEED_ID" "$redo_apply_tso" \
+ "$REDO_STORAGE_PATH" "$REDO_DOWNLOAD_PATH/meta"
+ count=$(probe_count)
+ if [ "$count" -ge "$expected_after" ]; then
+ echo "MySQL sink was not blocked before Redo recovery" >&2
+ return 1
+ fi
+
+ cleanup_process "$CDC_BINARY"
+ cdc redo apply --log-level debug --tmp-dir="$REDO_DOWNLOAD_PATH/apply" \
+ --storage="$REDO_STORAGE_PATH" \
+ --sink-uri="mysql://normal:123456@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/" >"$WORK_DIR/cdc_redo.log"
+ check_sync_diff "$WORK_DIR" "$CUR/conf/write_lease_diff_config.toml" 120
+}
+
+function run() {
+ rm -rf $WORK_DIR && mkdir -p $WORK_DIR
+
+ start_tidb_cluster --workdir $WORK_DIR
+
+ # record tso before we create tables to skip the system table DDLs
+ start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1})
+
+ # create $DB_COUNT databases and import initial workload
+ for i in $(seq $DB_COUNT); do
+ db="capture_write_lease_$i"
+ run_sql "CREATE DATABASE $db;" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
+ go-ycsb load mysql -P $CUR/conf/workload1 -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=$db
+ done
+
+ export GO_FAILPOINTS='github.com/pingcap/ticdc/utils/dynstream/InjectDropEvent=10%return(true)'
+ # start $CDC_COUNT cdc servers, and create a changefeed
+ for i in $(seq $CDC_COUNT); do
+ run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --logsuffix "$i" --addr "127.0.0.1:$((CDC_BASE_PORT + i))" --pd "http://${UP_PD_HOST_1}:${UP_PD_PORT_1}"
+ done
+
+ TOPIC_NAME="ticdc-capture-write-lease-test-$RANDOM"
+ case $SINK_TYPE in
+ kafka) SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-version=${KAFKA_VERSION}&max-message-bytes=10485760" ;;
+ storage) SINK_URI="file://$WORK_DIR/storage_test/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true" ;;
+ pulsar)
+ run_pulsar_cluster $WORK_DIR normal
+ SINK_URI="pulsar://127.0.0.1:6650/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true"
+ ;;
+ *) SINK_URI="mysql://normal:123456@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/" ;;
+ esac
+ if [ "$SINK_TYPE" = mysql ]; then
+ cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" \
+ --changefeed-id="$MAIN_CHANGEFEED_ID" --config="$CUR/conf/changefeed-main.toml" \
+ --server="127.0.0.1:$((CDC_BASE_PORT + 1))"
+ else
+ cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --server="127.0.0.1:$((CDC_BASE_PORT + 1))"
+ fi
+ case $SINK_TYPE in
+ kafka) run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&version=${KAFKA_VERSION}&max-message-bytes=10485760" ;;
+ storage) run_storage_consumer $WORK_DIR $SINK_URI "" "" ;;
+ pulsar) run_pulsar_consumer --upstream-uri $SINK_URI ;;
+ esac
+
+ # check tables are created and data is synchronized
+ for i in $(seq $DB_COUNT); do
+ check_table_exists "capture_write_lease_$i.usertable" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+ done
+ check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
+
+ # add more data in upstream and check again
+ for i in $(seq $DB_COUNT); do
+ db="capture_write_lease_$i"
+ go-ycsb load mysql -P $CUR/conf/workload2 -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=$db
+ done
+ check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
+ run_write_lease_test
+ run_redo_apply_test
+
+ cleanup_process $CDC_BINARY
+}
+
+function cleanup() {
+ disable_lease_failpoint_best_effort "$LEASE_DELAY_FAILPOINT"
+ disable_lease_failpoint_best_effort "$LEASE_DROP_FAILPOINT"
+ disable_lease_failpoint_best_effort "$LEASE_DUPLICATE_FAILPOINT"
+ disable_lease_failpoint_best_effort "$MYSQL_HANG_FAILPOINT"
+ stop_test "$WORK_DIR"
+}
+
+trap 'cleanup' EXIT
+run $*
+check_logs $WORK_DIR
+echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"
diff --git a/tests/integration_tests/maintainer_failover_when_operator/run.sh b/tests/integration_tests/maintainer_failover_when_operator/run.sh
index 9bc867f262..480a05e91f 100755
--- a/tests/integration_tests/maintainer_failover_when_operator/run.sh
+++ b/tests/integration_tests/maintainer_failover_when_operator/run.sh
@@ -538,10 +538,11 @@ function run_impl() {
new_maintainer_addr=$(wait_for_maintainer_move "$api_addr" "$changefeed_id" "$maintainer_addr")
echo "maintainer moved to $new_maintainer_addr"
- disable_failpoint --addr "$origin_addr" --name "$FAILPOINT_NOT_READY_TO_CLOSE_DISPATCHER"
+ # Let dispatcher managers answer bootstrap, but keep the merge unfinished
+ # until the new maintainer has restored its operator from that snapshot.
disable_failpoint_on_all_addrs_best_effort "$FAILPOINT_BLOCK_CREATE_DISPATCHER"
-
wait_for_restored_merge_operator_in_logs "$work_dir" 60
+ disable_failpoint --addr "$origin_addr" --name "$FAILPOINT_NOT_READY_TO_CLOSE_DISPATCHER"
wait_for_table_replication_count "$api_addr" "$changefeed_id" "$table_id_6" "$((merge_replication_count_before - 1))" eq "$mode" 60
set +e
diff --git a/tests/integration_tests/multi_capture/run.sh b/tests/integration_tests/multi_capture/run.sh
deleted file mode 100755
index a2257283f6..0000000000
--- a/tests/integration_tests/multi_capture/run.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-source $CUR/../_utils/test_prepare
-WORK_DIR=$OUT_DIR/$TEST_NAME
-CDC_BINARY=cdc.test
-SINK_TYPE=$1
-
-CDC_COUNT=3
-DB_COUNT=4
-
-function run() {
- rm -rf $WORK_DIR && mkdir -p $WORK_DIR
-
- start_tidb_cluster --workdir $WORK_DIR
-
- # record tso before we create tables to skip the system table DDLs
- start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1})
-
- # create $DB_COUNT databases and import initial workload
- for i in $(seq $DB_COUNT); do
- db="multi_capture_$i"
- run_sql "CREATE DATABASE $db;"
- go-ycsb load mysql -P $CUR/conf/workload1 -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=$db
- done
-
- export GO_FAILPOINTS='github.com/pingcap/ticdc/utils/dynstream/InjectDropEvent=10%return(true)'
- # start $CDC_COUNT cdc servers, and create a changefeed
- for i in $(seq $CDC_COUNT); do
- run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --logsuffix "$i" --addr "127.0.0.1:830${i}"
- done
-
- TOPIC_NAME="ticdc-multi-capture-test-$RANDOM"
- case $SINK_TYPE in
- kafka) SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-version=${KAFKA_VERSION}&max-message-bytes=10485760" ;;
- storage) SINK_URI="file://$WORK_DIR/storage_test/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true" ;;
- pulsar)
- run_pulsar_cluster $WORK_DIR normal
- SINK_URI="pulsar://127.0.0.1:6650/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true"
- ;;
- *) SINK_URI="mysql://normal:123456@127.0.0.1:3306/" ;;
- esac
- cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --server="127.0.0.1:8301"
- case $SINK_TYPE in
- kafka) run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&version=${KAFKA_VERSION}&max-message-bytes=10485760" ;;
- storage) run_storage_consumer $WORK_DIR $SINK_URI "" "" ;;
- pulsar) run_pulsar_consumer --upstream-uri $SINK_URI ;;
- esac
-
- # check tables are created and data is synchronized
- for i in $(seq $DB_COUNT); do
- check_table_exists "multi_capture_$i.usertable" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
- done
- check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
-
- # add more data in upstream and check again
- for i in $(seq $DB_COUNT); do
- db="multi_capture_$i"
- go-ycsb load mysql -P $CUR/conf/workload2 -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=$db
- done
- check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
-
- cleanup_process $CDC_BINARY
-}
-
-trap 'stop_test $WORK_DIR' EXIT
-run $*
-check_logs $WORK_DIR
-echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"
diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh
index 789310dd61..0d25acbf24 100755
--- a/tests/integration_tests/run_light_it_in_ci.sh
+++ b/tests/integration_tests/run_light_it_in_ci.sh
@@ -40,7 +40,7 @@ mysql_groups=(
# G03
'capture_suicide_while_balance_table capture_local_fence_on_session_done kv_client_stream_reconnect ddl_default_current_timestamp fail_over_ddl_D'
# G04
- 'multi_capture ci_collation_compatibility resourcecontrol fail_over_ddl_E'
+ 'capture_write_lease ci_collation_compatibility resourcecontrol fail_over_ddl_E'
# G05
'vector simple partition_table fail_over_ddl_F conflict_key_generated_column wide_table'
# G06
@@ -79,7 +79,7 @@ kafka_groups=(
# G03
'kv_client_stream_reconnect fail_over_ddl_D'
# G04
- 'multi_capture ci_collation_compatibility resourcecontrol fail_over_ddl_E'
+ 'capture_write_lease ci_collation_compatibility resourcecontrol fail_over_ddl_E'
# G05
'vector simple partition_table fail_over_ddl_F conflict_key_generated_column wide_table'
# G06
@@ -117,7 +117,7 @@ pulsar_groups=(
# G03
'kv_client_stream_reconnect fail_over_ddl_D'
# G04
- 'multi_capture ci_collation_compatibility resourcecontrol fail_over_ddl_E'
+ 'capture_write_lease ci_collation_compatibility resourcecontrol fail_over_ddl_E'
# G05
'vector simple partition_table fail_over_ddl_F conflict_key_generated_column wide_table'
# G06
@@ -155,7 +155,7 @@ storage_groups=(
# G03
'kv_client_stream_reconnect fail_over_ddl_D'
# G04
- 'multi_capture ci_collation_compatibility resourcecontrol fail_over_ddl_E'
+ 'capture_write_lease ci_collation_compatibility resourcecontrol fail_over_ddl_E'
# G05
'vector simple partition_table fail_over_ddl_F conflict_key_generated_column wide_table'
# G06
diff --git a/tests/integration_tests/synced_status/run.sh b/tests/integration_tests/synced_status/run.sh
index 9e3515865c..8b8cbb5c87 100755
--- a/tests/integration_tests/synced_status/run.sh
+++ b/tests/integration_tests/synced_status/run.sh
@@ -141,14 +141,11 @@ function run_normal_case_and_unavailable_pd() {
fi
#==========
- # case 2: test with unavailable pd, query will not get the available response
+ # case 2: when PD is unavailable, the capture loses its etcd write proof and
+ # exits instead of continuing to serve a potentially stale synced response.
+ cdc_pid=$(get_cdc_pid "$CDC_HOST" "$CDC_PORT")
kill_pd
-
- sleep 20
-
- synced_status=$(curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced?keyspace=$KEYSPACE_NAME)
- echo "synced_status: $synced_status"
- error_code=$(echo $synced_status | jq -r '.error_code')
+ ensure 30 "! kill -0 $cdc_pid > /dev/null 2>&1"
cleanup_process $CDC_BINARY
stop_tidb_cluster
}
diff --git a/tests/integration_tests/synced_status_with_redo/run.sh b/tests/integration_tests/synced_status_with_redo/run.sh
index 541b951ec5..3f1f327c98 100755
--- a/tests/integration_tests/synced_status_with_redo/run.sh
+++ b/tests/integration_tests/synced_status_with_redo/run.sh
@@ -145,13 +145,15 @@ function run_normal_case_and_unavailable_pd() {
fi
#==========
- # case 2: test with unavailable pd, query will not get the available response
+ # case 2: when PD is unavailable, the capture loses its etcd write proof and
+ # exits instead of continuing to serve a potentially stale synced response.
+ cdc_pid=$(get_cdc_pid "$CDC_HOST" "$CDC_PORT")
+ if [ -z "$cdc_pid" ] || [ "$cdc_pid" = "null" ]; then
+ echo "failed to get a valid cdc pid"
+ exit 1
+ fi
kill_pd
-
- sleep 20
-
- synced_status=$(curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced?keyspace=$KEYSPACE_NAME)
- error_code=$(echo $synced_status | jq -r '.error_code')
+ ensure 30 "! kill -0 $cdc_pid > /dev/null 2>&1"
cleanup_process $CDC_BINARY
stop_tidb_cluster
}