diff --git a/cmd/raftadmin/main.go b/cmd/raftadmin/main.go index 4caac23f2..fc9f54f18 100644 --- a/cmd/raftadmin/main.go +++ b/cmd/raftadmin/main.go @@ -256,6 +256,8 @@ func printStatus(ctx context.Context, client pb.RaftAdminClient) error { fmt.Printf("fsm_pending: %d\n", resp.FsmPending) fmt.Printf("num_peers: %d\n", resp.NumPeers) fmt.Printf("last_contact_nanos: %d\n", resp.LastContactNanos) + fmt.Printf("configuration_index: %d\n", resp.ConfigurationIndex) + fmt.Printf("pending_conf_change: %t\n", resp.PendingConfChange) return nil } diff --git a/cmd/raftadmin/main_test.go b/cmd/raftadmin/main_test.go index 7d1f9e67c..8247c849e 100644 --- a/cmd/raftadmin/main_test.go +++ b/cmd/raftadmin/main_test.go @@ -54,9 +54,11 @@ func (fakeRaftAdminClient) TransferLeadership(context.Context, *pb.RaftAdminTran func TestExecuteCommandLeaderAndStateOutput(t *testing.T) { client := fakeRaftAdminClient{ statusResp: &pb.RaftAdminStatusResponse{ - State: pb.RaftAdminState_RAFT_ADMIN_STATE_LEADER, - LeaderId: "n1", - LeaderAddress: "127.0.0.1:50051", + State: pb.RaftAdminState_RAFT_ADMIN_STATE_LEADER, + LeaderId: "n1", + LeaderAddress: "127.0.0.1:50051", + ConfigurationIndex: 7, + PendingConfChange: true, }, } @@ -78,6 +80,8 @@ func TestExecuteCommandLeaderAndStateOutput(t *testing.T) { require.Contains(t, out, "leader_id: \"n1\"") require.Contains(t, out, "leader_address: \"127.0.0.1:50051\"") require.Contains(t, out, "commit_index: 0") + require.Contains(t, out, "configuration_index: 7") + require.Contains(t, out, "pending_conf_change: true") } func TestRPCTimeoutHonorsEnvSeconds(t *testing.T) { diff --git a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md new file mode 100644 index 000000000..ddc257ea0 --- /dev/null +++ b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md @@ -0,0 +1,276 @@ +# Fenced Raft voter replacement operation + +## 1. Status + +Implemented by `scripts/raft-member-replace.sh` as the operational companion to +the fresh learner join mode. The command automates one same-ID voter +replacement in one Raft group while a surviving quorum remains available. + +This is not an automatic repair loop. A candidate state, failed health check, +or missing process never authorizes data deletion or membership mutation. + +## 2. Motivation + +Removing a damaged data directory and running the normal deployment command is +not a valid Raft repair. Membership survives on the other voters in committed +Raft configuration, snapshots, WAL, and peer metadata. The wiped process has +none of that authority and may campaign with an incompatible local view. + +The safe operation already had to be performed manually: + +1. fence the old ID owner; +2. commit `RemoveServer`; +3. archive the target data; +4. start a fresh process in discovery-only join mode; +5. commit `AddLearner`; +6. wait for catch-up; +7. commit `PromoteLearner`; and +8. restart without temporary join flags. + +Repeated manual execution makes ordering, stale configuration indexes, and +partial failure the dominant risks. The implementation makes those boundaries +explicit and resumable. + +## 3. Scope and non-goals + +The v1 command supports: + +- one current voter selected by exact Raft ID; +- one Raft group exposed at the configured RaftAdmin endpoint; +- same-ID replacement at the same advertised address; +- external fencing or an explicitly selected container fence; +- archive-by-default data reset; +- the existing `rolling-update.sh` fresh learner mode; and +- durable local resume state on the control host. + +The command deliberately does not support: + +- no-quorum recovery or forced configuration rewrite; +- automatic execution based on CANDIDATE state; +- simultaneous replacement of multiple members; +- changing a member ID or advertised address; +- multi-group process replacement; or +- proving that a stopped VM can never be powered on again. + +No-quorum recovery needs a separate disaster-recovery design with an explicit +choice of authoritative log and accepted data-loss boundary. + +## 4. Safety invariants + +### 4.1 Explicit authorization + +`--execute` is required and `REPLACEMENT_CONFIRM` must exactly equal +`TARGET_NODE`. `--dry-run` performs no build, RPC, SSH, fence, or deployment. +For `--execute`, `TARGET_NODE`, `REPLACEMENT_CONFIRM`, +`REPLACEMENT_VERIFY_COMMAND`, and the external fence command must be supplied +on the invocation. Replacement controls are never read from the deployment env; +that file is inventory only, so a stale recovery value cannot authorize a new +operation. + +The target must be present in `NODES`, be a current voter, and have exactly the +advertised `NODES` address. A learner, absent ID, or address mismatch aborts. + +### 4.2 Surviving quorum before fencing + +The command reads the leader's authoritative configuration and computes +`floor(voters/2)+1`. At least that many non-target voters must answer RaftAdmin +status before fencing starts. This is intentionally stricter than counting the +target: removal must still commit after the target is stopped. + +The leader must be fully applied, expose a non-zero configuration index, and +have no pending configuration change. If the target is leader, leadership is +transferred only to a non-target follower whose applied index has reached the +leader's observed commit index. + +### 4.3 One live owner of a Raft ID + +The old process is fenced before `RemoveServer`. External mode requires an +operator-supplied command that fences the VM/process from the cluster network. +Container mode is opt-in and is valid only when host lifecycle policy prevents +the stopped container from independently returning. + +After the fence command, the target RaftAdmin endpoint must remain unreachable +for the configured consecutive verification interval. It is checked again +before removal and before the fresh deployment. Membership is never removed +while the old endpoint still answers. + +External mode also stops and verifies the target container over the management +SSH path after the Raft fence is observed. If that management action is not +available, the operation stops before `RemoveServer` rather than shrinking the +group with a still-running process. Docker inspection must itself succeed; +missing containers, daemon failures, and permission errors are not interpreted +as proof that the process is stopped. + +### 4.4 Serialized membership changes + +RaftAdmin status now reports: + +- `configuration_index`, the current durable membership index; and +- `pending_conf_change`, whether an unapplied change is in flight. + +Preflight records both the complete member signature and its non-zero +configuration index. Every `remove_server`, `add_learner`, and +`promote_learner` call passes that recorded index as `previous_index`; it never +adopts a newer index merely because another topology change committed. After +each successful change, the command verifies the exact expected post-change +signature before persisting the returned index for the next stage. The engine +rejects every new add, promote, or remove proposal while an earlier +configuration entry remains unapplied, so etcd/raft cannot silently rewrite a +racing proposal to a no-op. A resume after a lost RPC response is accepted only +when the complete member signature matches the one transition that was in +flight. It also accepts an equal saved/live configuration index when that +signature already proves the metadata write completed before the stage write. + +### 4.5 Catch-up before promotion + +Immediately before `AddLearner`, the command records the leader's commit index +as a durable `min_catch_up_index`. It writes that value to the resume file +before proposing the change so a control-host crash after commit cannot lose +the promotion bound. + +The learner must report `FOLLOWER`, a known leader, and +`applied_index >= min_catch_up_index`. Promotion passes the same non-zero value +to `promote_learner` and explicitly leaves `skip_min_applied_check=false`. + +### 4.6 Durable membership restart and application verification + +After promotion, the target is redeployed without `RAFT_JOIN_NODE`. This proves +it can start from its WAL, snapshot, and persisted peer metadata rather than +temporary discovery intent. + +Completion requires all original voter IDs to be present as voters, every +voter to answer as LEADER or FOLLOWER with no local apply lag, and one leader to +remain stable for the configured interval. The operator must also provide +`REPLACEMENT_VERIFY_COMMAND`; a successful application-level write/read check +is mandatory. + +## 5. Resume state machine + +The local state file is mode `0600` and is updated by write-then-rename. An +atomic directory lock prevents two control processes from operating on the +same state file. + +| Stage | Durable fact | +| --- | --- | +| 0 | Target identity, address, complete member signature and configuration index, original voter set, operation ID, canonical data directory, and archive path recorded | +| 1 | Preflight and any leadership transfer completed | +| 2 | Old ID owner fenced and endpoint observed down | +| 3 | Target absent from committed membership | +| 4 | Only the target data directory archived or deleted | +| 5 | Target deployed with fresh learner join intent | +| 6 | Target committed as learner; catch-up floor persisted | +| 7 | Target committed as voter | +| 8 | Target redeployed without join intent | +| 9 | Stable voter set and application write/read verification completed | + +Operations around a stage boundary are idempotent. The chosen data mode and +canonical data directory are stored with the operation and cannot change on +resume. For example, a resumed stage 2 accepts that removal already committed, +stage 4 safely reruns the target-only learner deployment after a lost rollout +response, stage 5 accepts an already committed learner only when its catch-up +floor exists, and stage 6 accepts an already promoted voter. A conflicting +suffrage, voter set, or data directory aborts instead of guessing what +happened. + +The state file is retained by default as operational evidence. A retained +stage-9 file is never treated as authorization for a later incident: a new +replacement must archive that evidence or select a new +`REPLACEMENT_STATE_FILE`. Set `REPLACEMENT_KEEP_STATE=false` only when another +durable audit system records the completed operation. + +## 6. Data handling + +Archive mode is the default. The deterministic archive path includes the +target ID and operation ID, making a retry distinguishable from a new +replacement. If both the live data path and archive path exist, the command +fails rather than overwriting either. + +Trailing slashes are removed from `DATA_DIR` before the sibling archive path is +derived, and the filesystem root is rejected. The archive can therefore never +be constructed as a child of the directory being moved. + +Delete mode must be explicitly selected. Both modes execute only after the ID +is absent from committed membership and verify the target container is not +running. The command never iterates over multiple data hosts. + +## 7. Operator interface + +Validate a plan without network or filesystem changes: + +```sh +TARGET_NODE=n2 \ +ROLLING_UPDATE_ENV_FILE=~/dump/elastickv-deploy/deploy.env \ +./scripts/raft-member-replace.sh --dry-run +``` + +Execute with an external hypervisor or network fence: + +```sh +TARGET_NODE=n2 \ +REPLACEMENT_CONFIRM=n2 \ +ROLLING_UPDATE_ENV_FILE=~/dump/elastickv-deploy/deploy.env \ +REPLACEMENT_FENCE_COMMAND='fence-n2-and-block-raft' \ +REPLACEMENT_VERIFY_COMMAND='redis-cli -h 192.168.0.64 SET replacement-check ok && redis-cli -h 192.168.0.64 GET replacement-check | grep -Fx ok' \ +./scripts/raft-member-replace.sh --execute +``` + +Use container mode only when the host cannot revive the old container during +the operation: + +```sh +REPLACEMENT_FENCE_MODE=container +``` + +The replacement command sources the deployment env for inventory and then +invokes `rolling-update.sh` twice with only `TARGET_NODE` selected. Bootstrap +membership is removed from both invocations. The first deploy includes +`RAFT_JOIN_NODE`; the post-promotion deploy clears it. Both invocations force +`DRY_RUN=false`, so a stale deployment-env dry-run setting cannot advance the +replacement state without starting the target. + +## 8. Failure handling + +| Failure | Required response | +| --- | --- | +| No leader or insufficient surviving quorum | Stop. Restore an existing voter or use a separately reviewed disaster-recovery procedure. | +| Target endpoint remains reachable after fence | Strengthen the fence. Do not remove membership. | +| Recorded member signature or `previous_index` mismatch | Inspect the concurrent topology change and reconcile the state file with the authoritative configuration. Do not retry blindly. | +| Existing state is already at stage 9 | Archive the completed evidence or choose a new state-file path before authorizing a new replacement. | +| Data and deterministic archive both exist | Inspect the target host. Do not delete either path automatically. | +| Learner catch-up timeout | Leave it a learner, diagnose transport/disk/snapshot transfer, and resume with the same state file. | +| Promotion committed but normal restart fails | Keep the surviving voters running. Repair the target from its existing post-join data; do not wipe it again. | +| Final voter set differs from the recorded set | Stop and audit membership. The command does not normalize unexpected changes. | +| Application verification fails | Replacement remains incomplete even if Raft health checks pass. Diagnose the public protocol path and resume verification. | + +## 9. Tests and acceptance + +`raft_member_replace_script_test.go` runs the complete shell workflow against a +stateful fake RaftAdmin, SSH transport, and rolling deploy command. It verifies +the exact voter-to-absent-to-learner-to-voter sequence, non-zero catch-up floor, +both deploy modes, forced execute rollout, application verification, durable +stage 9, canonical data-directory persistence, external-fence process stop, +and rejection of a completed state as a new operation. + +A negative test makes one surviving voter unreachable in a three-voter group +and proves the command aborts before the fence action. Another test loses the +`AddLearner` RPC response after commit and proves the persisted catch-up floor +allows a safe resume without a duplicate add. A separate lost-response test +proves a completed learner rollout is safely repeated from stage 4 rather than +leaving the group shrunken. Lock ownership is tested so a losing control +process cannot remove the winner's directory lock. RaftAdmin server and CLI +tests cover propagation and rendering of the configuration index and pending +change fields. Existing engine membership tests cover stale previous indexes, +learner catch-up enforcement, same-ID replacement, and persisted membership +restart. + +Additional negative tests prove that deployment-env replacement controls +cannot authorize an operation, a configuration-index change after preflight +aborts before removal, invocation-level timing controls override stale env +values, dot-segment data paths canonicalize to a sibling archive, and a changed +data directory is rejected on resume. Engine coverage also proves a membership +proposal is rejected while the pending configuration fence is active; the +shell resume test covers the equal-index metadata-before-stage crash window. + +Production acceptance still requires recording the state file, fence evidence, +final `raftadmin status`/`configuration`, and the output of the configured +application verification command. diff --git a/docs/raft_learner_operations.md b/docs/raft_learner_operations.md index d5cc138a0..3f75c1b8f 100644 --- a/docs/raft_learner_operations.md +++ b/docs/raft_learner_operations.md @@ -233,6 +233,32 @@ The script derives `--raftJoinMembers` from `NODES` and rejects a join rollout that contains another node. After promotion, clear `RAFT_JOIN_NODE` and run the same single-node rollout once more to remove the temporary join flags. +For recurring production replacements, use the fenced, resumable wrapper +instead of issuing those steps independently: + +```sh +TARGET_NODE=n4 \ +REPLACEMENT_CONFIRM=n4 \ +ROLLING_UPDATE_ENV_FILE=/path/to/deploy.env \ +REPLACEMENT_FENCE_COMMAND='fence-n4-and-block-raft' \ +REPLACEMENT_VERIFY_COMMAND='verify-a-real-write-and-read' \ +./scripts/raft-member-replace.sh --execute +``` + +Run the same command with `--dry-run` first. The wrapper verifies surviving +quorum before fencing, records the complete member signature and +`configuration_index`, records a non-zero learner catch-up floor, restarts +without join flags, and requires the application verification command to +succeed. Any unrelated membership change aborts the operation. It supports one +Raft group with a surviving quorum; it is not a forced no-quorum recovery tool. +Supply `TARGET_NODE` and all `REPLACEMENT_*` values on the invocation, not in +`deploy.env`; the deployment env is inventory only. A retained completed state +file is audit evidence, so archive it or choose a new state path for a later +incident. +See +`docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md` for the +state machine, fencing contract, and failure handling. + ## Removing a learner If a learner needs to be detached (e.g., the operator decided not to diff --git a/internal/raftadmin/server.go b/internal/raftadmin/server.go index bf6b54c9b..a2f893db0 100644 --- a/internal/raftadmin/server.go +++ b/internal/raftadmin/server.go @@ -3,6 +3,7 @@ package raftadmin import ( "context" stderrors "errors" + "sync" "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" @@ -19,6 +20,11 @@ type Server struct { // aware adapter via NewServerWithInterceptor; encryption-unaware // builds keep this nil. interceptor MembershipChangeInterceptor + // membershipMu serializes the pre-add interceptor and the configuration + // proposal through completion. This closes the window where a concurrent + // membership RPC could mutate encryption metadata before the engine exposes + // the first request's pending configuration change in Status. + membershipMu sync.Mutex pb.UnimplementedRaftAdminServer } @@ -47,17 +53,19 @@ func (s *Server) Status(context.Context, *pb.RaftAdminStatusRequest) (*pb.RaftAd } current := s.engine.Status() return &pb.RaftAdminStatusResponse{ - State: stateToProto(current.State), - LeaderId: current.Leader.ID, - LeaderAddress: current.Leader.Address, - Term: current.Term, - CommitIndex: current.CommitIndex, - AppliedIndex: current.AppliedIndex, - LastLogIndex: current.LastLogIndex, - LastSnapshotIndex: current.LastSnapshotIndex, - FsmPending: current.FSMPending, - NumPeers: current.NumPeers, - LastContactNanos: current.LastContact.Nanoseconds(), + State: stateToProto(current.State), + LeaderId: current.Leader.ID, + LeaderAddress: current.Leader.Address, + Term: current.Term, + CommitIndex: current.CommitIndex, + AppliedIndex: current.AppliedIndex, + LastLogIndex: current.LastLogIndex, + LastSnapshotIndex: current.LastSnapshotIndex, + FsmPending: current.FSMPending, + NumPeers: current.NumPeers, + LastContactNanos: current.LastContact.Nanoseconds(), + ConfigurationIndex: current.ConfigurationIndex, + PendingConfChange: current.PendingConfChange, }, nil } @@ -89,20 +97,7 @@ func (s *Server) AddVoter(ctx context.Context, req *pb.RaftAdminAddVoterRequest) if req == nil || req.Id == "" || req.Address == "" { return nil, grpcStatus(codes.InvalidArgument, "id and address are required") } - // Stage 7c §3.1 pre-step: run the encryption-aware adapter (if any) - // before the conf-change proposal so the new node's writer-registry - // row exists at apply time and any §6.1 uint16 collision halts here - // rather than after the conf-change is durable. - if s.interceptor != nil { - if err := s.interceptor.PreAddMember(ctx, req.Id); err != nil { - return nil, adminError(err) - } - } - index, err := s.admin.AddVoter(ctx, req.Id, req.Address, req.PreviousIndex) - if err != nil { - return nil, adminError(err) - } - return &pb.RaftAdminConfigurationChangeResponse{Index: index}, nil + return s.addMember(ctx, req.Id, req.Address, req.PreviousIndex, s.admin.AddVoter) } func (s *Server) AddLearner(ctx context.Context, req *pb.RaftAdminAddLearnerRequest) (*pb.RaftAdminConfigurationChangeResponse, error) { @@ -112,18 +107,42 @@ func (s *Server) AddLearner(ctx context.Context, req *pb.RaftAdminAddLearnerRequ if req == nil || req.Id == "" || req.Address == "" { return nil, grpcStatus(codes.InvalidArgument, "id and address are required") } + return s.addMember(ctx, req.Id, req.Address, req.PreviousIndex, s.admin.AddLearner) +} + +func (s *Server) addMember( + ctx context.Context, + id string, + address string, + previousIndex uint64, + propose func(context.Context, string, string, uint64) (uint64, error), +) (*pb.RaftAdminConfigurationChangeResponse, error) { + s.membershipMu.Lock() + defer s.membershipMu.Unlock() + if err := s.requireMembershipChangeReady(); err != nil { + return nil, err + } + // Stage 7c pre-registers encryption metadata before proposing membership. + // Keep that pre-step and the proposal inside the same serialized region. if s.interceptor != nil { - if err := s.interceptor.PreAddMember(ctx, req.Id); err != nil { + if err := s.interceptor.PreAddMember(ctx, id); err != nil { return nil, adminError(err) } } - index, err := s.admin.AddLearner(ctx, req.Id, req.Address, req.PreviousIndex) + index, err := propose(ctx, id, address, previousIndex) if err != nil { return nil, adminError(err) } return &pb.RaftAdminConfigurationChangeResponse{Index: index}, nil } +func (s *Server) requireMembershipChangeReady() error { + if s.engine.Status().PendingConfChange { + return adminError(raftengine.ErrMembershipChangePending) + } + return nil +} + func (s *Server) PromoteLearner(ctx context.Context, req *pb.RaftAdminPromoteLearnerRequest) (*pb.RaftAdminConfigurationChangeResponse, error) { if s == nil || s.admin == nil { return nil, grpcStatus(codes.Unimplemented, "promote learner is not supported by this raft engine") @@ -131,6 +150,8 @@ func (s *Server) PromoteLearner(ctx context.Context, req *pb.RaftAdminPromoteLea if req == nil || req.Id == "" { return nil, grpcStatus(codes.InvalidArgument, "id is required") } + s.membershipMu.Lock() + defer s.membershipMu.Unlock() index, err := s.admin.PromoteLearner(ctx, req.Id, req.PreviousIndex, req.MinAppliedIndex, req.SkipMinAppliedCheck) if err != nil { return nil, adminError(err) @@ -145,6 +166,8 @@ func (s *Server) RemoveServer(ctx context.Context, req *pb.RaftAdminRemoveServer if req == nil || req.Id == "" { return nil, grpcStatus(codes.InvalidArgument, "id is required") } + s.membershipMu.Lock() + defer s.membershipMu.Unlock() index, err := s.admin.RemoveServer(ctx, req.Id, req.PreviousIndex) if err != nil { return nil, adminError(err) diff --git a/internal/raftadmin/server_test.go b/internal/raftadmin/server_test.go index 4ec02aba6..ac576f91b 100644 --- a/internal/raftadmin/server_test.go +++ b/internal/raftadmin/server_test.go @@ -202,14 +202,16 @@ func TestServerMapsEngineAdminMethods(t *testing.T) { ID: "node-1", Address: "127.0.0.1:50051", }, - Term: 7, - CommitIndex: 10, - AppliedIndex: 9, - LastLogIndex: 12, - LastSnapshotIndex: 8, - FSMPending: 1, - NumPeers: 2, - LastContact: 0, + Term: 7, + CommitIndex: 10, + AppliedIndex: 9, + LastLogIndex: 12, + LastSnapshotIndex: 8, + FSMPending: 1, + NumPeers: 2, + LastContact: 0, + ConfigurationIndex: 6, + PendingConfChange: true, }, config: raftengine.Configuration{ Servers: []raftengine.Server{ @@ -225,6 +227,11 @@ func TestServerMapsEngineAdminMethods(t *testing.T) { require.Equal(t, pb.RaftAdminState_RAFT_ADMIN_STATE_LEADER, statusResp.State) require.Equal(t, "node-1", statusResp.LeaderId) require.Equal(t, uint64(10), statusResp.CommitIndex) + require.Equal(t, uint64(6), statusResp.ConfigurationIndex) + require.True(t, statusResp.PendingConfChange) + engine.mu.Lock() + engine.status.PendingConfChange = false + engine.mu.Unlock() cfgResp, err := server.Configuration(context.Background(), &pb.RaftAdminConfigurationRequest{}) require.NoError(t, err) @@ -495,6 +502,82 @@ func TestServer_AddVoter_NilInterceptorSkipsPreStep(t *testing.T) { require.Equal(t, 1, len(engine.addVoterCalls)) } +func TestServer_AddMemberRejectsPendingConfigBeforeInterceptor(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + call func(*Server) error + }{ + { + name: "voter", + call: func(server *Server) error { + _, err := server.AddVoter(context.Background(), &pb.RaftAdminAddVoterRequest{Id: "n42", Address: "127.0.0.1:9999"}) + return err + }, + }, + { + name: "learner", + call: func(server *Server) error { + _, err := server.AddLearner(context.Background(), &pb.RaftAdminAddLearnerRequest{Id: "n42", Address: "127.0.0.1:9999"}) + return err + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + engine := &fakeEngine{status: raftengine.Status{PendingConfChange: true}} + interceptor := &recordingInterceptor{} + server := NewServerWithInterceptor(engine, interceptor) + + err := test.call(server) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, raftengine.ErrMembershipChangePending.Error()) + require.Empty(t, interceptor.calls) + require.Empty(t, engine.addVoterCalls) + require.Empty(t, engine.addLearnerCalls) + }) + } +} + +func TestServerSerializesMembershipInterceptorAndProposal(t *testing.T) { + t.Parallel() + + firstProposalEntered := make(chan struct{}) + releaseFirstProposal := make(chan struct{}) + interceptorCalls := make(chan string, 2) + engine := &fakeEngine{addVoterHook: func() { + close(firstProposalEntered) + <-releaseFirstProposal + }} + interceptor := &recordingInterceptor{preHook: func(id string) { interceptorCalls <- id }} + server := NewServerWithInterceptor(engine, interceptor) + + firstDone := make(chan error, 1) + go func() { + _, err := server.AddVoter(context.Background(), &pb.RaftAdminAddVoterRequest{Id: "n1", Address: "127.0.0.1:9001"}) + firstDone <- err + }() + require.Equal(t, "n1", <-interceptorCalls) + <-firstProposalEntered + + secondDone := make(chan error, 1) + go func() { + _, err := server.AddLearner(context.Background(), &pb.RaftAdminAddLearnerRequest{Id: "n2", Address: "127.0.0.1:9002"}) + secondDone <- err + }() + select { + case id := <-interceptorCalls: + require.Failf(t, "concurrent interceptor ran", "interceptor for %s ran before the first membership proposal completed", id) + case <-time.After(100 * time.Millisecond): + } + + close(releaseFirstProposal) + require.NoError(t, <-firstDone) + require.Equal(t, "n2", <-interceptorCalls) + require.NoError(t, <-secondDone) +} + // TestServer_AddLearner_InterceptorContract is the symmetric set for // AddLearner — combined into one test since the behavior mirrors // AddVoter exactly. diff --git a/internal/raftengine/engine.go b/internal/raftengine/engine.go index 75ceacc8c..7b010a3b4 100644 --- a/internal/raftengine/engine.go +++ b/internal/raftengine/engine.go @@ -35,6 +35,9 @@ var ( // unapplied configuration change, so transfer is rejected until the // membership transition settles. ErrLeadershipTransferConfChangePending = errors.New("raft engine: leadership transfer blocked by pending config change") + // ErrMembershipChangePending indicates that a membership proposal is + // already present in the local Raft log and has not settled yet. + ErrMembershipChangePending = errors.New("raft engine: membership change blocked by pending config change") // ErrEnvelopeCutoverInProgress indicates the §7.1 raft-envelope // cutover barrier is open on this leader and rejecting fresh // USER proposals on the Propose path. The error is the step-1 @@ -92,6 +95,10 @@ type Status struct { FSMPending uint64 NumPeers uint64 LastContact time.Duration + // ConfigurationIndex is the highest committed membership index durably + // published by this node. Operators pass it back as previous_index on the + // next membership RPC to reject concurrent topology changes. + ConfigurationIndex uint64 // LeadTransferee is non-zero on the current leader while a leadership // transfer is in progress, and zero otherwise (including on followers). // Writers should hold new proposals while this is non-zero, since etcd/raft diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 3fa139c8e..15a2bfc26 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -143,6 +143,7 @@ var ( errLeadershipTransferNoHealthyTarget = errors.Mark(errors.New("etcd raft leadership transfer has no healthy target"), raftengine.ErrLeadershipTransferNoHealthyTarget) errLeadershipTransferTargetNotCaughtUp = errors.Mark(errors.New("etcd raft leadership transfer target is not caught up"), raftengine.ErrLeadershipTransferTargetNotCaughtUp) errLeadershipTransferConfChangePending = errors.Mark(errors.New("etcd raft leadership transfer blocked by pending config change"), raftengine.ErrLeadershipTransferConfChangePending) + errMembershipConfChangePending = errors.Mark(errors.New("etcd raft membership change blocked by pending config change"), raftengine.ErrMembershipChangePending) errTooManyPendingConfigs = errors.New("etcd raft engine has too many pending config changes") errPromoteLearnerNotLearner = errors.New("etcd raft promote-learner target is not a learner") errPromoteLearnerNoProgress = errors.New("etcd raft promote-learner target has no leader-side progress entry") @@ -2030,9 +2031,17 @@ func (e *Engine) handlePromoteLearner(req adminRequest) { // proposeMembershipChange wraps the encode + storePendingConfig + // ProposeConfChange dance shared by AddVoter / AddLearner / -// PromoteLearner. The caller has already validated the leader and +// PromoteLearner / RemoveServer. The caller has already validated the leader and // prevIndex preconditions in handleAdmin. func (e *Engine) proposeMembershipChange(req adminRequest, changeType raftpb.ConfChangeType, peer Peer) { + if err := e.reconcilePendingConfChangeFence(); err != nil { + req.done <- adminResult{err: err} + return + } + if e.hasPendingConfChange() { + req.done <- adminResult{err: errors.WithStack(errMembershipConfChangePending)} + return + } contextBytes, err := encodeConfChangeContext(req.id, peer) if err != nil { req.done <- adminResult{err: err} @@ -2062,24 +2071,7 @@ func (e *Engine) handleRemoveServer(req adminRequest) { req.done <- adminResult{err: errors.Wrapf(errTransportPeerUnknown, "id=%q", req.peer.ID)} return } - contextBytes, err := encodeConfChangeContext(req.id, peer) - if err != nil { - req.done <- adminResult{err: err} - return - } - cc := raftpb.ConfChange{ - Type: confChangeTypePtr(raftpb.ConfChangeRemoveNode), - NodeId: uint64Ptr(peer.NodeID), - Context: contextBytes, - } - if err := e.storePendingConfig(req); err != nil { - req.done <- adminResult{err: err} - return - } - if err := e.rawNode.ProposeConfChange(&cc); err != nil { - e.cancelPendingConfig(req.id) - req.done <- adminResult{err: errors.WithStack(err)} - } + e.proposeMembershipChange(req, raftpb.ConfChangeRemoveNode, peer) } func (e *Engine) handleTransferLeadership(req adminRequest) { @@ -2552,12 +2544,15 @@ func (e *Engine) applyReadyEntries(entries []*raftpb.Entry) error { if len(entries) == 0 { return nil } + if err := e.storage.Append(entries); err != nil { + return errors.WithStack(err) + } for _, entry := range entries { if entry.GetType() == raftpb.EntryConfChange || entry.GetType() == raftpb.EntryConfChangeV2 { e.markPendingConfChange(entry.GetIndex()) } } - return errors.WithStack(e.storage.Append(entries)) + return e.reconcilePendingConfChangeFence() } func (e *Engine) restorePendingConfChangeFenceFromStorage(applied uint64) error { @@ -3564,18 +3559,19 @@ func (e *Engine) refreshStatus() { leader := e.leaderInfo(basic.Lead) status := raftengine.Status{ - State: state, - Leader: leader, - Term: basic.GetTerm(), - CommitIndex: basic.GetCommit(), - AppliedIndex: e.applied, - LastLogIndex: lastLogIndex, - LastSnapshotIndex: snapshot.GetMetadata().GetIndex(), - FSMPending: pendingEntries(basic.GetCommit(), e.applied), - NumPeers: numRemoteServers(config.Servers, e.localID), - LastContact: lastContactFor(state, basic.Lead, e.lastLeaderContactFrom, e.lastLeaderContactAt), - LeadTransferee: basic.LeadTransferee, - PendingConfChange: e.hasPendingConfChange(), + State: state, + Leader: leader, + Term: basic.GetTerm(), + CommitIndex: basic.GetCommit(), + AppliedIndex: e.applied, + LastLogIndex: lastLogIndex, + LastSnapshotIndex: snapshot.GetMetadata().GetIndex(), + FSMPending: pendingEntries(basic.GetCommit(), e.applied), + NumPeers: numRemoteServers(config.Servers, e.localID), + LastContact: lastContactFor(state, basic.Lead, e.lastLeaderContactFrom, e.lastLeaderContactAt), + ConfigurationIndex: e.currentConfigIndex(), + LeadTransferee: basic.LeadTransferee, + PendingConfChange: e.hasPendingConfChange(), } e.mu.Lock() @@ -3927,6 +3923,64 @@ func (e *Engine) hasPendingConfChange() bool { return e.pendingConfChangeIndex.Load() != 0 } +// reconcilePendingConfChangeFence drops a volatile fence when a later Ready +// proves that the proposal at that index was overwritten after leadership +// loss. A missing index is not enough: the proposal may still live only in +// raft's unstable log, so the fence remains until storage covers the index. +func (e *Engine) reconcilePendingConfChangeFence() error { + pending := e.pendingConfChangeIndex.Load() + if pending == 0 { + return nil + } + if e.storage == nil { + return nil + } + if e.appliedIndex.Load() >= pending { + e.clearPendingConfChange(pending) + return nil + } + covered, isConfChange, err := e.persistedPendingConfChange(pending) + if err != nil { + return err + } + if !covered { + return nil + } + if !isConfChange { + e.pendingConfChangeIndex.CompareAndSwap(pending, 0) + } + return nil +} + +func (e *Engine) persistedPendingConfChange(index uint64) (bool, bool, error) { + firstIndex, err := e.storage.FirstIndex() + if err != nil { + return false, false, errors.Wrap(err, "reconcile pending conf-change fence: first index") + } + lastIndex, err := e.storage.LastIndex() + if err != nil { + return false, false, errors.Wrap(err, "reconcile pending conf-change fence: last index") + } + if lastIndex < index { + return false, false, nil + } + if index < firstIndex { + return true, false, nil + } + if index == math.MaxUint64 { + return false, false, errors.New("reconcile pending conf-change fence: index overflows half-open range") + } + entries, err := e.storage.Entries(index, index+1, math.MaxUint64) + if err != nil { + return false, false, errors.Wrapf(err, "reconcile pending conf-change fence: entry %d", index) + } + if len(entries) != 1 { + return false, false, errors.Errorf("reconcile pending conf-change fence: expected one entry at %d, got %d", index, len(entries)) + } + entryType := entries[0].GetType() + return true, entryType == raftpb.EntryConfChange || entryType == raftpb.EntryConfChangeV2, nil +} + func (e *Engine) shouldCampaignSingleNode() bool { // Count VOTERS in the current configuration, not all servers -- // a learner does not vote and must not block the single-voter diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 7a4ec6518..22beaf800 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -420,6 +420,10 @@ func TestOpenInitializesAppliedIndexFromPersistedSnapshot(t *testing.T) { }) require.GreaterOrEqual(t, engine.Status().AppliedIndex, uint64(5)) + require.Eventually(t, func() bool { + return engine.Status().ConfigurationIndex == engine.currentConfigIndex() && + engine.Status().ConfigurationIndex > 0 + }, time.Second, 10*time.Millisecond) } func TestOpenRestoresPeersFromPersistedMetadata(t *testing.T) { @@ -1633,6 +1637,44 @@ func TestPendingConfChangeFenceTracksUnappliedConfig(t *testing.T) { require.False(t, engine.hasPendingConfChange()) } +func TestProposeMembershipChangeRejectsPendingConfig(t *testing.T) { + engine := &Engine{} + engine.markPendingConfChange(12) + done := make(chan adminResult, 1) + req := adminRequest{id: 17, done: done} + + engine.proposeMembershipChange(req, raftpb.ConfChangeRemoveNode, Peer{NodeID: 2, ID: "n2"}) + + result := <-done + require.ErrorIs(t, result.err, errMembershipConfChangePending) + require.Empty(t, engine.pendingConfigs) +} + +func TestReconcilePendingConfChangeFenceClearsOverwrittenProposal(t *testing.T) { + storage := committedTailStorageWithEntries(t, 10, 12, map[uint64]raftpb.Entry{ + 12: { + Type: entryTypePtr(raftpb.EntryNormal), + Data: []byte("replacement leader entry"), + }, + }) + engine := &Engine{storage: storage} + engine.appliedIndex.Store(10) + engine.markPendingConfChange(12) + + require.NoError(t, engine.reconcilePendingConfChangeFence()) + require.False(t, engine.hasPendingConfChange()) +} + +func TestReconcilePendingConfChangeFenceKeepsUnstableProposal(t *testing.T) { + storage := committedTailStorageWithEntries(t, 10, 11, nil) + engine := &Engine{storage: storage} + engine.appliedIndex.Store(10) + engine.markPendingConfChange(12) + + require.NoError(t, engine.reconcilePendingConfChangeFence()) + require.True(t, engine.hasPendingConfChange()) +} + func TestRestorePendingConfChangeFenceFromStorage(t *testing.T) { storage := committedTailStorageWithEntries(t, 100, 150, map[uint64]raftpb.Entry{ 130: { diff --git a/proto/service.pb.go b/proto/service.pb.go index 00000ae2f..0b54bfdab 100644 --- a/proto/service.pb.go +++ b/proto/service.pb.go @@ -1537,20 +1537,22 @@ func (*RaftAdminStatusRequest) Descriptor() ([]byte, []int) { } type RaftAdminStatusResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - State RaftAdminState `protobuf:"varint,1,opt,name=state,proto3,enum=RaftAdminState" json:"state,omitempty"` - LeaderId string `protobuf:"bytes,2,opt,name=leader_id,json=leaderId,proto3" json:"leader_id,omitempty"` - LeaderAddress string `protobuf:"bytes,3,opt,name=leader_address,json=leaderAddress,proto3" json:"leader_address,omitempty"` - Term uint64 `protobuf:"varint,4,opt,name=term,proto3" json:"term,omitempty"` - CommitIndex uint64 `protobuf:"varint,5,opt,name=commit_index,json=commitIndex,proto3" json:"commit_index,omitempty"` - AppliedIndex uint64 `protobuf:"varint,6,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` - LastLogIndex uint64 `protobuf:"varint,7,opt,name=last_log_index,json=lastLogIndex,proto3" json:"last_log_index,omitempty"` - LastSnapshotIndex uint64 `protobuf:"varint,8,opt,name=last_snapshot_index,json=lastSnapshotIndex,proto3" json:"last_snapshot_index,omitempty"` - FsmPending uint64 `protobuf:"varint,9,opt,name=fsm_pending,json=fsmPending,proto3" json:"fsm_pending,omitempty"` - NumPeers uint64 `protobuf:"varint,10,opt,name=num_peers,json=numPeers,proto3" json:"num_peers,omitempty"` - LastContactNanos int64 `protobuf:"varint,11,opt,name=last_contact_nanos,json=lastContactNanos,proto3" json:"last_contact_nanos,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + State RaftAdminState `protobuf:"varint,1,opt,name=state,proto3,enum=RaftAdminState" json:"state,omitempty"` + LeaderId string `protobuf:"bytes,2,opt,name=leader_id,json=leaderId,proto3" json:"leader_id,omitempty"` + LeaderAddress string `protobuf:"bytes,3,opt,name=leader_address,json=leaderAddress,proto3" json:"leader_address,omitempty"` + Term uint64 `protobuf:"varint,4,opt,name=term,proto3" json:"term,omitempty"` + CommitIndex uint64 `protobuf:"varint,5,opt,name=commit_index,json=commitIndex,proto3" json:"commit_index,omitempty"` + AppliedIndex uint64 `protobuf:"varint,6,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` + LastLogIndex uint64 `protobuf:"varint,7,opt,name=last_log_index,json=lastLogIndex,proto3" json:"last_log_index,omitempty"` + LastSnapshotIndex uint64 `protobuf:"varint,8,opt,name=last_snapshot_index,json=lastSnapshotIndex,proto3" json:"last_snapshot_index,omitempty"` + FsmPending uint64 `protobuf:"varint,9,opt,name=fsm_pending,json=fsmPending,proto3" json:"fsm_pending,omitempty"` + NumPeers uint64 `protobuf:"varint,10,opt,name=num_peers,json=numPeers,proto3" json:"num_peers,omitempty"` + LastContactNanos int64 `protobuf:"varint,11,opt,name=last_contact_nanos,json=lastContactNanos,proto3" json:"last_contact_nanos,omitempty"` + ConfigurationIndex uint64 `protobuf:"varint,12,opt,name=configuration_index,json=configurationIndex,proto3" json:"configuration_index,omitempty"` + PendingConfChange bool `protobuf:"varint,13,opt,name=pending_conf_change,json=pendingConfChange,proto3" json:"pending_conf_change,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RaftAdminStatusResponse) Reset() { @@ -1660,6 +1662,20 @@ func (x *RaftAdminStatusResponse) GetLastContactNanos() int64 { return 0 } +func (x *RaftAdminStatusResponse) GetConfigurationIndex() uint64 { + if x != nil { + return x.ConfigurationIndex + } + return 0 +} + +func (x *RaftAdminStatusResponse) GetPendingConfChange() bool { + if x != nil { + return x.PendingConfChange + } + return false +} + type RaftAdminConfigurationRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -2350,7 +2366,7 @@ const file_service_proto_rawDesc = "" + "\bstart_ts\x18\x01 \x01(\x04R\astartTs\",\n" + "\x10RollbackResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\"\x18\n" + - "\x16RaftAdminStatusRequest\"\xa2\x03\n" + + "\x16RaftAdminStatusRequest\"\x83\x04\n" + "\x17RaftAdminStatusResponse\x12%\n" + "\x05state\x18\x01 \x01(\x0e2\x0f.RaftAdminStateR\x05state\x12\x1b\n" + "\tleader_id\x18\x02 \x01(\tR\bleaderId\x12%\n" + @@ -2364,7 +2380,9 @@ const file_service_proto_rawDesc = "" + "fsmPending\x12\x1b\n" + "\tnum_peers\x18\n" + " \x01(\x04R\bnumPeers\x12,\n" + - "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanos\"\x1f\n" + + "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanos\x12/\n" + + "\x13configuration_index\x18\f \x01(\x04R\x12configurationIndex\x12.\n" + + "\x13pending_conf_change\x18\r \x01(\bR\x11pendingConfChange\"\x1f\n" + "\x1dRaftAdminConfigurationRequest\"W\n" + "\x0fRaftAdminMember\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x18\n" + diff --git a/proto/service.proto b/proto/service.proto index 68d12d5f7..c13e2e582 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -195,6 +195,8 @@ message RaftAdminStatusResponse { uint64 fsm_pending = 9; uint64 num_peers = 10; int64 last_contact_nanos = 11; + uint64 configuration_index = 12; + bool pending_conf_change = 13; } message RaftAdminConfigurationRequest {} diff --git a/raft_member_replace_script_test.go b/raft_member_replace_script_test.go new file mode 100644 index 000000000..685e4943e --- /dev/null +++ b/raft_member_replace_script_test.go @@ -0,0 +1,639 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte(strings.Join([]string{ + "NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3", + "RAFT_PORT=50051", + "SSH_USER=tester", + "CONTAINER_NAME=elastickv", + "DATA_DIR=/var/lib/./elastickv/.", + "TARGET_NODE=n3", + "REPLACEMENT_CONFIRM=n3", + "REPLACEMENT_VERIFY_COMMAND=false", + "REPLACEMENT_FENCE_VERIFY_SECONDS=999", + "REPLACEMENT_STABILITY_SECONDS=999", + "REPLACEMENT_TIMEOUT_SECONDS=999", + "REPLACEMENT_POLL_SECONDS=999", + "REPLACEMENT_KEEP_STATE=false", + "DRY_RUN=true", + }, "\n")+"\n"), 0o600)) + + stateFile := filepath.Join(stateDir, "replace.state") + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=touch $FAKE_STATE_DIR/verified", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + }) + require.Contains(t, output, "replacement completed: target=n2") + require.GreaterOrEqual(t, strings.Count(mustReadTestFile(t, filepath.Join(stateDir, "stability-status-calls")), "\n"), 10, + "a one-second stability window with one-second polling requires two complete healthy observations") + require.FileExists(t, filepath.Join(stateDir, "verified")) + require.FileExists(t, filepath.Join(stateDir, "data-reset")) + require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) + require.Equal(t, "n2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts"))) + + state := mustReadTestFile(t, stateFile) + require.Contains(t, state, "stage=9\n") + require.Contains(t, state, "expected_voters=n1,n2,n3\n") + require.Contains(t, state, "expected_members=n1|127.0.0.1:50051|voter,n2|127.0.0.2:50051|voter,n3|127.0.0.3:50051|voter\n") + require.Contains(t, state, "expected_config_index=13\n") + require.Contains(t, state, "min_catch_up_index=100\n") + require.Contains(t, state, "data_mode=archive\n") + require.Contains(t, state, "data_dir=/var/lib/elastickv\n") + require.Contains(t, state, "archive_path=/var/lib/elastickv.replacement-n2-") + require.NotContains(t, state, "archive_path=/var/lib/elastickv/.replacement") + require.Equal(t, "stop\nreset-data\n", mustReadTestFile(t, filepath.Join(stateDir, "remote-actions"))) + + resumeCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + resumeCmd.Dir = repoRoot + resumeCmd.Env = append(os.Environ(), + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE="+stateFile, + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + "RAFTADMIN_BIN="+raftadmin, + "SSH_BIN="+ssh, + "FAKE_STATE_DIR="+stateDir, + ) + resumeOutput, resumeErr := resumeCmd.CombinedOutput() + require.Error(t, resumeErr) + require.Contains(t, string(resumeOutput), "replacement state is already complete") + require.Equal(t, "n2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts")), "completed resume must not repeat deployment") +} + +func TestRaftMemberReplaceScriptStopsProcessAfterExternalFence(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=external", + `REPLACEMENT_FENCE_COMMAND=touch "$FAKE_STATE_DIR/external-fenced"; touch "$FAKE_STATE_DIR/down-n2"`, + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + }) + require.Contains(t, output, "external fence verified; stopping the target process") + require.FileExists(t, filepath.Join(stateDir, "external-fenced")) + require.Equal(t, "stop\nreset-data\n", mustReadTestFile(t, filepath.Join(stateDir, "remote-actions"))) +} + +func TestRaftMemberReplaceScriptValidatesReportedLeader(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "stale-leader-once"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n3=127.0.0.3,n1=127.0.0.1,n2=127.0.0.2\n"), 0o600)) + + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + }) + require.Contains(t, output, "replacement completed: target=n2") +} + +func TestRaftMemberReplaceScriptAdoptsConfigIndexAfterLeadershipTransfer(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + stateFile := filepath.Join(stateDir, "replace.state") + + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + "FAKE_TARGET_STARTS_LEADER=true", + "FAKE_BUMP_CONFIG_ON_TRANSFER=true", + }) + require.Contains(t, output, "transferring leadership from n2 to n1") + require.Contains(t, output, "replacement completed: target=n2") + require.Contains(t, mustReadTestFile(t, stateFile), "expected_config_index=14\n") +} + +func TestRaftMemberReplaceScriptRequiresInvocationControls(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + rolling := writeExecutableTestFile(t, stateDir, "rolling-update", "#!/usr/bin/env bash\nexit 0\n") + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte(strings.Join([]string{ + "NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3", + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_VERIFY_COMMAND=true", + }, "\n")+"\n"), 0o600)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "TARGET_NODE is required") +} + +func TestRaftMemberReplaceScriptRejectsConfigChangeAfterPreflight(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE="+filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + "RAFTADMIN_BIN="+raftadmin, + "SSH_BIN="+ssh, + "FAKE_STATE_DIR="+stateDir, + "FAKE_BUMP_CONFIG_ON_FENCE=true", + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "remove_server configuration index changed: expected=10 actual=11") + require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) +} + +func TestRaftMemberReplaceScriptRejectsInsufficientSurvivingQuorum(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "candidate-n3"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STATE_FILE="+filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + "RAFTADMIN_BIN="+raftadmin, + "SSH_BIN="+ssh, + "FAKE_STATE_DIR="+stateDir, + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "only 1 non-target voters are reachable; 2 are required") + require.NoFileExists(t, filepath.Join(stateDir, "down-n2"), "fencing must not start without surviving quorum") +} + +func TestRaftMemberReplaceScriptResumesAfterLearnerCommitResponseLoss(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "fail-add-once"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + stateFile := filepath.Join(stateDir, "replace.state") + commandEnv := []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + } + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), commandEnv...) + output, err := cmd.CombinedOutput() + require.Error(t, err, "%s", output) + require.Equal(t, "learner\n", mustReadTestFile(t, filepath.Join(stateDir, "member")), "the simulated RPC response is lost after commit") + interruptedState := mustReadTestFile(t, stateFile) + require.Contains(t, interruptedState, "stage=5\n") + require.Contains(t, interruptedState, "min_catch_up_index=100\n", "catch-up floor must precede the membership proposal") + require.Contains(t, interruptedState, "data_dir=/var/lib/elastickv\n") + + // Simulate the old two-write window where the membership metadata reached + // disk but the stage advance did not. Equal index plus the expected + // signature must be accepted so an interrupted replacement remains resumable. + interruptedState = strings.Replace(interruptedState, "expected_config_index=11\n", "expected_config_index=12\n", 1) + interruptedState = strings.Replace( + interruptedState, + "expected_members=n1|127.0.0.1:50051|voter,n3|127.0.0.3:50051|voter\n", + "expected_members=n1|127.0.0.1:50051|voter,n2|127.0.0.2:50051|learner,n3|127.0.0.3:50051|voter\n", + 1, + ) + require.NoError(t, os.WriteFile(stateFile, []byte(interruptedState), 0o600)) + + staleMetadataState := strings.Replace(interruptedState, "expected_config_index=12\n", "expected_config_index=11\n", 1) + require.NoError(t, os.WriteFile(stateFile, []byte(staleMetadataState), 0o600)) + staleMetadataCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + staleMetadataCmd.Dir = repoRoot + staleMetadataCmd.Env = append(os.Environ(), commandEnv...) + staleMetadataOutput, staleMetadataErr := staleMetadataCmd.CombinedOutput() + require.Error(t, staleMetadataErr) + require.Contains(t, string(staleMetadataOutput), "saved post-change configuration index changed: expected=11 actual=12") + require.NoError(t, os.WriteFile(stateFile, []byte(interruptedState), 0o600)) + + changedDataDirCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + changedDataDirCmd.Dir = repoRoot + changedDataDirEnv := append(append([]string{}, commandEnv...), "DATA_DIR=/srv/elastickv") + changedDataDirCmd.Env = append(os.Environ(), changedDataDirEnv...) + changedDataDirOutput, changedDataDirErr := changedDataDirCmd.CombinedOutput() + require.Error(t, changedDataDirErr) + require.Contains(t, string(changedDataDirOutput), "DATA_DIR changed from /var/lib/elastickv to /srv/elastickv") + + resumeOutput := runReplacementScript(t, repoRoot, commandEnv) + require.Contains(t, resumeOutput, "target is already a learner; resuming after committed add") + require.Contains(t, resumeOutput, "replacement completed: target=n2") + require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) +} + +func TestRaftMemberReplaceScriptResumesAfterJoinRolloutResponseLoss(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "fail-join-rollout-once"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + stateFile := filepath.Join(stateDir, "replace.state") + commandEnv := []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + } + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), commandEnv...) + output, err := cmd.CombinedOutput() + require.Error(t, err, "%s", output) + require.Contains(t, mustReadTestFile(t, stateFile), "stage=4\n") + require.Equal(t, "absent\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) + require.Equal(t, "n2:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts"))) + + resumeOutput := runReplacementScript(t, repoRoot, commandEnv) + require.Contains(t, resumeOutput, "replacement completed: target=n2") + require.Equal(t, "n2:n2\nn2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts"))) +} + +func TestRaftMemberReplaceScriptDoesNotRemoveAnotherProcessLock(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + rolling := writeExecutableTestFile(t, stateDir, "rolling-update", "#!/usr/bin/env bash\nexit 0\n") + stateFile := filepath.Join(stateDir, "replace.state") + lockDir := stateFile + ".lock" + require.NoError(t, os.Mkdir(lockDir, 0o700)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "TARGET_NODE=n2", + "NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STATE_FILE="+stateFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "another replacement process holds") + info, statErr := os.Stat(lockDir) + require.NoError(t, statErr, "a process that failed to acquire the lock must not remove it") + require.True(t, info.IsDir()) +} + +func runReplacementScript(t *testing.T, repoRoot string, env []string) string { + t.Helper() + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), env...) + output, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", output) + return string(output) +} + +func writeExecutableTestFile(t *testing.T, dir, name, contents string) string { + t.Helper() + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) + require.NoError(t, os.Chmod(path, 0o700)) + return path +} + +func mustReadTestFile(t *testing.T, path string) string { + t.Helper() + contents, err := os.ReadFile(path) + require.NoError(t, err) + return string(contents) +} + +const fakeRaftadminScript = `#!/usr/bin/env bash +set -euo pipefail +addr="$1" +command="$2" +state_dir="$FAKE_STATE_DIR" + +if [[ "$command" == "status" ]]; then + if [[ -f "$state_dir/stability-phase" ]]; then + printf '%s\n' "$addr" >> "$state_dir/stability-status-calls" + fi + if [[ "$addr" == "127.0.0.2:50051" && -f "$state_dir/down-n2" ]]; then + exit 1 + fi + leader_id=n1 + leader_address=127.0.0.1:50051 + if [[ "${FAKE_TARGET_STARTS_LEADER:-false}" == "true" && ! -f "$state_dir/transferred" ]]; then + leader_id=n2 + leader_address=127.0.0.2:50051 + fi + if [[ "$addr" == "$leader_address" ]]; then + raft_state=LEADER + elif [[ "$addr" == "127.0.0.3:50051" && -f "$state_dir/candidate-n3" ]]; then + raft_state=CANDIDATE + else + raft_state=FOLLOWER + fi + if [[ "$addr" == "127.0.0.3:50051" && -f "$state_dir/stale-leader-once" ]]; then + rm -f "$state_dir/stale-leader-once" + raft_state=FOLLOWER + leader_id=n2 + leader_address=127.0.0.2:50051 + fi + cat < "$state_dir/config-index" +} + +case "$command" in + remove_server) + [[ "$3" == "n2" && "$4" == "$(cat "$state_dir/config-index")" ]] + printf 'absent\n' > "$state_dir/member" + increment_index + ;; + add_learner) + [[ "$3" == "n2" && "$4" == "127.0.0.2:50051" && "$5" == "$(cat "$state_dir/config-index")" ]] + printf 'learner\n' > "$state_dir/member" + increment_index + if [[ -f "$state_dir/fail-add-once" ]]; then + rm -f "$state_dir/fail-add-once" + exit 73 + fi + ;; + promote_learner) + [[ "$3" == "n2" && "$4" == "$(cat "$state_dir/config-index")" && "$5" == "100" && "$6" == "false" ]] + printf 'voter\n' > "$state_dir/member" + increment_index + ;; + leadership_transfer_to_server) + touch "$state_dir/transferred" + if [[ "${FAKE_BUMP_CONFIG_ON_TRANSFER:-false}" == "true" ]]; then + increment_index + fi + ;; + *) + echo "unexpected raftadmin command: $*" >&2 + exit 1 + ;; +esac +printf 'index: %s\n' "$(cat "$state_dir/config-index")" +` + +const fakeReplacementSSHScript = `#!/usr/bin/env bash +set -euo pipefail +cat >/dev/null +while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done +[[ "$#" -ge 2 ]] +action="$2" +printf '%s\n' "$action" >> "$FAKE_STATE_DIR/remote-actions" +case "$action" in + stop) + touch "$FAKE_STATE_DIR/down-n2" + if [[ "${FAKE_BUMP_CONFIG_ON_FENCE:-false}" == "true" ]]; then + current="$(cat "$FAKE_STATE_DIR/config-index")" + printf '%s\n' "$((current + 1))" > "$FAKE_STATE_DIR/config-index" + fi + ;; + reset-data) touch "$FAKE_STATE_DIR/data-reset" ;; + *) echo "unexpected SSH action: $action" >&2; exit 1 ;; +esac +` + +const fakeReplacementRollingScript = `#!/usr/bin/env bash +set -euo pipefail +[[ "${DRY_RUN:-}" == "false" ]] +printf '%s:%s\n' "${RAFT_JOIN_NODE:-normal}" "$ROLLING_ORDER" >> "$FAKE_STATE_DIR/rollouts" +if [[ "${RAFT_JOIN_NODE:-}" == "n2" ]]; then + rm -f "$FAKE_STATE_DIR/down-n2" + if [[ -f "$FAKE_STATE_DIR/fail-join-rollout-once" ]]; then + rm -f "$FAKE_STATE_DIR/fail-join-rollout-once" + exit 74 + fi +else + touch "$FAKE_STATE_DIR/stability-phase" +fi +` diff --git a/scripts/raft-member-replace.sh b/scripts/raft-member-replace.sh new file mode 100755 index 000000000..e15970f52 --- /dev/null +++ b/scripts/raft-member-replace.sh @@ -0,0 +1,1259 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +usage() { + cat <<'EOF' +Usage: + TARGET_NODE=n2 ROLLING_UPDATE_ENV_FILE=deploy.env \ + REPLACEMENT_CONFIRM=n2 ./scripts/raft-member-replace.sh --execute + TARGET_NODE=n2 ROLLING_UPDATE_ENV_FILE=deploy.env \ + ./scripts/raft-member-replace.sh --dry-run + +This command replaces one voter in one Raft group by fencing the old process, +removing its membership, archiving its data, joining a fresh learner, waiting +for catch-up, promoting it, and restarting it without join flags. + +Required environment: + TARGET_NODE + Raft ID to replace. It must be a current voter and exist in NODES. + NODES + Same ID-to-host map used by scripts/rolling-update.sh. It may be supplied + through ROLLING_UPDATE_ENV_FILE. + REPLACEMENT_CONFIRM + Must exactly match TARGET_NODE for --execute. + REPLACEMENT_VERIFY_COMMAND + Application-level write/read verification command. It runs after the + replacement is a stable voter. The command must return zero. + +Fencing: + REPLACEMENT_FENCE_MODE=external (default) + Requires REPLACEMENT_FENCE_COMMAND. The command must prevent the old VM or + process from returning with stale data. The target Raft RPC must become + unreachable before membership removal. Management SSH must remain available + so the command can stop and verify CONTAINER_NAME before removal. + REPLACEMENT_FENCE_MODE=container + Stops CONTAINER_NAME over SSH. Use only when the host lifecycle guarantees + the stopped container cannot restart independently during the operation. + +Optional environment: + ROLLING_UPDATE_ENV_FILE Env file used by rolling-update.sh. + It supplies inventory only; replacement + controls from this file are ignored. + ROLLING_UPDATE_SCRIPT Default: scripts/rolling-update.sh. + RAFTADMIN_BIN Default: build ./cmd/raftadmin locally. + SSH_BIN Default: ssh. + SSH_TARGETS, SSH_USER Same meaning as rolling-update.sh. + RAFT_PORT Default: 50051. + CONTAINER_NAME Default: elastickv. + DATA_DIR Default: /var/lib/elastickv. + REPLACEMENT_STATE_FILE Durable local resume state. + REPLACEMENT_DATA_MODE archive (default) or delete. + REPLACEMENT_FENCE_COMMAND Required for external fencing. + REPLACEMENT_FENCE_VERIFY_SECONDS + Consecutive target downtime required (default: 3). + REPLACEMENT_STABILITY_SECONDS Default: 10. + REPLACEMENT_TIMEOUT_SECONDS Default: 120. + REPLACEMENT_POLL_SECONDS Default: 1. + REPLACEMENT_KEEP_STATE Keep completed state when true (default). + RAFTADMIN_ALLOW_INSECURE Default: true. + RAFTADMIN_RPC_TIMEOUT_SECONDS Default: 5. + SSH_CONNECT_TIMEOUT_SECONDS Default: 10. + SSH_STRICT_HOST_KEY_CHECKING Default: accept-new. + +The state file makes the sequence resumable. A resumed run validates the live +membership against its recorded stage and fails closed on conflicting changes. +An existing completed state is rejected; archive it or choose a new state path +before authorizing a later replacement of the same node. +No-quorum recovery and multi-group replacement are outside this command. +EOF +} + +MODE="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --execute) + [[ -z "$MODE" ]] || { echo "choose exactly one mode" >&2; exit 1; } + MODE="execute" + ;; + --dry-run) + [[ -z "$MODE" ]] || { echo "choose exactly one mode" >&2; exit 1; } + MODE="dry-run" + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac + shift +done + +if [[ -z "$MODE" ]]; then + echo "--execute or --dry-run is required" >&2 + usage >&2 + exit 1 +fi + +# The deployment env is inventory, while explicitly exported replacement +# controls are the operator's command for this run. Preserve the latter across +# sourcing so an old commented-in recovery value cannot retarget an operation. +INPUT_TARGET_NODE_SET="${TARGET_NODE+x}" +INPUT_TARGET_NODE="${TARGET_NODE:-}" +INPUT_REPLACEMENT_CONFIRM_SET="${REPLACEMENT_CONFIRM+x}" +INPUT_REPLACEMENT_CONFIRM="${REPLACEMENT_CONFIRM:-}" +INPUT_REPLACEMENT_STATE_FILE_SET="${REPLACEMENT_STATE_FILE+x}" +INPUT_REPLACEMENT_STATE_FILE="${REPLACEMENT_STATE_FILE:-}" +INPUT_REPLACEMENT_DATA_MODE_SET="${REPLACEMENT_DATA_MODE+x}" +INPUT_REPLACEMENT_DATA_MODE="${REPLACEMENT_DATA_MODE:-}" +INPUT_REPLACEMENT_FENCE_MODE_SET="${REPLACEMENT_FENCE_MODE+x}" +INPUT_REPLACEMENT_FENCE_MODE="${REPLACEMENT_FENCE_MODE:-}" +INPUT_REPLACEMENT_FENCE_COMMAND_SET="${REPLACEMENT_FENCE_COMMAND+x}" +INPUT_REPLACEMENT_FENCE_COMMAND="${REPLACEMENT_FENCE_COMMAND:-}" +INPUT_REPLACEMENT_VERIFY_COMMAND_SET="${REPLACEMENT_VERIFY_COMMAND+x}" +INPUT_REPLACEMENT_VERIFY_COMMAND="${REPLACEMENT_VERIFY_COMMAND:-}" +INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS_SET="${REPLACEMENT_FENCE_VERIFY_SECONDS+x}" +INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS="${REPLACEMENT_FENCE_VERIFY_SECONDS:-}" +INPUT_REPLACEMENT_STABILITY_SECONDS_SET="${REPLACEMENT_STABILITY_SECONDS+x}" +INPUT_REPLACEMENT_STABILITY_SECONDS="${REPLACEMENT_STABILITY_SECONDS:-}" +INPUT_REPLACEMENT_TIMEOUT_SECONDS_SET="${REPLACEMENT_TIMEOUT_SECONDS+x}" +INPUT_REPLACEMENT_TIMEOUT_SECONDS="${REPLACEMENT_TIMEOUT_SECONDS:-}" +INPUT_REPLACEMENT_POLL_SECONDS_SET="${REPLACEMENT_POLL_SECONDS+x}" +INPUT_REPLACEMENT_POLL_SECONDS="${REPLACEMENT_POLL_SECONDS:-}" +INPUT_REPLACEMENT_KEEP_STATE_SET="${REPLACEMENT_KEEP_STATE+x}" +INPUT_REPLACEMENT_KEEP_STATE="${REPLACEMENT_KEEP_STATE:-}" + +if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then + if [[ ! -f "$ROLLING_UPDATE_ENV_FILE" ]]; then + echo "ROLLING_UPDATE_ENV_FILE not found: $ROLLING_UPDATE_ENV_FILE" >&2 + exit 1 + fi + # shellcheck disable=SC1090 + source "$ROLLING_UPDATE_ENV_FILE" +fi + +restore_invocation_control() { + local name="$1" + local was_set="$2" + local value="$3" + if [[ -n "$was_set" ]]; then + printf -v "$name" '%s' "$value" + else + unset "$name" + fi +} + +# Replacement controls are never inherited from the deployment inventory. +# An unset invocation value therefore resolves to the script default below. +restore_invocation_control TARGET_NODE "$INPUT_TARGET_NODE_SET" "$INPUT_TARGET_NODE" +restore_invocation_control REPLACEMENT_CONFIRM "$INPUT_REPLACEMENT_CONFIRM_SET" "$INPUT_REPLACEMENT_CONFIRM" +restore_invocation_control REPLACEMENT_STATE_FILE "$INPUT_REPLACEMENT_STATE_FILE_SET" "$INPUT_REPLACEMENT_STATE_FILE" +restore_invocation_control REPLACEMENT_DATA_MODE "$INPUT_REPLACEMENT_DATA_MODE_SET" "$INPUT_REPLACEMENT_DATA_MODE" +restore_invocation_control REPLACEMENT_FENCE_MODE "$INPUT_REPLACEMENT_FENCE_MODE_SET" "$INPUT_REPLACEMENT_FENCE_MODE" +restore_invocation_control REPLACEMENT_FENCE_COMMAND "$INPUT_REPLACEMENT_FENCE_COMMAND_SET" "$INPUT_REPLACEMENT_FENCE_COMMAND" +restore_invocation_control REPLACEMENT_VERIFY_COMMAND "$INPUT_REPLACEMENT_VERIFY_COMMAND_SET" "$INPUT_REPLACEMENT_VERIFY_COMMAND" +restore_invocation_control REPLACEMENT_FENCE_VERIFY_SECONDS "$INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS_SET" "$INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS" +restore_invocation_control REPLACEMENT_STABILITY_SECONDS "$INPUT_REPLACEMENT_STABILITY_SECONDS_SET" "$INPUT_REPLACEMENT_STABILITY_SECONDS" +restore_invocation_control REPLACEMENT_TIMEOUT_SECONDS "$INPUT_REPLACEMENT_TIMEOUT_SECONDS_SET" "$INPUT_REPLACEMENT_TIMEOUT_SECONDS" +restore_invocation_control REPLACEMENT_POLL_SECONDS "$INPUT_REPLACEMENT_POLL_SECONDS_SET" "$INPUT_REPLACEMENT_POLL_SECONDS" +restore_invocation_control REPLACEMENT_KEEP_STATE "$INPUT_REPLACEMENT_KEEP_STATE_SET" "$INPUT_REPLACEMENT_KEEP_STATE" + +TARGET_NODE="${TARGET_NODE:-}" +NODES="${NODES:-}" +SSH_TARGETS="${SSH_TARGETS:-}" +SSH_USER="${SSH_USER:-${USER:-$(id -un)}}" +RAFT_PORT="${RAFT_PORT:-50051}" +CONTAINER_NAME="${CONTAINER_NAME:-elastickv}" +DATA_DIR="${DATA_DIR:-/var/lib/elastickv}" +RAFTADMIN_BIN="${RAFTADMIN_BIN:-}" +RAFTADMIN_ALLOW_INSECURE="${RAFTADMIN_ALLOW_INSECURE:-true}" +RAFTADMIN_RPC_TIMEOUT_SECONDS="${RAFTADMIN_RPC_TIMEOUT_SECONDS:-5}" +ROLLING_UPDATE_SCRIPT="${ROLLING_UPDATE_SCRIPT:-${SCRIPT_DIR}/rolling-update.sh}" +SSH_BIN="${SSH_BIN:-ssh}" +SSH_CONNECT_TIMEOUT_SECONDS="${SSH_CONNECT_TIMEOUT_SECONDS:-10}" +SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}" +REPLACEMENT_CONFIRM="${REPLACEMENT_CONFIRM:-}" +REPLACEMENT_FENCE_MODE="${REPLACEMENT_FENCE_MODE:-external}" +REPLACEMENT_FENCE_COMMAND="${REPLACEMENT_FENCE_COMMAND:-}" +REPLACEMENT_FENCE_VERIFY_SECONDS="${REPLACEMENT_FENCE_VERIFY_SECONDS:-3}" +REPLACEMENT_DATA_MODE="${REPLACEMENT_DATA_MODE:-archive}" +REPLACEMENT_VERIFY_COMMAND="${REPLACEMENT_VERIFY_COMMAND:-}" +REPLACEMENT_STABILITY_SECONDS="${REPLACEMENT_STABILITY_SECONDS:-10}" +REPLACEMENT_TIMEOUT_SECONDS="${REPLACEMENT_TIMEOUT_SECONDS:-120}" +REPLACEMENT_POLL_SECONDS="${REPLACEMENT_POLL_SECONDS:-1}" +REPLACEMENT_KEEP_STATE="${REPLACEMENT_KEEP_STATE:-true}" + +NODE_IDS=() +NODE_HOSTS=() +STATE_STAGE=0 +STATE_TARGET="" +STATE_TARGET_ADDRESS="" +STATE_OPERATION_ID="" +STATE_ARCHIVE_PATH="" +STATE_EXPECTED_VOTERS="" +STATE_EXPECTED_MEMBERS="" +STATE_EXPECTED_CONFIG_INDEX=0 +STATE_MIN_CATCH_UP_INDEX=0 +STATE_DATA_MODE="" +STATE_DATA_DIR="" +VIEW_LEADER_ID="" +VIEW_LEADER_ADDRESS="" +VIEW_CONFIG_INDEX=0 +VIEW_COMMIT_INDEX=0 +VIEW_APPLIED_INDEX=0 +VIEW_CONFIGURATION="" +RAFTADMIN_TMP_DIR="" +LOCK_DIR="" +LOCK_ACQUIRED=false + +log() { + printf '[raft-member-replace] %s\n' "$*" +} + +fail() { + printf '[raft-member-replace] error: %s\n' "$*" >&2 + exit 1 +} + +is_uint() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +canonicalize_absolute_path() { + local path="$1" + local component last + local -a components=() + local -a normalized=() + [[ "$path" == /* ]] || return 1 + IFS='/' read -r -a components <<< "${path#/}" + for component in "${components[@]}"; do + case "$component" in + ""|.) ;; + ..) + ((${#normalized[@]} > 0)) || return 1 + last=$((${#normalized[@]} - 1)) + unset "normalized[$last]" + ;; + *) normalized+=("$component") ;; + esac + done + if ((${#normalized[@]} == 0)); then + printf '/\n' + return 0 + fi + local IFS=/ + printf '/%s\n' "${normalized[*]}" +} + +validate_settings() { + local canonical_data_dir + [[ -n "$TARGET_NODE" ]] || fail "TARGET_NODE is required" + [[ "$TARGET_NODE" =~ ^[A-Za-z0-9._-]+$ ]] || fail "TARGET_NODE contains unsupported characters" + [[ -n "$NODES" ]] || fail "NODES is required" + [[ "$DATA_DIR" == /* ]] || fail "DATA_DIR must be absolute" + canonical_data_dir="$(canonicalize_absolute_path "$DATA_DIR")" || fail "DATA_DIR escapes the filesystem root" + DATA_DIR="$canonical_data_dir" + [[ "$DATA_DIR" != "/" ]] || fail "DATA_DIR must not be the filesystem root" + state_value_valid "$DATA_DIR" || fail "DATA_DIR contains characters unsupported by the resume state" + [[ -x "$ROLLING_UPDATE_SCRIPT" ]] || fail "ROLLING_UPDATE_SCRIPT is not executable: $ROLLING_UPDATE_SCRIPT" + is_uint "$RAFT_PORT" || fail "RAFT_PORT must be an unsigned integer" + is_uint "$REPLACEMENT_STABILITY_SECONDS" || fail "REPLACEMENT_STABILITY_SECONDS must be an unsigned integer" + is_uint "$REPLACEMENT_FENCE_VERIFY_SECONDS" || fail "REPLACEMENT_FENCE_VERIFY_SECONDS must be an unsigned integer" + is_uint "$REPLACEMENT_TIMEOUT_SECONDS" || fail "REPLACEMENT_TIMEOUT_SECONDS must be an unsigned integer" + is_uint "$REPLACEMENT_POLL_SECONDS" || fail "REPLACEMENT_POLL_SECONDS must be an unsigned integer" + (( REPLACEMENT_TIMEOUT_SECONDS > 0 )) || fail "REPLACEMENT_TIMEOUT_SECONDS must be positive" + (( REPLACEMENT_POLL_SECONDS > 0 )) || fail "REPLACEMENT_POLL_SECONDS must be positive" + (( REPLACEMENT_FENCE_VERIFY_SECONDS > 0 )) || fail "REPLACEMENT_FENCE_VERIFY_SECONDS must be positive" + case "$REPLACEMENT_FENCE_MODE" in + external) + if [[ "$MODE" == "execute" && -z "$REPLACEMENT_FENCE_COMMAND" ]]; then + fail "REPLACEMENT_FENCE_COMMAND is required for external fencing" + fi + ;; + container) ;; + *) fail "REPLACEMENT_FENCE_MODE must be external or container" ;; + esac + case "$REPLACEMENT_DATA_MODE" in + archive|delete) ;; + *) fail "REPLACEMENT_DATA_MODE must be archive or delete" ;; + esac + if [[ "$MODE" == "execute" ]]; then + [[ -n "$INPUT_TARGET_NODE_SET" && -n "$INPUT_TARGET_NODE" ]] || fail "TARGET_NODE must be supplied explicitly for --execute" + [[ -n "$INPUT_REPLACEMENT_CONFIRM_SET" ]] || fail "REPLACEMENT_CONFIRM must be supplied explicitly for --execute" + [[ -n "$INPUT_REPLACEMENT_VERIFY_COMMAND_SET" && -n "$INPUT_REPLACEMENT_VERIFY_COMMAND" ]] || \ + fail "REPLACEMENT_VERIFY_COMMAND must be supplied explicitly for --execute" + [[ "$REPLACEMENT_CONFIRM" == "$TARGET_NODE" ]] || fail "REPLACEMENT_CONFIRM must exactly match TARGET_NODE" + [[ -n "$REPLACEMENT_VERIFY_COMMAND" ]] || fail "REPLACEMENT_VERIFY_COMMAND is required for --execute" + if [[ "$REPLACEMENT_FENCE_MODE" == "external" ]]; then + [[ -n "$INPUT_REPLACEMENT_FENCE_COMMAND_SET" && -n "$INPUT_REPLACEMENT_FENCE_COMMAND" ]] || \ + fail "REPLACEMENT_FENCE_COMMAND must be supplied explicitly for external fencing" + fi + fi +} + +contains_value() { + local needle="$1" + shift + local value + for value in "$@"; do + [[ "$value" == "$needle" ]] && return 0 + done + return 1 +} + +lookup_mapping() { + local key="$1" + local mapping="$2" + local pair entry_key entry_value + local -a pairs=() + + [[ -n "$mapping" ]] || return 1 + IFS=',' read -r -a pairs <<< "$mapping" + for pair in "${pairs[@]}"; do + pair="${pair//[[:space:]]/}" + [[ -n "$pair" && "$pair" == *=* ]] || continue + entry_key="${pair%%=*}" + entry_value="${pair#*=}" + if [[ "$entry_key" == "$key" ]]; then + printf '%s\n' "$entry_value" + return 0 + fi + done + return 1 +} + +parse_nodes() { + local pair node_id node_host + local -a pairs=() + + IFS=',' read -r -a pairs <<< "$NODES" + for pair in "${pairs[@]}"; do + pair="${pair//[[:space:]]/}" + [[ -n "$pair" ]] || continue + [[ "$pair" == *=* ]] || fail "invalid NODES entry: $pair" + node_id="${pair%%=*}" + node_host="${pair#*=}" + [[ -n "$node_id" && -n "$node_host" ]] || fail "invalid NODES entry: $pair" + [[ "$node_id" =~ ^[A-Za-z0-9._-]+$ ]] || fail "raft ID contains unsupported characters: $node_id" + [[ "$node_host" != *'|'* ]] || fail "raft host contains unsupported characters: $node_host" + if (( ${#NODE_IDS[@]} > 0 )) && contains_value "$node_id" "${NODE_IDS[@]}"; then + fail "duplicate raft ID in NODES: $node_id" + fi + NODE_IDS+=("$node_id") + NODE_HOSTS+=("$node_host") + done + (( ${#NODE_IDS[@]} > 0 )) || fail "NODES did not contain any nodes" + contains_value "$TARGET_NODE" "${NODE_IDS[@]}" || fail "TARGET_NODE is not present in NODES: $TARGET_NODE" +} + +node_host_by_id() { + local wanted="$1" + local i + for i in "${!NODE_IDS[@]}"; do + if [[ "${NODE_IDS[$i]}" == "$wanted" ]]; then + printf '%s\n' "${NODE_HOSTS[$i]}" + return 0 + fi + done + return 1 +} + +ssh_target_by_id() { + local node_id="$1" + local target + target="$(lookup_mapping "$node_id" "$SSH_TARGETS" || true)" + [[ -n "$target" ]] || target="$(node_host_by_id "$node_id")" + if [[ "$target" == *@* ]]; then + printf '%s\n' "$target" + else + printf '%s@%s\n' "$SSH_USER" "$target" + fi +} + +cleanup() { + [[ -z "$RAFTADMIN_TMP_DIR" || ! -d "$RAFTADMIN_TMP_DIR" ]] || rm -rf "$RAFTADMIN_TMP_DIR" + [[ "$LOCK_ACQUIRED" != "true" || -z "$LOCK_DIR" || ! -d "$LOCK_DIR" ]] || rmdir "$LOCK_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +ensure_raftadmin() { + if [[ -n "$RAFTADMIN_BIN" ]]; then + [[ -x "$RAFTADMIN_BIN" ]] || fail "RAFTADMIN_BIN is not executable: $RAFTADMIN_BIN" + return 0 + fi + RAFTADMIN_TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/raft-member-replace.XXXXXX")" + RAFTADMIN_BIN="${RAFTADMIN_TMP_DIR}/raftadmin" + log "building raftadmin" + (cd "$REPO_ROOT" && go build -o "$RAFTADMIN_BIN" ./cmd/raftadmin) +} + +raftadmin() { + RAFTADMIN_ALLOW_INSECURE="$RAFTADMIN_ALLOW_INSECURE" \ + RAFTADMIN_RPC_TIMEOUT_SECONDS="$RAFTADMIN_RPC_TIMEOUT_SECONDS" \ + "$RAFTADMIN_BIN" "$@" +} + +field_value() { + local field="$1" + awk -F ': ' -v field="$field" '$1 == field { value=$2; gsub(/^"|"$/, "", value); print value; exit }' +} + +configuration_rows() { + awk ' + /^servers \{/ { id=""; address=""; suffrage=""; in_server=1; next } + in_server && /^[[:space:]]+id:/ { + value=$0; sub(/^[^:]*:[[:space:]]*"/, "", value); sub(/"[[:space:]]*$/, "", value); id=value; next + } + in_server && /^[[:space:]]+address:/ { + value=$0; sub(/^[^:]*:[[:space:]]*"/, "", value); sub(/"[[:space:]]*$/, "", value); address=value; next + } + in_server && /^[[:space:]]+suffrage:/ { + value=$0; sub(/^[^:]*:[[:space:]]*"/, "", value); sub(/"[[:space:]]*$/, "", value); suffrage=value; next + } + in_server && /^}/ { print id "|" address "|" suffrage; in_server=0 } + ' +} + +configuration_signature() { + local config="$1" + printf '%s\n' "$config" | configuration_rows | LC_ALL=C sort | \ + awk 'NF { if (result != "") result=result ","; result=result $0 } END { print result }' +} + +configuration_signature_with_target() { + local signature="$1" + local desired="$2" + local entry id + local -a entries=() + { + IFS=',' read -r -a entries <<< "$signature" + for entry in "${entries[@]}"; do + [[ -n "$entry" ]] || continue + id="${entry%%|*}" + [[ "$id" == "$TARGET_NODE" ]] || printf '%s\n' "$entry" + done + if [[ "$desired" != "absent" ]]; then + printf '%s|%s|%s\n' "$TARGET_NODE" "$STATE_TARGET_ADDRESS" "$desired" + fi + } | LC_ALL=C sort | awk 'NF { if (result != "") result=result ","; result=result $0 } END { print result }' +} + +status_for() { + raftadmin "$1" status 2>/dev/null +} + +load_leader_view() { + local leader before_ready after_ready before_index after_index + local after_commit after_applied + local before_status after_status + + leader="$(wait_for_leader)" || return 1 + VIEW_LEADER_ID="${leader%%|*}" + VIEW_LEADER_ADDRESS="${leader#*|}" + before_status="$(status_for "$VIEW_LEADER_ADDRESS")" || return 1 + [[ "$(printf '%s\n' "$before_status" | field_value state)" == "LEADER" ]] || return 1 + before_ready="$(status_ready_fields "$before_status")" || return 1 + before_index="${before_ready%%|*}" + VIEW_CONFIGURATION="$(raftadmin "$VIEW_LEADER_ADDRESS" configuration)" || return 1 + after_status="$(status_for "$VIEW_LEADER_ADDRESS")" || return 1 + [[ "$(printf '%s\n' "$after_status" | field_value state)" == "LEADER" ]] || return 1 + after_ready="$(status_ready_fields "$after_status")" || return 1 + IFS='|' read -r after_index after_commit after_applied <<< "$after_ready" + [[ "$before_index" == "$after_index" ]] || return 1 + VIEW_CONFIG_INDEX="$after_index" + VIEW_COMMIT_INDEX="$after_commit" + VIEW_APPLIED_INDEX="$after_applied" +} + +assert_expected_configuration() { + local context="$1" + local actual_signature + actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" + [[ "$VIEW_CONFIG_INDEX" == "$STATE_EXPECTED_CONFIG_INDEX" ]] || \ + fail "$context configuration index changed: expected=$STATE_EXPECTED_CONFIG_INDEX actual=$VIEW_CONFIG_INDEX" + [[ "$actual_signature" == "$STATE_EXPECTED_MEMBERS" ]] || \ + fail "$context membership changed outside this operation" +} + +adopt_committed_target_change() { + local desired="$1" + local context="$2" + local expected_signature actual_signature + expected_signature="$(configuration_signature_with_target "$STATE_EXPECTED_MEMBERS" "$desired")" + actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" + [[ "$actual_signature" == "$expected_signature" ]] || \ + fail "$context observed an unexpected membership after a lost response" + if [[ "$STATE_EXPECTED_MEMBERS" == "$actual_signature" ]]; then + [[ "$VIEW_CONFIG_INDEX" == "$STATE_EXPECTED_CONFIG_INDEX" ]] || \ + fail "$context saved post-change configuration index changed: expected=$STATE_EXPECTED_CONFIG_INDEX actual=$VIEW_CONFIG_INDEX" + else + (( VIEW_CONFIG_INDEX > STATE_EXPECTED_CONFIG_INDEX )) || \ + fail "$context did not advance the configuration index after the lost response" + fi + STATE_EXPECTED_MEMBERS="$actual_signature" + STATE_EXPECTED_CONFIG_INDEX="$VIEW_CONFIG_INDEX" + write_state +} + +record_committed_target_change() { + local desired="$1" + local change_index="$2" + local context="$3" + local expected_signature actual_signature + load_leader_view || fail "cannot read stable leader view after $context" + expected_signature="$(configuration_signature_with_target "$STATE_EXPECTED_MEMBERS" "$desired")" + actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" + [[ "$VIEW_CONFIG_INDEX" == "$change_index" ]] || \ + fail "$context configuration index was superseded: committed=$change_index actual=$VIEW_CONFIG_INDEX" + [[ "$actual_signature" == "$expected_signature" ]] || \ + fail "$context committed an unexpected membership" + STATE_EXPECTED_MEMBERS="$actual_signature" + STATE_EXPECTED_CONFIG_INDEX="$VIEW_CONFIG_INDEX" + write_state +} + +discover_leader() { + local address status state leader_address leader_id leader_status + local verified_state verified_id verified_address + local i + for i in "${!NODE_HOSTS[@]}"; do + address="${NODE_HOSTS[$i]}:${RAFT_PORT}" + status="$(status_for "$address" || true)" + [[ -n "$status" ]] || continue + state="$(printf '%s\n' "$status" | field_value state)" + if [[ "$state" == "LEADER" ]]; then + printf '%s|%s\n' "${NODE_IDS[$i]}" "$address" + return 0 + fi + leader_id="$(printf '%s\n' "$status" | field_value leader_id)" + leader_address="$(printf '%s\n' "$status" | field_value leader_address)" + if [[ -n "$leader_id" && -n "$leader_address" ]]; then + leader_status="$(status_for "$leader_address" || true)" + [[ -n "$leader_status" ]] || continue + verified_state="$(printf '%s\n' "$leader_status" | field_value state)" + verified_id="$(printf '%s\n' "$leader_status" | field_value leader_id)" + verified_address="$(printf '%s\n' "$leader_status" | field_value leader_address)" + if [[ "$verified_state" == "LEADER" && "$verified_id" == "$leader_id" && "$verified_address" == "$leader_address" ]]; then + printf '%s|%s\n' "$leader_id" "$leader_address" + return 0 + fi + fi + done + return 1 +} + +configuration_from_leader() { + local leader leader_address + leader="$(discover_leader)" || return 1 + leader_address="${leader#*|}" + raftadmin "$leader_address" configuration +} + +member_suffrage() { + local wanted="$1" + local config="$2" + local id address suffrage + while IFS='|' read -r id address suffrage; do + if [[ "$id" == "$wanted" ]]; then + printf '%s\n' "$suffrage" + return 0 + fi + done < <(printf '%s\n' "$config" | configuration_rows) + return 1 +} + +member_address() { + local wanted="$1" + local config="$2" + local id address suffrage + while IFS='|' read -r id address suffrage; do + if [[ "$id" == "$wanted" ]]; then + printf '%s\n' "$address" + return 0 + fi + done < <(printf '%s\n' "$config" | configuration_rows) + return 1 +} + +voter_csv() { + local config="$1" + local id address suffrage result="" + while IFS='|' read -r id address suffrage; do + [[ "$suffrage" == "voter" ]] || continue + result+="${result:+,}${id}" + done < <(printf '%s\n' "$config" | configuration_rows) + printf '%s\n' "$result" +} + +same_csv_set() { + local left="$1" + local right="$2" + [[ "$(printf '%s' "$left" | tr ',' '\n' | sed '/^$/d' | sort | tr '\n' ',')" == \ + "$(printf '%s' "$right" | tr ',' '\n' | sed '/^$/d' | sort | tr '\n' ',')" ]] +} + +status_ready_fields() { + local status="$1" + local pending config_index commit applied + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + config_index="$(printf '%s\n' "$status" | field_value configuration_index)" + commit="$(printf '%s\n' "$status" | field_value commit_index)" + applied="$(printf '%s\n' "$status" | field_value applied_index)" + [[ "$pending" == "false" ]] || return 1 + is_uint "$config_index" && (( config_index > 0 )) || return 1 + is_uint "$commit" && is_uint "$applied" && (( applied >= commit )) || return 1 + printf '%s|%s|%s\n' "$config_index" "$commit" "$applied" +} + +leader_status_ready() { + local address="$1" + local status + status="$(status_for "$address")" || return 1 + status_ready_fields "$status" +} + +select_transfer_candidate() { + local config="$1" + local required_index="$2" + local expected_leader="$3" + local id address suffrage status state applied observed_leader + while IFS='|' read -r id address suffrage; do + [[ "$suffrage" == "voter" && "$id" != "$TARGET_NODE" ]] || continue + status="$(status_for "$address" || true)" + [[ -n "$status" ]] || continue + state="$(printf '%s\n' "$status" | field_value state)" + applied="$(printf '%s\n' "$status" | field_value applied_index)" + observed_leader="$(printf '%s\n' "$status" | field_value leader_id)" + if [[ "$state" == "FOLLOWER" && "$observed_leader" == "$expected_leader" ]] && \ + is_uint "$applied" && (( applied >= required_index )); then + printf '%s|%s\n' "$id" "$address" + return 0 + fi + done < <(printf '%s\n' "$config" | configuration_rows) + return 1 +} + +preflight() { + local leader_id leader_address config target_address target_suffrage + local config_index commit applied voters quorum reachable=0 + local id address suffrage voter_status voter_state voter_leader + local initial_signature + + load_leader_view || fail "no stable fully-applied Raft leader; no-quorum recovery is not supported" + leader_id="$VIEW_LEADER_ID" + leader_address="$VIEW_LEADER_ADDRESS" + config_index="$VIEW_CONFIG_INDEX" + commit="$VIEW_COMMIT_INDEX" + applied="$VIEW_APPLIED_INDEX" + config="$VIEW_CONFIGURATION" + initial_signature="$(configuration_signature "$config")" + target_suffrage="$(member_suffrage "$TARGET_NODE" "$config" || true)" + target_address="$(member_address "$TARGET_NODE" "$config" || true)" + [[ "$target_suffrage" == "voter" ]] || fail "target must be a current voter; got ${target_suffrage:-absent}" + [[ "$target_address" == "$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" ]] || \ + fail "target address mismatch: membership=$target_address inventory=$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" + + voters=0 + while IFS='|' read -r id address suffrage; do + [[ "$suffrage" == "voter" ]] || continue + voters=$((voters + 1)) + [[ "$id" == "$TARGET_NODE" ]] && continue + voter_status="$(status_for "$address" || true)" + voter_state="$(printf '%s\n' "$voter_status" | field_value state)" + voter_leader="$(printf '%s\n' "$voter_status" | field_value leader_id)" + if [[ "$voter_state" == "LEADER" && "$id" == "$leader_id" ]] || \ + [[ "$voter_state" == "FOLLOWER" && "$voter_leader" == "$leader_id" ]]; then + reachable=$((reachable + 1)) + fi + done < <(printf '%s\n' "$config" | configuration_rows) + quorum=$((voters / 2 + 1)) + (( reachable >= quorum )) || fail "only $reachable non-target voters are reachable; $quorum are required to commit removal" + + if [[ "$leader_id" == "$TARGET_NODE" ]]; then + local candidate candidate_id candidate_address + candidate="$(select_transfer_candidate "$config" "$commit" "$leader_id")" || fail "target is leader and no caught-up transfer candidate exists" + candidate_id="${candidate%%|*}" + candidate_address="${candidate#*|}" + log "transferring leadership from $TARGET_NODE to $candidate_id" + raftadmin "$leader_address" leadership_transfer_to_server "$candidate_id" "$candidate_address" >/dev/null + wait_for_leader "$candidate_id" >/dev/null || fail "leadership did not transfer to $candidate_id" + load_leader_view || fail "cannot read stable leader view after leadership transfer" + [[ "$(configuration_signature "$VIEW_CONFIGURATION")" == "$initial_signature" ]] || \ + fail "membership changed during leadership transfer" + config_index="$VIEW_CONFIG_INDEX" + config="$VIEW_CONFIGURATION" + fi + + STATE_EXPECTED_VOTERS="$(voter_csv "$config")" + STATE_EXPECTED_MEMBERS="$initial_signature" + STATE_EXPECTED_CONFIG_INDEX="$config_index" + STATE_TARGET_ADDRESS="$target_address" + log "preflight passed: leader=$leader_id leader_applied=$applied voters=$voters non_target_reachable=$reachable quorum=$quorum config_index=$config_index" +} + +wait_for_leader() { + local expected_id="${1:-}" + local elapsed=0 leader leader_id + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(discover_leader || true)" + if [[ -n "$leader" ]]; then + leader_id="${leader%%|*}" + if [[ -z "$expected_id" || "$leader_id" == "$expected_id" ]]; then + printf '%s\n' "$leader" + return 0 + fi + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +wait_target_down() { + local elapsed=0 down_since=-1 + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + if ! status_for "$STATE_TARGET_ADDRESS" >/dev/null; then + if (( down_since < 0 )); then + down_since=$elapsed + elif (( elapsed - down_since >= REPLACEMENT_FENCE_VERIFY_SECONDS )); then + return 0 + fi + else + down_since=-1 + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +wait_target_up() { + local elapsed=0 + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + if status_for "$STATE_TARGET_ADDRESS" >/dev/null; then + return 0 + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +run_operator_command() { + local command="$1" + local operator_target_node="$TARGET_NODE" + local operator_target_address="$STATE_TARGET_ADDRESS" + local operator_target_ssh + operator_target_ssh="$(ssh_target_by_id "$TARGET_NODE")" + ( + if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ROLLING_UPDATE_ENV_FILE" + set +a + fi + env \ + TARGET_NODE="$operator_target_node" \ + TARGET_ADDRESS="$operator_target_address" \ + TARGET_SSH="$operator_target_ssh" \ + bash -lc "$command" + ) +} + +remote_target_action() { + local action="$1" + local ssh_target + ssh_target="$(ssh_target_by_id "$TARGET_NODE")" + "$SSH_BIN" \ + -o BatchMode=yes \ + -o ConnectTimeout="$SSH_CONNECT_TIMEOUT_SECONDS" \ + -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING" \ + "$ssh_target" bash -s -- "$action" "$CONTAINER_NAME" "$DATA_DIR" "$STATE_ARCHIVE_PATH" "$REPLACEMENT_DATA_MODE" <<'REMOTE' +set -euo pipefail +action="$1" +container="$2" +data_dir="$3" +archive_path="$4" +data_mode="$5" + +sudo_cmd=() +if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then + sudo_cmd=(sudo -n) +fi + +run_privileged() { + if (( ${#sudo_cmd[@]} > 0 )); then + "${sudo_cmd[@]}" "$@" + else + "$@" + fi +} + +container_running() { + local running + if ! running="$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null)"; then + echo "cannot inspect container state: $container" >&2 + return 1 + fi + case "$running" in + true|false) printf '%s\n' "$running" ;; + *) echo "invalid container running state: $running" >&2; return 1 ;; + esac +} + +case "$action" in + stop) + running="$(container_running)" + if [[ "$running" == "true" ]]; then + docker stop "$container" >/dev/null + fi + running="$(container_running)" + [[ "$running" == "false" ]] + ;; + reset-data) + docker_running="$(container_running)" + [[ "$docker_running" == "false" ]] + if [[ "$data_mode" == "archive" ]]; then + if [[ -e "$archive_path" && -e "$data_dir" ]]; then + echo "both data and archive paths exist: $data_dir $archive_path" >&2 + exit 1 + fi + if [[ -e "$data_dir" ]]; then + run_privileged mv "$data_dir" "$archive_path" + fi + [[ -e "$archive_path" ]] || { echo "target data directory was absent and no archive exists" >&2; exit 1; } + else + if [[ -e "$data_dir" ]]; then + run_privileged rm -rf --one-file-system "$data_dir" + fi + [[ ! -e "$data_dir" ]] + fi + ;; + *) + echo "unknown remote action: $action" >&2 + exit 1 + ;; +esac +REMOTE +} + +fence_target() { + case "$REPLACEMENT_FENCE_MODE" in + external) + log "running external fence for $TARGET_NODE" + run_operator_command "$REPLACEMENT_FENCE_COMMAND" + wait_target_down || fail "target Raft RPC is still reachable after external fencing" + log "external fence verified; stopping the target process before membership removal" + remote_target_action stop + ;; + container) + log "stopping target container; host lifecycle must keep it fenced" + remote_target_action stop + ;; + esac + wait_target_down || fail "target Raft RPC is still reachable after fencing" + log "fence verified: $STATE_TARGET_ADDRESS is unreachable" +} + +config_index_from_leader() { + local leader address ready + leader="$(wait_for_leader)" || return 1 + address="${leader#*|}" + ready="$(leader_status_ready "$address")" || return 1 + printf '%s\n' "${ready%%|*}" +} + +wait_member_state() { + local expected="$1" + local elapsed=0 leader address status config actual pending + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(discover_leader || true)" + if [[ -n "$leader" ]]; then + address="${leader#*|}" + status="$(status_for "$address" || true)" + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + config="$(raftadmin "$address" configuration 2>/dev/null || true)" + actual="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ "$pending" == "false" ]]; then + case "$expected" in + absent) [[ -z "$actual" ]] && return 0 ;; + *) [[ "$actual" == "$expected" ]] && return 0 ;; + esac + fi + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +wait_configuration_index() { + local minimum="$1" + local elapsed=0 leader address status current pending + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(discover_leader || true)" + if [[ -n "$leader" ]]; then + address="${leader#*|}" + status="$(status_for "$address" || true)" + current="$(printf '%s\n' "$status" | field_value configuration_index)" + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + if is_uint "$current" && (( current >= minimum )) && [[ "$pending" == "false" ]]; then + return 0 + fi + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +remove_target() { + local config current output change_index + load_leader_view || fail "cannot read stable leader view before removal" + config="$VIEW_CONFIGURATION" + current="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ -z "$current" ]]; then + adopt_committed_target_change absent "remove_server" + log "target is already absent; resuming after committed removal" + return 0 + fi + [[ "$current" == "voter" ]] || fail "target changed from voter to $current before removal" + assert_expected_configuration "remove_server" + ! status_for "$STATE_TARGET_ADDRESS" >/dev/null || fail "target returned after fencing" + log "removing $TARGET_NODE at configuration index $STATE_EXPECTED_CONFIG_INDEX" + output="$(raftadmin "$VIEW_LEADER_ADDRESS" remove_server "$TARGET_NODE" "$STATE_EXPECTED_CONFIG_INDEX")" + change_index="$(printf '%s\n' "$output" | field_value index)" + if ! is_uint "$change_index" || (( change_index <= STATE_EXPECTED_CONFIG_INDEX )); then + fail "remove_server returned an invalid configuration index" + fi + wait_member_state absent || fail "remove_server did not commit" + wait_configuration_index "$change_index" || fail "leader status did not publish committed removal index $change_index" + record_committed_target_change absent "$change_index" "remove_server" +} + +run_rollout() { + local join_node="$1" + local rollout_target="$TARGET_NODE" + ( + if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ROLLING_UPDATE_ENV_FILE" + set +a + fi + unset ROLLING_UPDATE_ENV_FILE RAFT_BOOTSTRAP_MEMBERS + export ROLLING_ORDER="$rollout_target" + export RAFT_JOIN_NODE="$join_node" + export DRY_RUN=false + exec "$ROLLING_UPDATE_SCRIPT" + ) +} + +add_learner() { + local config current commit output change_index + load_leader_view || fail "cannot read stable leader view before learner add" + config="$VIEW_CONFIGURATION" + current="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ "$current" == "learner" ]]; then + (( STATE_MIN_CATCH_UP_INDEX > 0 )) || fail "learner committed but catch-up floor is missing from replacement state" + adopt_committed_target_change learner "add_learner" + log "target is already a learner; resuming after committed add" + return 0 + fi + [[ -z "$current" ]] || fail "target unexpectedly has suffrage $current before learner add" + wait_target_up || fail "join deployment did not expose the target RaftAdmin endpoint" + load_leader_view || fail "cannot read stable leader view before learner add" + assert_expected_configuration "add_learner" + commit="$VIEW_COMMIT_INDEX" + STATE_MIN_CATCH_UP_INDEX="$commit" + # Persist the floor before proposing. If the RPC commits but the control + # process exits before advancing its stage, promotion still has a durable, + # non-zero catch-up bound on resume. + write_state + log "adding learner $TARGET_NODE at configuration index $STATE_EXPECTED_CONFIG_INDEX; catch-up floor=$commit" + output="$(raftadmin "$VIEW_LEADER_ADDRESS" add_learner "$TARGET_NODE" "$STATE_TARGET_ADDRESS" "$STATE_EXPECTED_CONFIG_INDEX")" + change_index="$(printf '%s\n' "$output" | field_value index)" + if ! is_uint "$change_index" || (( change_index <= STATE_EXPECTED_CONFIG_INDEX )); then + fail "add_learner returned an invalid configuration index" + fi + wait_member_state learner || fail "add_learner did not commit" + wait_configuration_index "$change_index" || fail "leader status did not publish committed learner index $change_index" + record_committed_target_change learner "$change_index" "add_learner" +} + +wait_for_catch_up() { + local elapsed=0 status state leader_id applied + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + status="$(status_for "$STATE_TARGET_ADDRESS" || true)" + state="$(printf '%s\n' "$status" | field_value state)" + leader_id="$(printf '%s\n' "$status" | field_value leader_id)" + applied="$(printf '%s\n' "$status" | field_value applied_index)" + if [[ "$state" == "FOLLOWER" && -n "$leader_id" ]] && is_uint "$applied" && (( applied >= STATE_MIN_CATCH_UP_INDEX )); then + log "learner caught up: applied_index=$applied floor=$STATE_MIN_CATCH_UP_INDEX" + return 0 + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +promote_learner() { + local config current output change_index + load_leader_view || fail "cannot read stable leader view before promotion" + config="$VIEW_CONFIGURATION" + current="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ "$current" == "voter" ]]; then + adopt_committed_target_change voter "promote_learner" + log "target is already a voter; resuming after committed promotion" + return 0 + fi + [[ "$current" == "learner" ]] || fail "target must be a learner before promotion; got ${current:-absent}" + (( STATE_MIN_CATCH_UP_INDEX > 0 )) || fail "catch-up floor is missing from replacement state" + assert_expected_configuration "promote_learner" + wait_for_catch_up || fail "learner did not reach catch-up floor" + load_leader_view || fail "cannot read stable leader view after learner catch-up" + assert_expected_configuration "promote_learner" + log "promoting $TARGET_NODE at configuration index $STATE_EXPECTED_CONFIG_INDEX" + output="$(raftadmin "$VIEW_LEADER_ADDRESS" promote_learner "$TARGET_NODE" "$STATE_EXPECTED_CONFIG_INDEX" "$STATE_MIN_CATCH_UP_INDEX" false)" + change_index="$(printf '%s\n' "$output" | field_value index)" + if ! is_uint "$change_index" || (( change_index <= STATE_EXPECTED_CONFIG_INDEX )); then + fail "promote_learner returned an invalid configuration index" + fi + wait_member_state voter || fail "promote_learner did not commit" + wait_configuration_index "$change_index" || fail "leader status did not publish committed promotion index $change_index" + record_committed_target_change voter "$change_index" "promote_learner" +} + +verify_cluster_once() { + local leader leader_id address status pending config voters signature id member_address suffrage member_status state member_leader applied commit + leader="$(discover_leader)" || return 1 + leader_id="${leader%%|*}" + address="${leader#*|}" + status="$(status_for "$address")" || return 1 + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + [[ "$pending" == "false" ]] || return 1 + config="$(raftadmin "$address" configuration 2>/dev/null)" || return 1 + signature="$(configuration_signature "$config")" + [[ "$signature" == "$STATE_EXPECTED_MEMBERS" ]] || return 1 + voters="$(voter_csv "$config")" + same_csv_set "$voters" "$STATE_EXPECTED_VOTERS" || return 1 + while IFS='|' read -r id member_address suffrage; do + [[ "$suffrage" == "voter" ]] || continue + member_status="$(status_for "$member_address" || true)" + [[ -n "$member_status" ]] || return 1 + state="$(printf '%s\n' "$member_status" | field_value state)" + member_leader="$(printf '%s\n' "$member_status" | field_value leader_id)" + if [[ "$state" == "LEADER" ]]; then + [[ "$id" == "$leader_id" ]] || return 1 + else + [[ "$state" == "FOLLOWER" && "$member_leader" == "$leader_id" ]] || return 1 + fi + applied="$(printf '%s\n' "$member_status" | field_value applied_index)" + commit="$(printf '%s\n' "$member_status" | field_value commit_index)" + is_uint "$applied" && is_uint "$commit" && (( applied >= commit )) || return 1 + done < <(printf '%s\n' "$config" | configuration_rows) + printf '%s\n' "${leader%%|*}" +} + +verify_stability() { + local elapsed=0 stable_since=-1 expected_leader="" leader stable + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(verify_cluster_once || true)" + if [[ -n "$leader" && ( -z "$expected_leader" || "$leader" == "$expected_leader" ) ]]; then + if [[ -z "$expected_leader" ]]; then + stable_since=$elapsed + fi + expected_leader="$leader" + stable=$((elapsed - stable_since)) + if (( stable >= REPLACEMENT_STABILITY_SECONDS )); then + log "cluster stable for ${stable}s with leader=$leader" + return 0 + fi + else + expected_leader="$leader" + if [[ -n "$leader" ]]; then + stable_since=$elapsed + else + stable_since=-1 + fi + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +state_value_valid() { + [[ "$1" =~ ^[A-Za-z0-9_./,:@|-]*$ ]] +} + +write_state() { + local tmp value + for value in "$STATE_STAGE" "$STATE_TARGET" "$STATE_TARGET_ADDRESS" "$STATE_OPERATION_ID" \ + "$STATE_ARCHIVE_PATH" "$STATE_EXPECTED_VOTERS" "$STATE_EXPECTED_MEMBERS" \ + "$STATE_EXPECTED_CONFIG_INDEX" "$STATE_MIN_CATCH_UP_INDEX" "$STATE_DATA_MODE" "$STATE_DATA_DIR"; do + state_value_valid "$value" || fail "replacement state contains an unsupported value" + done + mkdir -p "$(dirname "$REPLACEMENT_STATE_FILE")" + tmp="${REPLACEMENT_STATE_FILE}.tmp.$$" + { + printf 'stage=%s\n' "$STATE_STAGE" + printf 'target=%s\n' "$STATE_TARGET" + printf 'target_address=%s\n' "$STATE_TARGET_ADDRESS" + printf 'operation_id=%s\n' "$STATE_OPERATION_ID" + printf 'archive_path=%s\n' "$STATE_ARCHIVE_PATH" + printf 'expected_voters=%s\n' "$STATE_EXPECTED_VOTERS" + printf 'expected_members=%s\n' "$STATE_EXPECTED_MEMBERS" + printf 'expected_config_index=%s\n' "$STATE_EXPECTED_CONFIG_INDEX" + printf 'min_catch_up_index=%s\n' "$STATE_MIN_CATCH_UP_INDEX" + printf 'data_mode=%s\n' "$STATE_DATA_MODE" + printf 'data_dir=%s\n' "$STATE_DATA_DIR" + } > "$tmp" + chmod 600 "$tmp" + mv "$tmp" "$REPLACEMENT_STATE_FILE" +} + +load_state() { + local key value + [[ -f "$REPLACEMENT_STATE_FILE" ]] || return 1 + while IFS='=' read -r key value; do + state_value_valid "$value" || fail "invalid value in state file for $key" + case "$key" in + stage) STATE_STAGE="$value" ;; + target) STATE_TARGET="$value" ;; + target_address) STATE_TARGET_ADDRESS="$value" ;; + operation_id) STATE_OPERATION_ID="$value" ;; + archive_path) STATE_ARCHIVE_PATH="$value" ;; + expected_voters) STATE_EXPECTED_VOTERS="$value" ;; + expected_members) STATE_EXPECTED_MEMBERS="$value" ;; + expected_config_index) STATE_EXPECTED_CONFIG_INDEX="$value" ;; + min_catch_up_index) STATE_MIN_CATCH_UP_INDEX="$value" ;; + data_mode) STATE_DATA_MODE="$value" ;; + data_dir) STATE_DATA_DIR="$value" ;; + "") ;; + *) fail "unknown key in state file: $key" ;; + esac + done < "$REPLACEMENT_STATE_FILE" + if ! is_uint "$STATE_STAGE" || ! is_uint "$STATE_EXPECTED_CONFIG_INDEX" || \ + ! is_uint "$STATE_MIN_CATCH_UP_INDEX"; then + fail "invalid numeric value in state file" + fi + (( STATE_STAGE <= 9 )) || fail "state stage is outside the implemented range: $STATE_STAGE" + [[ "$STATE_TARGET" == "$TARGET_NODE" ]] || fail "state file belongs to target $STATE_TARGET, not $TARGET_NODE" + [[ "$STATE_TARGET_ADDRESS" == "$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" ]] || fail "state target address no longer matches NODES" + [[ -n "$STATE_OPERATION_ID" && -n "$STATE_ARCHIVE_PATH" && -n "$STATE_EXPECTED_VOTERS" && \ + -n "$STATE_EXPECTED_MEMBERS" && -n "$STATE_DATA_DIR" ]] || fail "state file is incomplete" + (( STATE_EXPECTED_CONFIG_INDEX > 0 )) || fail "state file has no expected configuration index" + [[ "$STATE_DATA_MODE" == "$REPLACEMENT_DATA_MODE" ]] || fail "REPLACEMENT_DATA_MODE changed from $STATE_DATA_MODE to $REPLACEMENT_DATA_MODE" + [[ "$STATE_DATA_DIR" == "$DATA_DIR" ]] || fail "DATA_DIR changed from $STATE_DATA_DIR to $DATA_DIR" + (( STATE_STAGE < 9 )) || fail "replacement state is already complete; archive it or choose a new REPLACEMENT_STATE_FILE" + log "resuming operation $STATE_OPERATION_ID at stage $STATE_STAGE" +} + +advance() { + STATE_STAGE="$1" + write_state +} + +validate_settings +parse_nodes +TARGET_HOST="$(node_host_by_id "$TARGET_NODE")" +REPLACEMENT_STATE_FILE="${REPLACEMENT_STATE_FILE:-${REPO_ROOT}/.state/raft-member-replacement-${TARGET_NODE}.state}" + +if [[ "$MODE" == "dry-run" ]]; then + cat </dev/null || fail "another replacement process holds $LOCK_DIR" +LOCK_ACQUIRED=true +ensure_raftadmin + +if ! load_state; then + STATE_TARGET="$TARGET_NODE" + STATE_TARGET_ADDRESS="${TARGET_HOST}:${RAFT_PORT}" + STATE_OPERATION_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" + STATE_DATA_DIR="$DATA_DIR" + STATE_ARCHIVE_PATH="${STATE_DATA_DIR}.replacement-${TARGET_NODE}-${STATE_OPERATION_ID}" + STATE_DATA_MODE="$REPLACEMENT_DATA_MODE" + preflight + advance 1 +fi + +if (( STATE_STAGE < 1 )); then + preflight + advance 1 +fi +if (( STATE_STAGE < 2 )); then + fence_target + advance 2 +elif (( STATE_STAGE < 4 )); then + wait_target_down || fail "fenced target is reachable again before removal" +fi +if (( STATE_STAGE < 3 )); then + remove_target + advance 3 +fi +if (( STATE_STAGE < 4 )); then + wait_member_state absent || fail "target membership returned before data reset" + log "resetting only $TARGET_NODE data at $DATA_DIR" + remote_target_action reset-data + advance 4 +fi +if (( STATE_STAGE < 5 )); then + wait_member_state absent || fail "target membership returned before join deployment" + log "deploying $TARGET_NODE in learner join mode" + run_rollout "$TARGET_NODE" + advance 5 +fi +if (( STATE_STAGE < 6 )); then + add_learner + advance 6 +fi +if (( STATE_STAGE < 7 )); then + promote_learner + advance 7 +fi +if (( STATE_STAGE < 8 )); then + log "redeploying $TARGET_NODE from durable membership without join flags" + run_rollout "" + advance 8 +fi +if (( STATE_STAGE < 9 )); then + verify_stability || fail "cluster did not reach a stable fully-applied voter configuration" + log "running application write/read verification" + run_operator_command "$REPLACEMENT_VERIFY_COMMAND" + advance 9 +fi + +log "replacement completed: target=$TARGET_NODE operation=$STATE_OPERATION_ID" +if [[ "$REPLACEMENT_KEEP_STATE" != "true" ]]; then + rm -f "$REPLACEMENT_STATE_FILE" +fi diff --git a/scripts/rolling-update.env.example b/scripts/rolling-update.env.example index a3864a649..2261987fd 100644 --- a/scripts/rolling-update.env.example +++ b/scripts/rolling-update.env.example @@ -27,6 +27,11 @@ RAFT_ENGINE="etcd" # ROLLING_ORDER="n2" # RAFT_JOIN_NODE="n2" +# Do not store fenced replacement controls in this inventory file. Pass +# TARGET_NODE and every REPLACEMENT_* authorization/tuning value directly to +# scripts/raft-member-replace.sh. The wrapper intentionally ignores replacement +# controls sourced from deployment inventory. + RAFT_PORT="50051" REDIS_PORT="6379" DYNAMO_PORT="8000"