Release 2.14.0

CHANGELOG.md

@@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+### Changed
+
+### Fixed
+
+## [2.14.0] - 2026-08-06
+
+This release introduces cluster backup and restore: `tt backup` plans a
+backup, takes it on the nodes, uploads the archives with a cluster manifest
+to file or S3 storage, and verifies and prunes that storage, while
+`tt restore` plans a recovery point and prepares the instance work
+directories for it. It also adds cluster topology discovery with
+`tt cluster topology` and the `\history` command in the interactive console,
+and fixes line loss and hangs in `tt log -f` around log rotation.
+
+### Added
+
 - `tt backup start` and `tt backup finalize`: add support for creating and
   finalizing local backup artifacts. `--backup-id` must be a single safe path
   component: it names a directory and a file on the node and an object key in
@@ -113,17 +129,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   restored into a healthy-looking instance sitting at the wrong position.
   Re-running for the same point is idempotent, and only the files a restore
   owns are cleared, so an instance config kept in the same directory survives.
-  Exits
-  with 2 when no xlog covers the point and 3 when an input is rejected, in
-  which case the work directory is left as it was. A `restore_state.json`
-  marker is written next to the work directory for the orchestrator to
-  compare across the restored nodes before the cluster is started.
+  Exits with 2 when no xlog covers the point and 3 when an input is
+  rejected, in which case the work directory is left as it was. A
+  `restore_state.json` marker is written next to the work directory for the
+  orchestrator to compare across the restored nodes before the cluster is
+  started.
 - `tt connect`: add `\history` command to display the last executed
   commands in the interactive console.
 
 ### Changed
 
-- `tt stop`: preliminary interrupts the processes to enable parallel termination.
+- `tt stop`: preliminary interrupts the processes to enable parallel
+  termination.
 - `tt create vshard_cluster`: the generated rockspec now pins `vshard 0.1.42`
   instead of `0.1.25`. 0.1.42 is the first release shipping the `vshard-router`
   backend of the `roles.recovery-point-manager` role, which is what lets a
@@ -224,8 +241,9 @@ docker container used by these commands is upgraded to Ubuntu 20.04.
 
 ## [2.11.0] - 2025-09-10
 
-The release supports Tarantool Config Storage in `tt cluster failover` commands and
-introduces templates to create Tarantool Config Storage and non-vshard cluster.
+The release supports Tarantool Config Storage in `tt cluster failover`
+commands and introduces templates to create Tarantool Config Storage and
+non-vshard cluster.
 
 ### Added
 

test/cartridge_helper.py

@@ -21,6 +21,10 @@
 cartridge_username = "admin"
 cartridge_password = "secret-cluster-cookie"
 
+# The default 10s is not enough for a two-phase commit over the whole cluster
+# on a loaded CI runner.
+bootstrap_timeout = 60
+
 instances = [
     "router",
     "s1-master",
@@ -111,6 +115,30 @@ def wait_inst_files(dir, inst):
     assert file != ""
 
 
+# Cartridge logs every confapplier state change, e.g.
+# "Instance state changed: ConfiguringRoles -> RolesConfigured".
+state_change_re = re.compile(r"Instance state changed: \S+ -> (\S+)")
+
+
+def wait_inst_roles_configured(dir, inst):
+    # The last recorded transition is checked instead of the mere presence of one:
+    # every clusterwide config apply drives the instance through ConfiguringRoles
+    # again, and a 2PC started before it settles is rejected with a Prepare2pcError.
+    log = os.path.join(dir, cartridge_name, log_path, inst, log_file)
+    state = None
+    trying = 0
+    while trying < 200:
+        with open(log, "r") as fp:
+            states = state_change_re.findall(fp.read())
+        if states:
+            state = states[-1]
+            if state == "RolesConfigured":
+                return
+        time.sleep(0.05)
+        trying = trying + 1
+    assert state == "RolesConfigured", f"{inst} is in state {state}, expected RolesConfigured"
+
+
 def wait_inst_start(dir, inst):
     wait_inst_files(dir, inst)
 
@@ -198,34 +226,28 @@ def start(self, bootstrap_vshard=True):
         self.bootstrap(bootstrap_vshard=bootstrap_vshard)
 
     def bootstrap(self, bootstrap_vshard=True):
-        cmd = [self.tt_cmd, "replicaset", "bootstrap", cartridge_name]
+        cmd = [
+            self.tt_cmd,
+            "replicaset",
+            "bootstrap",
+            "--timeout",
+            str(bootstrap_timeout),
+            cartridge_name,
+        ]
         if bootstrap_vshard:
             cmd.append("--bootstrap-vshard")
         rc, out = run_command_and_get_output(cmd, cwd=self.workdir)
         assert rc == 0, f"vshard bootstrap output: {out}"
         assert re.search(r"Done.", out)
 
         # Wait until the instances are configured.
+        self.wait_roles_configured()
+
+    def wait_roles_configured(self):
         for inst in self.instances:
             if inst == "stateboard":
                 continue
-            configured = False
-            log_dir = os.path.join(self.workdir, cartridge_name, log_path, inst)
-            trying = 0
-            while not configured and trying < 200:
-                with open(os.path.join(log_dir, log_file), "r") as fp:
-                    lines = fp.readlines()
-                    lines = [line.rstrip() for line in lines]
-                for line in lines:
-                    if re.search(
-                        r"Instance state changed: ConfiguringRoles -> RolesConfigured",
-                        line,
-                    ):
-                        configured = True
-                        break
-                time.sleep(0.05)
-                trying = trying + 1
-            assert configured is True
+            wait_inst_roles_configured(self.workdir, inst)
 
     def set_failover(self, data):
         with open(os.path.join(self.workdir, cartridge_name, "failover.yml"), "w") as f:
@@ -234,6 +256,10 @@ def set_failover(self, data):
         rc, out = run_command_and_get_output(cmd, cwd=os.path.join(self.workdir, cartridge_name))
         assert rc == 0
         assert re.search(r"Failover configured successfully", out)
+        # Failover setup patches the clusterwide config, so the instances reconfigure
+        # their roles once more. Without waiting for that, a command issued right away
+        # races with the reconfiguration and fails to start its own 2PC.
+        self.wait_roles_configured()
 
     def stop(self):
         cmd = [self.tt_cmd, "stop", "-y", cartridge_name]