diff --git a/CHANGELOG.md b/CHANGELOG.md index 16b3d31bf..46231ac35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +### Changed + +### Fixed + +## [2.14.0] - 2026-08-06 + +This release introduces cluster backup and restore: `tt backup` plans a +backup, takes it on the nodes, uploads the archives with a cluster manifest +to file or S3 storage, and verifies and prunes that storage, while +`tt restore` plans a recovery point and prepares the instance work +directories for it. It also adds cluster topology discovery with +`tt cluster topology` and the `\history` command in the interactive console, +and fixes line loss and hangs in `tt log -f` around log rotation. + +### Added + - `tt backup start` and `tt backup finalize`: add support for creating and finalizing local backup artifacts. `--backup-id` must be a single safe path component: it names a directory and a file on the node and an object key in @@ -113,17 +129,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. restored into a healthy-looking instance sitting at the wrong position. Re-running for the same point is idempotent, and only the files a restore owns are cleared, so an instance config kept in the same directory survives. - Exits - with 2 when no xlog covers the point and 3 when an input is rejected, in - which case the work directory is left as it was. A `restore_state.json` - marker is written next to the work directory for the orchestrator to - compare across the restored nodes before the cluster is started. + Exits with 2 when no xlog covers the point and 3 when an input is + rejected, in which case the work directory is left as it was. A + `restore_state.json` marker is written next to the work directory for the + orchestrator to compare across the restored nodes before the cluster is + started. - `tt connect`: add `\history` command to display the last executed commands in the interactive console. ### Changed -- `tt stop`: preliminary interrupts the processes to enable parallel termination. +- `tt stop`: preliminary interrupts the processes to enable parallel + termination. - `tt create vshard_cluster`: the generated rockspec now pins `vshard 0.1.42` instead of `0.1.25`. 0.1.42 is the first release shipping the `vshard-router` backend of the `roles.recovery-point-manager` role, which is what lets a @@ -224,8 +241,9 @@ docker container used by these commands is upgraded to Ubuntu 20.04. ## [2.11.0] - 2025-09-10 -The release supports Tarantool Config Storage in `tt cluster failover` commands and -introduces templates to create Tarantool Config Storage and non-vshard cluster. +The release supports Tarantool Config Storage in `tt cluster failover` +commands and introduces templates to create Tarantool Config Storage and +non-vshard cluster. ### Added diff --git a/test/cartridge_helper.py b/test/cartridge_helper.py index 88d76f335..ea587319f 100644 --- a/test/cartridge_helper.py +++ b/test/cartridge_helper.py @@ -21,6 +21,10 @@ cartridge_username = "admin" cartridge_password = "secret-cluster-cookie" +# The default 10s is not enough for a two-phase commit over the whole cluster +# on a loaded CI runner. +bootstrap_timeout = 60 + instances = [ "router", "s1-master", @@ -111,6 +115,30 @@ def wait_inst_files(dir, inst): assert file != "" +# Cartridge logs every confapplier state change, e.g. +# "Instance state changed: ConfiguringRoles -> RolesConfigured". +state_change_re = re.compile(r"Instance state changed: \S+ -> (\S+)") + + +def wait_inst_roles_configured(dir, inst): + # The last recorded transition is checked instead of the mere presence of one: + # every clusterwide config apply drives the instance through ConfiguringRoles + # again, and a 2PC started before it settles is rejected with a Prepare2pcError. + log = os.path.join(dir, cartridge_name, log_path, inst, log_file) + state = None + trying = 0 + while trying < 200: + with open(log, "r") as fp: + states = state_change_re.findall(fp.read()) + if states: + state = states[-1] + if state == "RolesConfigured": + return + time.sleep(0.05) + trying = trying + 1 + assert state == "RolesConfigured", f"{inst} is in state {state}, expected RolesConfigured" + + def wait_inst_start(dir, inst): wait_inst_files(dir, inst) @@ -198,7 +226,14 @@ def start(self, bootstrap_vshard=True): self.bootstrap(bootstrap_vshard=bootstrap_vshard) def bootstrap(self, bootstrap_vshard=True): - cmd = [self.tt_cmd, "replicaset", "bootstrap", cartridge_name] + cmd = [ + self.tt_cmd, + "replicaset", + "bootstrap", + "--timeout", + str(bootstrap_timeout), + cartridge_name, + ] if bootstrap_vshard: cmd.append("--bootstrap-vshard") rc, out = run_command_and_get_output(cmd, cwd=self.workdir) @@ -206,26 +241,13 @@ def bootstrap(self, bootstrap_vshard=True): assert re.search(r"Done.", out) # Wait until the instances are configured. + self.wait_roles_configured() + + def wait_roles_configured(self): for inst in self.instances: if inst == "stateboard": continue - configured = False - log_dir = os.path.join(self.workdir, cartridge_name, log_path, inst) - trying = 0 - while not configured and trying < 200: - with open(os.path.join(log_dir, log_file), "r") as fp: - lines = fp.readlines() - lines = [line.rstrip() for line in lines] - for line in lines: - if re.search( - r"Instance state changed: ConfiguringRoles -> RolesConfigured", - line, - ): - configured = True - break - time.sleep(0.05) - trying = trying + 1 - assert configured is True + wait_inst_roles_configured(self.workdir, inst) def set_failover(self, data): with open(os.path.join(self.workdir, cartridge_name, "failover.yml"), "w") as f: @@ -234,6 +256,10 @@ def set_failover(self, data): rc, out = run_command_and_get_output(cmd, cwd=os.path.join(self.workdir, cartridge_name)) assert rc == 0 assert re.search(r"Failover configured successfully", out) + # Failover setup patches the clusterwide config, so the instances reconfigure + # their roles once more. Without waiting for that, a command issued right away + # races with the reconfiguration and fails to start its own 2PC. + self.wait_roles_configured() def stop(self): cmd = [self.tt_cmd, "stop", "-y", cartridge_name]