From 7702adb5cdcdaa2b5c48d7ffa7ba63ffb3325694 Mon Sep 17 00:00:00 2001 From: Chris Kirkland Date: Mon, 22 Jun 2026 10:42:04 -0600 Subject: [PATCH 01/13] install failpoint --- go.mod | 1 + go.sum | 2 + go/base/context.go | 2 + go/cmd/gh-ost/main.go | 3 + go/logic/migrator.go | 12 + localtests/move-tables-test.sh | 23 + .../github.com/pingcap/failpoint/.codecov.yml | 39 ++ .../github.com/pingcap/failpoint/.gitignore | 27 + .../pingcap/failpoint/CONTRIBUTING.md | 94 +++ vendor/github.com/pingcap/failpoint/LICENSE | 201 +++++++ .../pingcap/failpoint/MAINTAINERS.md | 12 + vendor/github.com/pingcap/failpoint/Makefile | 78 +++ vendor/github.com/pingcap/failpoint/README.md | 557 ++++++++++++++++++ .../github.com/pingcap/failpoint/failpoint.go | 150 +++++ .../pingcap/failpoint/failpoints.go | 324 ++++++++++ vendor/github.com/pingcap/failpoint/http.go | 110 ++++ vendor/github.com/pingcap/failpoint/marker.go | 100 ++++ vendor/github.com/pingcap/failpoint/terms.go | 376 ++++++++++++ vendor/modules.txt | 3 + 19 files changed, 2114 insertions(+) create mode 100644 vendor/github.com/pingcap/failpoint/.codecov.yml create mode 100644 vendor/github.com/pingcap/failpoint/.gitignore create mode 100644 vendor/github.com/pingcap/failpoint/CONTRIBUTING.md create mode 100644 vendor/github.com/pingcap/failpoint/LICENSE create mode 100644 vendor/github.com/pingcap/failpoint/MAINTAINERS.md create mode 100644 vendor/github.com/pingcap/failpoint/Makefile create mode 100644 vendor/github.com/pingcap/failpoint/README.md create mode 100644 vendor/github.com/pingcap/failpoint/failpoint.go create mode 100644 vendor/github.com/pingcap/failpoint/failpoints.go create mode 100644 vendor/github.com/pingcap/failpoint/http.go create mode 100644 vendor/github.com/pingcap/failpoint/marker.go create mode 100644 vendor/github.com/pingcap/failpoint/terms.go diff --git a/go.mod b/go.mod index 11a57a8d8..764dd9a6f 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 // indirect + github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f // indirect github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 // indirect github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index dcb552f90..d4aae4ccd 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb h1:3pSi4EDG6hg0o github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 h1:cLgCk5mwDG9lDH+dPK8TmEliTjyGJwwKN0qevWAl8IY= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2/go.mod h1:ktAJCA9lxrHHjVyVl2pKJFvzBnq2eZbb+CUOjBRPlXo= +github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f h1:cDo4qNgaQc2POMWTXjNrMA7yySdIF/d1AaW8kOA7qOs= +github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f/go.mod h1:jimwlLpI/XtwQdlZML15HS+j4rirvwZM0GLY07wwgOo= github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g9Onkvu6UmuBBOeWRGQEjJaT/JY= github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 h1:A2bEfgSb7hLwR9mxDszgGKweF+xY9YoTDG+8RjdFjDE= diff --git a/go/base/context.go b/go/base/context.go index 747d23d01..ca24489c0 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -305,6 +305,8 @@ type MigrationContext struct { DrainGTID mysql.BinlogCoordinates // Source @@gtid_executed captured immediately after the source RENAME TABLE; the applier drains until it reaches this coordinate (move-tables only). } + UnsafeFailPointsEnabled bool + Log Logger } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 0cf4f6121..982c6d2fa 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -197,6 +197,9 @@ func main() { flag.StringVar(&migrationContext.MoveTables.TargetDatabase, "target-database", "", "Target MySQL database name for --move-tables mode. If not provided, uses the same database name as the source connection") flag.BoolVar(&migrationContext.MoveTables.AllowOnSourcePrimary, "allow-on-source-primary", false, "allow --move-tables to read (schema, row copy, binlog) from the source cluster's primary. By default gh-ost stops if --host is the primary; prefer pointing --host at a replica to spare the primary the copy load.") + // unsafe fail points, for integration testing purposes + flag.BoolVar(&migrationContext.UnsafeFailPointsEnabled, "unsafe-fail-points-enabled", false, "UNSAFE: Enable fail points for integration testing purposes. Do not use in production.") + flag.CommandLine.SetOutput(os.Stdout) flag.Parse() cutOverLockTimeoutUserSpecified := false diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f3658b397..c5b05083c 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -21,6 +21,8 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" + + "github.com/pingcap/failpoint" ) var ( @@ -2869,3 +2871,13 @@ func (mgtr *Migrator) teardown() { mgtr.sourcePrimaryDB.Close() } } + +func (mgtr *Migrator) newFailPoint(name string) { + if mgtr.migrationContext.UnsafeFailPointsEnabled { + mgtr.migrationContext.Log.Debugf("Injecting fail point: %s", name) + + failpoint.Inject(name, func(val failpoint.Value) { + panic(fmt.Sprintf("encountered fail point: '%s'", name)) + }) + } +} diff --git a/localtests/move-tables-test.sh b/localtests/move-tables-test.sh index e74db7545..b6c8e8c90 100755 --- a/localtests/move-tables-test.sh +++ b/localtests/move-tables-test.sh @@ -7,6 +7,8 @@ # Usage: localtests/test/sh [filter] # By default, runs all move-tables tests. Given filter, will only run tests matching given regep +set -x + repo_root=$(git rev-parse --show-toplevel) script_path="$repo_root/script/move-tables" tests_path=$(dirname $0)/move-tables @@ -453,6 +455,26 @@ test_single() { done } +install_failpoint() { + pushd $repo_root/../.. + + if [ ! -d "pingcap/failpoint" ]; then + echo "⚙️ Installing failpoint" + mkdir -p pingcap/failpoint + git clone https://github.com/pingcap/failpoint.git pingcap/failpoint + cd pingcap/failpoint + make + else + bin/failpoint-ctl disable + fi + + echo "⚙️ Enabling failpoint" + bin/failpoint-ctl enable + + echo "✅ Successfully enabled failpoint" + popd +} + build_binary() { echo "Building" rm -f $default_ghost_binary @@ -471,6 +493,7 @@ build_binary() { } test_all() { + install_failpoint build_binary test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) while read -r test_dir; do diff --git a/vendor/github.com/pingcap/failpoint/.codecov.yml b/vendor/github.com/pingcap/failpoint/.codecov.yml new file mode 100644 index 000000000..402988545 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/.codecov.yml @@ -0,0 +1,39 @@ +codecov: + notify: + require_ci_to_pass: yes + +coverage: + precision: 4 + round: down + range: "65...90" + + status: + project: + default: + threshold: 20 #Allow the coverage to drop by threshold%, and posting a success status. + patch: + default: + target: 0% # trial operation + changes: no + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "header, diff" + behavior: default + require_changes: no + +ignore: + - "LICENSES" + - "*_test.go" + - "marker.go" # This file only contains empty function stub + - "failpoint-ctl" # Ignore the `failpoint-ctl` command line tool + - ".git" + - "*.yml" + - "*.md" diff --git a/vendor/github.com/pingcap/failpoint/.gitignore b/vendor/github.com/pingcap/failpoint/.gitignore new file mode 100644 index 000000000..b1e5133d6 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/.gitignore @@ -0,0 +1,27 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +bin +coverage.out +.idea/ +*.iml +*.swp +*.txt +*.log +tags +profile.coverprofile +overalls.coverprofile +explain_test +*.fail.go +vendor +.DS_Store diff --git a/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md b/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md new file mode 100644 index 000000000..cfefb0ed5 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md @@ -0,0 +1,94 @@ +# How to contribute + +This document outlines some of the conventions on development workflow, commit +message formatting, contact points and other resources to make it easier to get +your contribution accepted. + +## Getting started + +- Fork the repository on GitHub. +- Read the README.md for build instructions. +- Play with the project, submit bugs, submit patches! + +## Building Failpoint + +Developing Failpoint requires: + +* [Go 1.13](http://golang.org/doc/code.html) +* An internet connection to download the dependencies + +Simply run `make` to build the program. + +```sh +make +``` + +### Running tests + +This project contains unit tests and integration tests with coverage collection. +See [tests/README.md](./tests/README.md) for how to execute and add tests. + +### Updating dependencies + +Failpoint manages dependencies using [Go module](https://github.com/golang/go/wiki/Modules). +To add or update a dependency, either + +* Use the `go mod edit` command to change the dependency, or +* Edit `go.mod` and then run `make update` to update the checksum. + +## Contribution flow + +This is a rough outline of what a contributor's workflow looks like: + +- Create a topic branch from where you want to base your work. This is usually `master`. +- Make commits of logical units and add test case if the change fixes a bug or adds new functionality. +- Run tests and make sure all the tests are passed. +- Make sure your commit messages are in the proper format (see below). +- Push your changes to a topic branch in your fork of the repository. +- Submit a pull request. +- Your PR must receive LGTMs from two maintainers. + +Thanks for your contributions! + +### Code style + +The coding style suggested by the Golang community is used in `failpoint`. +See the [style doc](https://github.com/golang/go/wiki/CodeReviewComments) for details. + +Please follow this style to makeg `failpoint` easy to review, maintain and develop. + +### Format of the Commit Message + +We follow a rough convention for commit messages that is designed to answer two +questions: what changed and why. The subject line should feature the what and +the body of the commit should describe the why. + +``` +restore: add comment for variable declaration + +Improve documentation. +``` + +The format can be described more formally as follows: + +``` +: + + + +