diff --git a/README.md b/README.md index bd56cbc1..ad4b43e0 100644 --- a/README.md +++ b/README.md @@ -244,8 +244,22 @@ $ example --format pretty --tags @smoke and not @ignore -- ./feature/acceptance # On windows you might need to enclose the tags in quotes: $ example --format pretty --tags "@smoke and not @ignore" -- ./feature/acceptance ./feature/integration + +# runs every matching scenario 3 times unconditionally (for sampling/measurement). +# Unlike --retry, it does not stop early on a pass; the run fails if any execution fails. +$ example --repeat 3 + +# only repeats scenarios matching the tag expression +$ example --repeat 3 --repeat-tag-filter @measure ``` +> ℹ️ `--repeat` and `--retry` are mutually exclusive. Aggregating the repeated +> executions (for example averaging a measurement) belongs in a `HOOK_AFTER_ALL` +> hook: record each execution's sample from a per-scenario hook and assert the +> aggregate once in `HOOK_AFTER_ALL`. A failing `HOOK_AFTER_ALL` fails the run; +> note that `HOOK_AFTER_FEATURE` failures do **not** affect the exit status, so +> the aggregate threshold check must not live there. + > ℹ️ use --help to see all available arguments and options ## Contributing diff --git a/compatibility/BaseCompatibility.cpp b/compatibility/BaseCompatibility.cpp index fb1da6e8..22b93a5c 100644 --- a/compatibility/BaseCompatibility.cpp +++ b/compatibility/BaseCompatibility.cpp @@ -232,6 +232,8 @@ namespace compatibility .retry = devkit.kitString.starts_with("retry") ? 2u : 0u, .strict = true, .retryTagExpression = cucumber_cpp::library::tag_expression::Parse(""), + .repeat = devkit.kitString.starts_with("repeat") ? 3u : 0u, + .repeatTagExpression = cucumber_cpp::library::tag_expression::Parse(""), }, }; diff --git a/compatibility/repeat/repeat.arguments.txt b/compatibility/repeat/repeat.arguments.txt new file mode 100644 index 00000000..dd2d6c40 --- /dev/null +++ b/compatibility/repeat/repeat.arguments.txt @@ -0,0 +1 @@ +--repeat 3 diff --git a/compatibility/repeat/repeat.cpp b/compatibility/repeat/repeat.cpp new file mode 100644 index 00000000..a4284705 --- /dev/null +++ b/compatibility/repeat/repeat.cpp @@ -0,0 +1,6 @@ +#include "cucumber_cpp/Steps.hpp" + +GIVEN(R"(a step that always passes)") +{ + // no-op +} diff --git a/compatibility/repeat/repeat.feature b/compatibility/repeat/repeat.feature new file mode 100644 index 00000000..dd3247ae --- /dev/null +++ b/compatibility/repeat/repeat.feature @@ -0,0 +1,8 @@ +Feature: Repeat + Some Cucumber implementations support running a scenario a fixed number of + times unconditionally, for sampling or measurement. Unlike Retry, every + execution runs regardless of the outcome, and the executions surface as + incrementing attempts on the same test case. + + Scenario: A repeated scenario runs the requested number of times + Given a step that always passes diff --git a/compatibility/repeat/repeat.ndjson b/compatibility/repeat/repeat.ndjson new file mode 100644 index 00000000..bb083995 --- /dev/null +++ b/compatibility/repeat/repeat.ndjson @@ -0,0 +1,19 @@ +{"source":{"data":"Feature: Repeat\n Some Cucumber implementations support running a scenario a fixed number of\n times unconditionally, for sampling or measurement. Unlike Retry, every\n execution runs regardless of the outcome, and the executions surface as\n incrementing attempts on the same test case.\n\n Scenario: A repeated scenario runs the requested number of times\n Given a step that always passes\n","mediaType":"text/x.cucumber.gherkin+plain","uri":"samples/repeat/repeat.feature"}} +{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"1","keyword":"Scenario","location":{"column":3},"name":"A repeated scenario runs the requested number of times","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":5},"text":"a step that always passes"}],"tags":[]}}],"description":" Some Cucumber implementations support running a scenario a fixed number of\n times unconditionally, for sampling or measurement. Unlike Retry, every\n execution runs regardless of the outcome, and the executions surface as\n incrementing attempts on the same test case.","keyword":"Feature","language":"en","location":{"column":1},"name":"Repeat","tags":[]},"uri":"samples/repeat/repeat.feature"}} +{"pickle":{"astNodeIds":["1"],"id":"3","language":"en","location":{"column":3},"name":"A repeated scenario runs the requested number of times","steps":[{"astNodeIds":["0"],"id":"2","text":"a step that always passes","type":"Context"}],"tags":[],"uri":"samples/repeat/repeat.feature"}} +{"stepDefinition":{"id":"4","pattern":{"source":"a step that always passes","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"location":{},"uri":"samples/repeat/repeat.cpp"}}} +{"testRunStarted":{"id":"5","timestamp":{"nanos":0,"seconds":0}}} +{"testCase":{"id":"6","pickleId":"3","testRunStartedId":"5","testSteps":[{"id":"7","pickleStepId":"2","stepDefinitionIds":["4"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}} +{"testCaseStarted":{"attempt":0,"id":"8","testCaseId":"6","timestamp":{"nanos":1000000,"seconds":0}}} +{"testStepStarted":{"testCaseStartedId":"8","testStepId":"7","timestamp":{"nanos":2000000,"seconds":0}}} +{"testStepFinished":{"testCaseStartedId":"8","testStepId":"7","testStepResult":{"duration":{"nanos":1000000,"seconds":0},"status":"PASSED"},"timestamp":{"nanos":3000000,"seconds":0}}} +{"testCaseFinished":{"testCaseStartedId":"8","timestamp":{"nanos":4000000,"seconds":0},"willBeRetried":true}} +{"testCaseStarted":{"attempt":1,"id":"9","testCaseId":"6","timestamp":{"nanos":5000000,"seconds":0}}} +{"testStepStarted":{"testCaseStartedId":"9","testStepId":"7","timestamp":{"nanos":6000000,"seconds":0}}} +{"testStepFinished":{"testCaseStartedId":"9","testStepId":"7","testStepResult":{"duration":{"nanos":1000000,"seconds":0},"status":"PASSED"},"timestamp":{"nanos":7000000,"seconds":0}}} +{"testCaseFinished":{"testCaseStartedId":"9","timestamp":{"nanos":8000000,"seconds":0},"willBeRetried":true}} +{"testCaseStarted":{"attempt":2,"id":"10","testCaseId":"6","timestamp":{"nanos":9000000,"seconds":0}}} +{"testStepStarted":{"testCaseStartedId":"10","testStepId":"7","timestamp":{"nanos":10000000,"seconds":0}}} +{"testStepFinished":{"testCaseStartedId":"10","testStepId":"7","testStepResult":{"duration":{"nanos":1000000,"seconds":0},"status":"PASSED"},"timestamp":{"nanos":11000000,"seconds":0}}} +{"testCaseFinished":{"testCaseStartedId":"10","timestamp":{"nanos":12000000,"seconds":0},"willBeRetried":false}} +{"testRunFinished":{"success":true,"testRunStartedId":"5","timestamp":{"nanos":13000000,"seconds":0}}} diff --git a/cucumber_cpp/library/Application.cpp b/cucumber_cpp/library/Application.cpp index 51516fcc..ff510712 100644 --- a/cucumber_cpp/library/Application.cpp +++ b/cucumber_cpp/library/Application.cpp @@ -115,6 +115,10 @@ namespace cucumber_cpp::library cli.add_option("--order", options.ordering, "Run scenarios in specificed order")->transform(CLI::CheckedTransformer(orderingMap, CLI::ignore_case))->default_val(options.ordering); auto* retryOpt = cli.add_option("--retry", options.retry, "Number of times to retry failed scenarios")->default_val(options.retry); cli.add_option("--retry-tag-filter", options.retryTagFilter, "Only retry scenarios matching this tag expression")->needs(retryOpt); + auto* repeatOpt = cli.add_option("--repeat", options.repeat, "Number of times to run matching scenarios unconditionally (for sampling/measurement); unlike --retry it does not stop early on pass")->default_val(options.repeat); + cli.add_option("--repeat-tag-filter", options.repeatTagFilter, "Only repeat scenarios matching this tag expression")->needs(repeatOpt); + repeatOpt->excludes(retryOpt); + retryOpt->excludes(repeatOpt); cli.add_flag("--strict,!--no-strict", options.strict, "Fail if there are pending steps")->default_val(options.strict); cli.add_flag("--feature-hooks,!--no-feature-hooks", options.featureHooks, "Run Before/After Feature hooks, note these are non-standard and are not supported by messages")->default_val(options.featureHooks); cli.add_flag("--recursive,!--no-recursive", options.recursive, "Search for feature files recursively")->default_val(options.recursive); @@ -197,6 +201,8 @@ namespace cucumber_cpp::library .retry = options.retry, .strict = options.strict, .retryTagExpression = tag_expression::Parse(fmt::to_string(fmt::join(options.retryTagFilter, " "))), + .repeat = options.repeat, + .repeatTagExpression = tag_expression::Parse(fmt::to_string(fmt::join(options.repeatTagFilter, " "))), .featureHooks = options.featureHooks, }, }; diff --git a/cucumber_cpp/library/Application.hpp b/cucumber_cpp/library/Application.hpp index 94cabed4..504c6c4b 100644 --- a/cucumber_cpp/library/Application.hpp +++ b/cucumber_cpp/library/Application.hpp @@ -48,6 +48,9 @@ namespace cucumber_cpp::library std::size_t retry{ 0 }; std::vector retryTagFilter{}; + std::size_t repeat{ 0 }; + std::vector repeatTagFilter{}; + bool strict{ true }; bool featureHooks{ false }; diff --git a/cucumber_cpp/library/runtime/TestCaseRunner.cpp b/cucumber_cpp/library/runtime/TestCaseRunner.cpp index f7756ac5..5efde6b9 100644 --- a/cucumber_cpp/library/runtime/TestCaseRunner.cpp +++ b/cucumber_cpp/library/runtime/TestCaseRunner.cpp @@ -79,13 +79,13 @@ namespace cucumber_cpp::library::runtime , testSuiteContext{ testSuiteContext } {} - cucumber::messages::test_step_result_status TestCaseRunner::Run() + cucumber::messages::test_step_result_status TestCaseRunner::Run(std::size_t attemptOffset, bool willRepeatAfter) { for (std::size_t attempt = 0; attempt < maximumAttempts; ++attempt) { testStepResults.clear(); - const auto willRetry = RunAttempt(attempt, (attempt + 1) < maximumAttempts); + const auto willRetry = RunAttempt(attemptOffset + attempt, (attempt + 1) < maximumAttempts, willRepeatAfter); if (willRetry) continue; @@ -96,7 +96,7 @@ namespace cucumber_cpp::library::runtime return cucumber::messages::test_step_result_status::UNKNOWN; } - bool TestCaseRunner::RunAttempt(std::size_t attempt, bool moreAttemptsAvailable) + bool TestCaseRunner::RunAttempt(std::size_t attempt, bool moreAttemptsAvailable, bool willRepeatAfter) { Context testCaseContext{ &testSuiteContext }; const auto currentTestCaseStartedId = idGenerator->next_id(); @@ -151,7 +151,7 @@ namespace cucumber_cpp::library::runtime broadcaster.BroadcastEvent(cucumber::messages::envelope{ .test_case_finished = cucumber::messages::test_case_finished{ .test_case_started_id = currentTestCaseStartedId, .timestamp = util::TimestampNow(), - .will_be_retried = willRetry, + .will_be_retried = willRetry || willRepeatAfter, } }); return willRetry; diff --git a/cucumber_cpp/library/runtime/TestCaseRunner.hpp b/cucumber_cpp/library/runtime/TestCaseRunner.hpp index 31dcb23e..5333597f 100644 --- a/cucumber_cpp/library/runtime/TestCaseRunner.hpp +++ b/cucumber_cpp/library/runtime/TestCaseRunner.hpp @@ -32,9 +32,9 @@ namespace cucumber_cpp::library::runtime support::SupportCodeLibrary& supportCodeLibrary, Context& testSuiteContext); - cucumber::messages::test_step_result_status Run(); + cucumber::messages::test_step_result_status Run(std::size_t attemptOffset = 0, bool willRepeatAfter = false); - bool RunAttempt(std::size_t attempt, bool moreAttemptsAvailable); + bool RunAttempt(std::size_t attempt, bool moreAttemptsAvailable, bool willRepeatAfter = false); cucumber::messages::test_step_result RunHook(const support::HookRegistry::Definition& hookDefinition, bool isBeforeHook, Context& testCaseContext, const cucumber::messages::test_step_started& testStepStarted, bool hasError); diff --git a/cucumber_cpp/library/runtime/Worker.cpp b/cucumber_cpp/library/runtime/Worker.cpp index b1a6e999..dfe45fcc 100644 --- a/cucumber_cpp/library/runtime/Worker.cpp +++ b/cucumber_cpp/library/runtime/Worker.cpp @@ -58,6 +58,16 @@ namespace cucumber_cpp::library::runtime return 0; } + std::size_t RepeatsForPickle(const cucumber::messages::pickle& pickle, const support::RunOptions::Runtime& options) + { + if (options.repeat == 0) + return 1; + else if (options.repeatTagExpression->Evaluate(util::TransformPickleTags(pickle.tags))) + return options.repeat; + else + return 1; + } + bool IsFailing(cucumber::messages::test_step_result_status status, bool dryRun) { if (dryRun) @@ -134,21 +144,32 @@ namespace cucumber_cpp::library::runtime bool Worker::RunTestCase(const cucumber::messages::gherkin_document& gherkinDocument, const assemble::AssembledTestCase& assembledTestCase, Context& testSuiteContext, bool failing) { - TestCaseRunner testCaseRunner{ - broadcaster, - idGenerator, - gherkinDocument, - assembledTestCase.pickle, - assembledTestCase.testCase, - RetriesForPickle(assembledTestCase.pickle, options), - options.dryRun || (options.failFast && failing), - supportCodeLibrary, - testSuiteContext, - }; + // A skipped test case (dry-run or fail-fast) never executes, so it must not repeat. + const bool skip = options.dryRun || (options.failFast && failing); + const auto repeats = skip ? 1 : RepeatsForPickle(assembledTestCase.pickle, options); + + bool allPassed = true; - const auto status = testCaseRunner.Run(); + for (std::size_t repeat = 0; repeat < repeats; ++repeat) + { + TestCaseRunner testCaseRunner{ + broadcaster, + idGenerator, + gherkinDocument, + assembledTestCase.pickle, + assembledTestCase.testCase, + RetriesForPickle(assembledTestCase.pickle, options), + skip, + supportCodeLibrary, + testSuiteContext, + }; + + const auto status = testCaseRunner.Run(repeat, (repeat + 1) < repeats); + + allPassed = allPassed && !IsStatusFailed(status); + } - return !IsStatusFailed(status); + return allPassed; } std::vector Worker::RunBeforeTestSuiteHooks(const cucumber::messages::feature& feature, Context& context) diff --git a/cucumber_cpp/library/support/Types.hpp b/cucumber_cpp/library/support/Types.hpp index 865aaa08..455bafc1 100644 --- a/cucumber_cpp/library/support/Types.hpp +++ b/cucumber_cpp/library/support/Types.hpp @@ -39,6 +39,8 @@ namespace cucumber_cpp::library::support std::size_t retry{ 0 }; bool strict{ true }; std::unique_ptr retryTagExpression{}; + std::size_t repeat{ 0 }; + std::unique_ptr repeatTagExpression{}; bool featureHooks{ false }; } runtime;