feat: add --repeat option to run scenarios a fixed number of times

README.md

@@ -244,8 +244,22 @@ $ example --format pretty --tags @smoke and not @ignore -- ./feature/acceptance
 
 # On windows you might need to enclose the tags in quotes:
 $ example --format pretty --tags "@smoke and not @ignore" -- ./feature/acceptance ./feature/integration
+
+# runs every matching scenario 3 times unconditionally (for sampling/measurement).
+# Unlike --retry, it does not stop early on a pass; the run fails if any execution fails.
+$ example --repeat 3
+
+# only repeats scenarios matching the tag expression
+$ example --repeat 3 --repeat-tag-filter @measure
 ```
 
+> ℹ️ `--repeat` and `--retry` are mutually exclusive. Aggregating the repeated
+> executions (for example averaging a measurement) belongs in a `HOOK_AFTER_ALL`
+> hook: record each execution's sample from a per-scenario hook and assert the
+> aggregate once in `HOOK_AFTER_ALL`. A failing `HOOK_AFTER_ALL` fails the run;
+> note that `HOOK_AFTER_FEATURE` failures do **not** affect the exit status, so
+> the aggregate threshold check must not live there.
+
 > ℹ️ use --help to see all available arguments and options
 
 ## Contributing

compatibility/BaseCompatibility.cpp

@@ -232,6 +232,8 @@ namespace compatibility
                 .retry = devkit.kitString.starts_with("retry") ? 2u : 0u,
                 .strict = true,
                 .retryTagExpression = cucumber_cpp::library::tag_expression::Parse(""),
+                .repeat = devkit.kitString.starts_with("repeat") ? 3u : 0u,
+                .repeatTagExpression = cucumber_cpp::library::tag_expression::Parse(""),
             },
         };
 

compatibility/repeat/repeat.arguments.txt

@@ -0,0 +1 @@
+--repeat 3

compatibility/repeat/repeat.cpp

@@ -0,0 +1,6 @@
+#include "cucumber_cpp/Steps.hpp"
+
+GIVEN(R"(a step that always passes)")
+{
+    // no-op
+}

compatibility/repeat/repeat.feature

@@ -0,0 +1,8 @@
+Feature: Repeat
+  Some Cucumber implementations support running a scenario a fixed number of
+  times unconditionally, for sampling or measurement. Unlike Retry, every
+  execution runs regardless of the outcome, and the executions surface as
+  incrementing attempts on the same test case.
+
+  Scenario: A repeated scenario runs the requested number of times
+    Given a step that always passes

compatibility/repeat/repeat.ndjson

@@ -0,0 +1,19 @@
+{"source":{"data":"Feature: Repeat\n  Some Cucumber implementations support running a scenario a fixed number of\n  times unconditionally, for sampling or measurement. Unlike Retry, every\n  execution runs regardless of the outcome, and the executions surface as\n  incrementing attempts on the same test case.\n\n  Scenario: A repeated scenario runs the requested number of times\n    Given a step that always passes\n","mediaType":"text/x.cucumber.gherkin+plain","uri":"samples/repeat/repeat.feature"}}
+{"gherkinDocument":{"comments":[],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"1","keyword":"Scenario","location":{"column":3},"name":"A repeated scenario runs the requested number of times","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":5},"text":"a step that always passes"}],"tags":[]}}],"description":"  Some Cucumber implementations support running a scenario a fixed number of\n  times unconditionally, for sampling or measurement. Unlike Retry, every\n  execution runs regardless of the outcome, and the executions surface as\n  incrementing attempts on the same test case.","keyword":"Feature","language":"en","location":{"column":1},"name":"Repeat","tags":[]},"uri":"samples/repeat/repeat.feature"}}
+{"pickle":{"astNodeIds":["1"],"id":"3","language":"en","location":{"column":3},"name":"A repeated scenario runs the requested number of times","steps":[{"astNodeIds":["0"],"id":"2","text":"a step that always passes","type":"Context"}],"tags":[],"uri":"samples/repeat/repeat.feature"}}
+{"stepDefinition":{"id":"4","pattern":{"source":"a step that always passes","type":"CUCUMBER_EXPRESSION"},"sourceReference":{"location":{},"uri":"samples/repeat/repeat.cpp"}}}
+{"testRunStarted":{"id":"5","timestamp":{"nanos":0,"seconds":0}}}
+{"testCase":{"id":"6","pickleId":"3","testRunStartedId":"5","testSteps":[{"id":"7","pickleStepId":"2","stepDefinitionIds":["4"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}]}}
+{"testCaseStarted":{"attempt":0,"id":"8","testCaseId":"6","timestamp":{"nanos":1000000,"seconds":0}}}
+{"testStepStarted":{"testCaseStartedId":"8","testStepId":"7","timestamp":{"nanos":2000000,"seconds":0}}}
+{"testStepFinished":{"testCaseStartedId":"8","testStepId":"7","testStepResult":{"duration":{"nanos":1000000,"seconds":0},"status":"PASSED"},"timestamp":{"nanos":3000000,"seconds":0}}}
+{"testCaseFinished":{"testCaseStartedId":"8","timestamp":{"nanos":4000000,"seconds":0},"willBeRetried":true}}
+{"testCaseStarted":{"attempt":1,"id":"9","testCaseId":"6","timestamp":{"nanos":5000000,"seconds":0}}}
+{"testStepStarted":{"testCaseStartedId":"9","testStepId":"7","timestamp":{"nanos":6000000,"seconds":0}}}
+{"testStepFinished":{"testCaseStartedId":"9","testStepId":"7","testStepResult":{"duration":{"nanos":1000000,"seconds":0},"status":"PASSED"},"timestamp":{"nanos":7000000,"seconds":0}}}
+{"testCaseFinished":{"testCaseStartedId":"9","timestamp":{"nanos":8000000,"seconds":0},"willBeRetried":true}}
+{"testCaseStarted":{"attempt":2,"id":"10","testCaseId":"6","timestamp":{"nanos":9000000,"seconds":0}}}
+{"testStepStarted":{"testCaseStartedId":"10","testStepId":"7","timestamp":{"nanos":10000000,"seconds":0}}}
+{"testStepFinished":{"testCaseStartedId":"10","testStepId":"7","testStepResult":{"duration":{"nanos":1000000,"seconds":0},"status":"PASSED"},"timestamp":{"nanos":11000000,"seconds":0}}}
+{"testCaseFinished":{"testCaseStartedId":"10","timestamp":{"nanos":12000000,"seconds":0},"willBeRetried":false}}
+{"testRunFinished":{"success":true,"testRunStartedId":"5","timestamp":{"nanos":13000000,"seconds":0}}}

cucumber_cpp/library/Application.cpp

@@ -115,6 +115,10 @@ namespace cucumber_cpp::library
             cli.add_option("--order", options.ordering, "Run scenarios in specificed order")->transform(CLI::CheckedTransformer(orderingMap, CLI::ignore_case))->default_val(options.ordering);
             auto* retryOpt = cli.add_option("--retry", options.retry, "Number of times to retry failed scenarios")->default_val(options.retry);
             cli.add_option("--retry-tag-filter", options.retryTagFilter, "Only retry scenarios matching this tag expression")->needs(retryOpt);
+            auto* repeatOpt = cli.add_option("--repeat", options.repeat, "Number of times to run matching scenarios unconditionally (for sampling/measurement); unlike --retry it does not stop early on pass")->default_val(options.repeat);
+            cli.add_option("--repeat-tag-filter", options.repeatTagFilter, "Only repeat scenarios matching this tag expression")->needs(repeatOpt);
+            repeatOpt->excludes(retryOpt);
+            retryOpt->excludes(repeatOpt);
             cli.add_flag("--strict,!--no-strict", options.strict, "Fail if there are pending steps")->default_val(options.strict);
             cli.add_flag("--feature-hooks,!--no-feature-hooks", options.featureHooks, "Run Before/After Feature hooks, note these are non-standard and are not supported by messages")->default_val(options.featureHooks);
             cli.add_flag("--recursive,!--no-recursive", options.recursive, "Search for feature files recursively")->default_val(options.recursive);
@@ -197,6 +201,8 @@ namespace cucumber_cpp::library
                 .retry = options.retry,
                 .strict = options.strict,
                 .retryTagExpression = tag_expression::Parse(fmt::to_string(fmt::join(options.retryTagFilter, " "))),
+                .repeat = options.repeat,
+                .repeatTagExpression = tag_expression::Parse(fmt::to_string(fmt::join(options.repeatTagFilter, " "))),
                 .featureHooks = options.featureHooks,
             },
         };

cucumber_cpp/library/Application.hpp

@@ -48,6 +48,9 @@ namespace cucumber_cpp::library
             std::size_t retry{ 0 };
             std::vector<std::string> retryTagFilter{};
 
+            std::size_t repeat{ 0 };
+            std::vector<std::string> repeatTagFilter{};
+
             bool strict{ true };
 
             bool featureHooks{ false };

cucumber_cpp/library/runtime/TestCaseRunner.cpp

@@ -79,13 +79,13 @@ namespace cucumber_cpp::library::runtime
         , testSuiteContext{ testSuiteContext }
     {}
 
-    cucumber::messages::test_step_result_status TestCaseRunner::Run()
+    cucumber::messages::test_step_result_status TestCaseRunner::Run(std::size_t attemptOffset, bool willRepeatAfter)
     {
         for (std::size_t attempt = 0; attempt < maximumAttempts; ++attempt)
         {
             testStepResults.clear();
 
-            const auto willRetry = RunAttempt(attempt, (attempt + 1) < maximumAttempts);
+            const auto willRetry = RunAttempt(attemptOffset + attempt, (attempt + 1) < maximumAttempts, willRepeatAfter);
 
             if (willRetry)
                 continue;
@@ -96,7 +96,7 @@ namespace cucumber_cpp::library::runtime
         return cucumber::messages::test_step_result_status::UNKNOWN;
     }
 
-    bool TestCaseRunner::RunAttempt(std::size_t attempt, bool moreAttemptsAvailable)
+    bool TestCaseRunner::RunAttempt(std::size_t attempt, bool moreAttemptsAvailable, bool willRepeatAfter)
     {
         Context testCaseContext{ &testSuiteContext };
         const auto currentTestCaseStartedId = idGenerator->next_id();
@@ -151,7 +151,7 @@ namespace cucumber_cpp::library::runtime
         broadcaster.BroadcastEvent(cucumber::messages::envelope{ .test_case_finished = cucumber::messages::test_case_finished{
                                                                      .test_case_started_id = currentTestCaseStartedId,
                                                                      .timestamp = util::TimestampNow(),
-                                                                     .will_be_retried = willRetry,
+                                                                     .will_be_retried = willRetry || willRepeatAfter,
                                                                  } });
 
         return willRetry;

cucumber_cpp/library/runtime/TestCaseRunner.hpp

@@ -32,9 +32,9 @@ namespace cucumber_cpp::library::runtime
             support::SupportCodeLibrary& supportCodeLibrary,
             Context& testSuiteContext);
 
-        cucumber::messages::test_step_result_status Run();
+        cucumber::messages::test_step_result_status Run(std::size_t attemptOffset = 0, bool willRepeatAfter = false);
 
-        bool RunAttempt(std::size_t attempt, bool moreAttemptsAvailable);
+        bool RunAttempt(std::size_t attempt, bool moreAttemptsAvailable, bool willRepeatAfter = false);
 
         cucumber::messages::test_step_result RunHook(const support::HookRegistry::Definition& hookDefinition, bool isBeforeHook, Context& testCaseContext, const cucumber::messages::test_step_started& testStepStarted, bool hasError);
 

cucumber_cpp/library/runtime/Worker.cpp

@@ -58,6 +58,16 @@ namespace cucumber_cpp::library::runtime
                 return 0;
         }
 
+        std::size_t RepeatsForPickle(const cucumber::messages::pickle& pickle, const support::RunOptions::Runtime& options)
+        {
+            if (options.repeat == 0)
+                return 1;
+            else if (options.repeatTagExpression->Evaluate(util::TransformPickleTags(pickle.tags)))
+                return options.repeat;
+            else
+                return 1;
+        }
+
         bool IsFailing(cucumber::messages::test_step_result_status status, bool dryRun)
         {
             if (dryRun)
@@ -134,21 +144,32 @@ namespace cucumber_cpp::library::runtime
 
     bool Worker::RunTestCase(const cucumber::messages::gherkin_document& gherkinDocument, const assemble::AssembledTestCase& assembledTestCase, Context& testSuiteContext, bool failing)
     {
-        TestCaseRunner testCaseRunner{
-            broadcaster,
-            idGenerator,
-            gherkinDocument,
-            assembledTestCase.pickle,
-            assembledTestCase.testCase,
-            RetriesForPickle(assembledTestCase.pickle, options),
-            options.dryRun || (options.failFast && failing),
-            supportCodeLibrary,
-            testSuiteContext,
-        };
+        // A skipped test case (dry-run or fail-fast) never executes, so it must not repeat.
+        const bool skip = options.dryRun || (options.failFast && failing);
+        const auto repeats = skip ? 1 : RepeatsForPickle(assembledTestCase.pickle, options);
+
+        bool allPassed = true;
 
-        const auto status = testCaseRunner.Run();
+        for (std::size_t repeat = 0; repeat < repeats; ++repeat)
+        {
+            TestCaseRunner testCaseRunner{
+                broadcaster,
+                idGenerator,
+                gherkinDocument,
+                assembledTestCase.pickle,
+                assembledTestCase.testCase,
+                RetriesForPickle(assembledTestCase.pickle, options),
+                skip,
+                supportCodeLibrary,
+                testSuiteContext,
+            };
+
+            const auto status = testCaseRunner.Run(repeat, (repeat + 1) < repeats);
+
+            allPassed = allPassed && !IsStatusFailed(status);
+        }
 
-        return !IsStatusFailed(status);
+        return allPassed;
     }
 
     std::vector<cucumber::messages::test_step_result> Worker::RunBeforeTestSuiteHooks(const cucumber::messages::feature& feature, Context& context)

cucumber_cpp/library/support/Types.hpp

@@ -39,6 +39,8 @@ namespace cucumber_cpp::library::support
             std::size_t retry{ 0 };
             bool strict{ true };
             std::unique_ptr<tag_expression::Expression> retryTagExpression{};
+            std::size_t repeat{ 0 };
+            std::unique_ptr<tag_expression::Expression> repeatTagExpression{};
             bool featureHooks{ false };
         } runtime;