Add incremental system test progress to job summary

changelog-entries/821.md

@@ -0,0 +1 @@
+- System tests now update the GitHub Actions job summary incrementally after each test finishes, with per-test status and timings, while keeping the final results table (fixes [#791](https://github.com/precice/tutorials/issues/791)).

tools/tests/systemtests.py

@@ -2,7 +2,13 @@
 import argparse
 from pathlib import Path
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table
+from systemtests.Systemtest import (
+    Systemtest,
+    GLOBAL_TIMEOUT,
+    append_systemtest_progress_to_github_summary,
+    display_systemtestresults_as_table,
+    initialize_systemtests_progress_in_github_summary,
+)
 from systemtests.TestSuite import TestSuites
 from metadata_parser.metdata import Tutorials, Case
 import logging
@@ -74,6 +80,7 @@ def main():
 
     logging.info(f"About to run the following systemtest in the directory {run_directory}:\n {systemtests_to_run}")
 
+    initialize_systemtests_progress_in_github_summary(len(systemtests_to_run))
     results = []
     for number, systemtest in enumerate(systemtests_to_run, start=1):
         logging.info(f"Started running {systemtest},  {number}/{len(systemtests_to_run)}")
@@ -82,6 +89,12 @@ def main():
         elapsed_time = time.perf_counter() - t
         logging.info(f"Running {systemtest} took {elapsed_time:^.1f} seconds")
         results.append(result)
+        append_systemtest_progress_to_github_summary(
+            result,
+            number,
+            len(systemtests_to_run),
+            elapsed_time,
+        )
 
     system_test_success = True
     for result in results:

tools/tests/systemtests/Systemtest.py

@@ -23,6 +23,8 @@
 SHORT_TIMEOUT = 10
 
 DIFF_RESULTS_DIR = "diff-results"
+GITHUB_SUMMARY_PROGRESS_HEADING = "## System tests progress"
+GITHUB_SUMMARY_RESULTS_HEADING = "## Final system test results"
 
 
 def slugify(value, allow_unicode=False):
@@ -76,6 +78,44 @@ class SystemtestResult:
     fieldcompare_time: float  # in seconds
 
 
+def _append_lines_to_github_summary(lines: List[str]):
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if not summary_path:
+        return
+
+    with open(summary_path, "a") as summary_file:
+        for line in lines:
+            print(line, file=summary_file)
+
+
+def initialize_systemtests_progress_in_github_summary(total_systemtests: int):
+    if total_systemtests <= 0:
+        return
+
+    _append_lines_to_github_summary([
+        "",
+        GITHUB_SUMMARY_PROGRESS_HEADING,
+        "",
+        f"- Starting execution of `{total_systemtests}` system tests.",
+    ])
+
+
+def append_systemtest_progress_to_github_summary(
+        result: SystemtestResult,
+        number: int,
+        total_systemtests: int,
+        elapsed_time: float):
+    status_checkbox = "x" if result.success else " "
+    status_text = "finished successfully" if result.success else "failed"
+    progress_line = (
+        f"- [{status_checkbox}] `{number}/{total_systemtests}` "
+        f"`{result.systemtest}` {status_text} in `{elapsed_time:.1f}s` "
+        f"(build `{result.build_time:.1f}s`, solver `{result.solver_time:.1f}s`, "
+        f"fieldcompare `{result.fieldcompare_time:.1f}s`)."
+    )
+    _append_lines_to_github_summary([progress_line])
+
+
 def display_systemtestresults_as_table(results: List[SystemtestResult]):
     """
     Prints the result in a nice tabluated way to get an easy overview
@@ -100,6 +140,9 @@ def _get_length_of_name(results: List[SystemtestResult]) -> int:
 
     if "GITHUB_STEP_SUMMARY" in os.environ:
         with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
+            print("", file=f)
+            print(GITHUB_SUMMARY_RESULTS_HEADING, file=f)
+            print("", file=f)
             print(header, file=f)
             print(separator_markdown, file=f)