ci: make critical artifact uploads resilient

.github/actionlint.yaml

@@ -0,0 +1,45 @@
+self-hosted-runner:
+  labels:
+    - amd64-mo-guangzhou-2xlarge16
+    - amd64-mo-guangzhou-2xlarge32
+    - amd64-mo-guangzhou-medium8
+    - amd64-mo-shanghai-4xlarge32
+    - amd64-mo-shanghai-dind
+    - arm64-mo-guangzhou-2xlarge16
+    - arm64-mo-shanghai-4c8g
+    - arm64-mo-shanghai-8c16g
+    - mac-arm64-mini
+
+paths:
+  .github/workflows/ci.yaml:
+    ignore:
+      - 'constant expression "false" in condition'
+  .github/workflows/coverage-ut.yaml:
+    ignore:
+      # Reusable-workflow-only contexts are supplied by MatrixOne callers.
+      - 'property "workflow_(repository|sha)" is not defined'
+  .github/workflows/e2e-compose-parallel.yaml:
+    ignore:
+      - 'constant expression "false" in condition'
+  .github/workflows/e2e-compose.yaml:
+    ignore:
+      - 'constant expression "false" in condition'
+  .github/workflows/e2e-standalone-parallel.yaml:
+    ignore:
+      - 'constant expression "false" in condition'
+  .github/workflows/e2e-standalone.yaml:
+    ignore:
+      - 'constant expression "false" in condition'
+  .github/workflows/e2e-upgrade.yaml:
+    ignore:
+      - 'constant expression "false" in condition'
+  .github/workflows/robot.yaml:
+    ignore:
+      # Existing legacy workflow; keep new changes clean without broadening
+      # this artifact-resilience PR into a robot migration.
+      - 'workflow command "set-output" was deprecated'
+  .github/workflows/utils.yaml:
+    ignore:
+      - 'constant expression "false" in condition'
+      # Legacy release workflow pinned from MatrixOne at CI@release/3.0-dev.
+      - 'property "setup_mo_test_env" is not defined'

.github/workflows/build-mo.yaml

@@ -205,9 +205,10 @@ jobs:
           echo "artifact_name=mo-shared-build-attempt-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT"
 
       - name: Upload build artifact
-        uses: actions/upload-artifact@v7
+        uses: matrixorigin/CI/actions/upload-artifact-with-retry@main
         with:
           name: mo-shared-build-attempt-${{ github.run_attempt }}
           path: ${{ runner.temp }}/mo-build/mo-build.tar.gz
           if-no-files-found: error
           retention-days: 3
+          compression-level: 0

.github/workflows/check-action-file.yaml

@@ -7,6 +7,12 @@ on:
       - 'release/*'
     paths:
       - '.github/workflows/*.yaml'
+      - '.github/workflows/*.yml'
+      - '.github/actionlint.yaml'
+      - 'actions/upload-artifact-with-retry/action.yaml'
+      - 'actions/upload-artifact-with-retry/state-machine.schema.json'
+      - 'scripts/select_coverage_artifacts.py'
+      - 'scripts/test_select_coverage_artifacts.py'
 
 permissions:
   contents: read
@@ -17,17 +23,103 @@ jobs:
     permissions:
       contents: read
     runs-on: arm64-mo-shanghai-4c8g
+    timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v6
-      - name: Check for any changed workflows
-        id: check-for-changed-workflows
-        uses: tj-actions/changed-files@v47
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
+          persist-credentials: false
+      - name: Validate all workflows
+        uses: dsanders11/json-schema-validate-action@eddf079f55830cc9a916a3c512ba9086240d2fea # v2.0.0
+        with:
+          schema: https://json.schemastore.org/github-workflow.json
           files: |
             .github/workflows/**.yaml
-      - name: Validate workflows
-        if: steps.check-for-changed-workflows.outputs.any_changed == 'true'
-        uses: dsanders11/json-schema-validate-action@v2.0.0
+            .github/workflows/**.yml
+      - name: Install actionlint
+        id: install-actionlint
+        env:
+          ACTIONLINT_VERSION: 1.7.12
+          ACTIONLINT_SHA256: 325e971b6ba9bfa504672e29be93c24981eeb1c07576d730e9f7c8805afff0c6
+        run: |
+          set -euo pipefail
+          archive="${RUNNER_TEMP}/actionlint.tar.gz"
+          install_dir="${RUNNER_TEMP}/actionlint-bin"
+          curl --fail --silent --show-error --location \
+            --connect-timeout 10 --max-time 60 --retry 5 --retry-all-errors \
+            "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_arm64.tar.gz" \
+            --output "${archive}"
+          echo "${ACTIONLINT_SHA256}  ${archive}" | sha256sum --check --strict
+          mkdir -p "${install_dir}"
+          tar -xzf "${archive}" -C "${install_dir}" actionlint
+          echo "executable=${install_dir}/actionlint" >> "$GITHUB_OUTPUT"
+      - name: Lint workflows
+        # Keep this gate focused on workflow/action semantics. The repository
+        # has a separate backlog of legacy shellcheck and pyflakes findings.
+        run: ${{ steps.install-actionlint.outputs.executable }} -shellcheck= -pyflakes= -color
+      - name: Test coverage artifact generation selection
+        run: python3 scripts/test_select_coverage_artifacts.py -v
+      - name: Guard deliberate workflow disablements
+        env:
+          ACTIONLINT: ${{ steps.install-actionlint.outputs.executable }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          # Record semantic locations and a short source hash, not line
+          # numbers, so unrelated edits do not churn the baseline.
+          expected="$(printf '%s\n' \
+            '.github/workflows/ci.yaml:ut-mac-x86:9:a81c7d5095cf' \
+            '.github/workflows/e2e-compose-parallel.yaml:bvt-docker-compose-push:9:57d5b9f8aae4' \
+            '.github/workflows/e2e-compose.yaml:bvt-docker-compose-push:9:57d5b9f8aae4' \
+            '.github/workflows/e2e-standalone-parallel.yaml:bvt-linux-x86:9:57d5b9f8aae4' \
+            '.github/workflows/e2e-standalone-parallel.yaml:multi-cn-proxy-bvt-linux-x86:9:57d5b9f8aae4' \
+            '.github/workflows/e2e-standalone.yaml:bvt-linux-x86:9:57d5b9f8aae4' \
+            '.github/workflows/e2e-upgrade.yaml:upgrade-ci-release-linux-amd64:9:57d5b9f8aae4' \
+            '.github/workflows/e2e-upgrade.yaml:upgrade-ci-target-linux-amd64:9:57d5b9f8aae4' \
+            '.github/workflows/utils.yaml:pr_basic_benchmarks:9:57d5b9f8aae4'
+          )"
+          findings="$(
+            "${ACTIONLINT}" -config-file /dev/null -shellcheck= -pyflakes= \
+              -format '{{range .}}{{.Filepath}}\t{{.Line}}\t{{.Column}}\t{{.Kind}}\t{{.Message}}\n{{end}}' \
+              2>/dev/null || true
+          )"
+          actual="$(
+            while IFS=$'\t' read -r file line column kind message; do
+              if [ "${kind}" != 'if-cond' ]; then
+                continue
+              fi
+              metadata="$(awk -v target="${line}" '
+                /^  [A-Za-z0-9_-]+:$/ {
+                  job = $1
+                  sub(/:$/, "", job)
+                }
+                NR == target {
+                  source = $0
+                  sub(/^[[:space:]]+/, "", source)
+                  sub(/[[:space:]]+$/, "", source)
+                  print job "\t" source
+                  exit
+                }
+              ' "${file}")"
+              job="${metadata%%$'\t'*}"
+              source="${metadata#*$'\t'}"
+              digest="$(printf '%s' "${source}" | sha256sum | cut -c1-12)"
+              printf '%s:%s:%s:%s\n' "${file}" "${job}" "${column}" "${digest}"
+            done <<< "${findings}" | LC_ALL=C sort
+          )"
+          if ! diff -u <(printf '%s\n' "${expected}") <(printf '%s\n' "${actual}"); then
+            echo '::error title=Workflow disablement drift::The set of constant-false conditions changed. Remove accidental disablements, or update the explicit baseline when the change is intentional.'
+            printf '%s\n' "${findings}" |
+              awk -F '\t' '$4 == "if-cond" { print $1 ":" $2 ":" $3 ": " $5 }'
+            exit 1
+          fi
+      - name: Validate artifact retry action metadata
+        uses: dsanders11/json-schema-validate-action@eddf079f55830cc9a916a3c512ba9086240d2fea # v2.0.0
         with:
-          schema: https://json.schemastore.org/github-workflow.json
-          files: .github/workflows/**.yaml 
+          schema: https://json.schemastore.org/github-action.json
+          files: actions/upload-artifact-with-retry/action.yaml
+      - name: Validate artifact retry state machine
+        uses: dsanders11/json-schema-validate-action@eddf079f55830cc9a916a3c512ba9086240d2fea # v2.0.0
+        with:
+          schema: ./actions/upload-artifact-with-retry/state-machine.schema.json
+          files: actions/upload-artifact-with-retry/action.yaml
+          all-errors: true

.github/workflows/ci.yaml

@@ -304,7 +304,7 @@ jobs:
     #if: ${{ !github.event.pull_request.draft }}
     if: ${{ false }}
     environment: ci
-    runs-on: macos-12
+    runs-on: macos-15-intel
     name: UT Test on Darwin/x86
 
     steps:

.github/workflows/coverage-merge.yaml

@@ -87,23 +87,35 @@ jobs:
               "https://api.github.com/repos/${pr_repo}/pulls/${pr_number}" -o diff.patch
           fi
       - name: Download test coverage artifacts
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
         with:
           # A full rerun rotates the shared generation. Filter out artifacts
           # left by previous attempts while retaining successful producers
           # from an earlier attempt during "Re-run failed jobs".
           pattern: ${{ inputs.expected_bvt_generation != '' && format('*coverage*generation-{0}-*', inputs.expected_bvt_generation) || '*coverage*' }}
-          merge-multiple: true
-          path: ${{ github.workspace }}/coverage-artifacts
+          # Keep attempts isolated. Concurrent extraction into one directory
+          # makes same-name profiles overwrite each other nondeterministically.
+          merge-multiple: false
+          path: ${{ runner.temp }}/coverage-artifacts
+      - name: Select latest coverage producer attempts
+        env:
+          EXPECTED_BVT_GENERATION: ${{ inputs.expected_bvt_generation }}
+        run: |
+          set -euo pipefail
+          find "${RUNNER_TEMP}/coverage-artifacts" -type f -printf '%P %s bytes\n' | sort \
+            | tee "${RUNNER_TEMP}/coverage-inputs.txt"
+          python3 "$GITHUB_WORKSPACE/CI/scripts/select_coverage_artifacts.py" \
+            --input "${RUNNER_TEMP}/coverage-artifacts" \
+            --output "${RUNNER_TEMP}/coverage-selected" \
+            --expected-generation "${EXPECTED_BVT_GENERATION}" \
+            --manifest "${RUNNER_TEMP}/coverage-selected-artifacts.json"
       - name: Merge profiles and check coverage rate
         id: merge_coverage
         run: |
           set -euo pipefail
-          artifact_dir="$GITHUB_WORKSPACE/coverage-artifacts"
-          processed_dir="$GITHUB_WORKSPACE/coverage-processed"
+          artifact_dir="${RUNNER_TEMP}/coverage-selected"
+          processed_dir="${RUNNER_TEMP}/coverage-processed"
           mkdir -p "${processed_dir}"
-          find "${artifact_dir}" -type f -printf '%P %s bytes\n' | sort \
-            | tee "${RUNNER_TEMP}/coverage-inputs.txt"
 
           mapfile -t ut_profiles < <(find "${artifact_dir}" -type f -name 'ut-coverage.out' | sort)
           mapfile -t bvt_profiles < <(find "${artifact_dir}" -type f -name 'bvt-*.out' | sort)
@@ -199,7 +211,11 @@ jobs:
           set -euo pipefail
           echo '### Coverage merge result' >> "$GITHUB_STEP_SUMMARY"
           echo "- Merge step: \`${{ steps.merge_coverage.outcome }}\`" >> "$GITHUB_STEP_SUMMARY"
-          echo '- Expected inputs: one UT profile and two BVT profiles.' >> "$GITHUB_STEP_SUMMARY"
+          if [ '${{ github.base_ref }}' = 'main' ]; then
+            echo '- Expected inputs: one UT profile and two BVT profiles.' >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo '- Expected inputs: one UT profile and either zero or two BVT profiles for legacy branches.' >> "$GITHUB_STEP_SUMMARY"
+          fi
           summary="$GITHUB_WORKSPACE/coverage-summary.json"
           if [ -s "${summary}" ]; then
             overall=$(jq -r '.overall_coverage' "${summary}")
@@ -246,19 +262,31 @@ jobs:
             echo '```' >> "$GITHUB_STEP_SUMMARY"
           fi
           echo '' >> "$GITHUB_STEP_SUMMARY"
-          echo '#### Downloaded profiles' >> "$GITHUB_STEP_SUMMARY"
+          echo '#### Downloaded coverage artifacts' >> "$GITHUB_STEP_SUMMARY"
           echo '' >> "$GITHUB_STEP_SUMMARY"
           echo '```text' >> "$GITHUB_STEP_SUMMARY"
           cat "${RUNNER_TEMP}/coverage-inputs.txt" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true
           echo '```' >> "$GITHUB_STEP_SUMMARY"
-          if [ "${{ steps.merge_coverage.outcome }}" = 'success' ]; then
-            echo '- Result: `final-result-files` contains the changed-code profile, merged profile, summary, and PR coverage HTML.' >> "$GITHUB_STEP_SUMMARY"
+          if [ -s "${RUNNER_TEMP}/coverage-selected-artifacts.json" ]; then
+            echo '' >> "$GITHUB_STEP_SUMMARY"
+            echo '#### Selected producer attempts' >> "$GITHUB_STEP_SUMMARY"
+            echo '' >> "$GITHUB_STEP_SUMMARY"
+            echo '```json' >> "$GITHUB_STEP_SUMMARY"
+            jq . "${RUNNER_TEMP}/coverage-selected-artifacts.json" >> "$GITHUB_STEP_SUMMARY"
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+          fi
+          merge_outcome='${{ steps.merge_coverage.outcome }}'
+          if [ "${merge_outcome}" = 'success' ]; then
+            echo '- Result prepared for publication as `final-result-files`: changed-code profile, merged profile, summary, and PR coverage HTML.' >> "$GITHUB_STEP_SUMMARY"
+          elif [ "${merge_outcome}" = 'failure' ]; then
+            echo '- Diagnostics artifact (best-effort): `coverage-merge-debug-attempt-${{ github.run_attempt }}` contains the input inventory and PR diff when publication succeeds; any merged outputs are prepared for publication as `final-result-files`.' >> "$GITHUB_STEP_SUMMARY"
           else
-            echo '- Diagnostics: `coverage-merge-debug` contains the input inventory and PR diff; `final-result-files` retains any merged outputs.' >> "$GITHUB_STEP_SUMMARY"
+            echo "- Result not prepared because the merge step was \`${merge_outcome}\`. See the first failed prerequisite or artifact-selection step." >> "$GITHUB_STEP_SUMMARY"
           fi
       - name: Upload coverage result
+        id: upload_coverage_result
         if: ${{ always() && !cancelled() && steps.merge_coverage.outputs.has_pr_coverage == 'true' }}
-        uses: actions/upload-artifact@v7
+        uses: matrixorigin/CI/actions/upload-artifact-with-retry@main
         with:
           name: final-result-files
           path: |
@@ -268,15 +296,34 @@ jobs:
             ${{ github.workspace }}/coverage-summary.json
           if-no-files-found: error
           retention-days: 7
+          overwrite: true
+      - name: Report coverage artifact publication
+        if: ${{ always() && !cancelled() && steps.merge_coverage.outputs.has_pr_coverage == 'true' }}
+        continue-on-error: true
+        env:
+          UPLOAD_OUTCOME: ${{ steps.upload_coverage_result.outcome }}
+          ARTIFACT_URL: ${{ steps.upload_coverage_result.outputs.artifact-url }}
+        run: |
+          if [ "${UPLOAD_OUTCOME}" = 'success' ]; then
+            if [ -n "${ARTIFACT_URL}" ]; then
+              echo "- Artifact publication: succeeded ([download \`final-result-files\`](${ARTIFACT_URL}))." >> "$GITHUB_STEP_SUMMARY"
+            else
+              echo '- Artifact publication: succeeded as `final-result-files`.' >> "$GITHUB_STEP_SUMMARY"
+            fi
+          else
+            echo '- Artifact publication: failed after 3 attempts; see `Upload coverage result` for the underlying upload errors.' >> "$GITHUB_STEP_SUMMARY"
+          fi
       - name: Upload coverage merge diagnostics
         if: ${{ failure() && !cancelled() }}
+        continue-on-error: true
         uses: actions/upload-artifact@v7
         with:
-          name: coverage-merge-debug
+          name: coverage-merge-debug-attempt-${{ github.run_attempt }}
           path: |
             ${{ runner.temp }}/coverage-inputs.txt
             ${{ runner.temp }}/coverage-parser.log
             ${{ runner.temp }}/bvt-group-manifests.json
+            ${{ runner.temp }}/coverage-selected-artifacts.json
             ${{ github.workspace }}/matrixone/diff.patch
             ${{ github.workspace }}/coverage-summary.json
           if-no-files-found: warn

.github/workflows/coverage-ut.yaml

@@ -427,14 +427,15 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
       - name: Upload UT coverage profile
         if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'success' }}
-        uses: actions/upload-artifact@v7
+        uses: matrixorigin/CI/actions/upload-artifact-with-retry@main
         with:
           name: ut-coverage-generation-${{ inputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }}
           path: ${{ runner.temp }}/ut-coverage.out
-          if-no-files-found: warn
+          if-no-files-found: error
           retention-days: 7
       - name: Upload failed UT diagnostics
         if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }}
+        continue-on-error: true
         uses: actions/upload-artifact@v7
         with:
           name: ut-coverage-diagnostics-attempt-${{ github.run_attempt }}