ci: build TKE images from shared ci-builder

.github/workflows/build-mo.yaml

@@ -123,9 +123,29 @@ jobs:
               # may lack passwordless sudo on self-hosted runners.
               trap "chown -R \"${HOST_UID}:${HOST_GID}\" /go/src/github.com/matrixorigin/matrixone >/dev/null 2>&1 || true" EXIT
               git config --global --add safe.directory /go/src/github.com/matrixorigin/matrixone
-              expected=$(printf "schema=1\nos=%s\narch=%s\nthirdparties_tree=%s" \
+              if [ -x ./optools/images/ci-builder-fingerprint.sh ]; then
+                expected=$(./optools/images/ci-builder-fingerprint.sh)
+              else
+                # Keep CI/MatrixOne merge order safe: older MatrixOne
+                # checkouts do not have the shared fingerprint helper yet.
+                expected=$(printf "schema=1\nos=%s\narch=%s\nthirdparties_tree=%s" \
+                  "$(uname -s)" "$(uname -m)" "$(git rev-parse HEAD:thirdparties)")
+              fi
+              printf '%s\n' "${expected}" > /tmp/expected-thirdparties.fingerprint
+              legacy=$(printf "schema=1\nos=%s\narch=%s\nthirdparties_tree=%s" \
                 "$(uname -s)" "$(uname -m)" "$(git rev-parse HEAD:thirdparties)")
-              if [ "${expected}" = "$(cat /mo-prebuilt/thirdparties.fingerprint 2>/dev/null)" ]; then
+              printf '%s\n' "${legacy}" > /tmp/legacy-thirdparties.fingerprint
+              echo "Computed CI builder fingerprint:"
+              cat /tmp/expected-thirdparties.fingerprint
+              if [ -r /mo-prebuilt/thirdparties.fingerprint ]; then
+                echo "Cached CI builder fingerprint:"
+                cat /mo-prebuilt/thirdparties.fingerprint
+              fi
+              # Accept only the pre-helper schema-1 metadata during rollout;
+              # the schema-3 contract covers the complete native input tree,
+              # while older partial fingerprints must rebuild safely.
+              if cmp -s /tmp/expected-thirdparties.fingerprint /mo-prebuilt/thirdparties.fingerprint 2>/dev/null \
+                || cmp -s /tmp/legacy-thirdparties.fingerprint /mo-prebuilt/thirdparties.fingerprint 2>/dev/null; then
                 rm -rf thirdparties/install
                 cp -r /mo-prebuilt/thirdparties/install thirdparties/install
               else

.github/workflows/image-build.yaml

@@ -34,6 +34,12 @@ on:
       ACR_TOKEN: 
         description: 'ACR_TOKEN '
         required: true
+      TCR_USERNAME:
+        description: 'Optional Tencent TCR username for the Shanghai CI-builder mirror'
+        required: false
+      TCR_TOKEN:
+        description: 'Optional Tencent TCR token for the Shanghai CI-builder mirror'
+        required: false
       AWS_ROLE: 
         description: 'AWS_ROLE'
         required: false
@@ -208,8 +214,8 @@ jobs:
     # arm64 SCA job is served by its own actions/cache pipeline (see
     # matrixorigin/matrixone sca-go-module-cache.yaml), so only amd64 is
     # published. Skipped for release tags so the caches always track the
-    # default branch, and skipped gracefully on branches that do not ship
-    # the Dockerfile yet.
+    # default branch. Non-main refs run only the cheap presence gate and
+    # report why publication is skipped.
     name: Build CI builder image
     if: ${{ github.event_name != 'release' }}
     # Best-effort: a builder-image problem must not turn the nightly product
@@ -218,8 +224,9 @@ jobs:
     continue-on-error: true
     environment: ci
     runs-on: ubuntu-22.04
-    # Each warm flavor is roughly a full-tree compile.
-    timeout-minutes: 120
+    # The coverage and plain Go flavors are each full-tree compiles; leave
+    # enough room for both nightly warms plus the UT cache seed steps.
+    timeout-minutes: 180
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -230,7 +237,10 @@ jobs:
       - name: Check Dockerfile presence
         id: gate
         run: |
-          if [ -f optools/images/Dockerfile.ci-builder ]; then
+          if [ "${GITHUB_REF}" != "refs/heads/main" ]; then
+            echo "::notice::ci-builder publication is restricted to refs/heads/main; ref=${GITHUB_REF}; skipping"
+            echo "present=false" >> "$GITHUB_OUTPUT"
+          elif [ -f optools/images/Dockerfile.ci-builder ]; then
             echo "present=true" >> "$GITHUB_OUTPUT"
           else
             echo "::notice::optools/images/Dockerfile.ci-builder not present; skipping builder image"
@@ -249,12 +259,27 @@ jobs:
       - name: Prepare
         if: ${{ steps.gate.outputs.present == 'true' }}
         id: prep
+        env:
+          TCR_USERNAME: ${{ secrets.TCR_USERNAME }}
+          TCR_TOKEN: ${{ secrets.TCR_TOKEN }}
         run: |
           DOCKER_IMAGE=${{ secrets.DOCKERHUB_USERNAME }}/${GITHUB_REPOSITORY#*/}
           ACR_DOCKER_IMAGE="registry.cn-shanghai.aliyuncs.com/${DOCKER_IMAGE}"
+          TCR_DOCKER_IMAGE="ccr.ccs.tencentyun.com/matrixone-dev/matrixone"
+          IMMUTABLE_TAG="ci-builder-${GITHUB_SHA::12}"
           echo "docker_image=${DOCKER_IMAGE}" >> "$GITHUB_OUTPUT"
           echo "tag=${DOCKER_IMAGE}:ci-builder" >> "$GITHUB_OUTPUT"
+          echo "immutable_tag=${DOCKER_IMAGE}:${IMMUTABLE_TAG}" >> "$GITHUB_OUTPUT"
           echo "acr_tag=${ACR_DOCKER_IMAGE}:ci-builder" >> "$GITHUB_OUTPUT"
+          echo "acr_immutable_tag=${ACR_DOCKER_IMAGE}:${IMMUTABLE_TAG}" >> "$GITHUB_OUTPUT"
+          echo "tcr_tag=${TCR_DOCKER_IMAGE}:ci-builder-amd64" >> "$GITHUB_OUTPUT"
+          echo "tcr_immutable_tag=${TCR_DOCKER_IMAGE}:${IMMUTABLE_TAG}-amd64" >> "$GITHUB_OUTPUT"
+          if [ -n "${TCR_USERNAME}" ] && [ -n "${TCR_TOKEN}" ]; then
+            echo "tcr_enabled=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "::warning::TCR credentials are not configured; the TKE ci-builder mirror will remain unavailable and consumers will use the source fallback"
+            echo "tcr_enabled=false" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Set up Docker Buildx
         if: ${{ steps.gate.outputs.present == 'true' }}
@@ -276,6 +301,14 @@ jobs:
           username: ${{ secrets.ACR_USERNAME }}
           password: ${{ secrets.ACR_TOKEN }}
 
+      - name: Login to Tencent Container Registry
+        if: ${{ steps.gate.outputs.present == 'true' && steps.prep.outputs.tcr_enabled == 'true' }}
+        uses: docker/login-action@v4
+        with:
+          registry: ccr.ccs.tencentyun.com
+          username: ${{ secrets.TCR_USERNAME }}
+          password: ${{ secrets.TCR_TOKEN }}
+
       # No registry layer cache on purpose: COPY . . changes nightly, which
       # invalidates the expensive warm-build layer anyway, and reusing a stale
       # cache would defeat the point of refreshing the Go caches.
@@ -290,9 +323,29 @@ jobs:
           push: true
           tags: |
             ${{ steps.prep.outputs.tag }}
+            ${{ steps.prep.outputs.immutable_tag }}
             ${{ steps.prep.outputs.acr_tag }}
+            ${{ steps.prep.outputs.acr_immutable_tag }}
           # Aliyun ACR rejects OCI attestation manifests; see the product
           # build step above.
           provenance: false
           sbom: false
 
+      # TKE runners use the Shanghai Tencent registry. Keep the DockerHub and
+      # ACR publications above for existing consumers, then mirror the exact
+      # immutable image and its stable alias when TCR credentials are
+      # available. This step is optional so other image-build callers are not
+      # forced to provision TCR secrets.
+      - name: Publish Tencent CI-builder mirror
+        if: ${{ steps.gate.outputs.present == 'true' && steps.prep.outputs.tcr_enabled == 'true' }}
+        env:
+          SOURCE_IMAGE: ${{ steps.prep.outputs.acr_immutable_tag }}
+          TCR_IMAGE: ${{ steps.prep.outputs.tcr_immutable_tag }}
+          TCR_ALIAS: ${{ steps.prep.outputs.tcr_tag }}
+        run: |
+          set -euo pipefail
+          docker pull "${SOURCE_IMAGE}"
+          docker tag "${SOURCE_IMAGE}" "${TCR_IMAGE}"
+          docker tag "${SOURCE_IMAGE}" "${TCR_ALIAS}"
+          docker push "${TCR_IMAGE}"
+          docker push "${TCR_ALIAS}"

.github/workflows/merge-trigger-tke.yaml

@@ -74,44 +74,165 @@ jobs:
           echo "VERSION: ${{ steps.prep.outputs.version }}"
           echo "COMMIT_ID: ${{ steps.prep.outputs.commit_id }}"
 
-      - name: Adjust Dockerfile
+      # ci-builder is produced only from trusted MatrixOne main and contains
+      # the stable Go/native inputs shared by source-only PRs. Pulling it is
+      # best effort: the old Dockerfile path below remains the correctness
+      # fallback when the mirror is unavailable.
+      - name: Resolve CI builder image
+        id: builder
+        env:
+          BASIC_IMAGE: ccr.ccs.tencentyun.com/matrixone-dev/matrixone:ci-builder-amd64
         run: |
-          cd $GITHUB_WORKSPACE/matrixone
+          set +e
+          PREBUILT_DOCKERFILE="$GITHUB_WORKSPACE/matrixone/optools/images/Dockerfile.prebuilt"
+          if [ ! -f "${PREBUILT_DOCKERFILE}" ]; then
+            echo "::notice::Dockerfile.prebuilt is not present in this checkout; using the source Dockerfile fallback"
+            echo "ref=" >> "$GITHUB_OUTPUT"
+            echo "tag=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # TCR is an optional optimization mirror. Keep a registry outage
+          # from adding minutes to every post-merge regression before the
+          # source Dockerfile fallback takes over.
+          if timeout 90 docker pull "${BASIC_IMAGE}"; then
+            repository="${BASIC_IMAGE%%:*}"
+            digest=$(docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BASIC_IMAGE}" |
+              awk -v prefix="${repository}@" 'index($0, prefix) == 1 { print; exit }')
+            if [ -n "${digest}" ]; then
+              echo "ref=${digest}" >> "$GITHUB_OUTPUT"
+              echo "tag=${BASIC_IMAGE}" >> "$GITHUB_OUTPUT"
+              echo "Using CI builder ${digest}"
+              exit 0
+            fi
+          fi
+          echo "::warning::CI builder is unavailable; using the source Dockerfile fallback"
+          docker system prune -f >/dev/null 2>&1 || true
+          echo "ref=" >> "$GITHUB_OUTPUT"
+          echo "tag=" >> "$GITHUB_OUTPUT"
+
+      # Build the PR in the prewarmed image. The mount path is part of the
+      # builder contract because cgo compile-cache keys include include paths.
+      - name: Build MatrixOne with CI builder
+        id: cached
+        if: ${{ steps.builder.outputs.ref != '' }}
+        continue-on-error: true
+        run: |
+          set -u +e -o pipefail
+          cd "$GITHUB_WORKSPACE"
+          docker run --rm -i \
+            -v "$GITHUB_WORKSPACE/matrixone":/go/src/github.com/matrixorigin/matrixone \
+            -w /go/src/github.com/matrixorigin/matrixone \
+            -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
+            -e GOPROXY="http://goproxy.goproxy.svc.cluster.local|https://goproxy.cn|direct" \
+            "${{ steps.builder.outputs.ref }}" bash -s <<'CONTAINER_SCRIPT'
+          set -euo pipefail
+          trap 'chown -R "${HOST_UID}:${HOST_GID}" /go/src/github.com/matrixorigin/matrixone >/dev/null 2>&1 || true' EXIT
+          git config --global --add safe.directory /go/src/github.com/matrixorigin/matrixone
+
+          if [ -x ./optools/images/ci-builder-fingerprint.sh ]; then
+            expected=$(./optools/images/ci-builder-fingerprint.sh)
+          else
+            # Keep the transition safe for already merged commits that do not
+            # contain the shared helper yet.
+            expected=$(printf "schema=1\nos=%s\narch=%s\nthirdparties_tree=%s" \
+              "$(uname -s)" "$(uname -m)" "$(git rev-parse HEAD:thirdparties)")
+          fi
+          printf '%s\n' "${expected}" > /tmp/expected-thirdparties.fingerprint
+          legacy=$(printf "schema=1\nos=%s\narch=%s\nthirdparties_tree=%s" \
+            "$(uname -s)" "$(uname -m)" "$(git rev-parse HEAD:thirdparties)")
+          printf '%s\n' "${legacy}" > /tmp/legacy-thirdparties.fingerprint
+          echo "Computed CI builder fingerprint:"
+          cat /tmp/expected-thirdparties.fingerprint
+          if [ -r /mo-prebuilt/thirdparties.fingerprint ]; then
+            echo "Cached CI builder fingerprint:"
+            cat /mo-prebuilt/thirdparties.fingerprint
+          fi
+          # Accept only the pre-helper schema-1 metadata during rollout.  The
+          # schema-3 contract covers the complete native input tree, so older
+          # partial fingerprints must rebuild rather than risk stale outputs.
+          if cmp -s /tmp/expected-thirdparties.fingerprint /mo-prebuilt/thirdparties.fingerprint 2>/dev/null \
+            || cmp -s /tmp/legacy-thirdparties.fingerprint /mo-prebuilt/thirdparties.fingerprint 2>/dev/null; then
+            echo "CI builder native fingerprint matched"
+            rm -rf thirdparties/install
+            cp -a /mo-prebuilt/thirdparties/install thirdparties/install
+          else
+            echo "CI builder native fingerprint mismatch; rebuilding native dependencies"
+            rm -rf thirdparties/install
+          fi
+          make build
+          CONTAINER_SCRIPT
+          status=$?
+          sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/matrixone" >/dev/null 2>&1 || true
+          if [ "${status}" -ne 0 ]; then
+            exit "${status}"
+          fi
+          # The cached build completed successfully.  Fail closed while
+          # staging its runtime artifacts so a missing library cannot be
+          # published as a successful image.
+          set -e
+
+          cd "$GITHUB_WORKSPACE/matrixone"
+          env -u LD_LIBRARY_PATH GOCOVERDIR="$(mktemp -d)" ./mo-service -h >/dev/null
+          rm -rf "$GITHUB_WORKSPACE/mo-runtime"
+          mkdir -p "$GITHUB_WORKSPACE/mo-runtime/lib"
+          cp ./mo-service "$GITHUB_WORKSPACE/mo-runtime/mo-service"
+          cp ./lib/*.so ./cgo/*.so "$GITHUB_WORKSPACE/mo-runtime/lib/"
+          cp -a ./dict "$GITHUB_WORKSPACE/mo-runtime/dict"
+          cp -a ./etc "$GITHUB_WORKSPACE/mo-runtime/etc"
+          {
+            echo "matrixone_commit=$(git rev-parse HEAD)"
+            echo "basic_image=${{ steps.builder.outputs.ref }}"
+            echo "gobuild_opt="
+          } > "$GITHUB_WORKSPACE/mo-runtime/BUILD_INFO"
+
+      - name: Build and Push Image from CI builder
+        if: ${{ steps.cached.outcome == 'success' }}
+        uses: matrixorigin/CI/actions/image-build@main
+        with:
+          tags: ${{ steps.prep.outputs.tags }}
+          file-path: ${{ github.workspace }}/matrixone/optools/images/Dockerfile.prebuilt
+          context: ${{ github.workspace }}/mo-runtime
+          # Keep the tiny prebuilt image cache separate from the expensive
+          # source-build cache. Otherwise a successful cached build erases the
+          # source layers needed by the fallback path.
+          cache-from: type=registry,ref=${{ steps.prep.outputs.docker_image }}:cache-prebuilt-${{ github.base_ref }}
+          cache-to: type=registry,ref=${{ steps.prep.outputs.docker_image }}:cache-prebuilt-${{ github.base_ref }},mode=max
+          platform: linux/amd64
+          build-args: |
+            RUNTIME_IMAGE=ccr.ccs.tencentyun.com/mo-infra/ubuntu:22.04
 
-          # # add tencentyun docker image mirror and tsinghua apt mirror
+      # Source fallback. It keeps the existing build path available when the
+      # Basic Image cannot be pulled or the cached build cannot start. A
+      # genuine source/compile error still fails this step and blocks BVT.
+      - name: Adjust source Dockerfile fallback
+        if: ${{ steps.cached.outcome != 'success' }}
+        run: |
+          cd "$GITHUB_WORKSPACE/matrixone"
+          # The fallback must quote GOPROXY because the resilient `|` proxy
+          # chain is interpreted as a shell pipeline by an old Dockerfile.
+          sed -i 's#RUN go env -w GOPROXY=${GOPROXY}#RUN go env -w GOPROXY="${GOPROXY}"#' optools/images/Dockerfile
           sed -i 's#^RUN apt-get update#RUN sed -i "s?http.*ubuntu\.com?http://mirrors\.tuna\.tsinghua\.edu\.cn?g" /etc/apt/sources.list  \&\& apt-get clean all \&\& apt-get update#g' optools/images/Dockerfile
           sed -i 's#FROM golang:1.21.5-bookworm#FROM ccr.ccs.tencentyun.com/cicd-runner/golang:1.21.5-bookworm#g' optools/images/Dockerfile
           sed -i 's#FROM ubuntu:22.04#FROM ccr.ccs.tencentyun.com/mo-infra/ubuntu:22.04#g' optools/images/Dockerfile
-
-          # adjust for matrixorigin/ubuntu:22.04
           sed -i 's#FROM matrixorigin/ubuntu:22.04#FROM ccr.ccs.tencentyun.com/mo-infra/ubuntu:22.04#g' optools/images/Dockerfile
-
-          # adjust for go 1.22.3
           sed -i 's#FROM golang:1.22.3-bookworm#FROM ccr.ccs.tencentyun.com/mo-infra/golang:1.22.3-bookworm#g' optools/images/Dockerfile
-
-          # adjust for go 1.23.0
           sed -i 's#FROM golang:1.23.0-bookworm#FROM ccr.ccs.tencentyun.com/mo-infra/golang:1.23.0-bookworm#g' optools/images/Dockerfile
-
-          # adjust for go 1.24
           sed -i 's#FROM matrixorigin/golang:1.24-ubuntu22#FROM ccr.ccs.tencentyun.com/mo-infra/golang:1.24-ubuntu22#g' optools/images/Dockerfile
-
-          # adjust for go 1.25
           sed -i 's#FROM matrixorigin/golang:1.25-ubuntu22#FROM ccr.ccs.tencentyun.com/mo-infra/golang:1.25-ubuntu22#g' optools/images/Dockerfile
-
-          # adjust for go 1.26.4
           sed -i 's#FROM matrixorigin/golang:1.26.4-ubuntu22.04#FROM ccr.ccs.tencentyun.com/mo-infra/golang:1.26.4-ubuntu22.04#g' optools/images/Dockerfile
 
-      - name: Build and Push Image
+      - name: Build and Push Source Image Fallback
+        if: ${{ steps.cached.outcome != 'success' }}
         uses: matrixorigin/CI/actions/image-build@main
         with:
           tags: ${{ steps.prep.outputs.tags }}
           file-path: ${{ github.workspace }}/matrixone/optools/images/Dockerfile
           context: ${{ github.workspace }}/matrixone
-          cache-from: type=registry,ref=${{ steps.prep.outputs.docker_image }}:cache-${{ github.base_ref }}
-          cache-to: type=registry,ref=${{ steps.prep.outputs.docker_image }}:cache-${{ github.base_ref }},mode=max
+          cache-from: type=registry,ref=${{ steps.prep.outputs.docker_image }}:cache-source-${{ github.base_ref }}
+          cache-to: type=registry,ref=${{ steps.prep.outputs.docker_image }}:cache-source-${{ github.base_ref }},mode=max
           platform: linux/amd64
           build-args: |
-            GOPROXY=http://goproxy.goproxy.svc.cluster.local
+            GOPROXY=http://goproxy.goproxy.svc.cluster.local|https://goproxy.cn|direct
   setup_mo_test_env:
     if: ${{ always() && !cancelled() && needs.docker_image_build.result == 'success' }}
     needs: docker_image_build