Skip to content

26.4 - CICD GH artifacts #1901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
MyroTk wants to merge 1 commit into
base: antalya-26.4
Choose a base branch
from
Draft

26.4 - CICD GH artifacts #1901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
364 changes: 364 additions & 0 deletions .github/workflows/master.yml
View file
Open in desktop

Large diffs are not rendered by default.

371 changes: 371 additions & 0 deletions .github/workflows/pull_request.yml
View file
Open in desktop

Large diffs are not rendered by default.

182 changes: 91 additions & 91 deletions .github/workflows/pull_request_community.yml
View file
Open in desktop

Large diffs are not rendered by default.

24 changes: 24 additions & 0 deletions .github/workflows/regression-reusable-suite.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ jobs:
build_sha: ${{ inputs.build_sha }}
pr_number: ${{ github.event.number }}
artifacts: builds
GH_BINARY_ARTIFACT_NAME: ${{ inputs.runner_arch == 'aarch64' && 'CH_ARM_BIN_GH' || 'CH_AMD_BINARY_GH' }}
# Args
args: --test-to-end
--no-colors
Expand Down Expand Up @@ -127,7 +128,30 @@ jobs:
- name: 🛠️ Setup
run: .github/setup.sh

- name: 📥 Try get binary from GH artifact
id: get_binary_gh
continue-on-error: true
uses: actions/download-artifact@v4
with:
name: ${{ env.GH_BINARY_ARTIFACT_NAME }}
path: ${{ runner.temp }}/gh_binary

- name: 🔎 Resolve binary source
id: resolve_binary_source
run: |
GH_BINARY="${{ runner.temp }}/gh_binary/clickhouse"
if [ -f "$GH_BINARY" ]; then
chmod +x "$GH_BINARY"
echo "clickhouse_path=$GH_BINARY" >> "$GITHUB_ENV"
echo "has_gh_binary=true" >> "$GITHUB_OUTPUT"
echo "Using GH artifact binary: $GH_BINARY"
else
echo "has_gh_binary=false" >> "$GITHUB_OUTPUT"
echo "GH artifact binary not found, fallback to S3 URL resolution"
fi

- name: 📦 Get deb url
if: ${{ steps.resolve_binary_source.outputs.has_gh_binary != 'true' }}
env:
S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
Expand Down
98 changes: 98 additions & 0 deletions .github/workflows/release_builds.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,18 @@ jobs:
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log

- name: Upload artifact CH_AMD_DEBUG_GH
id: upload_CH_AMD_DEBUG_GH
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: CH_AMD_DEBUG_GH
path: ci/tmp/build/programs/self-extracting/clickhouse
retention-days: 1
- name: Warn on failed upload of CH_AMD_DEBUG_GH
if: steps.upload_CH_AMD_DEBUG_GH.outcome == 'failure'
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."

build_amd_asan_ubsan:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
Expand Down Expand Up @@ -322,6 +334,18 @@ jobs:
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log

- name: Upload artifact CH_AMD_ASAN_UBSAN_GH
id: upload_CH_AMD_ASAN_UBSAN_GH
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: CH_AMD_ASAN_UBSAN_GH
path: ci/tmp/build/programs/self-extracting/clickhouse
retention-days: 1
- name: Warn on failed upload of CH_AMD_ASAN_UBSAN_GH
if: steps.upload_CH_AMD_ASAN_UBSAN_GH.outcome == 'failure'
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."

build_amd_tsan:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
Expand Down Expand Up @@ -369,6 +393,18 @@ jobs:
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log

- name: Upload artifact CH_AMD_TSAN_GH
id: upload_CH_AMD_TSAN_GH
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: CH_AMD_TSAN_GH
path: ci/tmp/build/programs/self-extracting/clickhouse
retention-days: 1
- name: Warn on failed upload of CH_AMD_TSAN_GH
if: steps.upload_CH_AMD_TSAN_GH.outcome == 'failure'
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."

build_amd_msan:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
Expand Down Expand Up @@ -416,6 +452,18 @@ jobs:
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log

- name: Upload artifact CH_AMD_MSAN_GH
id: upload_CH_AMD_MSAN_GH
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: CH_AMD_MSAN_GH
path: ci/tmp/build/programs/self-extracting/clickhouse
retention-days: 1
- name: Warn on failed upload of CH_AMD_MSAN_GH
if: steps.upload_CH_AMD_MSAN_GH.outcome == 'failure'
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."

build_amd_binary:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
Expand Down Expand Up @@ -463,6 +511,18 @@ jobs:
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log

- name: Upload artifact CH_AMD_BINARY_GH
id: upload_CH_AMD_BINARY_GH
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: CH_AMD_BINARY_GH
path: ci/tmp/build/programs/self-extracting/clickhouse
retention-days: 1
- name: Warn on failed upload of CH_AMD_BINARY_GH
if: steps.upload_CH_AMD_BINARY_GH.outcome == 'failure'
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_BINARY_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."

build_arm_debug:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
Expand Down Expand Up @@ -557,6 +617,18 @@ jobs:
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log

- name: Upload artifact CH_ARM_ASAN_UBSAN_GH
id: upload_CH_ARM_ASAN_UBSAN_GH
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: CH_ARM_ASAN_UBSAN_GH
path: ci/tmp/build/programs/self-extracting/clickhouse
retention-days: 1
- name: Warn on failed upload of CH_ARM_ASAN_UBSAN_GH
if: steps.upload_CH_ARM_ASAN_UBSAN_GH.outcome == 'failure'
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_ASAN_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."

build_arm_tsan:
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
Expand Down Expand Up @@ -745,6 +817,18 @@ jobs:
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log

- name: Upload artifact CH_ARM_BIN_GH
id: upload_CH_ARM_BIN_GH
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: CH_ARM_BIN_GH
path: ci/tmp/build/programs/self-extracting/clickhouse
retention-days: 1
- name: Warn on failed upload of CH_ARM_BIN_GH
if: steps.upload_CH_ARM_BIN_GH.outcome == 'failure'
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_BIN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."

build_amd_release:
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
Expand Down Expand Up @@ -1065,6 +1149,13 @@ jobs:
EOF
ENV_SETUP_SCRIPT_EOF

- name: Download artifact CH_ARM_BIN_GH
uses: actions/download-artifact@v8
continue-on-error: true
with:
name: CH_ARM_BIN_GH
path: ./ci/tmp

- name: Run
id: run
run: |
Expand Down Expand Up @@ -1112,6 +1203,13 @@ jobs:
EOF
ENV_SETUP_SCRIPT_EOF

- name: Download artifact CH_ARM_BIN_GH
uses: actions/download-artifact@v8
continue-on-error: true
with:
name: CH_ARM_BIN_GH
path: ./ci/tmp

- name: Run
id: run
run: |
Expand Down
30 changes: 30 additions & 0 deletions ci/defs/defs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,17 @@ class ArtifactNames:
CH_RISCV64 = "CH_RISCV64_BIN"
CH_S390X = "CH_S390X_BIN"
CH_LOONGARCH64 = "CH_LOONGARCH64_BIN"
CH_AMD_DEBUG_GH = "CH_AMD_DEBUG_GH"
CH_AMD_BINARY_GH = "CH_AMD_BINARY_GH"
CH_ARM_BINARY_GH = "CH_ARM_BIN_GH"
CH_AMD_ASAN_UBSAN_GH = "CH_AMD_ASAN_UBSAN_GH"
CH_AMD_ASAN_GH = "CH_AMD_ASAN_GH"
CH_AMD_TSAN_GH = "CH_AMD_TSAN_GH"
CH_AMD_MSAN_GH = "CH_AMD_MSAN_GH"
CH_AMD_UBSAN_GH = "CH_AMD_UBSAN_GH"
CH_ARM_ASAN_UBSAN_GH = "CH_ARM_ASAN_UBSAN_GH"
CH_ARM_ASAN_GH = "CH_ARM_ASAN_GH"
CH_ARM_TSAN_GH = "CH_ARM_TSAN_GH"

FAST_TEST = "FAST_TEST"
UNITTEST_AMD_ASAN_UBSAN = "UNITTEST_AMD_ASAN_UBSAN"
Expand Down Expand Up @@ -558,6 +569,25 @@ class ArtifactConfigs:
ArtifactNames.CH_LOONGARCH64,
]
)
clickhouse_binaries_gh = Artifact.Config(
name="...",
type=Artifact.Type.GH,
path=f"{TEMP_DIR}/build/programs/self-extracting/clickhouse",
).parametrize(
names=[
ArtifactNames.CH_AMD_DEBUG_GH,
ArtifactNames.CH_AMD_BINARY_GH,
ArtifactNames.CH_ARM_BINARY_GH,
ArtifactNames.CH_AMD_ASAN_UBSAN_GH,
ArtifactNames.CH_AMD_ASAN_GH,
ArtifactNames.CH_AMD_TSAN_GH,
ArtifactNames.CH_AMD_MSAN_GH,
ArtifactNames.CH_AMD_UBSAN_GH,
ArtifactNames.CH_ARM_ASAN_UBSAN_GH,
ArtifactNames.CH_ARM_ASAN_GH,
ArtifactNames.CH_ARM_TSAN_GH,
]
)
clickhouse_stripped_binaries = Artifact.Config(
name="...",
type=Artifact.Type.S3,
Expand Down
Loading
Loading