diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix-all.yml new file mode 100644 index 0000000..ce5f01d --- /dev/null +++ b/.github/workflows/matrix-all.yml @@ -0,0 +1,37 @@ +name: Build Matrix (All) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the full build matrix for Python & Exasol versions" + value: ${{ jobs.set-matrix-all.outputs.matrix }} + +jobs: + set-matrix-all: + runs-on: "ubuntu-24.04" + permissions: + contents: read + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v8 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Generate Matrix + id: generate-matrix + run: poetry run -- nox -s matrix:all + + - name: Set Matrix + id: set-matrix + run: | + echo "matrix=$(poetry run -- nox -s matrix:all)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-checks-itde.yml b/.github/workflows/slow-checks-itde.yml new file mode 100644 index 0000000..4b003a7 --- /dev/null +++ b/.github/workflows/slow-checks-itde.yml @@ -0,0 +1,68 @@ +name: Slow-Checks-ITDE + +on: + workflow_call: + +jobs: + build-matrix: + name: Build Matrix + uses: ./.github/workflows/matrix-all.yml + permissions: + contents: read + + run-integration-tests: + name: Run Integration Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + needs: + - build-matrix + runs-on: "ubuntu-24.04" + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + + steps: + - name: Allow unprivileged user namespaces + id: allow-unprivileged-user-namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + large-packages: false + + - name: Free disk space by removing large directories + run: | + sudo rm -rf /usr/local/graalvm/ + sudo rm -rf /usr/local/.ghcup/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /opt/ghc + + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v8 + with: + python-version: ${{ matrix.python-version }} + poetry-version: "2.3.0" + + - name: Run Integration Tests + id: run-integration-tests + run: > + poetry run -- nox -s test:integration -- -s --coverage + --backend=onprem + --db-version ${{ matrix.exasol-version }} + + - name: Upload Artifacts + id: upload-artifacts + uses: actions/upload-artifact@v7 + with: + name: coverage-python${{ matrix.python-version }}-exasol${{ matrix.exasol-version }}-slow-itde + path: .coverage + include-hidden-files: true diff --git a/.github/workflows/slow-checks-saas.yml b/.github/workflows/slow-checks-saas.yml new file mode 100644 index 0000000..9481ea5 --- /dev/null +++ b/.github/workflows/slow-checks-saas.yml @@ -0,0 +1,45 @@ +name: Slow-Checks-SaaS + +on: + workflow_call: + +jobs: + run-integration-tests: + name: Run Integration Tests with SaaS (Python-3.12) + runs-on: "ubuntu-24.04" + permissions: + contents: read + + steps: + - name: Allow unprivileged user namespaces + id: allow-unprivileged-user-namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v8 + with: + python-version: "3.12" + poetry-version: "2.3.0" + + - name: Run Integration Tests + id: run-integration-tests + run: > + poetry run -- nox -s test:integration -- -s --coverage + --backend=saas + env: + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} + + - name: Upload Artifacts + id: upload-artifacts + uses: actions/upload-artifact@v7 + with: + name: coverage-python3.12-slow-saas + path: .coverage + include-hidden-files: true diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 7c3c6eb..de34d1a 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -1,80 +1,19 @@ -# Please note that this workflow was manually adapted from the version generated -# by the Exasol Python Toolbox. -# -# 1. The matrix consists of backend type and Exasol version -# 2. Disabling AppArmor namespace restrictions is required to run dockerdb -# 3. To make sure space on the standard runner is freed up, -# unneeded large directories are removed -# -# Once the workflow patcher can do this modification without having to replace -# each workflow step individually, this workflow should be generated. - name: Slow-Checks on: workflow_call: jobs: - - run-integration-tests: - name: Integration Tests (${{ matrix.backend }}, ${{ matrix.exasol-version}}) - runs-on: "ubuntu-24.04" + run-slow-checks-itde: + name: Run Slow Checks with ITDE + uses: ./.github/workflows/slow-checks-itde.yml + secrets: inherit permissions: contents: read - strategy: - fail-fast: false - matrix: - include: - - backend: saas - exasol-version: provided - - backend: onprem - exasol-version: "2025.1.8" - - steps: - - name: Allow unprivileged user namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - - name: Free disk space - uses: jlumbroso/free-disk-space@main - with: - tool-cache: true - large-packages: false - - - name: Free disk space by removing large directories - run: | - sudo rm -rf /usr/local/graalvm/ - sudo rm -rf /usr/local/.ghcup/ - sudo rm -rf /usr/local/share/powershell - sudo rm -rf /usr/local/share/chromium - sudo rm -rf /usr/local/lib/node_modules - sudo rm -rf /opt/ghc - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v7 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run Integration Tests - id: run-integration-tests - run: > - poetry run -- nox -s test:integration -- --coverage - --backend ${{ matrix.backend }} - --db-version ${{ matrix.exasol-version }} - env: - SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} - SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} - SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} - - - name: Upload Artifacts - id: upload-artifacts - uses: actions/upload-artifact@v7 - with: - name: coverage-python3.10-${{ matrix.backend }}-${{ matrix.exasol-version }} - path: .coverage - include-hidden-files: true + run-slow-checks-saas: + name: Run Slow Checks with SaaS + uses: ./.github/workflows/slow-checks-saas.yml + secrets: inherit + permissions: + contents: read diff --git a/test/integration/connections/test_bucketfs_location.py b/test/integration/connections/test_bucketfs_location.py index 3739822..98f8656 100644 --- a/test/integration/connections/test_bucketfs_location.py +++ b/test/integration/connections/test_bucketfs_location.py @@ -54,7 +54,7 @@ def test_create_bucketfs_location_onprem(use_onprem, onprem_bfs_params): if not use_onprem: pytest.skip("The test is not configured to use ITDE.") - extra_params = {StdParams.path_in_bucket.name: "test_create_location"} + extra_params = {StdParams.path_in_bucket.name: "test_create_location.csv"} bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) with write_test_file(bfs_path): validate_test_file(bfs_path) @@ -64,7 +64,7 @@ def test_create_bucketfs_location_saas_db_id(use_saas, saas_params_id): if not use_saas: pytest.skip("The test is not configured to use SaaS.") - extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_id"} + extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_id.csv"} bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) with write_test_file(bfs_path): validate_test_file(bfs_path) @@ -74,7 +74,7 @@ def test_create_bucketfs_location_saas_db_name(use_saas, saas_params_name): if not use_saas: pytest.skip("The test is not configured to use SaaS.") - extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_name"} + extra_params = {StdParams.path_in_bucket.name: "test_create_location_with_name.csv"} bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) with write_test_file(bfs_path): validate_test_file(bfs_path) @@ -93,7 +93,7 @@ def test_create_bucketfs_conn_object_onprem( pytest.skip("The test is not configured to use ITDE.") write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object"} + extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object.csv"} bfs_path = create_bucketfs_location(**onprem_bfs_params, **extra_params) with write_test_file(bfs_path): # onprem_db_params and onprem_bfs_params have one item in common - @@ -109,7 +109,7 @@ def test_create_bucketfs_conn_object_saas_db_id(write_conn_object_mock, use_saas pytest.skip("The test is not configured to use SaaS.") write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_id"} + extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_id.csv"} bfs_path = create_bucketfs_location(**saas_params_id, **extra_params) with write_test_file(bfs_path): create_bucketfs_conn_object(conn_name="SAAS_TEST_BFS_ID", **saas_params_id, **extra_params) @@ -123,7 +123,7 @@ def test_create_bucketfs_conn_object_saas_db_name( pytest.skip("The test is not configured to use SaaS.") write_conn_object_mock.side_effect = validate_conn_object - extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_name"} + extra_params = {StdParams.path_in_bucket.name: "test_create_conn_object_with_name.csv"} bfs_path = create_bucketfs_location(**saas_params_name, **extra_params) with write_test_file(bfs_path): create_bucketfs_conn_object(