From c2cac77b9d6ed1715ae504b43de57115b39c65fe Mon Sep 17 00:00:00 2001 From: Ford544 Date: Tue, 15 Sep 2026 12:21:33 +0000 Subject: [PATCH 1/4] Unformatted commit --- cli/index_run_e2e_test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/index_run_e2e_test.ts b/cli/index_run_e2e_test.ts index 159e49fb2..ca8747204 100644 --- a/cli/index_run_e2e_test.ts +++ b/cli/index_run_e2e_test.ts @@ -42,14 +42,14 @@ select 1 as \${dataform.projectConfig.vars.testVar2} "--vars=testVar1=testValue1,testVar2=testValue2", "--schema-suffix=test_schema_suffix" ] - ); + ) expect(compileResult.exitCode).equals(0); expect(JSON.parse(compileResult.stdout)).deep.equals({ tables: [ { - type: "table", + type : "table", enumType: "TABLE", target: { database: INTEGRATION_TEST_PROJECT, From a56cecbe83a5457e31f88509637dd59fa185ac15 Mon Sep 17 00:00:00 2001 From: Ford544 Date: Tue, 15 Sep 2026 12:31:13 +0000 Subject: [PATCH 2/4] fixin --- .github/workflows/test.yaml | 2 +- scripts/lint | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9b38c4ed6..7bd4eeaae 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -28,7 +28,7 @@ jobs: - name: Run integration tests # 7.3.2 - uses: docker://gcr.io/cloud-builders/bazel@sha256:aa6c5ffd0d6d37960715d9c3392ba885e6484322c81c3604a90ccfb15c07c11 + uses: docker://gcr.io/cloud-builders/bazel@sha256:aa6c5ffd0d6d37960715d9c3392ba885e6484322c81c3604a90ccfb15c07c110 with: entrypoint: bash args: ./scripts/run_integration_tests diff --git a/scripts/lint b/scripts/lint index fe76746c4..573da2b41 100755 --- a/scripts/lint +++ b/scripts/lint @@ -6,6 +6,9 @@ bazel run @nodejs//:yarn # Run eslint. bazel build @npm//eslint/bin:eslint && bazel-bin/external/npm/eslint/bin/eslint.sh --rulesdir eslint-rules '**/*.ts' +# Fetch origin/main (unshallowing if running in a shallow CI clone like Cloud Build) +git fetch origin main --unshallow 2>/dev/null || git fetch origin main + # Run prettier on .ts files changed against origin/main CHANGED_TS_FILES=$(git diff --name-only --diff-filter=ACMR origin/main...HEAD -- '*.ts') From 0741d144042b7c31c17791c1bdff26d04cd6d511 Mon Sep 17 00:00:00 2001 From: Ford544 Date: Tue, 15 Sep 2026 12:38:10 +0000 Subject: [PATCH 3/4] fixin on --- scripts/lint | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/lint b/scripts/lint index 573da2b41..a6fc76afb 100755 --- a/scripts/lint +++ b/scripts/lint @@ -6,6 +6,9 @@ bazel run @nodejs//:yarn # Run eslint. bazel build @npm//eslint/bin:eslint && bazel-bin/external/npm/eslint/bin/eslint.sh --rulesdir eslint-rules '**/*.ts' +# Allow git to run inside Docker containers where container UID != workspace owner UID +git config --global --add safe.directory "$(pwd)" + # Fetch origin/main (unshallowing if running in a shallow CI clone like Cloud Build) git fetch origin main --unshallow 2>/dev/null || git fetch origin main From 80a48421879bdefb9664017d5be311d6f2cd8c0d Mon Sep 17 00:00:00 2001 From: Ford544 Date: Tue, 15 Sep 2026 12:44:08 +0000 Subject: [PATCH 4/4] Formatted commit --- cli/index_run_e2e_test.ts | 396 +++++++++++++++++--------------------- 1 file changed, 179 insertions(+), 217 deletions(-) diff --git a/cli/index_run_e2e_test.ts b/cli/index_run_e2e_test.ts index ca8747204..3f9522771 100644 --- a/cli/index_run_e2e_test.ts +++ b/cli/index_run_e2e_test.ts @@ -34,22 +34,19 @@ select 1 as \${dataform.projectConfig.vars.testVar2} ); // Compile the project using the CLI. - const compileResult = await runCli( - "compile", - [ - projectDir, - "--json", - "--vars=testVar1=testValue1,testVar2=testValue2", - "--schema-suffix=test_schema_suffix" - ] - ) + const compileResult = await runCli("compile", [ + projectDir, + "--json", + "--vars=testVar1=testValue1,testVar2=testValue2", + "--schema-suffix=test_schema_suffix" + ]); expect(compileResult.exitCode).equals(0); expect(JSON.parse(compileResult.stdout)).deep.equals({ tables: [ { - type : "table", + type: "table", enumType: "TABLE", target: { database: INTEGRATION_TEST_PROJECT, @@ -93,20 +90,17 @@ select 1 as \${dataform.projectConfig.vars.testVar2} }); // Dry run the project. - const runResult = await runCli( - "run", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--dry-run", - "--json", - "--vars=testVar1=testValue1,testVar2=testValue2", - "--default-location=europe", - "--tags=someTag,someOtherTag", - "--actions=example,someOtherAction" - ] - ); + const runResult = await runCli("run", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--dry-run", + "--json", + "--vars=testVar1=testValue1,testVar2=testValue2", + "--default-location=europe", + "--tags=someTag,someOtherTag", + "--actions=example,someOtherAction" + ]); if (runResult.exitCode !== 0 || runResult.stdout.trim().length === 0) { console.error("GOLDEN PATH FAILED. STDERR:", runResult.stderr); @@ -126,8 +120,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2} }, tasks: [ { - statement: - `create or replace table \`${INTEGRATION_TEST_PROJECT}.dataform.example\` as \n\nselect 1 as testValue2`, + statement: `create or replace table \`${INTEGRATION_TEST_PROJECT}.dataform.example\` as \n\nselect 1 as testValue2`, type: "statement" } ], @@ -199,8 +192,7 @@ SELECT 1 as id }, tasks: [ { - statement: - `create or replace table \`${INTEGRATION_TEST_PROJECT}.dataform.example_table\` as \n\nSELECT 1 as id`, + statement: `create or replace table \`${INTEGRATION_TEST_PROJECT}.dataform.example_table\` as \n\nSELECT 1 as id`, type: "statement" } ], @@ -236,18 +228,15 @@ SELECT 1 as id test("with --disable-assertions flag", async () => { alterWorkflowSettings(projectDir, { disableAssertions: false }); - const runResult = await runCli( - "run", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--dry-run", - "--json", - "--disable-assertions", - "--actions=test_assertion,example_table" - ] - ); + const runResult = await runCli("run", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--dry-run", + "--json", + "--disable-assertions", + "--actions=test_assertion,example_table" + ]); if (runResult.exitCode !== 0 || runResult.stdout.trim().length === 0) { console.error("ASSERTIONS TEST FAILED. STDERR:", runResult.stderr); @@ -259,17 +248,14 @@ SELECT 1 as id test("with disableAssertions set in workflow_settings.yaml", async () => { alterWorkflowSettings(projectDir, { disableAssertions: true }); - const runResult = await runCli( - "run", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--dry-run", - "--json", - "--actions=test_assertion,example_table" - ] - ); + const runResult = await runCli("run", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--dry-run", + "--json", + "--actions=test_assertion,example_table" + ]); if (runResult.exitCode !== 0 || runResult.stdout.trim().length === 0) { console.error("ASSERTIONS TEST FAILED. STDERR:", runResult.stderr); @@ -281,19 +267,16 @@ SELECT 1 as id test("with --job-labels flag", async () => { alterWorkflowSettings(projectDir, { disableAssertions: false }); - const runResult = await runCli( - "run", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--dry-run", - "--json", - "--disable-assertions", - "--actions=test_assertion,example_table", - "--job-labels=env=testing,team=dataform" - ] - ); + const runResult = await runCli("run", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--dry-run", + "--json", + "--disable-assertions", + "--actions=test_assertion,example_table", + "--job-labels=env=testing,team=dataform" + ]); if (runResult.exitCode !== 0 || runResult.stdout.trim().length === 0) { console.error("ASSERTIONS TEST FAILED. STDERR:", runResult.stderr); @@ -303,7 +286,6 @@ SELECT 1 as id }); }); - suite("--default-reservation flag", ({ beforeEach }) => { let projectDir: string; @@ -322,14 +304,11 @@ SELECT 1 as id }); test("--default-reservation flag is applied to projectConfig in compile output", async () => { - const compileResult = await runCli( - "compile", - [ - projectDir, - "--json", - `--default-reservation=${INTEGRATION_TEST_RESERVATION}` - ] - ); + const compileResult = await runCli("compile", [ + projectDir, + "--json", + `--default-reservation=${INTEGRATION_TEST_RESERVATION}` + ]); expect(compileResult.exitCode).equals(0); const compiledGraph = JSON.parse(compileResult.stdout); @@ -344,18 +323,15 @@ SELECT 1 as id }); test("--default-reservation flag is applied to projectConfig in run (dry-run) output", async () => { - const runResult = await runCli( - "run", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--dry-run", - "--json", - `--default-reservation=${INTEGRATION_TEST_RESERVATION}`, - "--actions=example_table" - ] - ); + const runResult = await runCli("run", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--dry-run", + "--json", + `--default-reservation=${INTEGRATION_TEST_RESERVATION}`, + "--actions=example_table" + ]); expect(runResult.exitCode).equals(0); const executionGraph = JSON.parse(runResult.stdout); @@ -378,9 +354,9 @@ SELECT 1 as id await setupProject(tmpDirFixture, projectDir); // Write a simple file to the project. writeDefinitionFile( - projectDir, - "example.sqlx", - ` + projectDir, + "example.sqlx", + ` config { type: "table" } select 1 ` @@ -392,30 +368,29 @@ select 1 writeDefinitionFile( projectDir, "example_test.sqlx", - ` + ` config { type: "test", dataset: "example" } select 1 ` ); // Run tests using the CLI. - const testResult = await runCli( - "test", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--json" - ] - ); + const testResult = await runCli("test", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--json" + ]); expect(testResult.exitCode).equals(0); - expect(JSON.parse(testResult.stdout)).deep.equals([ { - "name": "example_test", - "successful": true, - }]); - }); + expect(JSON.parse(testResult.stdout)).deep.equals([ + { + name: "example_test", + successful: true + } + ]); + }); test("golden with failed unit test", async () => { // Write a simple failing test to the project. @@ -429,32 +404,30 @@ select 2 ); // Run tests using the CLI. - const testResult = await runCli( - "test", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--json" - ] - ); + const testResult = await runCli("test", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--json" + ]); expect(testResult.exitCode).equals(1); - expect(JSON.parse(testResult.stdout)).deep.equals([{ - "name": "example_test", - "successful": false, - messages: [ - "For row 0 and column \"f0_\": expected \"2\", but saw \"1\"." - ] - }]); + expect(JSON.parse(testResult.stdout)).deep.equals([ + { + name: "example_test", + successful: false, + messages: ['For row 0 and column "f0_": expected "2", but saw "1".'] + } + ]); }); - }); suite("onSchemaChange", ({ beforeEach }) => { let projectDir: string; - const uniqueDataset = `dataform_e2e_osc_${Math.random().toString(36).substring(7)}`; + const uniqueDataset = `dataform_e2e_osc_${Math.random() + .toString(36) + .substring(7)}`; beforeEach("setup test project", async () => { projectDir = tmpDirFixture.createNewTmpDir(); @@ -495,91 +468,88 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p ); }); - test("generates dynamic SQL for EXTEND when table exists in BigQuery", { timeout: 120000 }, async () => { - try { - // Run setup operation to create the table in BigQuery. - // Dataform will automatically create the uniqueDataset schema. - await runCli( - "run", - [ + test( + "generates dynamic SQL for EXTEND when table exists in BigQuery", + { timeout: 120000 }, + async () => { + try { + // Run setup operation to create the table in BigQuery. + // Dataform will automatically create the uniqueDataset schema. + await runCli("run", [ projectDir, "--credentials", CREDENTIALS_PATH, "--actions=setup_table" - ] - ); - - // Run the incremental table in dry-run mode. - // Dataform will detect the table exists and generate the dynamic procedural SQL. - const runResult = await runCli( - "run", - [ + ]); + + // Run the incremental table in dry-run mode. + // Dataform will detect the table exists and generate the dynamic procedural SQL. + const runResult = await runCli("run", [ projectDir, "--credentials", CREDENTIALS_PATH, "--dry-run", "--json", "--actions=example_incremental" - ] - ); - - expect(runResult.exitCode).equals(0); - const executionGraph = JSON.parse(runResult.stdout); - const statement = executionGraph.actions[0].tasks[0].statement; - - const expectedRunResult = { - projectConfig: { - warehouse: "bigquery", - defaultSchema: uniqueDataset, - assertionSchema: "dataform_assertions", - defaultDatabase: INTEGRATION_TEST_PROJECT, - defaultLocation: INTEGRATION_TEST_LOCATION - }, - runConfig: { - actions: ["example_incremental"], - fullRefresh: false - }, - actions: [ - { - fileName: "definitions/example_incremental.sqlx", - hermeticity: "NON_HERMETIC", - tableType: "incremental", - target: { - database: INTEGRATION_TEST_PROJECT, - name: "example_incremental", - schema: uniqueDataset - }, - tasks: [ - { - statement, - type: "statement" - } - ], - type: "table" - } - ], - jitData: {}, - warehouseState: executionGraph.warehouseState - }; - - expect(executionGraph).deep.equals(expectedRunResult); - expect(statement).to.include("CREATE OR REPLACE PROCEDURE"); - expect(statement).to.include("Column removals are not allowed when on_schema_change = 'EXTEND'."); - expect(statement).to.include("ALTER TABLE"); - expect(statement).to.include("ADD COLUMN IF NOT EXISTS"); - } finally { - // Teardown the schema completely, regardless of test success or failure. - await runCli( - "run", - [ + ]); + + expect(runResult.exitCode).equals(0); + const executionGraph = JSON.parse(runResult.stdout); + const statement = executionGraph.actions[0].tasks[0].statement; + + const expectedRunResult = { + projectConfig: { + warehouse: "bigquery", + defaultSchema: uniqueDataset, + assertionSchema: "dataform_assertions", + defaultDatabase: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION + }, + runConfig: { + actions: ["example_incremental"], + fullRefresh: false + }, + actions: [ + { + fileName: "definitions/example_incremental.sqlx", + hermeticity: "NON_HERMETIC", + tableType: "incremental", + target: { + database: INTEGRATION_TEST_PROJECT, + name: "example_incremental", + schema: uniqueDataset + }, + tasks: [ + { + statement, + type: "statement" + } + ], + type: "table" + } + ], + jitData: {}, + warehouseState: executionGraph.warehouseState + }; + + expect(executionGraph).deep.equals(expectedRunResult); + expect(statement).to.include("CREATE OR REPLACE PROCEDURE"); + expect(statement).to.include( + "Column removals are not allowed when on_schema_change = 'EXTEND'." + ); + expect(statement).to.include("ALTER TABLE"); + expect(statement).to.include("ADD COLUMN IF NOT EXISTS"); + } finally { + // Teardown the schema completely, regardless of test success or failure. + await runCli("run", [ projectDir, "--credentials", CREDENTIALS_PATH, "--actions=teardown_schema" - ] - ); + ]); + } } - }); + ); }); suite("run --timeout deprecation", ({ beforeEach }) => { @@ -597,16 +567,13 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p // The notice fires in the run handler before compile. Yargs validation // requires workflow_settings.yaml to exist, but compile can fail after that // — we only assert on stderr for the notice line. - const runResult = await runCli( - "run", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--timeout", - "30s" - ] - ); + const runResult = await runCli("run", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--timeout", + "30s" + ]); expect(runResult.stderr).to.match( /--timeout only bounds project compilation[\s\S]*use --execution-timeout/ @@ -614,22 +581,17 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p }); test("--timeout on run does NOT emit notice when --execution-timeout is also set", async () => { - const runResult = await runCli( - "run", - [ - projectDir, - "--credentials", - CREDENTIALS_PATH, - "--timeout", - "30s", - "--execution-timeout", - "10m" - ] - ); + const runResult = await runCli("run", [ + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--timeout", + "30s", + "--execution-timeout", + "10m" + ]); - expect(runResult.stderr).to.not.match( - /--timeout only bounds project compilation/ - ); + expect(runResult.stderr).to.not.match(/--timeout only bounds project compilation/); }); - }) + }); });