From 361779c5a09ccc55795355b2e110ba100e4bc0fe Mon Sep 17 00:00:00 2001 From: Yohan Belleguic Date: Fri, 18 Sep 2026 12:35:15 +0000 Subject: [PATCH 1/5] Add universeDomain support to BigQuery and Lineage clients Add an optional universe_domain field to the BigQuery credentials proto and plumb it through to the BigQuery and data-lineage clients so the CLI can target non-default universes (e.g. Trusted Partner Cloud). The value is read from .df-credentials.json and can be entered during 'dataform init-creds'. When unset, clients keep defaulting to the Google Default Universe. Adds unit tests covering credential parsing and client construction. --- cli/api/BUILD | 1 + cli/api/commands/credentials_test.ts | 47 ++++++++++++++++++++++++++++ cli/api/dbadapters/bigquery.ts | 3 +- cli/api/dbadapters/bigquery_test.ts | 24 +++++++++++++- cli/api/lineage/emitter.ts | 3 +- cli/credentials.ts | 11 +++++-- protos/profiles.proto | 4 +++ 7 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 cli/api/commands/credentials_test.ts diff --git a/cli/api/BUILD b/cli/api/BUILD index 9b817c962..0d439e2fc 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -62,6 +62,7 @@ ts_test_suite( srcs = [ "tasks_test.ts", "utils_test.ts", + "commands/credentials_test.ts", "commands/jit/rpc_test.ts", "commands/prune_test.ts", "dbadapters/bigquery_test.ts", diff --git a/cli/api/commands/credentials_test.ts b/cli/api/commands/credentials_test.ts new file mode 100644 index 000000000..bc205d3a2 --- /dev/null +++ b/cli/api/commands/credentials_test.ts @@ -0,0 +1,47 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; + +import { read } from "df/cli/api/commands/credentials"; +import { suite, test } from "df/testing"; +import { TmpDirFixture } from "df/testing/fixtures"; + +suite("credentials", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + function writeCredentials(contents: object): string { + const credentialsPath = path.join(tmpDirFixture.createNewTmpDir(), ".df-credentials.json"); + fs.writeFileSync(credentialsPath, JSON.stringify(contents)); + return credentialsPath; + } + + test("read maps universeDomain when present", () => { + const credentialsPath = writeCredentials({ + projectId: "my-project", + location: "US", + universeDomain: "my-universe.example.com" + }); + + const credentials = read(credentialsPath); + + expect(credentials.projectId).to.equal("my-project"); + expect(credentials.location).to.equal("US"); + expect(credentials.universeDomain).to.equal("my-universe.example.com"); + }); + + test("read leaves universeDomain unset when omitted", () => { + const credentialsPath = writeCredentials({ projectId: "my-project", location: "US" }); + + const credentials = read(credentialsPath); + + expect(credentials.universeDomain).to.satisfy( + (value: string) => value === "" || value === undefined + ); + }); + + test("read rejects unknown fields", () => { + const credentialsPath = writeCredentials({ projectId: "my-project", notARealField: "x" }); + + expect(() => read(credentialsPath)).to.throw(/notARealField/); + }); +}); diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index 303e08024..102973a86 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -52,7 +52,8 @@ export function createBigQueryClientProvider( projectId, scopes: EXTRA_GOOGLE_SCOPES, location: credentials.location, - credentials: credentials.credentials && JSON.parse(credentials.credentials) + credentials: credentials.credentials && JSON.parse(credentials.credentials), + ...(credentials.universeDomain ? { universeDomain: credentials.universeDomain } : {}) }) ); } diff --git a/cli/api/dbadapters/bigquery_test.ts b/cli/api/dbadapters/bigquery_test.ts index 6659941e8..3915b61d7 100644 --- a/cli/api/dbadapters/bigquery_test.ts +++ b/cli/api/dbadapters/bigquery_test.ts @@ -2,7 +2,7 @@ import { Dataset, Table } from "@google-cloud/bigquery"; import { expect } from "chai"; import { anything, instance, mock, verify, when } from "ts-mockito"; -import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; +import { BigQueryDbAdapter, createBigQueryClientProvider } from "df/cli/api/dbadapters/bigquery"; import { dataform } from "df/protos/ts"; import { suite, test } from "df/testing"; @@ -146,4 +146,26 @@ suite("BigQueryDbAdapter", () => { await adapter.setMetadata(action); }); + + suite("createBigQueryClientProvider", () => { + test("passes universeDomain to the BigQuery client when set", () => { + const credentials = dataform.BigQuery.create({ + projectId: "project1", + location: "US", + universeDomain: "my-universe.example.com" + }); + + const client = createBigQueryClientProvider(credentials)(); + + expect(client.universeDomain).to.equal("my-universe.example.com"); + }); + + test("defaults to googleapis.com when universeDomain is unset", () => { + const credentials = dataform.BigQuery.create({ projectId: "project1", location: "US" }); + + const client = createBigQueryClientProvider(credentials)(); + + expect(client.universeDomain).to.equal("googleapis.com"); + }); + }); }); diff --git a/cli/api/lineage/emitter.ts b/cli/api/lineage/emitter.ts index eab5700e6..d06f48e05 100644 --- a/cli/api/lineage/emitter.ts +++ b/cli/api/lineage/emitter.ts @@ -42,7 +42,8 @@ export function createLineageClientProvider( apiEndpoint: endpoint, credentials: credentials.credentials && JSON.parse(credentials.credentials), libName: DATAFORM_CLI_LIB_NAME, - libVersion: version + libVersion: version, + ...(credentials.universeDomain ? { universeDomain: credentials.universeDomain } : {}) }) ); } diff --git a/cli/credentials.ts b/cli/credentials.ts index 12e55c5e2..ef8b54fd5 100644 --- a/cli/credentials.ts +++ b/cli/credentials.ts @@ -14,6 +14,11 @@ export function getBigQueryCredentials(): dataform.IBigQuery { if (locationIndex === 2) { location = question("Enter the location's region name (e.g. 'asia-south1'):"); } + const universeDomain = question( + "Enter the universe domain to connect to, or leave blank to use the default " + + "('googleapis.com'). Set this only when targeting a non-default universe such as a " + + "Trusted Partner Cloud (TPC):" + ).trim(); const isApplicationDefaultOrJSONKeyIndex = selectionQuestion( "Do you wish to use Application Default Credentials or JSON Key:", ["ADC (default)", "JSON Key"] @@ -22,7 +27,8 @@ export function getBigQueryCredentials(): dataform.IBigQuery { const projectId = question("Enter your billing project ID:"); return { projectId, - location + location, + ...(universeDomain ? { universeDomain } : {}) }; } const cloudCredentialsPath = actuallyResolve( @@ -40,6 +46,7 @@ export function getBigQueryCredentials(): dataform.IBigQuery { return { projectId: cloudCredentials.project_id, credentials: fs.readFileSync(cloudCredentialsPath, "utf8"), - location + location, + ...(universeDomain ? { universeDomain } : {}) }; } diff --git a/protos/profiles.proto b/protos/profiles.proto index 09eabcdb8..08d89cef4 100644 --- a/protos/profiles.proto +++ b/protos/profiles.proto @@ -11,6 +11,10 @@ message BigQuery { string credentials = 3; // Options are listed here: https://cloud.google.com/bigquery/docs/locations string location = 4; + // The universe domain to connect to (e.g. "googleapis.com"). Leave unset to + // use the default Google Default Universe (GDU). Set this when targeting a + // Trusted Partner Cloud (TPC) or another non-default universe. + string universe_domain = 5; reserved 2; } From 2f09b5cbae1759ef8a0f43cf170e8ae1762ee74d Mon Sep 17 00:00:00 2001 From: Yohan Belleguic Date: Fri, 18 Sep 2026 12:35:15 +0000 Subject: [PATCH 2/5] Document optional universeDomain credentials field --- contributing.md | 1 + 1 file changed, 1 insertion(+) diff --git a/contributing.md b/contributing.md index 96096a514..47901c578 100644 --- a/contributing.md +++ b/contributing.md @@ -57,6 +57,7 @@ To run the CLI integration test against your own GCP project: - `projectId`: your GCP project id - `credentials`: the entire content of your GCP service account key JSON file as a single string (you can generate it with `jq -Rsa < path/to/key.json`). - `location`: location to use in your project + - `universeDomain` (optional): the universe domain to connect to (e.g. `googleapis.com`). Leave unset to use the default Google Default Universe (GDU). Set this only when targeting a non-default universe such as a Trusted Partner Cloud (TPC). Example: From 2962ae0a097bfef4fbe5a73fe39b77aa22519b54 Mon Sep 17 00:00:00 2001 From: Yohan Belleguic Date: Mon, 21 Sep 2026 13:38:05 +0000 Subject: [PATCH 3/5] Stop prompting for universeDomain during init-creds Almost all users are on GDU; the proto field already lets TPC users set universeDomain by hand in .df-credentials.json. --- cli/credentials.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cli/credentials.ts b/cli/credentials.ts index ef8b54fd5..12e55c5e2 100644 --- a/cli/credentials.ts +++ b/cli/credentials.ts @@ -14,11 +14,6 @@ export function getBigQueryCredentials(): dataform.IBigQuery { if (locationIndex === 2) { location = question("Enter the location's region name (e.g. 'asia-south1'):"); } - const universeDomain = question( - "Enter the universe domain to connect to, or leave blank to use the default " + - "('googleapis.com'). Set this only when targeting a non-default universe such as a " + - "Trusted Partner Cloud (TPC):" - ).trim(); const isApplicationDefaultOrJSONKeyIndex = selectionQuestion( "Do you wish to use Application Default Credentials or JSON Key:", ["ADC (default)", "JSON Key"] @@ -27,8 +22,7 @@ export function getBigQueryCredentials(): dataform.IBigQuery { const projectId = question("Enter your billing project ID:"); return { projectId, - location, - ...(universeDomain ? { universeDomain } : {}) + location }; } const cloudCredentialsPath = actuallyResolve( @@ -46,7 +40,6 @@ export function getBigQueryCredentials(): dataform.IBigQuery { return { projectId: cloudCredentials.project_id, credentials: fs.readFileSync(cloudCredentialsPath, "utf8"), - location, - ...(universeDomain ? { universeDomain } : {}) + location }; } From 8bf34e9b90d4525151d2b567bcc658e468bed3b7 Mon Sep 17 00:00:00 2001 From: Yohan Belleguic Date: Mon, 21 Sep 2026 13:41:51 +0000 Subject: [PATCH 4/5] Fall back to the JSON key universe_domain in init-creds TPC service-account keys include universe_domain. Copy it into .df-credentials.json so the BigQuery client and google-auth agree instead of BigQuery defaulting to googleapis.com. --- cli/BUILD | 1 + cli/credentials.ts | 29 +++++++++++++++++++++-------- cli/credentials_test.ts | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 cli/credentials_test.ts diff --git a/cli/BUILD b/cli/BUILD index aa20288d4..6dd9a66d7 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -74,6 +74,7 @@ ts_library( ts_test_suite( name = "tests", srcs = [ + "credentials_test.ts", "index_help_test.ts", "index_init_test.ts", "index_project_test.ts", diff --git a/cli/credentials.ts b/cli/credentials.ts index 12e55c5e2..cb20cb988 100644 --- a/cli/credentials.ts +++ b/cli/credentials.ts @@ -8,7 +8,7 @@ export function getBigQueryCredentials(): dataform.IBigQuery { const locationIndex = selectionQuestion("Enter the location of your datasets:", [ "US (default)", "EU", - "other" + "other", ]); let location = locationIndex === 0 ? "US" : "EU"; if (locationIndex === 2) { @@ -16,13 +16,13 @@ export function getBigQueryCredentials(): dataform.IBigQuery { } const isApplicationDefaultOrJSONKeyIndex = selectionQuestion( "Do you wish to use Application Default Credentials or JSON Key:", - ["ADC (default)", "JSON Key"] + ["ADC (default)", "JSON Key"], ); if (isApplicationDefaultOrJSONKeyIndex === 0) { const projectId = question("Enter your billing project ID:"); return { projectId, - location + location, }; } const cloudCredentialsPath = actuallyResolve( @@ -30,16 +30,29 @@ export function getBigQueryCredentials(): dataform.IBigQuery { "Please follow the instructions at https://docs.dataform.co/dataform-cli#create-a-credentials-file/\n" + "to create and download a private key from the Google Cloud Console in JSON format.\n" + "(You can delete this file after credential initialization is complete.)\n\n" + - "Enter the path to your Google Cloud private key file:" - ) + "Enter the path to your Google Cloud private key file:", + ), ); if (!fs.existsSync(cloudCredentialsPath)) { throw new Error(`Google Cloud private key file "${cloudCredentialsPath}" does not exist!`); } - const cloudCredentials = JSON.parse(fs.readFileSync(cloudCredentialsPath, "utf8")); + return credentialsFromServiceAccountJson(fs.readFileSync(cloudCredentialsPath, "utf8"), location); +} + +// Copy universe_domain from the key so BigQuery and google-auth target the same universe. +export function credentialsFromServiceAccountJson( + keyJson: string, + location: string, +): dataform.IBigQuery { + const cloudCredentials = JSON.parse(keyJson); + const universeDomain = + typeof cloudCredentials.universe_domain === "string" + ? cloudCredentials.universe_domain.trim() + : ""; return { projectId: cloudCredentials.project_id, - credentials: fs.readFileSync(cloudCredentialsPath, "utf8"), - location + credentials: keyJson, + location, + ...(universeDomain ? { universeDomain } : {}), }; } diff --git a/cli/credentials_test.ts b/cli/credentials_test.ts new file mode 100644 index 000000000..4c1fd6680 --- /dev/null +++ b/cli/credentials_test.ts @@ -0,0 +1,39 @@ +import { expect } from "chai"; + +import { credentialsFromServiceAccountJson } from "df/cli/credentials"; +import { suite, test } from "df/testing"; + +suite("credentialsFromServiceAccountJson", () => { + function keyJson(overrides: object = {}): string { + return JSON.stringify({ + type: "service_account", + project_id: "my-project", + private_key: "fake-key", + ...overrides, + }); + } + + test("copies universe_domain from the key so BigQuery matches google-auth", () => { + const credentials = credentialsFromServiceAccountJson( + keyJson({ universe_domain: "my-universe.example.com" }), + "EU", + ); + + expect(credentials.projectId).to.equal("my-project"); + expect(credentials.location).to.equal("EU"); + expect(credentials.universeDomain).to.equal("my-universe.example.com"); + expect(JSON.parse(credentials.credentials).universe_domain).to.equal("my-universe.example.com"); + }); + + test("omits universeDomain when the key has no universe_domain", () => { + const credentials = credentialsFromServiceAccountJson(keyJson(), "US"); + + expect(credentials).to.not.have.property("universeDomain"); + }); + + test("omits universeDomain when universe_domain is blank", () => { + const credentials = credentialsFromServiceAccountJson(keyJson({ universe_domain: " " }), "US"); + + expect(credentials).to.not.have.property("universeDomain"); + }); +}); From d99767c73a6604e5962996f54b2c5489f2c2cf43 Mon Sep 17 00:00:00 2001 From: Yohan Belleguic Date: Mon, 21 Sep 2026 15:11:03 +0000 Subject: [PATCH 5/5] Simplify BigQuery universeDomain option passing @google-cloud/bigquery already falsy-checks options.universeDomain, so pass credentials.universeDomain || undefined. Keep the omit-empty guard in the lineage emitter, where gax uses ?? instead. --- cli/api/dbadapters/bigquery.ts | 2 +- cli/api/lineage/emitter.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index 102973a86..3b33ecfca 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -53,7 +53,7 @@ export function createBigQueryClientProvider( scopes: EXTRA_GOOGLE_SCOPES, location: credentials.location, credentials: credentials.credentials && JSON.parse(credentials.credentials), - ...(credentials.universeDomain ? { universeDomain: credentials.universeDomain } : {}) + universeDomain: credentials.universeDomain || undefined }) ); } diff --git a/cli/api/lineage/emitter.ts b/cli/api/lineage/emitter.ts index d06f48e05..08debf488 100644 --- a/cli/api/lineage/emitter.ts +++ b/cli/api/lineage/emitter.ts @@ -43,6 +43,8 @@ export function createLineageClientProvider( credentials: credentials.credentials && JSON.parse(credentials.credentials), libName: DATAFORM_CLI_LIB_NAME, libVersion: version, + // Omit empty universeDomain: gax uses ??, so "" would skip the default. (BigQuery + // falsy-checks options.universeDomain, so this guard is not needed there.) ...(credentials.universeDomain ? { universeDomain: credentials.universeDomain } : {}) }) );