diff --git a/cli/vm/BUILD b/cli/vm/BUILD index 4adc5f391..a8fc54948 100644 --- a/cli/vm/BUILD +++ b/cli/vm/BUILD @@ -1,5 +1,6 @@ package(default_visibility = ["//visibility:public"]) +load("//testing:index.bzl", "ts_test_suite") load("//tools:ts_library.bzl", "ts_library") ts_library( @@ -10,6 +11,7 @@ ts_library( ], deps = [ "//common/protos", + "//common/vm:vm_runner", "//core", "//protos:ts", "@npm//@types/glob", @@ -17,7 +19,6 @@ ts_library( "@npm//@types/semver", "@npm//glob", "@npm//semver", - "@npm//vm2", ], ) diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 5ae3fe93d..ba74c3596 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -2,22 +2,26 @@ import * as fs from "fs"; import * as glob from "glob"; import * as path from "path"; import * as semver from "semver"; -import { CompilerFunction, NodeVM } from "vm2"; import { encode64 } from "df/common/protos"; +import { CompilerFunction, VmRunner } from "df/common/vm/vm_runner"; import { dataform } from "df/protos/ts"; export function compile(compileConfig: dataform.ICompileConfig) { compileConfig.projectDir = fs.realpathSync(path.resolve(compileConfig.projectDir)); const coreBundlePath = path.join( - compileConfig.projectDir, "node_modules", "@dataform", "core", "bundle.js" + compileConfig.projectDir, + "node_modules", + "@dataform", + "core", + "bundle.js", ); if (!fs.existsSync(coreBundlePath)) { throw new Error( "Could not find a recent installed version of @dataform/core in the project. Check that " + "either `dataformCoreVersion` is specified in `workflow_settings.yaml`, or " + "`@dataform/core` is specified in `package.json`. If using `package.json`, then run " + - "`dataform install`." + "`dataform install`.", ); } @@ -27,22 +31,17 @@ export function compile(compileConfig: dataform.ICompileConfig) { // through Node's resolver inside the vm covers every install layout // (package.json, workflow_settings.yaml, JiT) and matches what the user's // code will see. require() caches the bundle so the second call is free. - const indexGeneratorVm = new NodeVM({ - wrapper: "none", - require: { - context: "sandbox", - root: compileConfig.projectDir, - external: true, - builtin: ["path"] - } + const indexGeneratorVm = new VmRunner({ + projectDir: compileConfig.projectDir, + builtinModules: ["path"], }); const compiler: CompilerFunction = indexGeneratorVm.run( 'return require("@dataform/core").compiler', - vmIndexFileName + vmIndexFileName, ); const dataformCoreVersion: string = indexGeneratorVm.run( 'return require("@dataform/core").version || "0.0.0"', - vmIndexFileName + vmIndexFileName, ); const cliVersion = readCliVersion(); @@ -61,35 +60,32 @@ export function compile(compileConfig: dataform.ICompileConfig) { `${cliVersion}. The CLI requires @dataform/core >= ${minCoreVersion} ` + `(matching major.minor). Set \`dataformCoreVersion: ${cliVersion}\` in ` + `workflow_settings.yaml (or pin @dataform/core in package.json), then run ` + - `\`dataform install\`.` + `\`dataform install\`.`, ); } } const needsCallerFileShim = semver.lt(dataformCoreVersion, "3.0.57"); - // vm2 strips file paths from V8 CallSite objects inside the sandbox, so - // getCallerFile() in @dataform/core needs a fallback. Track the currently + // While VmRunner preserves V8 CallSite file paths natively, older @dataform/core + // versions check global.__dataform_current_file as a fallback. Track the currently // executing file via a host-side stack exposed through sandbox helpers, and // expose it as a getter on `global.__dataform_current_file`. const fileStack: string[] = []; - // Then use vm2's native compiler integration to apply the compiler to files. - const userCodeVm = new NodeVM({ - wrapper: "none", + // Then use VmRunner to apply the compiler to files. + const userCodeVm = new VmRunner({ + projectDir: compileConfig.projectDir, sandbox: { - __df_enter: (p: string) => { fileStack.push(p); }, - __df_exit: () => { fileStack.pop(); }, - __df_current: () => fileStack.length > 0 ? fileStack[fileStack.length - 1] : null - }, - require: { - builtin: ["path"], - context: "sandbox", - external: true, - root: compileConfig.projectDir, - resolve: (moduleName, parentDirName) => - path.join(parentDirName, path.relative(parentDirName, compileConfig.projectDir), moduleName) + __df_enter: (p: string) => { + fileStack.push(p); + }, + __df_exit: () => { + fileStack.pop(); + }, + __df_current: () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null), }, - sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml"], + builtinModules: ["path"], + sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml", "ipynb", "md"], compiler: (code, filePath) => { let source = code; if (needsCallerFileShim && filePath === coreBundlePath) { @@ -104,15 +100,13 @@ export function compile(compileConfig: dataform.ICompileConfig) { __df_exit(); } `; - } + }, }); const hasWorkflowSettingsYaml = fs.existsSync( - path.join(compileConfig.projectDir, "workflow_settings.yaml") - ); - const hasDataformJson = fs.existsSync( - path.join(compileConfig.projectDir, "dataform.json") + path.join(compileConfig.projectDir, "workflow_settings.yaml"), ); + const hasDataformJson = fs.existsSync(path.join(compileConfig.projectDir, "dataform.json")); return userCodeVm.run( ` @@ -120,15 +114,15 @@ export function compile(compileConfig: dataform.ICompileConfig) { configurable: true, get: function() { return __df_current(); } }); - ${hasWorkflowSettingsYaml - ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' - : ''} - ${hasDataformJson - ? 'global.dataformJson = require("./dataform.json");' - : ''} + ${ + hasWorkflowSettingsYaml + ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' + : "" + } + ${hasDataformJson ? 'global.dataformJson = require("./dataform.json");' : ""} return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}") `, - vmIndexFileName + vmIndexFileName, ); } @@ -162,9 +156,7 @@ if (require.main === module) { // by pkg_json(version = DF_VERSION). Returns "0.0.0" when unreadable. function readCliVersion(): string { try { - const pkg = JSON.parse( - fs.readFileSync(path.join(__dirname, "package.json"), "utf8") - ); + const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, "package.json"), "utf8")); return pkg.version || "0.0.0"; } catch { return "0.0.0"; @@ -179,7 +171,7 @@ function readCliVersion(): string { const OLD_CORE_THROW = 'if(!t)throw new Error("Unable to find valid caller file; please report this issue.")'; const OLD_CORE_WITH_FALLBACK = - 'if(!t){if(global.__dataform_current_file){t=global.__dataform_current_file}' + + "if(!t){if(global.__dataform_current_file){t=global.__dataform_current_file}" + 'else{throw new Error("Unable to find valid caller file; please report this issue.")}}'; function patchOldCoreCallerFile(source: string): string { @@ -191,11 +183,11 @@ function patchOldCoreCallerFile(source: string): string { */ function createCoreExecutionRequest(compileConfig: dataform.ICompileConfig): string { const filePaths = Array.from( - new Set(glob.sync("!(node_modules)/**/*.*", { cwd: compileConfig.projectDir })) + new Set(glob.sync("!(node_modules)/**/*.*", { cwd: compileConfig.projectDir })), ); return encode64(dataform.CoreExecutionRequest, { // Add the list of file paths to the compile config if not already set. - compile: { compileConfig: { filePaths, ...compileConfig } } + compile: { compileConfig: { filePaths, ...compileConfig } }, }); } diff --git a/cli/vm/jit_worker.ts b/cli/vm/jit_worker.ts index 86c603923..1b48b7ec3 100644 --- a/cli/vm/jit_worker.ts +++ b/cli/vm/jit_worker.ts @@ -1,10 +1,13 @@ import * as fs from "fs"; import * as path from "path"; -import { NodeVM } from "vm2"; +import { VmRunner } from "df/common/vm/vm_runner"; import { dataform } from "df/protos/ts"; -const pendingRpcCallbacks = new Map void>(); +const pendingRpcCallbacks = new Map< + string, + (err: string | null, resBytes: Uint8Array | null) => void +>(); export function registerRpcResponseHandler() { process.on("message", (res: any) => { @@ -25,7 +28,8 @@ export function registerJitCompileHandler() { if (hasStartedProcessing) { process.send({ type: "jit_error", - error: "Worker process received multiple JiT compilation requests. Subsequent requests are rejected." + error: + "Worker process received multiple JiT compilation requests. Subsequent requests are rejected.", }); return; } @@ -35,26 +39,36 @@ export function registerJitCompileHandler() { }); } -export async function handleJitRequest(message: { - request: any; - projectDir: string; -}) { +export async function handleJitRequest(message: { request: any; projectDir: string }) { try { const { request, projectDir } = message; - const projectLocalCorePath = path.join(projectDir, "node_modules", "@dataform", "core", "bundle.js"); + const projectLocalCorePath = path.join( + projectDir, + "node_modules", + "@dataform", + "core", + "bundle.js", + ); const hasProjectLocalCore = fs.existsSync(projectLocalCorePath); - if (!hasProjectLocalCore && !fs.existsSync(path.join(projectDir, "node_modules", "@dataform", "core", "package.json"))) { + if ( + !hasProjectLocalCore && + !fs.existsSync(path.join(projectDir, "node_modules", "@dataform", "core", "package.json")) + ) { throw new Error( "Could not find a recent installed version of @dataform/core in the project. Check that " + "either `dataformCoreVersion` is specified in `workflow_settings.yaml`, or " + "`@dataform/core` is specified in `package.json`. If using `package.json`, then run " + - "`dataform install`." + "`dataform install`.", ); } - const rpcCallback = (method: string, reqBytes: Uint8Array, callback: (err: string | null, resBytes: Uint8Array | null) => void) => { + const rpcCallback = ( + method: string, + reqBytes: Uint8Array, + callback: (err: string | null, resBytes: Uint8Array | null) => void, + ) => { const correlationId = Math.random().toString(36).substring(7); pendingRpcCallbacks.set(correlationId, callback); @@ -62,7 +76,7 @@ export async function handleJitRequest(message: { type: "rpc_request", method, request: reqBytes, - correlationId + correlationId, }); }; @@ -71,20 +85,19 @@ export async function handleJitRequest(message: { const vmFileName = path.resolve(projectDir, "index.js"); - const vm = new NodeVM({ - require: { - builtin: [], - context: "sandbox", - external: { modules: ["@dataform/*"], transitive: false }, - root: projectDir, - mock: hasProjectLocalCore ? {} : { - "@dataform/core": require("@dataform/core") - } - }, - sourceExtensions: ["js", "json", "yaml", "yml"] + const vm = new VmRunner({ + projectDir, + builtinModules: [], + mockModules: hasProjectLocalCore + ? {} + : { + "@dataform/core": require("@dataform/core"), + }, + sourceExtensions: ["js", "json", "yaml", "yml"], }); - const jitCompileInVm = vm.run(` + const jitCompileInVm = vm.run( + ` const { jitCompiler } = require("@dataform/core"); global.require = require; @@ -103,10 +116,14 @@ export async function handleJitRequest(message: { const compilerInstance = jitCompiler(internalRpcCallback); return await compilerInstance.compile(requestBytesTyped); }; - `, vmFileName); + `, + vmFileName, + ); const responseBytes = await jitCompileInVm(requestBytes, rpcCallback); - const response = dataform.JitCompilationResponse.decode(new Uint8Array(responseBytes as number[])); + const response = dataform.JitCompilationResponse.decode( + new Uint8Array(responseBytes as number[]), + ); process.send({ type: "jit_response", response: response.toJSON() }); } catch (e) { diff --git a/common/vm/BUILD b/common/vm/BUILD new file mode 100644 index 000000000..7203034bf --- /dev/null +++ b/common/vm/BUILD @@ -0,0 +1,50 @@ +package(default_visibility = ["//visibility:public"]) + +load("//testing:index.bzl", "ts_test_suite") +load("//tools:ts_library.bzl", "ts_library") + +ts_library( + name = "vm_runner", + srcs = [ + "vm_runner.ts", + ], + deps = [ + "@npm//@types/node", + ], +) + +ts_test_suite( + name = "tests", + srcs = ["vm_runner_test.ts"], + deps = [ + ":vm_runner", + "//testing", + "@npm//@types/chai", + "@npm//@types/node", + "@npm//chai", + ], +) + +load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary") + +ts_library( + name = "benchmark_lib", + srcs = ["vm_runner_benchmark.ts"], + deps = [ + ":vm_runner", + "@npm//@types/node", + ], +) + +nodejs_binary( + name = "benchmark", + data = [ + ":benchmark_lib", + "@npm//source-map-support", + ], + entry_point = ":vm_runner_benchmark.ts", + templated_args = [ + "--node_options=--require=source-map-support/register", + "--bazel_patch_module_resolver", + ], +) diff --git a/common/vm/vm_runner.ts b/common/vm/vm_runner.ts new file mode 100644 index 000000000..902c7e3bd --- /dev/null +++ b/common/vm/vm_runner.ts @@ -0,0 +1,461 @@ +import * as fs from "fs"; +import { builtinModules as nodeBuiltins, createRequire } from "module"; +import * as path from "path"; +import * as vm from "vm"; + +export type CompilerFunction = (code: string, filePath: string) => string; + +export interface VmRunnerOptions { + projectDir: string; + sourceExtensions?: string[]; + compiler?: CompilerFunction; + sandbox?: Record; + builtinModules?: string[]; + mockModules?: Record; + resolve?: (moduleName: string, parentDirName: string) => string; + console?: "inherit" | "off"; + env?: Record; + envAllowlist?: string[]; + allowedExternalPaths?: string[]; +} + +export class VmRunner { + private readonly projectDir: string; + private readonly allowedExternalPaths: string[]; + private readonly sourceExtensions: Set; + private readonly allExtensions: string[]; + private readonly compiler?: CompilerFunction; + private readonly builtinModules: Set; + private readonly mockModules: Record; + private readonly customResolve?: (moduleName: string, parentDirName: string) => string; + private readonly context: vm.Context; + private readonly moduleCache = new Map< + string, + { exports: any; id: string; filename: string; loaded: boolean } + >(); + private readonly resolveCache = new Map(); + private readonly nodeBuiltinSet: Set; + + constructor(options: VmRunnerOptions) { + this.projectDir = this.getRealPath(options.projectDir); + this.allowedExternalPaths = (options.allowedExternalPaths || []).map((p) => + this.getRealPath(p), + ); + const rawExtensions = options.sourceExtensions || ["js", "json"]; + this.sourceExtensions = new Set( + rawExtensions.map((ext) => + ext.startsWith(".") ? ext.slice(1).toLowerCase() : ext.toLowerCase(), + ), + ); + this.allExtensions = Array.from( + new Set([ + ".js", + ".json", + ...rawExtensions.map((ext) => (ext.startsWith(".") ? ext : `.${ext}`)), + ]), + ); + this.compiler = options.compiler; + this.builtinModules = new Set( + options.builtinModules !== undefined ? options.builtinModules : ["path"], + ); + this.mockModules = options.mockModules || {}; + this.customResolve = options.resolve; + this.nodeBuiltinSet = new Set(nodeBuiltins); + + let env: Record; + if (options.env !== undefined) { + env = { ...options.env }; + } else if (options.envAllowlist !== undefined) { + env = {}; + for (const key of options.envAllowlist) { + if (key in process.env) { + env[key] = process.env[key]; + } + } + } else { + env = { ...process.env }; + } + + const sandbox: Record = { + console: + options.console === "off" + ? { log: () => {}, error: () => {}, warn: () => {}, info: () => {} } + : console, + process: { + env, + cwd: () => this.projectDir, + version: process.version, + versions: process.versions, + platform: process.platform, + arch: process.arch, + }, + Buffer, + Uint8Array, + ArrayBuffer, + setTimeout, + clearTimeout, + setInterval, + clearInterval, + setImmediate, + clearImmediate, + URL, + URLSearchParams, + TextEncoder, + TextDecoder, + ...(options.sandbox || {}), + }; + + this.context = vm.createContext(sandbox); + sandbox.global = this.context; + sandbox.globalThis = this.context; + } + + public run(code: string, filename: string = path.join(this.projectDir, "index.js")): any { + let source = code; + const ext = path.extname(filename).toLowerCase().replace(/^\./, ""); + if (ext === "json") { + const module = { + exports: JSON.parse(source), + id: filename, + filename, + loaded: true, + }; + this.moduleCache.set(filename, module); + return module.exports; + } + + if (this.compiler && this.sourceExtensions.has(ext)) { + source = this.compiler(source, filename); + } + + const fn = vm.compileFunction( + source, + ["exports", "require", "module", "__filename", "__dirname"], + { + filename, + parsingContext: this.context, + }, + ); + + const module = { + exports: {}, + id: filename, + filename, + loaded: false, + }; + this.moduleCache.set(filename, module); + + try { + const scopedRequire = this.createRequire(filename); + const result = fn.call( + module.exports, + module.exports, + scopedRequire, + module, + filename, + path.dirname(filename), + ); + module.loaded = true; + + return result !== undefined ? result : module.exports; + } catch (e) { + this.moduleCache.delete(filename); + throw e; + } + } + + public require( + moduleName: string, + fromPath: string = path.join(this.projectDir, "index.js"), + ): any { + if (Object.prototype.hasOwnProperty.call(this.mockModules, moduleName)) { + return this.mockModules[moduleName]; + } + + const cleanBuiltinName = moduleName.startsWith("node:") ? moduleName.slice(5) : moduleName; + if (this.nodeBuiltinSet.has(cleanBuiltinName)) { + if (this.builtinModules.has(cleanBuiltinName) || this.builtinModules.has(moduleName)) { + return require(moduleName); + } + const err: any = new Error(`Access to built-in module '${moduleName}' is not allowed`); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + + const resolvedPath = this.resolve(moduleName, fromPath); + if (this.moduleCache.has(resolvedPath)) { + return this.moduleCache.get(resolvedPath)!.exports; + } + + const module = { + exports: {}, + id: resolvedPath, + filename: resolvedPath, + loaded: false, + }; + this.moduleCache.set(resolvedPath, module); + + try { + const ext = path.extname(resolvedPath).toLowerCase().replace(/^\./, ""); + if (ext === "json") { + const content = fs.readFileSync(resolvedPath, "utf8"); + module.exports = JSON.parse(content); + module.loaded = true; + return module.exports; + } + + let source = fs.readFileSync(resolvedPath, "utf8"); + if (this.compiler && this.sourceExtensions.has(ext)) { + source = this.compiler(source, resolvedPath); + } + + const fn = vm.compileFunction( + source, + ["exports", "require", "module", "__filename", "__dirname"], + { + filename: resolvedPath, + parsingContext: this.context, + }, + ); + + const scopedRequire = this.createRequire(resolvedPath); + fn.call( + module.exports, + module.exports, + scopedRequire, + module, + resolvedPath, + path.dirname(resolvedPath), + ); + module.loaded = true; + + return module.exports; + } catch (e) { + this.moduleCache.delete(resolvedPath); + throw e; + } + } + + public resolve(moduleName: string, fromPath: string): string { + const cacheKey = `${fromPath}\0${moduleName}`; + if (this.resolveCache.has(cacheKey)) { + return this.resolveCache.get(cacheKey)!; + } + + const parentDir = path.dirname(fromPath); + + // Check custom resolve function if provided + if (this.customResolve) { + try { + const candidate = this.customResolve(moduleName, parentDir); + const resolved = this.tryResolvePath(candidate); + if (resolved) { + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolved); + return resolved; + } + } catch (e) { + if (e && e.code === "MODULE_NOT_FOUND") { + throw e; + } + } + } + + // Relative or absolute path + if ( + moduleName.startsWith("./") || + moduleName.startsWith("../") || + path.isAbsolute(moduleName) + ) { + const candidate = path.resolve(parentDir, moduleName); + const resolved = this.tryResolvePath(candidate); + if (resolved) { + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolved); + return resolved; + } + } else { + // Project-relative path (e.g. require("includes/helpers")) + const projectRelative = path.resolve(this.projectDir, moduleName); + const resolvedProjectRelative = this.tryResolvePath(projectRelative); + if (resolvedProjectRelative) { + if (!this.isPathContained(resolvedProjectRelative)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolvedProjectRelative); + return resolvedProjectRelative; + } + + // Check project node_modules directory directly (e.g. @dataform/core) + const nodeModulesCandidate = path.resolve(this.projectDir, "node_modules", moduleName); + const resolvedNodeModules = this.tryResolvePath(nodeModulesCandidate); + if (resolvedNodeModules) { + this.resolveCache.set(cacheKey, resolvedNodeModules); + return resolvedNodeModules; + } + + // Fallback to standard Node.js require.resolve resolution + let nodeReqError: any; + try { + const nodeReq = createRequire(fromPath); + const resolved = nodeReq.resolve(moduleName); + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolved); + return resolved; + } catch (e) { + if ( + e && + e.code === "MODULE_NOT_FOUND" && + e.message && + e.message.includes("outside of project directory") + ) { + throw e; + } + nodeReqError = e; + } + + let projectReqError: any; + try { + const projectReq = createRequire(path.join(this.projectDir, "index.js")); + const resolved = projectReq.resolve(moduleName); + if (!this.isPathContained(resolved)) { + const err: any = new Error( + `Cannot require '${moduleName}' outside of project directory '${this.projectDir}'`, + ); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + this.resolveCache.set(cacheKey, resolved); + return resolved; + } catch (e) { + if ( + e && + e.code === "MODULE_NOT_FOUND" && + e.message && + e.message.includes("outside of project directory") + ) { + throw e; + } + projectReqError = e; + } + + const err: any = new Error(`Cannot find module '${moduleName}' from '${fromPath}'`); + err.code = "MODULE_NOT_FOUND"; + if (nodeReqError || projectReqError) { + err.cause = nodeReqError || projectReqError; + } + throw err; + } + + const err: any = new Error(`Cannot find module '${moduleName}' from '${fromPath}'`); + err.code = "MODULE_NOT_FOUND"; + throw err; + } + + private getRealPath(targetPath: string): string { + try { + return fs.realpathSync(targetPath); + } catch { + return path.resolve(targetPath); + } + } + + private isPathContained(targetPath: string): boolean { + const realTarget = this.getRealPath(targetPath); + const isContainedIn = (parentDir: string) => { + const rel = path.relative(parentDir, realTarget); + return !rel.startsWith("..") && !path.isAbsolute(rel); + }; + + if (isContainedIn(this.projectDir)) { + return true; + } + return this.allowedExternalPaths.some((allowed) => isContainedIn(allowed)); + } + + private getStat(targetPath: string): fs.Stats | null { + try { + return fs.statSync(targetPath); + } catch { + return null; + } + } + + private tryResolvePath(candidatePath: string): string | null { + const stat = this.getStat(candidatePath); + if (stat) { + if (stat.isFile()) { + return candidatePath; + } + if (stat.isDirectory()) { + const pkgPath = path.join(candidatePath, "package.json"); + const pkgStat = this.getStat(pkgPath); + if (pkgStat && pkgStat.isFile()) { + try { + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + if (pkg.main) { + const mainPath = path.resolve(candidatePath, pkg.main); + const resolvedMain = this.tryResolvePath(mainPath); + if (resolvedMain) { + return resolvedMain; + } + } + } catch {} + } + for (const ext of this.allExtensions) { + const indexPath = path.join(candidatePath, `index${ext}`); + const indexStat = this.getStat(indexPath); + if (indexStat && indexStat.isFile()) { + return indexPath; + } + } + } + } + + for (const ext of this.allExtensions) { + const withExt = candidatePath.endsWith(ext) ? candidatePath : `${candidatePath}${ext}`; + const withExtStat = this.getStat(withExt); + if (withExtStat && withExtStat.isFile()) { + return withExt; + } + } + + return null; + } + + private createRequire(fromPath: string): NodeJS.Require { + const requireFn = ((moduleName: string) => { + return this.require(moduleName, fromPath); + }) as any; + + requireFn.resolve = (moduleName: string) => { + return this.resolve(moduleName, fromPath); + }; + requireFn.extensions = {}; + requireFn.main = undefined; + + return requireFn; + } +} diff --git a/common/vm/vm_runner_benchmark.ts b/common/vm/vm_runner_benchmark.ts new file mode 100644 index 000000000..91f1567f5 --- /dev/null +++ b/common/vm/vm_runner_benchmark.ts @@ -0,0 +1,48 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { VmRunner } from "df/common/vm/vm_runner"; + +function runBenchmark() { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "vm-runner-benchmark-")); + try { + fs.mkdirSync(path.join(tmpDir, "includes")); + fs.writeFileSync( + path.join(tmpDir, "includes", "helpers.js"), + "module.exports = { format: (x) => 'formatted_' + x };", + ); + + for (let i = 0; i < 50; i++) { + fs.writeFileSync( + path.join(tmpDir, `table_${i}.js`), + `const { format } = require("./includes/helpers"); + module.exports = { name: format("table_${i}"), query: "SELECT ${i}" };`, + ); + } + + const runner = new VmRunner({ projectDir: tmpDir }); + + const iterations = 500; + const start = process.hrtime.bigint(); + + for (let iter = 0; iter < iterations; iter++) { + const idx = iter % 50; + runner.run(`require("./table_${idx}");`); + } + + const end = process.hrtime.bigint(); + const durationMs = Number(end - start) / 1e6; + const opsPerSec = Math.round(iterations / (durationMs / 1000)); + + // eslint-disable-next-line no-console + console.log("VmRunner Benchmark Results:"); + // eslint-disable-next-line no-console + console.log( + ` Evaluated ${iterations} module requires in ${durationMs.toFixed(2)} ms (${opsPerSec} ops/sec)`, + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} + +runBenchmark(); diff --git a/common/vm/vm_runner_test.ts b/common/vm/vm_runner_test.ts new file mode 100644 index 000000000..33f1e3754 --- /dev/null +++ b/common/vm/vm_runner_test.ts @@ -0,0 +1,398 @@ +import { expect } from "chai"; +import * as fs from "fs"; +import * as path from "path"; +import { VmRunner } from "df/common/vm/vm_runner"; +import { suite, test } from "df/testing"; +import { TmpDirFixture } from "df/testing/fixtures"; + +suite("VmRunner", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + test("executes basic script and returns return value", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run("return 40 + 2;"); + expect(result).to.equal(42); + }); + + test("returns module.exports when no explicit return statement exists", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run("module.exports = { value: 'hello' };"); + expect(result).to.deep.equal({ value: "hello" }); + }); + + test("parses JSON files in run method when filename has .json extension", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run( + JSON.stringify({ name: "dataform-test", active: true }), + path.join(tmpDir, "config.json"), + ); + expect(result).to.deep.equal({ name: "dataform-test", active: true }); + }); + + test("resolves relative requires and json files", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(tmpDir, "config.json"), JSON.stringify({ name: "test-project" })); + fs.mkdirSync(path.join(tmpDir, "sub")); + fs.writeFileSync( + path.join(tmpDir, "sub", "helper.js"), + "module.exports = { greet: (x) => `Hello ${x}` };", + ); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run(` + const config = require("./config.json"); + const { greet } = require("./sub/helper"); + return greet(config.name); + `); + expect(result).to.equal("Hello test-project"); + }); + + test("resolves project-relative requires (without leading ./)", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.mkdirSync(path.join(tmpDir, "includes")); + fs.writeFileSync( + path.join(tmpDir, "includes", "math.js"), + "module.exports = { add: (a, b) => a + b };", + ); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run(` + const math = require("includes/math"); + return math.add(10, 20); + `); + expect(result).to.equal(30); + }); + + test("applies compiler hook to custom sourceExtensions", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync(path.join(tmpDir, "model.sqlx"), "SELECT 1 AS id"); + + const runner = new VmRunner({ + projectDir: tmpDir, + sourceExtensions: ["js", "sqlx"], + compiler: (code, filePath) => { + if (filePath.endsWith(".sqlx")) { + return `module.exports = { query: ${JSON.stringify(code.trim())}, file: ${JSON.stringify(filePath)} };`; + } + return code; + }, + }); + + const result = runner.run(` + const model = require("./model.sqlx"); + return model; + `); + expect(result.query).to.equal("SELECT 1 AS id"); + expect(result.file).to.equal(path.join(tmpDir, "model.sqlx")); + }); + + test("handles circular require without crashing", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(tmpDir, "a.js"), + ` + exports.name = "moduleA"; + const b = require("./b"); + exports.getBName = () => b.name; + `, + ); + fs.writeFileSync( + path.join(tmpDir, "b.js"), + ` + exports.name = "moduleB"; + const a = require("./a"); + exports.getAName = () => a.name; + `, + ); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.run(` + const a = require("./a"); + const b = require("./b"); + return { aToB: a.getBName(), bToA: b.getAName() }; + `); + expect(result.aToB).to.equal("moduleB"); + expect(result.bToA).to.equal("moduleA"); + }); + + test("allows configured builtin modules and rejects unallowed ones", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + builtinModules: ["path"], + }); + + const pathResult = runner.run(` + const path = require("path"); + return path.join("foo", "bar"); + `); + expect(pathResult).to.equal(path.join("foo", "bar")); + + expect(() => { + runner.run(`require("fs");`); + }).to.throw(/Access to built-in module 'fs' is not allowed/); + }); + + test("intercepts mockModules", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const mockCore = { + version: "9.9.9", + compiler: () => "compiled", + }; + + const runner = new VmRunner({ + projectDir: tmpDir, + mockModules: { + "@dataform/core": mockCore, + }, + }); + + const result = runner.run(` + const core = require("@dataform/core"); + return core.version; + `); + expect(result).to.equal("9.9.9"); + }); + + test("does not pollute host global", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ projectDir: tmpDir }); + + runner.run(`global.pollutedState = "in-sandbox";`); + expect((global as any).pollutedState).to.equal(undefined); + }); + + test("retains context global state across run calls", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const runner = new VmRunner({ + projectDir: tmpDir, + sandbox: { + injectedValue: 123, + }, + }); + + const result = runner.run(` + global.customState = "active"; + return injectedValue + 1; + `); + expect(result).to.equal(124); + + const state = runner.run("return global.customState;"); + expect(state).to.equal("active"); + }); + + test("removes module from cache when module execution throws and re-throws on next require", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const brokenFile = path.join(tmpDir, "broken.js"); + fs.writeFileSync(brokenFile, "throw new Error('boom');"); + + const runner = new VmRunner({ projectDir: tmpDir }); + + expect(() => runner.run(`require("./broken");`)).to.throw("boom"); + // Ensure second require also throws and does not return an empty cached exports object + expect(() => runner.run(`require("./broken");`)).to.throw("boom"); + }); + + test("rejects requires that escape projectDir via relative or absolute path traversal", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const outsideDir = tmpDirFixture.createNewTmpDir(); + const secretFile = path.join(outsideDir, "secret.json"); + fs.writeFileSync(secretFile, JSON.stringify({ secret: "sensitive" })); + + const runner = new VmRunner({ projectDir: tmpDir }); + + // Relative path traversal + const relativePath = path.relative(tmpDir, secretFile); + expect(() => runner.run(`require(${JSON.stringify(relativePath)});`)).to.throw( + /outside of project directory/, + ); + + // Absolute path traversal + expect(() => runner.run(`require(${JSON.stringify(secretFile)});`)).to.throw( + /outside of project directory/, + ); + }); + + test("rejects requires that escape projectDir via customResolve", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const outsideDir = tmpDirFixture.createNewTmpDir(); + const secretFile = path.join(outsideDir, "secret.json"); + fs.writeFileSync(secretFile, JSON.stringify({ secret: "sensitive" })); + + const runner = new VmRunner({ + projectDir: tmpDir, + resolve: (moduleName) => path.resolve(outsideDir, moduleName), + }); + + expect(() => runner.run(`require("secret.json");`)).to.throw(/outside of project directory/); + }); + + test("resolves relative paths from subfolders relative to caller directory without custom resolve", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const subDir = path.join(tmpDir, "models", "sub"); + fs.mkdirSync(subDir, { recursive: true }); + + const rootHelper = path.join(tmpDir, "helper.js"); + fs.writeFileSync(rootHelper, "module.exports = 'root';"); + + const subHelper = path.join(subDir, "helper.js"); + fs.writeFileSync(subHelper, "module.exports = 'sub';"); + + const subCaller = path.join(subDir, "caller.js"); + fs.writeFileSync(subCaller, "module.exports = require('./helper');"); + + const runner = new VmRunner({ projectDir: tmpDir }); + const result = runner.require("./models/sub/caller"); + expect(result).to.equal("sub"); + }); + + test("allows requiring external files when explicitly permitted via allowedExternalPaths", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const sharedDir = tmpDirFixture.createNewTmpDir(); + const sharedFile = path.join(sharedDir, "shared.json"); + fs.writeFileSync(sharedFile, JSON.stringify({ shared: "data" })); + + const runner = new VmRunner({ + projectDir: tmpDir, + allowedExternalPaths: [sharedDir], + }); + + const result = runner.run(` + const data = require(${JSON.stringify(sharedFile)}); + return data.shared; + `); + expect(result).to.equal("data"); + }); + + test("supports custom env and envAllowlist", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + process.env.TEST_HOST_SECRET = "secret_123"; + process.env.TEST_PUBLIC_VAR = "public_abc"; + + try { + // With envAllowlist + const allowlistRunner = new VmRunner({ + projectDir: tmpDir, + envAllowlist: ["TEST_PUBLIC_VAR"], + }); + const allowlistEnv = allowlistRunner.run("return process.env;"); + expect(allowlistEnv.TEST_PUBLIC_VAR).to.equal("public_abc"); + expect(allowlistEnv.TEST_HOST_SECRET).to.equal(undefined); + + // With custom env record + const customRunner = new VmRunner({ + projectDir: tmpDir, + env: { CUSTOM_KEY: "custom_value" }, + }); + const customEnv = customRunner.run("return process.env;"); + expect(customEnv.CUSTOM_KEY).to.equal("custom_value"); + expect(customEnv.TEST_PUBLIC_VAR).to.equal(undefined); + } finally { + delete process.env.TEST_HOST_SECRET; + delete process.env.TEST_PUBLIC_VAR; + } + }); + + test("compiles and requires .ipynb and .md files via compiler hook", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(tmpDir, "notebook.ipynb"), + JSON.stringify({ cells: [{ cell_type: "code", source: ["print('hello')"] }] }), + ); + fs.writeFileSync(path.join(tmpDir, "doc.md"), "# Hello Documentation"); + + const runner = new VmRunner({ + projectDir: tmpDir, + sourceExtensions: ["js", "json", "ipynb", "md"], + compiler: (code, filePath) => { + if (filePath.endsWith(".ipynb")) { + return `module.exports = { asJson: JSON.parse(${JSON.stringify(code)}) };`; + } + if (filePath.endsWith(".md")) { + return `module.exports = { asMarkdown: ${JSON.stringify(code)} };`; + } + return code; + }, + }); + + const result = runner.run(` + const notebook = require("./notebook.ipynb"); + const doc = require("./doc.md"); + return { + cellType: notebook.asJson.cells[0].cell_type, + docTitle: doc.asMarkdown.trim() + }; + `); + expect(result.cellType).to.equal("code"); + expect(result.docTitle).to.equal("# Hello Documentation"); + }); + + test("preserves V8 CallSite file paths and line numbers in stack traces", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const errorFile = path.join(tmpDir, "faulty.sqlx"); + fs.writeFileSync(errorFile, "throw new Error('boom');"); + + const runner = new VmRunner({ + projectDir: tmpDir, + sourceExtensions: ["sqlx"], + compiler: (code) => code, + }); + + let caughtError: Error | null = null; + try { + runner.run(`require("./faulty.sqlx");`); + } catch (e) { + caughtError = e; + } + + expect(caughtError).to.not.equal(null); + expect(caughtError!.stack).to.include(errorFile); + }); + + test("caches module resolution results across multiple requires", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + const helperFile = path.join(tmpDir, "helper.js"); + fs.writeFileSync(helperFile, "module.exports = { count: 1 };"); + + const runner = new VmRunner({ projectDir: tmpDir }); + const resolvedFirst = runner.resolve("./helper", path.join(tmpDir, "index.js")); + const resolvedSecond = runner.resolve("./helper", path.join(tmpDir, "index.js")); + expect(resolvedFirst).to.equal(helperFile); + expect(resolvedSecond).to.equal(helperFile); + }); + + test("shares Uint8Array constructor with host across realm boundary", () => { + const tmpDir = tmpDirFixture.createNewTmpDir(); + let receivedBytes: any = null; + const runner = new VmRunner({ + projectDir: tmpDir, + mockModules: { + "@dataform/core": { + jitCompiler: () => ({ + compile: (bytes: Uint8Array) => { + receivedBytes = bytes; + return new Uint8Array([bytes[0] + 1, bytes[1] + 1]); + }, + }), + }, + }, + }); + + const result = runner.run(` + const { jitCompiler } = require("@dataform/core"); + const compiler = jitCompiler(); + const input = new Uint8Array([10, 20]); + const output = compiler.compile(input); + module.exports = { input, output }; + `); + + expect(receivedBytes).to.be.an.instanceOf(Uint8Array); + expect(receivedBytes[0]).to.equal(10); + expect(result.input).to.be.an.instanceOf(Uint8Array); + expect(result.output).to.be.an.instanceOf(Uint8Array); + expect(Array.from(result.output)).to.deep.equal([11, 21]); + }); +}); diff --git a/core/main_property_graphs_test.ts b/core/main_property_graphs_test.ts index 43c91e45d..468b9ccb6 100644 --- a/core/main_property_graphs_test.ts +++ b/core/main_property_graphs_test.ts @@ -8,24 +8,24 @@ import { suite, test, writeDefinitionFile, - writeWorkflowSettingsFile + writeWorkflowSettingsFile, } from "df/testing"; import { TmpDirFixture } from "df/testing/fixtures"; import { coreExecutionRequestFromPath, runMainInVm, - VALID_WORKFLOW_SETTINGS_YAML + VALID_WORKFLOW_SETTINGS_YAML, } from "df/testing/run_core"; interface TestCase { - testName: string, - workflowSettings: string, + testName: string; + workflowSettings: string; definitionFiles: { - name: string, - contents: string - }[], - expectedGraph? : dataform.ICompiledGraph, - expectedPropertyGraphs?: dataform.IPropertyGraph[] + name: string; + contents: string; + }[]; + expectedGraph?: dataform.ICompiledGraph; + expectedPropertyGraphs?: dataform.IPropertyGraph[]; } suite("property graphs", ({ afterEach }) => { @@ -34,32 +34,38 @@ suite("property graphs", ({ afterEach }) => { warehouse: "bigquery", defaultSchema: "defaultDataset", defaultDatabase: "defaultProject", - defaultLocation: "US" + defaultLocation: "US", }; - const graphStackTail = "\n at CallSite {}".repeat(10); const graphError = (fileName: string, message: string, extra: object = {}) => ({ fileName, message, - stack: `Error: ${message}${graphStackTail}`, - ...extra + ...extra, }); + const asPlainGraph = (graph: dataform.ICompiledGraph) => { + const plain = asPlainObject(graph); + plain.graphErrors?.compilationErrors?.forEach((e: any) => { + expect(e.stack).to.include(`Error: ${e.message}`); + delete e.stack; + }); + return plain; + }; const missingRefTarget = { schema: "defaultDataset", name: "MissingRefGraph", - database: "defaultProject" + database: "defaultProject", }; const declOneTarget = { schema: "one", name: "books", database: "defaultProject" }; const declTwoTarget = { schema: "two", name: "books", database: "defaultProject" }; const graphTarget = { schema: "defaultDataset", name: "AmbiguousRefGraph", - database: "defaultProject" + database: "defaultProject", }; const collisionTarget = { schema: "defaultDataset", name: "CollisionName", - database: "defaultProject" + database: "defaultProject", }; const collisionActionName = "defaultProject.defaultDataset.CollisionName"; const collisionTargetJson = `{"schema":"defaultDataset","name":"CollisionName","database":"defaultProject"}`; @@ -86,8 +92,8 @@ entities: dataSourceString: defaultProject.defaultDataset.customers keys: - id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -100,12 +106,12 @@ entities: target: { schema: "defaultDataset", name: "SimpleGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "SimpleGraph", - database: "defaultProject" + database: "defaultProject", }, fileName: "definitions/graph.yaml", description: "", @@ -116,18 +122,18 @@ entities: dataSource: { schema: "defaultDataset", name: "customers", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset.customers` AS Customer KEY (id)\n" + - ")" - } - ] - } + ")", + }, + ], + }, }, { testName: "graph.yaml tags propagate into the compiled proto", @@ -145,8 +151,8 @@ entities: dataSourceString: defaultProject.defaultDataset.customers keys: - id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -159,12 +165,12 @@ entities: target: { schema: "defaultDataset", name: "TaggedGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "TaggedGraph", - database: "defaultProject" + database: "defaultProject", }, fileName: "definitions/graph.yaml", description: "", @@ -176,18 +182,18 @@ entities: dataSource: { schema: "defaultDataset", name: "customers", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset.customers` AS Customer KEY (id)\n" + - ")" - } - ] - } + ")", + }, + ], + }, }, { testName: "nodes-only graph compiles without EDGE TABLES", @@ -206,8 +212,8 @@ entities: dataSourceString: defaultProject.defaultDataset.products keys: - sku -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -220,12 +226,12 @@ entities: target: { schema: "defaultDataset", name: "NodesOnly", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "NodesOnly", - database: "defaultProject" + database: "defaultProject", }, fileName: "definitions/graph.yaml", description: "", @@ -236,28 +242,28 @@ entities: dataSource: { schema: "defaultDataset", name: "customers", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] + keys: ["id"], }, { name: "Product", dataSource: { schema: "defaultDataset", name: "products", - database: "defaultProject" + database: "defaultProject", }, - keys: ["sku"] - } + keys: ["sku"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset.customers` AS Customer KEY (id),\n" + " `defaultProject.defaultDataset.products` AS Product KEY (sku)\n" + - ")" - } - ] - } + ")", + }, + ], + }, }, { testName: "targetDataset overrides the schema on the graph target", @@ -274,8 +280,8 @@ entities: dataSourceString: defaultProject.defaultDataset.customers keys: - id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -288,12 +294,12 @@ entities: target: { schema: "customDs", name: "CustomDsGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "customDs", name: "CustomDsGraph", - database: "defaultProject" + database: "defaultProject", }, fileName: "definitions/graph.yaml", description: "", @@ -304,18 +310,18 @@ entities: dataSource: { schema: "defaultDataset", name: "customers", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset.customers` AS Customer KEY (id)\n" + - ")" - } - ] - } + ")", + }, + ], + }, }, { testName: "empty graph.yaml produces a compilation error", @@ -323,8 +329,8 @@ entities: definitionFiles: [ { name: "graph.yaml", - contents: "" - } + contents: "", + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -333,13 +339,13 @@ entities: graphError( "definitions/graph.yaml", "Property graph config is empty or malformed. Expected a top-level " + - "object with 'name' and 'entities'." - ) - ] + "object with 'name' and 'entities'.", + ), + ], }, dataformCoreVersion: version, - jitData: {} - } + jitData: {}, + }, }, { testName: "graph.yaml with only a comment produces a compilation error", @@ -347,8 +353,8 @@ entities: definitionFiles: [ { name: "graph.yaml", - contents: "# nothing here\n" - } + contents: "# nothing here\n", + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -357,13 +363,13 @@ entities: graphError( "definitions/graph.yaml", "Property graph config is empty or malformed. Expected a top-level " + - "object with 'name' and 'entities'." - ) - ] + "object with 'name' and 'entities'.", + ), + ], }, dataformCoreVersion: version, - jitData: {} - } + jitData: {}, + }, }, { testName: "graph.yaml with a top-level scalar produces a compilation error", @@ -371,8 +377,8 @@ entities: definitionFiles: [ { name: "graph.yaml", - contents: "just a string\n" - } + contents: "just a string\n", + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -381,13 +387,13 @@ entities: graphError( "definitions/graph.yaml", "Property graph config is empty or malformed. Expected a top-level " + - "object with 'name' and 'entities'." - ) - ] + "object with 'name' and 'entities'.", + ), + ], }, dataformCoreVersion: version, - jitData: {} - } + jitData: {}, + }, }, { testName: "graph.yaml missing entities produces a compilation error", @@ -397,8 +403,8 @@ entities: name: "graph.yaml", contents: ` name: EmptyGraph -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -406,13 +412,13 @@ name: EmptyGraph compilationErrors: [ graphError( "definitions/graph.yaml", - "Property graph 'EmptyGraph' must declare at least one entity." - ) - ] + "Property graph 'EmptyGraph' must declare at least one entity.", + ), + ], }, dataformCoreVersion: version, - jitData: {} - } + jitData: {}, + }, }, { testName: "graph with relationships emits EDGE TABLES", @@ -442,8 +448,8 @@ relationships: entity: Customer joinKeys: - customer_id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -456,12 +462,12 @@ relationships: target: { schema: "defaultDataset", name: "RelGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "RelGraph", - database: "defaultProject" + database: "defaultProject", }, fileName: "definitions/graph.yaml", description: "", @@ -472,19 +478,19 @@ relationships: dataSource: { schema: "defaultDataset", name: "customers", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] + keys: ["id"], }, { name: "Order", dataSource: { schema: "defaultDataset", name: "orders", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], relationships: [ { @@ -492,19 +498,19 @@ relationships: dataSource: { schema: "defaultDataset", name: "orders", - database: "defaultProject" + database: "defaultProject", }, source: { entity: "Order", relationshipColumns: ["order_id"], - entityColumns: ["id"] + entityColumns: ["id"], }, destination: { entity: "Customer", relationshipColumns: ["customer_id"], - entityColumns: ["id"] - } - } + entityColumns: ["id"], + }, + }, ], graphBody: "NODE TABLES (\n" + @@ -515,10 +521,10 @@ relationships: " `defaultProject.defaultDataset.orders` AS PlacedBy " + "SOURCE KEY (order_id) REFERENCES Order (id) " + "DESTINATION KEY (customer_id) REFERENCES Customer (id)\n" + - ")" - } - ] - } + ")", + }, + ], + }, }, { testName: "ref to declaration resolves entity dataSource and renders graphBody", @@ -530,7 +536,7 @@ relationships: actions: - declaration: name: books -` +`, }, { name: "graph.yaml", @@ -541,27 +547,27 @@ entities: ref: books keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "RefGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "RefGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { database: "defaultProject", schema: "defaultDataset", - name: "books" - } + name: "books", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -572,15 +578,15 @@ entities: dataSource: { schema: "defaultDataset", name: "books", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: - "NODE TABLES (\n" + " `defaultProject.defaultDataset.books` AS Book KEY (id)\n" + ")" - } - ] + "NODE TABLES (\n" + " `defaultProject.defaultDataset.books` AS Book KEY (id)\n" + ")", + }, + ], }, { testName: "ref with schema override resolves the matching declaration", @@ -595,7 +601,7 @@ actions: dataset: alt - declaration: name: books -` +`, }, { name: "graph.yaml", @@ -608,27 +614,27 @@ entities: schema: alt keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "RefWithSchemaGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "RefWithSchemaGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { database: "defaultProject", schema: "alt", - name: "books" - } + name: "books", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -639,14 +645,14 @@ entities: dataSource: { schema: "alt", name: "books", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], - graphBody: "NODE TABLES (\n" + " `defaultProject.alt.books` AS Book KEY (id)\n" + ")" - } - ] + graphBody: "NODE TABLES (\n" + " `defaultProject.alt.books` AS Book KEY (id)\n" + ")", + }, + ], }, { testName: "ref with includeDependentAssertions pulls the dependency's assertions", @@ -658,7 +664,7 @@ entities: type: "table", assertions: { rowConditions: ["id > 0"] } } -select 1 as id` +select 1 as id`, }, { name: "graph.yaml", @@ -671,32 +677,32 @@ entities: includeDependentAssertions: true keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "AssertRefGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "AssertRefGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { database: "defaultProject", schema: "defaultDataset", - name: "books" + name: "books", }, { database: "defaultProject", schema: "defaultDataset", - name: "defaultDataset_books_assertions_rowConditions" - } + name: "defaultDataset_books_assertions_rowConditions", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -707,15 +713,15 @@ entities: dataSource: { schema: "defaultDataset", name: "books", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: - "NODE TABLES (\n" + " `defaultProject.defaultDataset.books` AS Book KEY (id)\n" + ")" - } - ] + "NODE TABLES (\n" + " `defaultProject.defaultDataset.books` AS Book KEY (id)\n" + ")", + }, + ], }, { testName: "graph-level dependOnDependencyAssertions pulls every ref's assertions", @@ -727,7 +733,7 @@ entities: type: "table", assertions: { rowConditions: ["id > 0"] } } -select 1 as id` +select 1 as id`, }, { name: "graph.yaml", @@ -739,32 +745,32 @@ entities: ref: books keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "GraphAssertDefaultGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "GraphAssertDefaultGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { database: "defaultProject", schema: "defaultDataset", - name: "books" + name: "books", }, { database: "defaultProject", schema: "defaultDataset", - name: "defaultDataset_books_assertions_rowConditions" - } + name: "defaultDataset_books_assertions_rowConditions", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -775,15 +781,15 @@ entities: dataSource: { schema: "defaultDataset", name: "books", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: - "NODE TABLES (\n" + " `defaultProject.defaultDataset.books` AS Book KEY (id)\n" + ")" - } - ] + "NODE TABLES (\n" + " `defaultProject.defaultDataset.books` AS Book KEY (id)\n" + ")", + }, + ], }, { testName: "missing ref emits a compilation error and leaves graphBody empty", @@ -798,8 +804,8 @@ entities: ref: nonexistent keys: - id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -813,10 +819,10 @@ entities: "which does not exist", { actionName: "defaultProject.defaultDataset.MissingRefGraph", - actionTarget: missingRefTarget - } - ) - ] + actionTarget: missingRefTarget, + }, + ), + ], }, dataformCoreVersion: version, targets: [missingRefTarget], @@ -831,12 +837,12 @@ entities: entities: [ { name: "Book", - keys: ["id"] - } - ] - } - ] - } + keys: ["id"], + }, + ], + }, + ], + }, }, { testName: "ref to a table respects datasetSuffix on the resolved dependency", @@ -850,7 +856,7 @@ datasetSuffix: dev { name: "books.sqlx", contents: `config {type: "table"} -select 1 as id` +select 1 as id`, }, { name: "graph.yaml", @@ -861,27 +867,27 @@ entities: ref: books keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset_dev", name: "SuffixRefGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "SuffixRefGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { schema: "defaultDataset_dev", name: "books", - database: "defaultProject" - } + database: "defaultProject", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -892,17 +898,17 @@ entities: dataSource: { schema: "defaultDataset_dev", name: "books", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset_dev.books` AS Book KEY (id)\n" + - ")" - } - ] + ")", + }, + ], }, { testName: "ref with database override resolves the matching declaration", @@ -917,7 +923,7 @@ actions: project: otherProject - declaration: name: books -` +`, }, { name: "graph.yaml", @@ -930,27 +936,27 @@ entities: database: otherProject keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "RefWithDatabaseGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "RefWithDatabaseGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { database: "otherProject", schema: "defaultDataset", - name: "books" - } + name: "books", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -961,15 +967,15 @@ entities: dataSource: { database: "otherProject", schema: "defaultDataset", - name: "books" + name: "books", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: - "NODE TABLES (\n" + " `otherProject.defaultDataset.books` AS Book KEY (id)\n" + ")" - } - ] + "NODE TABLES (\n" + " `otherProject.defaultDataset.books` AS Book KEY (id)\n" + ")", + }, + ], }, { testName: "relationship ref resolves through the full pipeline", @@ -981,7 +987,7 @@ entities: actions: - declaration: name: wrote -` +`, }, { name: "graph.yaml", @@ -1010,27 +1016,27 @@ relationships: entity: Author joinKeys: - author_id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "RelationshipRefGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "RelationshipRefGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { database: "defaultProject", schema: "defaultDataset", - name: "wrote" - } + name: "wrote", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -1041,19 +1047,19 @@ relationships: dataSource: { database: "defaultProject", schema: "defaultDataset", - name: "books" + name: "books", }, - keys: ["id"] + keys: ["id"], }, { name: "Author", dataSource: { database: "defaultProject", schema: "defaultDataset", - name: "authors" + name: "authors", }, - keys: ["id"] - } + keys: ["id"], + }, ], relationships: [ { @@ -1061,20 +1067,20 @@ relationships: dataSource: { database: "defaultProject", schema: "defaultDataset", - name: "wrote" + name: "wrote", }, keys: ["author_id", "book_id"], source: { entity: "Book", relationshipColumns: ["book_id"], - entityColumns: ["id"] + entityColumns: ["id"], }, destination: { entity: "Author", relationshipColumns: ["author_id"], - entityColumns: ["id"] - } - } + entityColumns: ["id"], + }, + }, ], graphBody: "NODE TABLES (\n" + @@ -1086,9 +1092,9 @@ relationships: "KEY (author_id, book_id) " + "SOURCE KEY (book_id) REFERENCES Book (id) " + "DESTINATION KEY (author_id) REFERENCES Author (id)\n" + - ")" - } - ] + ")", + }, + ], }, { testName: "ref to a view resolves and picks up datasetSuffix", @@ -1102,7 +1108,7 @@ datasetSuffix: dev { name: "books.sqlx", contents: `config {type: "view"} -select 1 as id` +select 1 as id`, }, { name: "graph.yaml", @@ -1113,27 +1119,27 @@ entities: ref: books keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset_dev", name: "ViewRefGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "ViewRefGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { database: "defaultProject", schema: "defaultDataset_dev", - name: "books" - } + name: "books", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -1144,17 +1150,17 @@ entities: dataSource: { database: "defaultProject", schema: "defaultDataset_dev", - name: "books" + name: "books", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset_dev.books` AS Book KEY (id)\n" + - ")" - } - ] + ")", + }, + ], }, { testName: "ambiguous ref emits a compilation error", @@ -1170,7 +1176,7 @@ actions: - declaration: name: books dataset: two -` +`, }, { name: "graph.yaml", @@ -1181,8 +1187,8 @@ entities: ref: books keys: - id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -1194,17 +1200,17 @@ entities: "Did you mean one of: one.books, two.books.", { actionName: "defaultProject.defaultDataset.AmbiguousRefGraph", - actionTarget: graphTarget - } - ) - ] + actionTarget: graphTarget, + }, + ), + ], }, dataformCoreVersion: version, targets: [declOneTarget, declTwoTarget, graphTarget], jitData: {}, declarations: [ { target: declOneTarget, canonicalTarget: declOneTarget }, - { target: declTwoTarget, canonicalTarget: declTwoTarget } + { target: declTwoTarget, canonicalTarget: declTwoTarget }, ], propertyGraphs: [ { @@ -1213,10 +1219,10 @@ entities: fileName: "definitions/graph.yaml", description: "", disabled: false, - entities: [{ name: "Book", keys: ["id"] }] - } - ] - } + entities: [{ name: "Book", keys: ["id"] }], + }, + ], + }, }, { testName: "ref to a table respects projectSuffix on the resolved dependency", @@ -1230,7 +1236,7 @@ projectSuffix: dev { name: "books.sqlx", contents: `config {type: "table"} -select 1 as id` +select 1 as id`, }, { name: "graph.yaml", @@ -1241,27 +1247,27 @@ entities: ref: books keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "ProjectSuffixRefGraph", - database: "defaultProject_dev" + database: "defaultProject_dev", }, canonicalTarget: { schema: "defaultDataset", name: "ProjectSuffixRefGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { schema: "defaultDataset", name: "books", - database: "defaultProject_dev" - } + database: "defaultProject_dev", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -1272,17 +1278,17 @@ entities: dataSource: { schema: "defaultDataset", name: "books", - database: "defaultProject_dev" + database: "defaultProject_dev", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject_dev.defaultDataset.books` AS Book KEY (id)\n" + - ")" - } - ] + ")", + }, + ], }, { testName: "ref to a table respects namePrefix on the resolved dependency", @@ -1296,7 +1302,7 @@ namePrefix: pfx { name: "books.sqlx", contents: `config {type: "table"} -select 1 as id` +select 1 as id`, }, { name: "graph.yaml", @@ -1307,27 +1313,27 @@ entities: ref: books keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "pfx_NamePrefixRefGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "NamePrefixRefGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ { schema: "defaultDataset", name: "pfx_books", - database: "defaultProject" - } + database: "defaultProject", + }, ], fileName: "definitions/graph.yaml", description: "", @@ -1338,17 +1344,17 @@ entities: dataSource: { schema: "defaultDataset", name: "pfx_books", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset.pfx_books` AS Book KEY (id)\n" + - ")" - } - ] + ")", + }, + ], }, { testName: "graph target colliding with a table target is flagged as duplicate", @@ -1357,7 +1363,7 @@ entities: { name: "collision.sqlx", contents: `config {type: "table", name: "CollisionName"} -select 1 as a` +select 1 as a`, }, { name: "graph.yaml", @@ -1368,8 +1374,8 @@ entities: dataSourceString: defaultProject.defaultDataset.customers keys: - id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -1377,26 +1383,26 @@ entities: compilationErrors: [ graphError("definitions/collision.sqlx", duplicateActionMessage, { actionName: collisionActionName, - actionTarget: collisionTarget + actionTarget: collisionTarget, }), graphError("definitions/collision.sqlx", duplicateCanonicalMessage, { actionName: collisionActionName, - actionTarget: collisionTarget + actionTarget: collisionTarget, }), graphError("definitions/graph.yaml", duplicateActionMessage, { actionName: collisionActionName, - actionTarget: collisionTarget + actionTarget: collisionTarget, }), graphError("definitions/graph.yaml", duplicateCanonicalMessage, { actionName: collisionActionName, - actionTarget: collisionTarget - }) - ] + actionTarget: collisionTarget, + }), + ], }, dataformCoreVersion: version, targets: [collisionTarget, collisionTarget], - jitData: {} - } + jitData: {}, + }, }, { testName: "graph.yaml accepts snake_case keys per BQ spec", @@ -1432,8 +1438,8 @@ relationships: join_keys: relationship_columns: - owned_id -` - } +`, + }, ], expectedGraph: { projectConfig: graphProjectConfig, @@ -1446,12 +1452,12 @@ relationships: target: { schema: "defaultDataset", name: "SnakeGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "SnakeGraph", - database: "defaultProject" + database: "defaultProject", }, fileName: "definitions/graph.yaml", description: "end to end snake case", @@ -1462,7 +1468,7 @@ relationships: dataSource: { schema: "defaultDataset", name: "accounts", - database: "defaultProject" + database: "defaultProject", }, keys: ["id"], labels: [ @@ -1471,10 +1477,10 @@ relationships: description: "", importAll: true, importExcept: ["secret"], - isDefault: true - } - ] - } + isDefault: true, + }, + ], + }, ], relationships: [ { @@ -1482,19 +1488,19 @@ relationships: dataSource: { schema: "defaultDataset", name: "ownership", - database: "defaultProject" + database: "defaultProject", }, source: { entity: "Account", relationshipColumns: ["owner_id"], - entityColumns: ["id"] + entityColumns: ["id"], }, destination: { entity: "Account", relationshipColumns: ["owned_id"], - entityColumns: ["id"] - } - } + entityColumns: ["id"], + }, + }, ], graphBody: "NODE TABLES (\n" + @@ -1505,10 +1511,10 @@ relationships: " `defaultProject.defaultDataset.ownership` AS Owns " + "SOURCE KEY (owner_id) REFERENCES Account (id) " + "DESTINATION KEY (owned_id) REFERENCES Account (id)\n" + - ")" - } - ] - } + ")", + }, + ], + }, }, { testName: "mixed ref and dataSourceString: only ref target appears in dependencyTargets", @@ -1517,12 +1523,12 @@ relationships: { name: "books.sqlx", contents: `config {type: "table"} -select 1 as id` +select 1 as id`, }, { name: "authors.sqlx", contents: `config {type: "table"} -select 1 as id` +select 1 as id`, }, { name: "graph.yaml", @@ -1537,23 +1543,23 @@ entities: dataSourceString: defaultProject.defaultDataset.authors keys: - id -` - } +`, + }, ], expectedPropertyGraphs: [ { target: { schema: "defaultDataset", name: "MixedRefStringGraph", - database: "defaultProject" + database: "defaultProject", }, canonicalTarget: { schema: "defaultDataset", name: "MixedRefStringGraph", - database: "defaultProject" + database: "defaultProject", }, dependencyTargets: [ - { database: "defaultProject", schema: "defaultDataset", name: "books" } + { database: "defaultProject", schema: "defaultDataset", name: "books" }, ], fileName: "definitions/graph.yaml", description: "", @@ -1564,35 +1570,35 @@ entities: dataSource: { schema: "defaultDataset", name: "books", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] + keys: ["id"], }, { name: "Author", dataSource: { schema: "defaultDataset", name: "authors", - database: "defaultProject" + database: "defaultProject", }, - keys: ["id"] - } + keys: ["id"], + }, ], graphBody: "NODE TABLES (\n" + " `defaultProject.defaultDataset.books` AS Book KEY (id),\n" + " `defaultProject.defaultDataset.authors` AS Author KEY (id)\n" + - ")" - } - ] - } + ")", + }, + ], + }, ]; - - testCases.forEach(testParameters => { + + testCases.forEach((testParameters) => { test(testParameters.testName, () => { const projectDir = tmpDirFixture.createNewTmpDir(); writeWorkflowSettingsFile(projectDir, testParameters.workflowSettings); - testParameters.definitionFiles.forEach(file => { + testParameters.definitionFiles.forEach((file) => { writeDefinitionFile(projectDir, file.name, file.contents); }); @@ -1600,19 +1606,19 @@ entities: if (!testParameters.expectedGraph && !testParameters.expectedPropertyGraphs) { throw new Error( - `Test case "${testParameters.testName}" must specify either expectedGraph or expectedPropertyGraphs` + `Test case "${testParameters.testName}" must specify either expectedGraph or expectedPropertyGraphs`, ); } if (testParameters.expectedGraph) { - expect(asPlainObject(result.compile?.compiledGraph)).deep.equals( - asPlainObject(testParameters.expectedGraph) + expect(asPlainGraph(result.compile?.compiledGraph)).deep.equals( + asPlainObject(testParameters.expectedGraph), ); } if (testParameters.expectedPropertyGraphs) { expect(result.compile?.compiledGraph?.graphErrors?.compilationErrors).deep.equals([]); expect(asPlainObject(result.compile?.compiledGraph?.propertyGraphs)).deep.equals( - asPlainObject(testParameters.expectedPropertyGraphs) + asPlainObject(testParameters.expectedPropertyGraphs), ); } }); @@ -1639,15 +1645,15 @@ entities: filePaths: [ "workflow_settings.yaml", "definitions/graph.yaml", - "definitions/subdir/graph.yaml" - ] - } - } + "definitions/subdir/graph.yaml", + ], + }, + }, }); const result = runMainInVm(request); - expect(asPlainObject(result.compile?.compiledGraph)).deep.equals( + expect(asPlainGraph(result.compile?.compiledGraph)).deep.equals( asPlainObject({ projectConfig: graphProjectConfig, graphErrors: { @@ -1656,13 +1662,13 @@ entities: "definitions/graph.yaml", "At most one graph.yaml is allowed per project (found 2: " + "definitions/graph.yaml, definitions/subdir/graph.yaml). This " + - "restriction may be relaxed in a future version." - ) - ] + "restriction may be relaxed in a future version.", + ), + ], }, dataformCoreVersion: version, - jitData: {} - }) + jitData: {}, + }), ); }); }); diff --git a/package.json b/package.json index 2c0d939aa..db3dd941e 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,6 @@ "uglify-js": "^3.7.7", "untildify": "^4.0.0", "url": "^0.11.0", - "vm2": "3.11.6", "vsce": "^1.79.5", "vscode-jsonrpc": "^5.0.1", "vscode-languageclient": "^6.1.3", diff --git a/packages/@dataform/cli/BUILD b/packages/@dataform/cli/BUILD index 31309c581..cf4eea43b 100644 --- a/packages/@dataform/cli/BUILD +++ b/packages/@dataform/cli/BUILD @@ -51,7 +51,6 @@ externals = [ "tmp", "typeid-js", "untildify", - "vm2", "yargs", ] diff --git a/packages/rollup.config.js b/packages/rollup.config.js index 3ac33cdec..a88828070 100644 --- a/packages/rollup.config.js +++ b/packages/rollup.config.js @@ -20,7 +20,9 @@ const knownNodeBuiltins = [ "events", "long", "https", - "net" + "net", + "module", + "vm" ].map(moduleName => convertToRegex(moduleName)); const importsToBundle = ["df", /df\/.*$/, /^bazel\-.*$/]; diff --git a/testing/BUILD b/testing/BUILD index ad2d9cd6a..571e48efe 100644 --- a/testing/BUILD +++ b/testing/BUILD @@ -41,12 +41,12 @@ ts_library( ], deps = [ "//common/protos", + "//common/vm:vm_runner", "//core", "//protos:ts", "@npm//@types/fs-extra", "@npm//@types/node", "@npm//fs-extra", - "@npm//vm2", ], ) diff --git a/testing/run_core.ts b/testing/run_core.ts index f12c77a18..3c5f79a13 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -1,8 +1,8 @@ import * as fs from "fs-extra"; import * as path from "path"; -import { CompilerFunction, NodeVM } from "vm2"; import { decode64, encode64 } from "df/common/protos"; +import { VmRunner } from "df/common/vm/vm_runner"; import { compile } from "df/core/compilers"; import { dataform } from "df/protos/ts"; @@ -23,35 +23,35 @@ defaultLocation: US export class WorkflowSettingsTemplates { public static bigquery = dataform.WorkflowSettings.create({ defaultDataset: "defaultDataset", - defaultLocation: "US" + defaultLocation: "US", }); public static bigqueryWithDefaultProject = dataform.WorkflowSettings.create({ ...WorkflowSettingsTemplates.bigquery, - defaultProject: "defaultProject" + defaultProject: "defaultProject", }); public static bigqueryWithDatasetSuffix = dataform.WorkflowSettings.create({ ...WorkflowSettingsTemplates.bigquery, - datasetSuffix: "suffix" + datasetSuffix: "suffix", }); public static bigqueryWithDefaultProjectAndDataset = dataform.WorkflowSettings.create({ ...WorkflowSettingsTemplates.bigqueryWithDefaultProject, - projectSuffix: "suffix" + projectSuffix: "suffix", }); public static bigqueryWithNamePrefix = dataform.WorkflowSettings.create({ ...WorkflowSettingsTemplates.bigquery, - namePrefix: "prefix" + namePrefix: "prefix", }); } -const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb","md"]; +const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb", "md"]; export function coreExecutionRequestFromPath( projectDir: string, - projectConfigOverride?: dataform.ProjectConfig + projectConfigOverride?: dataform.ProjectConfig, ): dataform.CoreExecutionRequest { const resolvedProjectDir = fs.realpathSync(path.resolve(projectDir)); return dataform.CoreExecutionRequest.create({ @@ -59,45 +59,41 @@ export function coreExecutionRequestFromPath( compileConfig: { projectDir: resolvedProjectDir, filePaths: walkDirectoryForFilenames(resolvedProjectDir), - projectConfigOverride - } - } + projectConfigOverride, + }, + }, }); } // A VM is needed when running main because Node functions like `require` are overridden. export function runMainInVm( - coreExecutionRequest: dataform.CoreExecutionRequest + coreExecutionRequest: dataform.CoreExecutionRequest, ): dataform.CoreExecutionResponse { const projectDir = coreExecutionRequest.compile.compileConfig.projectDir; // Copy over the build Dataform Core that is set up as a node_modules directory. fs.copySync(`${process.cwd()}/core/node_modules`, `${projectDir}/node_modules`); - const compiler = compile as CompilerFunction; + const compiler = compile; // See cli/vm/compile.ts for why we use a host-side stack + enter/exit helpers // instead of writing to `global.__dataform_current_file` inside every module. const fileStack: string[] = []; - // Then use vm2's native compiler integration to apply the compiler to files. - const nodeVm = new NodeVM({ + const vmRunner = new VmRunner({ + projectDir, // Inheriting the console makes console.logs show when tests are running, which is useful for // debugging. console: "inherit", - wrapper: "none", sandbox: { - __df_enter: (p: string) => { fileStack.push(p); }, - __df_exit: () => { fileStack.pop(); }, - __df_current: () => fileStack.length > 0 ? fileStack[fileStack.length - 1] : null - }, - require: { - builtin: ["path"], - context: "sandbox", - external: true, - root: projectDir, - resolve: (moduleName, parentDirName) => - path.join(parentDirName, path.relative(parentDirName, projectDir), moduleName) + __df_enter: (p: string) => { + fileStack.push(p); + }, + __df_exit: () => { + fileStack.pop(); + }, + __df_current: () => (fileStack.length > 0 ? fileStack[fileStack.length - 1] : null), }, + builtinModules: ["path"], sourceExtensions: SOURCE_EXTENSIONS, compiler: (code, filePath) => { const compiledCode = compiler(code, filePath); @@ -109,22 +105,25 @@ export function runMainInVm( __df_exit(); } `; - } + }, }); + const hasWorkflowSettingsYaml = fs.existsSync(path.join(projectDir, "workflow_settings.yaml")); + const hasDataformJson = fs.existsSync(path.join(projectDir, "dataform.json")); + const encodedCoreExecutionRequest = encode64(dataform.CoreExecutionRequest, coreExecutionRequest); const vmIndexFileName = path.resolve(path.join(projectDir, "index.js")); - const encodedCoreExecutionResponse = nodeVm.run( + const encodedCoreExecutionResponse = vmRunner.run( ` Object.defineProperty(global, '__dataform_current_file', { configurable: true, get: function() { return __df_current(); } }); - global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed run_core:", e); } })(); - global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })(); + ${hasWorkflowSettingsYaml ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' : ""} + ${hasDataformJson ? 'global.dataformJson = require("./dataform.json");' : ""} return require("@dataform/core").main("${encodedCoreExecutionRequest}") `, - vmIndexFileName + vmIndexFileName, ); return decode64(dataform.CoreExecutionResponse, encodedCoreExecutionResponse); } @@ -132,8 +131,8 @@ export function runMainInVm( function walkDirectoryForFilenames(projectDir: string, relativePath: string = ""): string[] { let paths: string[] = []; fs.readdirSync(path.join(projectDir, relativePath), { withFileTypes: true }) - .filter(directoryEntry => directoryEntry.name !== "node_modules") - .forEach(directoryEntry => { + .filter((directoryEntry) => directoryEntry.name !== "node_modules") + .forEach((directoryEntry) => { if (directoryEntry.isDirectory()) { paths = paths.concat(walkDirectoryForFilenames(projectDir, directoryEntry.name)); return; @@ -143,5 +142,5 @@ function walkDirectoryForFilenames(projectDir: string, relativePath: string = "" paths.push(directoryEntry.name); } }); - return paths.map(filename => path.join(relativePath, filename)); + return paths.map((filename) => path.join(relativePath, filename)); } diff --git a/yarn.lock b/yarn.lock index c6125b440..ce7c5a42d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -847,19 +847,12 @@ acorn-jsx@^5.3.1, acorn-jsx@^5.3.2: resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity "sha1-ftW7VZCLOy8bxVxq8WU7rafweTc= sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==" -acorn-walk@^8.3.4: - version "8.3.4" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.4.tgz#794dd169c3977edf4ba4ea47583587c5866236b7" - integrity sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g== - dependencies: - acorn "^8.11.0" - acorn@^7.4.0: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity "sha1-/q7SVZc9LndVW4PbwIhRpsY1IPo= sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==" -acorn@^8.11.0, acorn@^8.15.0, acorn@^8.7.1, acorn@^8.8.2, acorn@^8.9.0: +acorn@^8.15.0, acorn@^8.7.1, acorn@^8.8.2, acorn@^8.9.0: version "8.17.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.17.0.tgz#1785adb84faf8d8add10369b93826fc2bd08f1fe" integrity sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg== @@ -4577,14 +4570,6 @@ uuidv7@^0.4.4: resolved "https://registry.yarnpkg.com/uuidv7/-/uuidv7-0.4.4.tgz#e7ffd7981f590c478fb8868eff4bb3bc55fa90e6" integrity sha512-jjRGChg03uGp9f6wQYSO8qXkweJwRbA5WRuEQE8xLIiehIzIIi23qZSzsyvZPCPoFqkeLtZuz7Plt1LGukAInA== -vm2@3.11.6: - version "3.11.6" - resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.11.6.tgz#044ddbbd68c0157bc07b2e2fca20f38d3d673be7" - integrity sha512-35hVTcKieg7jJMntHhgWT5c2a1J2vmpXm66Xs1Z8ayHOJTj8rzIlXKzba3nO1Om/sonmnkjlAOMt9p8lYJXsWw== - dependencies: - acorn "^8.15.0" - acorn-walk "^8.3.4" - vsce@^1.79.5: version "1.88.0" resolved "https://registry.yarnpkg.com/vsce/-/vsce-1.88.0.tgz#748dc9f75996d97a5953408848c56c4c1b4dca6b"