From c3ddc1bc6ba843fd6dfadf01ca990fb2058a1749 Mon Sep 17 00:00:00 2001 From: Keber Flores <6089594+keber@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:53:42 -0300 Subject: [PATCH 1/3] feat(upgrade): carry the v1.12.0 scaffold into existing projects upgrade force-copied only .github/skills/, copilot-instructions.md and QA-STRUCTURE-GUIDE.md, so everything v1.12.0 changed under templates/ reached new projects through init.js alone. Existing installations kept the old scaffold, including the login that types the password with fill(). Section 9 applies the principle per file: overwrite what is framework-owned, warn and continue where overwriting would destroy working user code, and treat a dependency group as all-or-nothing. playwright.config.ts and global-setup.ts both require() e2e/scripts/*.js, so those five files install together or not at all - a partial install leaves a project that cannot start Playwright. Skipping the group is a consistent state; the project keeps working on what it has. Two assumptions did not survive contact with the live projects. The lane scripts are not new files free of user edits: two projects already ship hand-written lane-lock.js with their own tests, 400+ lines each. And frameworkVersion cannot select a baseline - the recorded values are 1.10.0, 1.1.3, 1.1.3 and 1.0.0, two of them malformed. Detection is content-hash based instead, over every hash shipped since v1.8.0 plus a runtime hash of the current template, so a freshly scaffolded project is not flagged user-owned and the table does not go stale. Files left user-owned are scanned for .fill(password) and reported with file and line. No textual patch: the shapes differ across projects (passwordSelector vs PASSWORD_SEL) and a wrong edit to a working login is the breakage this avoids. Verified against a copy of the most customised project: its base.ts stayed byte-identical while auth.ts and test-helpers.ts upgraded, the lane group was withheld with an explanation, and trace safety reported base.ts:156. --- scripts/upgrade.js | 308 ++++++++++++++++++++++++++++++++++++- test/upgrade.test.js | 356 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 663 insertions(+), 1 deletion(-) create mode 100644 test/upgrade.test.js diff --git a/scripts/upgrade.js b/scripts/upgrade.js index 7794883..6773790 100644 --- a/scripts/upgrade.js +++ b/scripts/upgrade.js @@ -27,7 +27,37 @@ * - qa/02-test-plans/manual/ -> qa/02-test-plans/sprints/legacy-manual/ * - qa/02-test-plans/*.md -> qa/02-test-plans/sprints/legacy/ * - Creates qa/02-test-plans/sprints/ if absent - * - Creates qa/03-test-cases/README.md optional marker if absent * + * - Creates qa/03-test-cases/README.md optional marker if absent + * + * MIGRATES (v1.11.x -> v1.12.0, lane-aware automation scaffold): + * Installs the lane scaffold only where it cannot break working user code. + * A file counts as framework-owned ("pristine") when its normalised sha256 + * matches one this framework actually shipped; otherwise it is user-owned. + * + * ATOMIC GROUP (all installed, or none - config/global-setup require() the scripts): + * - qa/07-automation/e2e/scripts/lane-config.js (new in v1.12.0) + * - qa/07-automation/e2e/scripts/lane-lock.js (new in v1.12.0) + * - qa/07-automation/e2e/scripts/global-setup-guards.js (new in v1.12.0) + * - qa/07-automation/e2e/playwright.config.ts + * - qa/07-automation/e2e/global-setup.ts + * If any member is user-owned the whole group is skipped with a warning and + * the upgrade continues; the project keeps its working setup rather than + * being left half-migrated. + * + * PER FILE (leaf modules - a mixed state is still coherent): + * - qa/07-automation/e2e/fixtures/{auth,base,test-helpers}.ts + * - qa/07-automation/integration/{playwright.config.ts,global-setup.ts} + * - .env.example in both scaffolds (reference template only; your .env + * is never touched) + * Pristine -> overwritten. User-owned -> kept, with a warning. + * + * REPORTS only (never patched): + * - .fill(password) in any .ts under e2e/ or integration/ - leaks the + * password into Playwright traces. See "Pattern 7" in + * .github/skills/qa-automation/references/patterns.md. Not auto-fixed: + * the surrounding code differs per project and a bad rewrite of a + * working login is worse than the finding. + * * NEVER touches (project-owned): * - qa/01-specifications/ <- Your specs * - qa/02-test-plans/ <- Your test plans @@ -73,6 +103,52 @@ const githubDir = path.join(cwd, '.github'); const skillsDest = path.join(githubDir, 'skills'); const skillsSrc = path.resolve(__dirname, '..', 'skills'); +// Normalised sha256 of every version of these scaffold files that this framework has +// shipped (v1.8.0 through v1.11.3). A project file matching one of these was written +// by the framework and never edited, so v1.12.0 may safely replace it. Anything else +// is user-owned. Regenerate by hashing `git show :templates/` for each tag. +// +// Deliberately hash-based rather than version-based: the frameworkVersion recorded in +// real installations is unreliable (observed values include 1.1.3 and 1.0.0, neither +// of which corresponds to a released scaffold). +const SHIPPED_SCAFFOLD_HASHES = { + 'automation-scaffold/playwright.config.ts': [ + '948635c118553e293447930ba3dcff9155446718554dd3bbcffac2720baebba1', + 'f145fda0a7215942b5431a779188609f8958e51a048b161ede605576efba3ef8', + ], + 'automation-scaffold/global-setup.ts': [ + '077580914881ee71ac082d47579dc4cffa01e8348b0744fced6b8de2863e0bd1', + '7b3e7ca54929e525055d19ba8cd9643de722c7092c67b1a2e273072dd712ec7a', + ], + 'automation-scaffold/.env.example': [ + '18fab7c165085a4a585a6911b42c49b97177265d46261c8362c3508a7436c65c', + '29dc3a97e42350fbe7a16681837ba8db2a539574f8aa86fe269204bc3081bff9', + ], + 'automation-scaffold/fixtures/auth.ts': [ + 'a856df26d80d521a0ad349879096698191ed6a2f919fac96d85980ab1d435c3e', + ], + 'automation-scaffold/fixtures/base.ts': [ + '7c52ea17fad732825e327af59a7f0f3bf53db166f7191932ae51c97fd2825a39', + ], + 'automation-scaffold/fixtures/test-helpers.ts': [ + 'e2298972dea50e7df54265a7a94a232c0ff35c969f1b8778929079cfa2a1a541', + ], + // Introduced in v1.12.0 - never shipped before, so an existing copy in a project is + // always user-authored (two live installations hand-wrote their own lane-lock.js). + 'automation-scaffold/scripts/lane-config.js': [], + 'automation-scaffold/scripts/lane-lock.js': [], + 'automation-scaffold/scripts/global-setup-guards.js': [], + 'integration-scaffold/playwright.config.ts': [ + '7276ed4fda85f80e37381257bc559f4bb48055d359858f69f6b838f3671a9e2d', + ], + 'integration-scaffold/global-setup.ts': [ + 'f63f9588c4746a16e46345d1d4b781f2905b652481d563cfc075d9d8c8481bde', + ], + 'integration-scaffold/.env.example': [ + '671461d7dc9883505cec314aefa1129a336b6a03b31a0eb2eb2ede2adda7c4ac', + ], +}; + const updated = []; const skipped = []; const warnings = []; @@ -471,6 +547,146 @@ if (fs.existsSync(memoryDir)) { } } + +// --------------------------------------------------------------------------- +// 9. Migration v1.11.x -> v1.12.0: lane-aware automation scaffold +// +// v1.12.0 rewrote the automation scaffold around a lane table: playwright.config.ts +// and global-setup.ts now require() three CommonJS helpers under e2e/scripts/. +// init.js installs those for NEW projects only, so existing installations never +// receive them. This section carries them across. +// +// Classification per file (the governing rule is: never break working user code): +// +// a) Lane scripts (scripts/lane-config.js, lane-lock.js, global-setup-guards.js) +// plus playwright.config.ts and global-setup.ts form ONE ATOMIC GROUP, +// because config/global-setup require() the scripts by path. Installing a +// subset yields a project where Playwright cannot start at all. The group is +// installed only when EVERY member is either absent or provably pristine; +// otherwise the whole group is skipped with a warning and the rest of the +// upgrade continues. This is a coherent skip, not an abort: the project keeps +// the working setup it already had, so it is never left half-migrated. +// +// b) fixtures/{auth,base,test-helpers}.ts are per-file. They are leaf modules - +// nothing in the scaffold require()s them by path - so a mixed state is +// still coherent. +// +// c) .env.example is a reference template, never live config (the real file is +// .env, which this migration never touches). Overwritten when pristine. +// +// "Pristine" means the file's normalised sha256 matches a hash this framework +// actually shipped in some earlier release (SHIPPED_SCAFFOLD_HASHES). Anything +// else is user-owned and is only ever warned about. The project's recorded +// frameworkVersion is deliberately NOT used: live installations report values +// like 1.1.3 and 1.0.0 that do not correspond to any real scaffold release. +// +// Files that stay user-owned are additionally scanned for the trace-safety +// defect (.fill(password) leaks the password into Playwright traces). That is +// reported with file and line only - never patched, because the surrounding +// shapes differ per project (passwordSelector vs PASSWORD_SEL) and a bad regex +// edit to a working login is exactly the breakage this section must avoid. +// --------------------------------------------------------------------------- +const AUTOMATION_LANE_SCRIPTS = ['lane-config.js', 'lane-lock.js', 'global-setup-guards.js']; +const AUTOMATION_FIXTURES = ['auth.ts', 'base.ts', 'test-helpers.ts']; + +// A bare e2e/ directory is not evidence of an automation setup: section 5 above +// creates e2e/tests/{helpers/debug,seeds}/ unconditionally. Only migrate a project +// that actually has a scaffold, i.e. one of the files this migration would replace. +const hasAutomationScaffold = ['playwright.config.ts', 'global-setup.ts', 'package.json'] + .some((f) => fs.existsSync(path.join(e2eDir, f))); + +if (hasAutomationScaffold) { + const scaffoldSrc = path.resolve(__dirname, '..', 'templates', 'automation-scaffold'); + + // --- (a) the atomic lane group ------------------------------------------- + const groupMembers = [ + ...AUTOMATION_LANE_SCRIPTS.map((f) => ({ + rel: `scripts/${f}`, + src: path.join(scaffoldSrc, 'scripts', f), + dest: path.join(e2eDir, 'scripts', f), + key: `automation-scaffold/scripts/${f}`, + })), + ...['playwright.config.ts', 'global-setup.ts'].map((f) => ({ + rel: f, + src: path.join(scaffoldSrc, f), + dest: path.join(e2eDir, f), + key: `automation-scaffold/${f}`, + })), + ]; + + const blockers = groupMembers.filter((m) => fs.existsSync(m.dest) && !isPristine(m.dest, m.key)); + + if (blockers.length === 0) { + for (const m of groupMembers) { + forceWrite(m.dest, fs.readFileSync(m.src, 'utf8')); + console.log(` [updated] e2e/${m.rel}`); + } + } else { + warnings.push( + `Lane-aware automation scaffold (v1.12.0) NOT installed - these files are user-owned:\n` + + blockers.map((m) => ` e2e/${m.rel}`).join('\n') + + `\n playwright.config.ts and global-setup.ts require() e2e/scripts/*.js, so the\n` + + ` group is installed all-or-nothing. Your current setup is left working and\n` + + ` untouched. To adopt lanes, merge these by hand from:\n` + + ` node_modules/@keber/qa-framework/templates/automation-scaffold/` + ); + for (const m of groupMembers) skipped.push(m.dest); + } + + // --- (b) fixtures, per file ---------------------------------------------- + for (const file of AUTOMATION_FIXTURES) { + const dest = path.join(e2eDir, 'fixtures', file); + const key = `automation-scaffold/fixtures/${file}`; + if (!fs.existsSync(dest) || isPristine(dest, key)) { + forceWrite(dest, fs.readFileSync(path.join(scaffoldSrc, 'fixtures', file), 'utf8')); + console.log(` [updated] e2e/fixtures/${file}`); + } else { + skipped.push(dest); + warnings.push( + `e2e/fixtures/${file} is user-owned - kept as is (v1.12.0 version not applied).` + ); + } + } + + // --- (c) .env.example ------------------------------------------------------ + upgradeReferenceEnvExample( + path.join(e2eDir, '.env.example'), + path.join(scaffoldSrc, '.env.example'), + 'automation-scaffold/.env.example', + 'e2e/.env.example' + ); + + // --- trace safety on everything left user-owned --------------------------- + reportTraceSafety(e2eDir); +} + +// --- integration scaffold: no lane coupling, so purely per-file ------------- +// Same reasoning as above: require a real scaffold file, not just the folder. +const hasIntegrationScaffold = ['playwright.config.ts', 'global-setup.ts', 'package.json'] + .some((f) => fs.existsSync(path.join(integrationDir, f))); + +if (hasIntegrationScaffold) { + const integrationSrc = path.resolve(__dirname, '..', 'templates', 'integration-scaffold'); + for (const file of ['playwright.config.ts', 'global-setup.ts']) { + const dest = path.join(integrationDir, file); + const key = `integration-scaffold/${file}`; + if (!fs.existsSync(dest) || isPristine(dest, key)) { + forceWrite(dest, fs.readFileSync(path.join(integrationSrc, file), 'utf8')); + console.log(` [updated] integration/${file}`); + } else { + skipped.push(dest); + warnings.push(`integration/${file} is user-owned - kept as is (v1.12.0 version not applied).`); + } + } + upgradeReferenceEnvExample( + path.join(integrationDir, '.env.example'), + path.join(integrationSrc, '.env.example'), + 'integration-scaffold/.env.example', + 'integration/.env.example' + ); + reportTraceSafety(integrationDir); +} + // --------------------------------------------------------------------------- // Summary // --------------------------------------------------------------------------- @@ -521,3 +737,93 @@ function copyDirForce(srcDir, destDir) { } } + +// Normalised content hash: strips a UTF-8 BOM and CRLF so that a file which only +// differs by line endings or a BOM (both of which tooling rewrites silently) is +// still recognised as the framework's own output. +function scaffoldHash(content) { + const normalised = content.replace(/^/, '').replace(/\r\n/g, '\n'); + return require('crypto').createHash('sha256').update(normalised, 'utf8').digest('hex'); +} + +// A file is "pristine" when its content is byte-identical (after normalisation) to +// something this framework actually shipped. A key with no recorded hashes - a file +// introduced in the current release - can never be pristine, so an existing copy is +// always treated as user-owned. +function isPristine(filePath, key) { + let content; + try { + content = fs.readFileSync(filePath, 'utf8'); + } catch { + return false; + } + const hash = scaffoldHash(content); + + // Already identical to what this release ships: nothing to do, and certainly not a + // user edit. Computed rather than tabulated so the table never goes stale on release. + try { + const current = path.resolve(__dirname, '..', 'templates', ...key.split('/')); + if (fs.existsSync(current) && scaffoldHash(fs.readFileSync(current, 'utf8')) === hash) return true; + } catch { + /* fall through to the historical table */ + } + + const known = SHIPPED_SCAFFOLD_HASHES[key]; + return Boolean(known) && known.includes(hash); +} + +// .env.example is a reference template, never live configuration - the real values +// live in .env, which is never touched here. Overwrite it when pristine; when the +// user has edited it, keep theirs and say so. +function upgradeReferenceEnvExample(dest, src, key, label) { + if (!fs.existsSync(src)) return; + if (!fs.existsSync(dest) || isPristine(dest, key)) { + forceWrite(dest, fs.readFileSync(src, 'utf8')); + console.log(` [updated] ${label}`); + } else { + skipped.push(dest); + warnings.push( + `${label} is user-owned - kept as is. New v1.12.0 keys may be missing; compare against\n` + + ` node_modules/@keber/qa-framework/templates/${key}` + ); + } +} + +// Trace safety (see skills/qa-automation/references/patterns.md, "Pattern 7"): +// page.locator(...).fill(password) records the password in the Playwright trace. +// Report location only. The surrounding code differs per project, so an automated +// rewrite of a working login is more dangerous than the finding itself. +function reportTraceSafety(rootDir) { + const offenders = []; + const FILL_PASSWORD = /\.fill\(\s*(?:password|pwd|[A-Za-z_$]*(?:[Pp]ass(?:word)?|PASSWORD)[A-Za-z_$]*)\s*\)/; + + (function walk(dir) { + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(full); + } else if (/\.ts$/.test(entry.name)) { + const lines = fs.readFileSync(full, 'utf8').split(/\r?\n/); + lines.forEach((line, i) => { + if (FILL_PASSWORD.test(line)) offenders.push(`${path.relative(cwd, full)}:${i + 1}`); + }); + } + } + })(rootDir); + + if (offenders.length) { + warnings.push( + `Trace safety: the password is written with .fill(), which stores it in the\n` + + ` Playwright trace. Replace with the page.evaluate() form ("Pattern 7" in\n` + + ` .github/skills/qa-automation/references/patterns.md):\n` + + offenders.map((o) => ` ${o}`).join('\n') + ); + } +} diff --git a/test/upgrade.test.js b/test/upgrade.test.js new file mode 100644 index 0000000..eef65c4 --- /dev/null +++ b/test/upgrade.test.js @@ -0,0 +1,356 @@ +'use strict'; + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const crypto = require('node:crypto'); + +const REPO_ROOT = path.join(__dirname, '..'); +const INIT_PATH = path.join(REPO_ROOT, 'scripts', 'init.js'); +const UPGRADE_PATH = path.join(REPO_ROOT, 'scripts', 'upgrade.js'); + +const AUTOMATION_SRC = path.join(REPO_ROOT, 'templates', 'automation-scaffold'); +const INTEGRATION_SRC = path.join(REPO_ROOT, 'templates', 'integration-scaffold'); + +const LANE_SCRIPTS = ['lane-config.js', 'lane-lock.js', 'global-setup-guards.js']; + +function run(script, args, cwd) { + return spawnSync(process.execPath, [script, ...args], { + cwd, + env: { ...process.env, INIT_CWD: cwd }, + encoding: 'utf8', + }); +} + +const runInit = (cwd, args = []) => run(INIT_PATH, args, cwd); +const runUpgrade = (cwd, args = []) => run(UPGRADE_PATH, args, cwd); + +function tmpProject() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'qa-framework-upgrade-')); +} + +const e2e = (root) => path.join(root, 'qa', '07-automation', 'e2e'); +const integration = (root) => path.join(root, 'qa', '07-automation', 'integration'); + +/** Snapshot every file under a directory as relpath -> sha256. */ +function snapshot(dir) { + const out = {}; + if (!fs.existsSync(dir)) return out; + (function walk(d) { + for (const entry of fs.readdirSync(d, { withFileTypes: true })) { + const full = path.join(d, entry.name); + if (entry.isDirectory()) walk(full); + else { + out[path.relative(dir, full)] = crypto + .createHash('sha256') + .update(fs.readFileSync(full)) + .digest('hex'); + } + } + })(dir); + return out; +} + +/** + * Build a project that looks like a v1.11.3 installation: init it with the current + * templates, then overwrite the scaffold files with the versions git shipped at + * v1.11.3 and delete everything v1.12.0 introduced. + */ +function makeLegacyProject() { + const root = tmpProject(); + const init = runInit(root); + assert.equal(init.status, 0, init.stderr); + + for (const rel of [ + 'playwright.config.ts', + 'global-setup.ts', + '.env.example', + 'fixtures/auth.ts', + 'fixtures/base.ts', + 'fixtures/test-helpers.ts', + ]) { + const shipped = spawnSync( + 'git', + ['show', `v1.11.3:templates/automation-scaffold/${rel}`], + { cwd: REPO_ROOT, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 } + ); + assert.equal(shipped.status, 0, `git show failed for ${rel}: ${shipped.stderr}`); + const dest = path.join(e2e(root), ...rel.split('/')); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, shipped.stdout, 'utf8'); + } + + // v1.12.0 introduced the lane scripts; a v1.11.3 project has none. + fs.rmSync(path.join(e2e(root), 'scripts'), { recursive: true, force: true }); + return root; +} + +test('upgrade: a pristine v1.11.3 project receives the whole v1.12.0 lane group', () => { + const root = makeLegacyProject(); + try { + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + + for (const file of LANE_SCRIPTS) { + const dest = path.join(e2e(root), 'scripts', file); + assert.ok(fs.existsSync(dest), `${file} should have been installed`); + assert.equal( + fs.readFileSync(dest, 'utf8'), + fs.readFileSync(path.join(AUTOMATION_SRC, 'scripts', file), 'utf8') + ); + } + for (const file of ['playwright.config.ts', 'global-setup.ts']) { + assert.equal( + fs.readFileSync(path.join(e2e(root), file), 'utf8'), + fs.readFileSync(path.join(AUTOMATION_SRC, file), 'utf8'), + `${file} should have been updated to the v1.12.0 template` + ); + } + // The config is only usable if its require() target really landed. + assert.match( + fs.readFileSync(path.join(e2e(root), 'playwright.config.ts'), 'utf8'), + /require\('\.\/scripts\/lane-lock\.js'\)/ + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: pristine fixtures and .env.example are refreshed to v1.12.0', () => { + const root = makeLegacyProject(); + try { + assert.equal(runUpgrade(root).status, 0); + for (const file of ['auth.ts', 'base.ts', 'test-helpers.ts']) { + assert.equal( + fs.readFileSync(path.join(e2e(root), 'fixtures', file), 'utf8'), + fs.readFileSync(path.join(AUTOMATION_SRC, 'fixtures', file), 'utf8'), + `fixtures/${file} should have been refreshed` + ); + } + assert.equal( + fs.readFileSync(path.join(e2e(root), '.env.example'), 'utf8'), + fs.readFileSync(path.join(AUTOMATION_SRC, '.env.example'), 'utf8') + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: a customised fixture is kept and warned about, not clobbered', () => { + const root = makeLegacyProject(); + try { + // Replicate the QA_PortalProductores shape: a diverged base.ts using PASSWORD_SEL. + const custom = [ + "import { test as base } from '@playwright/test';", + "const PASSWORD_SEL = '#pwd';", + 'export async function login(page, password) {', + ' await page.locator(PASSWORD_SEL).fill(password);', + '}', + '// project-specific helper the framework must never remove', + ].join('\n'); + const target = path.join(e2e(root), 'fixtures', 'base.ts'); + fs.writeFileSync(target, custom, 'utf8'); + + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + + assert.equal(fs.readFileSync(target, 'utf8'), custom, 'customised base.ts must survive'); + assert.match(result.stdout, /fixtures\/base\.ts is user-owned/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: the lane group is all-or-nothing when one member is user-owned', () => { + const root = makeLegacyProject(); + try { + const configPath = path.join(e2e(root), 'playwright.config.ts'); + // No testDir/testIgnore here, so the older v1.6.0 patcher has nothing to touch + // and this test isolates the v1.12.0 group decision. + const custom = '// hand-tuned by the team\nexport default { fullyParallel: true };\n'; + fs.writeFileSync(configPath, custom, 'utf8'); + + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + + // The user's config survives... + assert.equal(fs.readFileSync(configPath, 'utf8'), custom); + // ...and because it did, none of the scripts it would need were installed either. + for (const file of LANE_SCRIPTS) { + assert.ok( + !fs.existsSync(path.join(e2e(root), 'scripts', file)), + `${file} must NOT be installed when the group is skipped` + ); + } + assert.equal( + fs + .readFileSync(path.join(e2e(root), 'global-setup.ts'), 'utf8') + .includes('./scripts/lane-lock.js'), + false, + 'global-setup.ts must not be upgraded to require scripts that were not installed' + ); + assert.match(result.stdout, /Lane-aware automation scaffold \(v1\.12\.0\) NOT installed/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: a hand-written lane-lock.js also blocks the whole group', () => { + const root = makeLegacyProject(); + try { + // QA_Nexus / QA_Sispro shape: their own lane-lock.js, no lane-config.js. + const scriptsDir = path.join(e2e(root), 'scripts'); + fs.mkdirSync(scriptsDir, { recursive: true }); + const mine = 'module.exports = { acquire() { /* our own implementation */ } };\n'; + fs.writeFileSync(path.join(scriptsDir, 'lane-lock.js'), mine, 'utf8'); + + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + + assert.equal(fs.readFileSync(path.join(scriptsDir, 'lane-lock.js'), 'utf8'), mine); + assert.ok(!fs.existsSync(path.join(scriptsDir, 'lane-config.js'))); + assert.match(result.stdout, /NOT installed/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: --dry-run reports the migration but writes nothing', () => { + const root = makeLegacyProject(); + try { + const before = snapshot(path.join(root, 'qa')); + const result = runUpgrade(root, ['--dry-run']); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(snapshot(path.join(root, 'qa')), before, 'dry-run must not change any file'); + assert.ok(!fs.existsSync(path.join(e2e(root), 'scripts', 'lane-lock.js'))); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: a project without qa/07-automation/e2e/ is skipped cleanly', () => { + const root = tmpProject(); + try { + assert.equal(runInit(root).status, 0); + fs.rmSync(path.join(root, 'qa', '07-automation'), { recursive: true, force: true }); + + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + + // The v1.12.0 migration must install none of the lane scaffold and must not warn + // about it. (Earlier sections still recreate some empty e2e/ subfolders; that is + // pre-existing v1.6.0 behaviour and deliberately not asserted here.) + for (const file of LANE_SCRIPTS) { + assert.ok(!fs.existsSync(path.join(e2e(root), 'scripts', file)), `${file} must not appear`); + } + assert.ok(!fs.existsSync(path.join(e2e(root), 'playwright.config.ts'))); + assert.ok(!fs.existsSync(path.join(e2e(root), 'fixtures', 'base.ts'))); + assert.doesNotMatch(result.stdout, /NOT installed/); + assert.doesNotMatch(result.stdout, /Trace safety/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: trace-safety warns on .fill(password) with file and line', () => { + const root = makeLegacyProject(); + try { + const target = path.join(e2e(root), 'fixtures', 'base.ts'); + fs.writeFileSync( + target, + [ + '// custom login', + 'export async function login(page, password) {', + ' await page.locator(PASSWORD_SEL).fill(password);', + '}', + ].join('\n'), + 'utf8' + ); + + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Trace safety/); + assert.match(result.stdout, /Pattern 7/); + assert.match(result.stdout, /fixtures[\\/]base\.ts:3/, 'must report the exact line'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: trace-safety does not fire on the safe page.evaluate() pattern', () => { + const root = makeLegacyProject(); + try { + const target = path.join(e2e(root), 'fixtures', 'base.ts'); + fs.writeFileSync( + target, + [ + 'export async function login(page, password) {', + ' await page.evaluate(([sel, pwd]) => {', + ' (document.querySelector(sel)).value = pwd;', + ' }, [PASSWORD_SEL, password]);', + '}', + ].join('\n'), + 'utf8' + ); + + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stdout, /Trace safety/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: a user-owned integration config is kept, a pristine one is refreshed', () => { + const root = makeLegacyProject(); + try { + const custom = '// our integration config\nexport default {};\n'; + fs.writeFileSync(path.join(integration(root), 'playwright.config.ts'), custom, 'utf8'); + + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + + assert.equal( + fs.readFileSync(path.join(integration(root), 'playwright.config.ts'), 'utf8'), + custom + ); + assert.match(result.stdout, /integration\/playwright\.config\.ts is user-owned/); + // global-setup.ts was left pristine, so it still gets the new version. + assert.equal( + fs.readFileSync(path.join(integration(root), 'global-setup.ts'), 'utf8'), + fs.readFileSync(path.join(INTEGRATION_SRC, 'global-setup.ts'), 'utf8') + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: a file already identical to the current template is not called user-owned', () => { + // init.js writes the CURRENT templates, so a freshly initialised project must not be + // reported as user-owned just because that hash is not in the historical table. + const root = tmpProject(); + try { + assert.equal(runInit(root).status, 0); + const result = runUpgrade(root); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stdout, /is user-owned/); + assert.doesNotMatch(result.stdout, /NOT installed/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('upgrade: is idempotent - a second run changes nothing further', () => { + const root = makeLegacyProject(); + try { + assert.equal(runUpgrade(root).status, 0); + const after1 = snapshot(path.join(root, 'qa')); + assert.equal(runUpgrade(root).status, 0); + assert.deepEqual(snapshot(path.join(root, 'qa')), after1); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); From f52257a72db320547c1d65d8b64245ae447f68e4 Mon Sep 17 00:00:00 2001 From: Keber Flores <6089594+keber@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:55:08 -0300 Subject: [PATCH 2/3] chore(release): 1.12.0 Minor, not major: a project with no parallelLanes config behaves exactly as before, PENDING-BROWSER adds an enum value without removing any, and the spec path change affects what init generates from now on, not what exists. The one user-visible change is engines moving from >=18 to >=20. Node 18 left maintenance in April 2025 and was never covered by CI, so this aligns the declared floor with what is actually verified. --- .github/copilot-instructions.md | 2 +- README.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index efd8e01..efd3733 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,6 +1,6 @@ ## About this repository -This is the **source code** of `@keber/qa-framework` v1.11.3 - a spec-driven, agent-oriented QA framework published to npm. +This is the **source code** of `@keber/qa-framework` v1.12.0 - a spec-driven, agent-oriented QA framework published to npm. ### What this repo is diff --git a/README.md b/README.md index 94f31c5..376bf36 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # qa-framework > Reusable, installable, agent-oriented QA framework for spec-driven automated testing. -> Current version: **v1.11.3** +> Current version: **v1.12.0** --- diff --git a/package-lock.json b/package-lock.json index 3cfe456..f9a351c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@keber/qa-framework", - "version": "1.11.3", + "version": "1.12.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@keber/qa-framework", - "version": "1.11.3", + "version": "1.12.0", "hasInstallScript": true, "license": "MIT", "bin": { diff --git a/package.json b/package.json index 4951365..ac630ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@keber/qa-framework", - "version": "1.11.3", + "version": "1.12.0", "description": "Reusable spec-driven QA framework for IDE-agent-assisted automated testing. Installable as an npm package. Provides structure, templates, agent instructions, and optional integrations for Playwright and Azure DevOps.", "keywords": [ "qa", From 01cbbbacf2ea166fadffda18db58e9b148e018f9 Mon Sep 17 00:00:00 2001 From: Keber Flores <6089594+keber@users.noreply.github.com> Date: Wed, 9 Sep 2026 00:38:27 -0300 Subject: [PATCH 3/3] test(upgrade): decouple legacy fixtures from git history makeLegacyProject() rebuilt a v1.11.3 installation by shelling out to `git show v1.11.3:templates/automation-scaffold/`. That coupled the suite to repository history and broke in GitHub Actions, where actions/checkout does a shallow clone that fetches no tags: 10 of 12 tests failed with "fatal: invalid object name 'v1.11.3'". It would equally break inside an npm pack tarball or any exported copy of the tree. Commit the six v1.11.3 scaffold files as static fixtures under test/fixtures/legacy-v1.11.3/ and copy from there instead. Bytes are unchanged, so every fixture still hashes into SHIPPED_SCAFFOLD_HASHES and the upgrade path under test is exercised exactly as before. No production logic changed. Add a guard test asserting each fixture is both in the shipped hash table and different from the current template. Without it, a drifted fixture would be read as user-owned and silently turn the "pristine file is refreshed" tests into vacuous passes. Exclude legacy-v1.11.3/ from check-forbidden-chars.js: the archived content contains em-dashes and arrows that must not be rewritten, since editing a byte changes the hash and voids the pristine match. --- scripts/check-forbidden-chars.js | 9 +- test/fixtures/legacy-v1.11.3/.env.example | 75 +++++++ test/fixtures/legacy-v1.11.3/fixtures/auth.ts | 77 +++++++ test/fixtures/legacy-v1.11.3/fixtures/base.ts | 108 ++++++++++ .../legacy-v1.11.3/fixtures/test-helpers.ts | 85 ++++++++ test/fixtures/legacy-v1.11.3/global-setup.ts | 196 ++++++++++++++++++ .../legacy-v1.11.3/playwright.config.ts | 122 +++++++++++ test/upgrade.test.js | 82 ++++++-- 8 files changed, 736 insertions(+), 18 deletions(-) create mode 100644 test/fixtures/legacy-v1.11.3/.env.example create mode 100644 test/fixtures/legacy-v1.11.3/fixtures/auth.ts create mode 100644 test/fixtures/legacy-v1.11.3/fixtures/base.ts create mode 100644 test/fixtures/legacy-v1.11.3/fixtures/test-helpers.ts create mode 100644 test/fixtures/legacy-v1.11.3/global-setup.ts create mode 100644 test/fixtures/legacy-v1.11.3/playwright.config.ts diff --git a/scripts/check-forbidden-chars.js b/scripts/check-forbidden-chars.js index 59aa924..60296ec 100644 --- a/scripts/check-forbidden-chars.js +++ b/scripts/check-forbidden-chars.js @@ -110,11 +110,18 @@ function scanFile(file) { return scanText(fs.readFileSync(file, 'utf8'), file); } +// Archived copies of content this framework shipped in an earlier release. Their bytes +// are the point: test/upgrade.test.js matches them against SHIPPED_SCAFFOLD_HASHES to +// prove a real legacy installation is recognised as pristine. Rewriting a character in +// one of them changes its hash and silently voids that proof, so they are read-only +// history rather than authored content and the rule does not apply to them. +const SKIP_DIRS = new Set(['node_modules', '.git', 'legacy-v1.11.3']); + function collect(target) { const stat = fs.statSync(target); if (stat.isFile()) return EXTENSIONS.includes(path.extname(target)) ? [target] : []; return fs.readdirSync(target).flatMap((entry) => { - if (entry === 'node_modules' || entry === '.git') return []; + if (SKIP_DIRS.has(entry)) return []; return collect(path.join(target, entry)); }); } diff --git a/test/fixtures/legacy-v1.11.3/.env.example b/test/fixtures/legacy-v1.11.3/.env.example new file mode 100644 index 0000000..337e9ba --- /dev/null +++ b/test/fixtures/legacy-v1.11.3/.env.example @@ -0,0 +1,75 @@ +# ----------------------------------------------------------------------- +# keber/qa-framework — Environment Variables Reference +# ----------------------------------------------------------------------- +# Copy this file to .env and fill in real values. +# NEVER commit .env to source control. +# ----------------------------------------------------------------------- + +# ----------------------------------------------------------------------- +# Application Under Test +# ----------------------------------------------------------------------- +QA_BASE_URL=https://your-app.qa.example.com + +# ----------------------------------------------------------------------- +# Session and login resilience +# ----------------------------------------------------------------------- +QA_SESSION_TTL_MS=7200000 +# Optional path to a server-side logout endpoint if the app exposes one. +# QA_LOGOUT_PATH=/logout + +# When parallel user mode is enabled, limit auth to the matching project(s). +# QA_AUTH_USER=1 + +# Optional second test user for 2-project parallel execution. +# QA_USER2_EMAIL=qa-user2@example.com +# QA_USER2_PASSWORD=CHANGE_ME + +# ----------------------------------------------------------------------- +# Login Configuration +# These selectors allow the framework to work with any login form. +# ----------------------------------------------------------------------- +QA_LOGIN_PATH=/login +QA_LOGIN_EMAIL_SELECTOR=input[type="email"] +QA_LOGIN_PASSWORD_SELECTOR=input[type="password"] +QA_LOGIN_SUBMIT_SELECTOR=button[type="submit"] +# Selector that appears AFTER successful login (used by global-setup to confirm): +QA_LOGIN_SUCCESS_SELECTOR=.dashboard + +# ----------------------------------------------------------------------- +# Test Users +# Add one block per role your project uses. +# ----------------------------------------------------------------------- + +# Default / Standard user +QA_USER_EMAIL=qa-user@example.com +QA_USER_PASSWORD=CHANGE_ME + +# Admin user (uncomment if multi-role setup is needed) +# QA_ADMIN_EMAIL=qa-admin@example.com +# QA_ADMIN_PASSWORD=CHANGE_ME + +# Read-only / viewer user +# QA_READONLY_EMAIL=qa-readonly@example.com +# QA_READONLY_PASSWORD=CHANGE_ME + +# If your app uses RUT-based login instead of email: +# QA_USER_RUT=12345678-9 + +# ----------------------------------------------------------------------- +# Azure DevOps Integration (optional — leave blank to disable) +# ----------------------------------------------------------------------- +# ADO_ORG_URL=https://dev.azure.com/your-org +# ADO_PROJECT_NAME=YourProject +# ADO_PLAN_ID=00000 +# AZURE_TOKEN=your-pat-token-here # Personal Access Token — NEVER commit + +# ----------------------------------------------------------------------- +# CI / Pipeline overrides +# ----------------------------------------------------------------------- +# CI=true # Set automatically by most CI systems +# ADO_SYNC_DISABLED=false # Set to true to skip ADO reporter in local runs + +# ----------------------------------------------------------------------- +# Playwright Service (optional - only needed for cloud or self-hosted grid) +# ----------------------------------------------------------------------- +# PLAYWRIGHT_SERVICE_URL=wss://eastus.api.playwright.microsoft.com/accounts//browsers diff --git a/test/fixtures/legacy-v1.11.3/fixtures/auth.ts b/test/fixtures/legacy-v1.11.3/fixtures/auth.ts new file mode 100644 index 0000000..ef34344 --- /dev/null +++ b/test/fixtures/legacy-v1.11.3/fixtures/auth.ts @@ -0,0 +1,77 @@ +/** + * fixtures/auth.ts + * + * Provides a `loginAs(page, role)` helper that navigates the login form + * and waits for successful authentication. + * + * Usage in spec files: + * import { loginAs } from '../../fixtures/auth'; + * // inside a test: + * await loginAs(page, 'admin'); + * + * Alternatively, use storageState via global-setup.ts (recommended for + * performance — avoids repeating the login flow on every test). + * Use loginAs() only in tests that need to validate the login flow itself + * or switch users mid-test. + * + * Supported roles are driven by env vars — see .env.example. + */ + +import { Page } from '@playwright/test'; + +export type QARole = 'default' | 'admin' | 'readonly' | string; + +interface LoginConfig { + email: string; + password: string; +} + +/** Map role name → credentials from environment variables */ +function getCredentials(role: QARole): LoginConfig { + switch (role) { + case 'admin': + return { + email: process.env.QA_ADMIN_EMAIL ?? '', + password: process.env.QA_ADMIN_PASSWORD ?? '', + }; + case 'readonly': + return { + email: process.env.QA_READONLY_EMAIL ?? '', + password: process.env.QA_READONLY_PASSWORD ?? '', + }; + case 'default': + default: + return { + email: process.env.QA_USER_EMAIL ?? '', + password: process.env.QA_USER_PASSWORD ?? '', + }; + } +} + +/** + * Navigate to the login page and authenticate as the given role. + * Waits for the post-login success selector before resolving. + */ +export async function loginAs(page: Page, role: QARole = 'default'): Promise { + const { email, password } = getCredentials(role); + + if (!email || !password) { + throw new Error( + `[qa-framework/auth] Missing credentials for role "${role}". ` + + `Check your .env file — expected ${role.toUpperCase()}_EMAIL and ${role.toUpperCase()}_PASSWORD.` + ); + } + + const loginPath = process.env.QA_LOGIN_PATH ?? '/login'; + const emailSelector = process.env.QA_LOGIN_EMAIL_SELECTOR ?? 'input[type="email"]'; + const passwordSelector = process.env.QA_LOGIN_PASSWORD_SELECTOR ?? 'input[type="password"]'; + const submitSelector = process.env.QA_LOGIN_SUBMIT_SELECTOR ?? 'button[type="submit"]'; + const successSelector = process.env.QA_LOGIN_SUCCESS_SELECTOR ?? '.dashboard'; + const baseURL = process.env.QA_BASE_URL ?? ''; + + await page.goto(`${baseURL}${loginPath}`, { waitUntil: 'domcontentloaded' }); + await page.locator(emailSelector).fill(email); + await page.locator(passwordSelector).fill(password); + await page.locator(submitSelector).click(); + await page.waitForSelector(successSelector, { timeout: 15_000 }); +} diff --git a/test/fixtures/legacy-v1.11.3/fixtures/base.ts b/test/fixtures/legacy-v1.11.3/fixtures/base.ts new file mode 100644 index 0000000..4714557 --- /dev/null +++ b/test/fixtures/legacy-v1.11.3/fixtures/base.ts @@ -0,0 +1,108 @@ +/** + * fixtures/base.ts + * + * Playwright test wrapper with TTL-aware session refresh. + * Import this file instead of `@playwright/test` when the app uses a + * server-side session that may expire during long runs. + */ + +import { test as base, devices } from '@playwright/test'; +import { SESSION_TTL_MS } from '../playwright.config'; +import * as dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +dotenv.config(); + +const authTimestamps = new Map(); + +function stateFileForProject(storageState: unknown): string { + if (typeof storageState === 'string' && storageState) { + return path.resolve(__dirname, '..', storageState); + } + return path.resolve(__dirname, '../.auth/user-default.json'); +} + +function credentialsForState(stateFile: string): { email: string; password: string } { + if (stateFile.includes('user-2')) { + return { + email: process.env.QA_USER2_EMAIL ?? '', + password: process.env.QA_USER2_PASSWORD ?? '', + }; + } + + return { + email: process.env.QA_USER_EMAIL ?? '', + password: process.env.QA_USER_PASSWORD ?? '', + }; +} + +function lastAuthTime(stateFile: string): number { + if (!authTimestamps.has(stateFile)) { + try { + authTimestamps.set(stateFile, fs.statSync(stateFile).mtimeMs); + } catch { + authTimestamps.set(stateFile, 0); + } + } + return authTimestamps.get(stateFile) ?? 0; +} + +async function reauth(browser: import('@playwright/test').Browser, stateFile: string): Promise { + const { email, password } = credentialsForState(stateFile); + const baseURL = process.env.QA_BASE_URL ?? ''; + const loginPath = process.env.QA_LOGIN_PATH ?? '/login'; + const emailSelector = process.env.QA_LOGIN_EMAIL_SELECTOR ?? 'input[type="email"]'; + const passwordSelector = process.env.QA_LOGIN_PASSWORD_SELECTOR ?? 'input[type="password"]'; + const submitSelector = process.env.QA_LOGIN_SUBMIT_SELECTOR ?? 'button[type="submit"]'; + const successSelector = process.env.QA_LOGIN_SUCCESS_SELECTOR ?? '.dashboard, .main-content, [data-testid="app-shell"]'; + + const context = await browser.newContext({ ...devices['Desktop Chrome'], baseURL }); + const page = await context.newPage(); + + try { + await page.goto(loginPath, { waitUntil: 'load', timeout: 60_000 }); + await page.locator(emailSelector).fill(email); + await page.locator(passwordSelector).fill(password); + await page.locator(submitSelector).click(); + await page.waitForSelector(successSelector, { timeout: 30_000 }); + + const dir = path.dirname(stateFile); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + await context.storageState({ path: stateFile }); + authTimestamps.set(stateFile, Date.now()); + console.log(`[session-refresh] storageState renewed: ${stateFile}`); + } finally { + await context.close(); + } +} + +export const test = base.extend<{ authStatePath: string }>({ + authStatePath: async ({}, use, testInfo) => { + await use(stateFileForProject(testInfo.project.use.storageState)); + }, + + context: async ({ browser }, use, testInfo) => { + const stateFile = stateFileForProject(testInfo.project.use.storageState); + if (Date.now() - lastAuthTime(stateFile) > SESSION_TTL_MS) { + await reauth(browser, stateFile).catch((error) => { + console.error(`[session-refresh] reauth failed: ${(error as Error).message}`); + }); + } + + const context = await browser.newContext({ + ...devices['Desktop Chrome'], + storageState: stateFile, + baseURL: process.env.QA_BASE_URL, + }); + + await use(context); + await context.close(); + }, +}); + +export { expect } from '@playwright/test'; +export type { Page, Browser, BrowserContext } from '@playwright/test'; \ No newline at end of file diff --git a/test/fixtures/legacy-v1.11.3/fixtures/test-helpers.ts b/test/fixtures/legacy-v1.11.3/fixtures/test-helpers.ts new file mode 100644 index 0000000..698919a --- /dev/null +++ b/test/fixtures/legacy-v1.11.3/fixtures/test-helpers.ts @@ -0,0 +1,85 @@ +/** + * fixtures/test-helpers.ts + * + * Utility functions used across all spec files. + * Import only what you need; keep specs self-contained. + */ + +// ----------------------------------------------------------------------- +// EXEC_IDX — unique numeric seed per minute-long execution window +// ----------------------------------------------------------------------- +// Use this wherever test data needs to be unique (e.g., names, emails, +// codes) to avoid collisions between parallel or consecutive test runs +// without relying on random values that are impossible to correlate. +// +// Example: +// const idx = execIdx(); // e.g., 42591 +// const name = `QA-Supplier-${idx}`; +// ----------------------------------------------------------------------- +export function execIdx(): number { + return Math.floor(Date.now() / 60_000) % 100_000; +} + +/** Date string formatted as YYYY-MM-DD in local time. */ +export function todayISO(): string { + const d = new Date(); + return d.toISOString().slice(0, 10); +} + +/** Date string formatted as DD/MM/YYYY (common in Spanish-language UIs). */ +export function todayDMY(): string { + const d = new Date(); + const dd = String(d.getDate()).padStart(2, '0'); + const mm = String(d.getMonth() + 1).padStart(2, '0'); + return `${dd}/${mm}/${d.getFullYear()}`; +} + +/** Future date offset by `days` from today, formatted as YYYY-MM-DD. */ +export function futureDateISO(days: number): string { + const d = new Date(); + d.setDate(d.getDate() + days); + return d.toISOString().slice(0, 10); +} + +// ----------------------------------------------------------------------- +// 3-layer email assertion helper +// ----------------------------------------------------------------------- +// Use this when asserting an email field value to avoid brittle exact-match +// assertions that break when display format changes. +// +// Example: +// await assertEmailContains( +// await page.locator('#email-cell').textContent() ?? '', +// process.env.QA_USER_EMAIL! +// ); +// ----------------------------------------------------------------------- +export function assertEmailContains( + actual: string, + expected: string +): void { + const localPart = expected.split('@')[0]; + const domain = expected.split('@')[1]; + if (!actual.includes(localPart)) { + throw new Error(`Email assertion failed: expected local part "${localPart}" in "${actual}"`); + } + if (!actual.includes(domain)) { + throw new Error(`Email assertion failed: expected domain "${domain}" in "${actual}"`); + } + if (!actual.includes('@')) { + throw new Error(`Email assertion failed: no "@" symbol found in "${actual}"`); + } +} + +// ----------------------------------------------------------------------- +// Unique test string builder +// ----------------------------------------------------------------------- +// Combines a readable prefix with EXEC_IDX for easy triage in QA data. +// ----------------------------------------------------------------------- +export function uniqueName(prefix: string): string { + return `${prefix}-${execIdx()}`; +} + +/** Unique email address for test isolation (uses execIdx). */ +export function uniqueEmail(domain = 'qa-test.example.com'): string { + return `qa-user-${execIdx()}@${domain}`; +} diff --git a/test/fixtures/legacy-v1.11.3/global-setup.ts b/test/fixtures/legacy-v1.11.3/global-setup.ts new file mode 100644 index 0000000..76e0965 --- /dev/null +++ b/test/fixtures/legacy-v1.11.3/global-setup.ts @@ -0,0 +1,196 @@ +/** + * global-setup.ts + * + * Runs ONCE before all test files. + * Logs in as the default QA user and saves storageState so each test + * doesn't need to repeat the login flow. + * + * For multi-role projects: + * - Add additional loginAs() calls below, one per role. + * - Save each to `.auth/user-{role}.json`. + * - Reference the matching storageState in playwright.config.ts projects[]. + * + * Environment variables required: + * QA_BASE_URL — Base URL of the application under test + * QA_USER_EMAIL — Default QA user email (or username/RUT) + * QA_USER_PASSWORD — Default QA user password + * QA_LOGIN_PATH — Relative path to the login page (default: /login) + * QA_LOGIN_EMAIL_SELECTOR — CSS selector for the username/email input + * QA_LOGIN_PASSWORD_SELECTOR — CSS selector for the password input + * QA_LOGIN_SUBMIT_SELECTOR — CSS selector for the submit button + * QA_LOGIN_SUCCESS_SELECTOR — CSS selector that confirms successful login + * + * See .env.example for all supported variables. + */ + +import { chromium, FullConfig } from '@playwright/test'; +import * as dotenv from 'dotenv'; +import * as path from 'path'; +import * as fs from 'fs'; + +dotenv.config(); + +function resolveVar(param: string | undefined, envKey: string, defaultValue: string): [string, string] { + if (param !== undefined) { + return [param, 'param']; + } + if (process.env[envKey] !== undefined) { + return [process.env[envKey]!, `env:${envKey}`]; + } + return [defaultValue, 'default']; +} + +function resolveAuthUsers(config: FullConfig): Set<'1' | '2'> { + const explicit = process.env.QA_AUTH_USER; + if (explicit) { + const resolved = new Set<'1' | '2'>(); + for (const token of explicit.split(',').map((value) => value.trim())) { + if (token === '1' || token === '2') { + resolved.add(token); + } + } + if (resolved.size) { + return resolved; + } + } + + const fromProjects = new Set<'1' | '2'>(); + for (const project of config.projects) { + const storageState = (project.use as { storageState?: unknown } | undefined)?.storageState; + if (typeof storageState === 'string') { + if (storageState.includes('user-2')) { + fromProjects.add('2'); + } else if (storageState.includes('user-1')) { + fromProjects.add('1'); + } + } + } + + if (fromProjects.size) { + return fromProjects; + } + + return new Set(['1']); +} + +async function dismissOnboardingFlow(page: import('@playwright/test').Page): Promise { + // Override this with your app's onboarding dismissal logic. + // Use QA_DISMISS_ONBOARDING_LABELS (comma-separated button labels) to enable + // without modifying this file. Example: QA_DISMISS_ONBOARDING_LABELS=Skip,Got it + const raw = process.env.QA_DISMISS_ONBOARDING_LABELS ?? ''; + const labels = raw.split(',').map((l: string) => l.trim()).filter(Boolean); + if (!labels.length) return; + + for (let round = 0; round < 3; round++) { + let dismissed = false; + for (const label of labels) { + const button = page.getByRole('button', { name: label, exact: true }).first(); + const visible = await button.isVisible({ timeout: 1_000 }).catch(() => false); + if (visible) { + await button.click({ timeout: 2_000 }).catch(() => {}); + await page.waitForTimeout(250); + dismissed = true; + } + } + if (!dismissed) { + break; + } + } +} + +/** Perform login and persist storageState to disk. */ +async function loginAs(params: { + email: string; + password: string; + baseURL: string; + loginPath?: string; + emailSelector?: string; + passwordSelector?: string; + submitSelector?: string; + successSelector?: string; + stateFile: string; +}): Promise { + const { + email, + password, + baseURL, + loginPath = process.env.QA_LOGIN_PATH ?? '/login', + emailSelector = process.env.QA_LOGIN_EMAIL_SELECTOR ?? 'input[type="email"]', + passwordSelector = process.env.QA_LOGIN_PASSWORD_SELECTOR ?? 'input[type="password"]', + submitSelector = process.env.QA_LOGIN_SUBMIT_SELECTOR ?? 'button[type="submit"]', + successSelector = process.env.QA_LOGIN_SUCCESS_SELECTOR ?? '.dashboard, .main-content, [data-testid="app-shell"]', + stateFile, + } = params; + + const browser = await chromium.launch(); + const context = await browser.newContext(); + const page = await context.newPage(); + + try { + const loginUrl = `${baseURL}${loginPath.startsWith('/') ? loginPath : `/${loginPath}`}`; + let signedIn = false; + + for (let attempt = 1; attempt <= 3 && !signedIn; attempt++) { + await page.goto(loginUrl, { waitUntil: 'load', timeout: 60_000 }); + await page.locator(emailSelector).fill(email); + await page.locator(passwordSelector).fill(password); + await page.locator(submitSelector).click(); + + const result = await Promise.race([ + page.waitForSelector(successSelector, { timeout: 30_000 }).then(() => 'ok' as const), + page.locator('text=An unhandled error has occurred.').waitFor({ state: 'visible', timeout: 30_000 }).then(() => 'crash' as const), + ]).catch(() => 'timeout' as const); + + if (result === 'ok') { + signedIn = true; + } else if (result === 'crash') { + await page.getByText('Reload').click({ timeout: 5_000 }).catch(() => {}); + await page.waitForTimeout(2_000); + } + } + + if (!signedIn) { + throw new Error('[qa-framework] Login did not reach the success selector after 3 attempts.'); + } + + await dismissOnboardingFlow(page); + + // Ensure .auth/ directory exists + const dir = path.dirname(stateFile); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + + await context.storageState({ path: stateFile }); + console.log(`[global-setup] storageState saved: ${stateFile}`); + } finally { + await browser.close(); + } +} + +export default async function globalSetup(_config: FullConfig): Promise { + const baseURL = process.env.QA_BASE_URL!; + const authUsers = resolveAuthUsers(_config); + + if (authUsers.has('1')) { + const [email, emailSource] = resolveVar(process.env.QA_USER_EMAIL, 'QA_USER_EMAIL', 'qa-user@example.com'); + const [password, passwordSource] = resolveVar(process.env.QA_USER_PASSWORD, 'QA_USER_PASSWORD', 'CHANGE_ME'); + console.log(`[global-setup] user-1 credentials from ${emailSource} / ${passwordSource}`); + await loginAs({ + email, + password, + baseURL, + stateFile: '.auth/user-default.json', + }); + } + + if (authUsers.has('2') && process.env.QA_USER2_EMAIL && process.env.QA_USER2_PASSWORD) { + const [email, emailSource] = resolveVar(process.env.QA_USER2_EMAIL, 'QA_USER2_EMAIL', 'qa-user2@example.com'); + const [password, passwordSource] = resolveVar(process.env.QA_USER2_PASSWORD, 'QA_USER2_PASSWORD', 'CHANGE_ME'); + console.log(`[global-setup] user-2 credentials from ${emailSource} / ${passwordSource}`); + await loginAs({ + email, + password, + baseURL, + stateFile: '.auth/user-2.json', + }); + } +} diff --git a/test/fixtures/legacy-v1.11.3/playwright.config.ts b/test/fixtures/legacy-v1.11.3/playwright.config.ts new file mode 100644 index 0000000..e9c9a51 --- /dev/null +++ b/test/fixtures/legacy-v1.11.3/playwright.config.ts @@ -0,0 +1,122 @@ +import { defineConfig, devices } from '@playwright/test'; +import * as dotenv from 'dotenv'; +dotenv.config(); + +export const SESSION_TTL_MS = Number(process.env.QA_SESSION_TTL_MS ?? 2 * 60 * 60 * 1000); + +// ------------------------------------------------------------------- +// Validate required environment variables at config load time +// ------------------------------------------------------------------- +const required = ['QA_BASE_URL', 'QA_USER_EMAIL', 'QA_USER_PASSWORD']; +for (const key of required) { + if (!process.env[key]) { + throw new Error(`[qa-framework] Missing required env var: ${key}. Check your .env file.`); + } +} + +// Optional: Azure DevOps reporter configuration +// To enable: set env vars ADO_ORG, ADO_PROJECT, ADO_PAT, ADO_PLAN_ID and CI=true, +// then uncomment the reporter entry in the `reporter` array below. +// Install: npm install @alex_neo/playwright-azure-reporter --save-dev +// import { AzureReporter } from '@alex_neo/playwright-azure-reporter'; +// const adoReporterConfig = { +// orgUrl: `https://dev.azure.com/${process.env.ADO_ORG}`, +// // In CI it uses System.AccessToken (pipeline's OAuth, not subject to Conditional Access Policy). +// // Locally, if you want to test the reporter, you can set ADO_PAT manually. +// token: process.env.SYSTEM_ACCESSTOKEN ?? process.env.ADO_PAT, +// planId: Number(process.env.ADO_PLAN_ID), +// projectName: process.env.ADO_PROJECT!, +// testRunTitle: `[Auto] Sprint {{NNN}} — ${new Date().toISOString().slice(0, 10)}`, +// publishTestResultsMode: 'testRun' as const, +// uploadAttachments: true, +// attachmentsType: ['screenshot', 'video', 'trace'] as const, +// isDisabled: !process.env.CI, // only publishes when CI=true +// autoMarkTestCasesAsAutomated: { +// enabled: true, +// updateAutomatedTestName: true, // saves test title in AutomatedTestName +// updateAutomatedTestStorage: true, // saves spec file name in AutomatedTestStorage +// }, +// }; + +export default defineConfig({ + // ------ Test discovery ------ + testDir: './tests', + // Exclude non-suite directories from test runs + testIgnore: ['**/helpers/debug/**', '**/seeds/**'], + // Use any subdir pattern your project standardizes on, e.g.: + // testMatch: ['**/*.spec.ts'], + + // ------ Parallelism ------ + // Keep fullyParallel:false when tests share storageState / session data. + fullyParallel: false, + workers: 1, + + // ------ Retry strategy ------ + retries: process.env.CI ? 1 : 0, + + // ------ Reporter ------ + reporter: [ + ['html', { open: 'never' }], + ['list'], + // Uncomment for ADO: + // ['@alex_neo/playwright-azure-reporter', adoReporterConfig], + ], + + // ------ Global settings ------ + use: { + baseURL: process.env.QA_BASE_URL, + headless: true, + screenshot: 'only-on-failure', + video: 'retain-on-failure', + trace: 'retain-on-failure', + actionTimeout: 15_000, + navigationTimeout: 30_000, + }, + + // ------ Auth setup ------ + // global-setup.ts logs in once and saves storageState per role. + globalSetup: './global-setup.ts', + + // ------ Projects ------ + projects: [ + { + name: 'setup', + use: { ...devices['Desktop Chrome'] }, + testMatch: /global-setup\.ts/, + }, + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + storageState: '.auth/user-default.json', + }, + dependencies: ['setup'], + }, + ], + + // ------ Parallel 2-user variant ------ + // Uncomment and replace the single chromium project above to run two parallel + // workers with separate credentials. Also set QA_USER2_EMAIL and + // QA_USER2_PASSWORD in .env. + // + // { + // name: 'chromium-user1', + // use: { + // ...devices['Desktop Chrome'], + // storageState: '.auth/user-1.json', + // }, + // dependencies: ['setup'], + // }, + // { + // name: 'chromium-user2', + // use: { + // ...devices['Desktop Chrome'], + // storageState: '.auth/user-2.json', + // }, + // dependencies: ['setup'], + // }, + + // ------ Output directories ------ + outputDir: 'test-results/', + snapshotPathTemplate: '{testDir}/__snapshots__/{testFilePath}/{arg}{ext}', +}); diff --git a/test/upgrade.test.js b/test/upgrade.test.js index eef65c4..a6cd0a7 100644 --- a/test/upgrade.test.js +++ b/test/upgrade.test.js @@ -15,6 +15,22 @@ const UPGRADE_PATH = path.join(REPO_ROOT, 'scripts', 'upgrade.js'); const AUTOMATION_SRC = path.join(REPO_ROOT, 'templates', 'automation-scaffold'); const INTEGRATION_SRC = path.join(REPO_ROOT, 'templates', 'integration-scaffold'); +// Byte-for-byte copies of the automation scaffold as it shipped in v1.11.3, committed +// as static fixtures. They used to be recovered with `git show v1.11.3:...`, which made +// the suite depend on repository history: it broke under the shallow clone that +// actions/checkout performs (no tags fetched), inside an `npm pack` tarball (no .git) +// and in any exported copy of the tree. Nothing here needs git any more. +const LEGACY_SRC = path.join(__dirname, 'fixtures', 'legacy-v1.11.3'); + +const LEGACY_SCAFFOLD_FILES = [ + 'playwright.config.ts', + 'global-setup.ts', + '.env.example', + 'fixtures/auth.ts', + 'fixtures/base.ts', + 'fixtures/test-helpers.ts', +]; + const LANE_SCRIPTS = ['lane-config.js', 'lane-lock.js', 'global-setup-guards.js']; function run(script, args, cwd) { @@ -54,33 +70,40 @@ function snapshot(dir) { return out; } +/** Normalised content hash - must mirror scaffoldHash() in scripts/upgrade.js. */ +function scaffoldHash(content) { + return crypto + .createHash('sha256') + .update(content.replace(/^/, '').replace(/\r\n/g, '\n'), 'utf8') + .digest('hex'); +} + +/** The SHIPPED_SCAFFOLD_HASHES table as scripts/upgrade.js declares it. */ +function shippedScaffoldHashes() { + const source = fs.readFileSync(UPGRADE_PATH, 'utf8'); + const match = source.match(/const SHIPPED_SCAFFOLD_HASHES = (\{[\s\S]*?\n\});/); + assert.ok(match, 'SHIPPED_SCAFFOLD_HASHES not found in scripts/upgrade.js'); + // The literal holds only strings and arrays; parsing it keeps the fixture guard + // honest without exporting production internals purely for the tests. + return new Function(`return ${match[1]};`)(); +} + /** * Build a project that looks like a v1.11.3 installation: init it with the current - * templates, then overwrite the scaffold files with the versions git shipped at - * v1.11.3 and delete everything v1.12.0 introduced. + * templates, then overwrite the scaffold files with the versions shipped at v1.11.3 + * (from test/fixtures/legacy-v1.11.3/) and delete everything v1.12.0 introduced. */ function makeLegacyProject() { const root = tmpProject(); const init = runInit(root); assert.equal(init.status, 0, init.stderr); - for (const rel of [ - 'playwright.config.ts', - 'global-setup.ts', - '.env.example', - 'fixtures/auth.ts', - 'fixtures/base.ts', - 'fixtures/test-helpers.ts', - ]) { - const shipped = spawnSync( - 'git', - ['show', `v1.11.3:templates/automation-scaffold/${rel}`], - { cwd: REPO_ROOT, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 } - ); - assert.equal(shipped.status, 0, `git show failed for ${rel}: ${shipped.stderr}`); + for (const rel of LEGACY_SCAFFOLD_FILES) { + const src = path.join(LEGACY_SRC, ...rel.split('/')); + assert.ok(fs.existsSync(src), `missing legacy fixture: ${src}`); const dest = path.join(e2e(root), ...rel.split('/')); fs.mkdirSync(path.dirname(dest), { recursive: true }); - fs.writeFileSync(dest, shipped.stdout, 'utf8'); + fs.copyFileSync(src, dest); } // v1.12.0 introduced the lane scripts; a v1.11.3 project has none. @@ -88,6 +111,31 @@ function makeLegacyProject() { return root; } +test('fixtures: every legacy v1.11.3 file is pristine-but-outdated for upgrade.js', () => { + // Guards what makeLegacyProject() silently assumes. Without this, a fixture that + // drifted out of SHIPPED_SCAFFOLD_HASHES would turn every "pristine file is + // refreshed" test into a vacuous "user-owned file is kept" test. + const table = shippedScaffoldHashes(); + for (const rel of LEGACY_SCAFFOLD_FILES) { + const key = `automation-scaffold/${rel}`; + const legacy = scaffoldHash(fs.readFileSync(path.join(LEGACY_SRC, ...rel.split('/')), 'utf8')); + const current = scaffoldHash( + fs.readFileSync(path.join(AUTOMATION_SRC, ...rel.split('/')), 'utf8') + ); + assert.ok( + Array.isArray(table[key]) && table[key].includes(legacy), + `${key}: fixture hash ${legacy} is not in SHIPPED_SCAFFOLD_HASHES, so upgrade.js ` + + 'would treat it as user-owned and the refresh tests would prove nothing' + ); + assert.notEqual( + legacy, + current, + `${key}: fixture is identical to the current template, so "refreshed to v1.12.0" ` + + 'would pass without the upgrade doing anything' + ); + } +}); + test('upgrade: a pristine v1.11.3 project receives the whole v1.12.0 lane group', () => { const root = makeLegacyProject(); try {