diff --git a/src/common/inlineScript/cacheLayout.ts b/src/common/inlineScript/cacheLayout.ts index fdc31bf9f..13af989e3 100644 --- a/src/common/inlineScript/cacheLayout.ts +++ b/src/common/inlineScript/cacheLayout.ts @@ -22,8 +22,12 @@ export const META_JSON_FILENAME = '.meta.json'; * Schema version embedded in every {@link InlineScriptEnvMeta}. */ export const META_SCHEMA_VERSION = 1 as const; +export const SOURCE_METADATA_IDENTITY_HASH_HEX_LENGTH = 64; +export const MAX_SOURCE_METADATA_IDENTITY_HASHES = 128; const MAX_META_JSON_BYTES = 1024 * 1024; +const META_JSON_BACKUP_FILENAME_RE = /^\.meta\.json\.backup-[0-9a-f]{12}$/; +const pendingMetaJsonWrites = new Map>(); /** * Validated on-disk schema for a cached inline-script environment's @@ -38,11 +42,13 @@ export interface InlineScriptEnvMeta { readonly baseInterpreterVersion: string; /** Last successful use as a canonical UTC string produced by `Date.toISOString()`. */ readonly lastUsedAt: string; + /** Bounded SHA-256 hashes of metadata identities proven for this cache entry. */ + readonly sourceMetadataIdentityHashes?: readonly string[]; } export type InlineScriptMetaReadResult = | { readonly kind: 'valid'; readonly metadata: InlineScriptEnvMeta } - | { readonly kind: 'missing' | 'invalid' | 'unavailable' }; + | { readonly kind: 'missing' | 'invalid' | 'unsupported' | 'unavailable' }; export type BaseInterpreterStatus = 'available' | 'missing' | 'unavailable'; export type CacheEnvironmentInspection = 'expected' | 'stale' | 'uncertain'; @@ -119,8 +125,73 @@ export async function readMetaJson(envDir: Uri): Promise { - const metaPath = getMetaJsonPath(envDir).fsPath; + return inspectMetaJsonFile(getMetaJsonPath(envDir).fsPath); +} + +/** + * Restore the most recently-used compatible sidecar backup while the caller + * owns the cache-entry lock. General readers must use {@link inspectMetaJson}. + */ +export async function restoreMetaJsonBackupUnderLock( + envDir: Uri, + isCompatible: (metadata: InlineScriptEnvMeta) => boolean = () => true, +): Promise { + const finalPath = getMetaJsonPath(envDir).fsPath; + const initial = await inspectMetaJsonFile(finalPath); + if (initial.kind !== 'missing') { + return initial; + } + let entries: string[]; + try { + entries = await fsapi.readdir(envDir.fsPath); + } catch (error) { + traceWarn(`inline-script meta: failed to scan backup sidecars in ${envDir.fsPath}:`, error); + return { kind: 'unavailable' }; + } + + const validBackups: Array<{ readonly path: string; readonly metadata: InlineScriptEnvMeta }> = []; + for (const entry of entries.filter((name) => META_JSON_BACKUP_FILENAME_RE.test(name))) { + const result = await inspectMetaJsonFile(path.join(envDir.fsPath, entry)); + if (result.kind === 'valid' && isCompatible(result.metadata)) { + validBackups.push({ path: path.join(envDir.fsPath, entry), metadata: result.metadata }); + } else if (result.kind === 'unavailable' || result.kind === 'missing') { + // A listed candidate changing or becoming unreadable is an + // uncertain scan; preserve the entry rather than rebuilding it. + return { kind: 'unavailable' }; + } + } + + if (validBackups.length === 0) { + return { kind: 'missing' }; + } + + // `lastUsedAt` is schema-validated canonical ISO text. Prefer the newest + // compatible backup; use the path as a stable tie-breaker. + validBackups.sort((a, b) => { + if (a.metadata.lastUsedAt !== b.metadata.lastUsedAt) { + return a.metadata.lastUsedAt < b.metadata.lastUsedAt ? 1 : -1; + } + return a.path < b.path ? -1 : a.path > b.path ? 1 : 0; + }); + const selected = validBackups[0]; + + // Native rename may replace an existing destination on some platforms, + // so recheck under the caller's entry lock before restoring. + const current = await inspectMetaJsonFile(finalPath); + if (current.kind !== 'missing') { + return current; + } + try { + await fsapi.rename(selected.path, finalPath); + } catch (error) { + traceWarn(`inline-script meta: failed to restore backup ${selected.path}:`, error); + return { kind: 'unavailable' }; + } + return { kind: 'valid', metadata: selected.metadata }; +} + +async function inspectMetaJsonFile(metaPath: string): Promise { try { const stat = await fsapi.lstat(metaPath); if (!stat.isFile()) { @@ -160,6 +231,10 @@ export async function inspectMetaJson(envDir: Uri): Promise { - await fsapi.ensureDir(envDir.fsPath); +export function writeMetaJson(envDir: Uri, meta: InlineScriptEnvMeta): Promise { const finalPath = getMetaJsonPath(envDir).fsPath; + const key = normalizePath(path.resolve(finalPath)); + const previous = pendingMetaJsonWrites.get(key) ?? Promise.resolve(); + const operation = previous.catch(() => undefined).then(() => writeMetaJsonOnce(envDir, meta, finalPath)); + let queued: Promise; + queued = operation.finally(() => { + if (pendingMetaJsonWrites.get(key) === queued) { + pendingMetaJsonWrites.delete(key); + } + }); + pendingMetaJsonWrites.set(key, queued); + return queued; +} + +async function writeMetaJsonOnce(envDir: Uri, meta: InlineScriptEnvMeta, finalPath: string): Promise { + await fsapi.ensureDir(envDir.fsPath); const tmpSuffix = crypto.randomBytes(6).toString('hex'); const tmpPath = `${finalPath}.tmp-${tmpSuffix}`; + const backupPath = `${finalPath}.backup-${tmpSuffix}`; const payload = JSON.stringify(meta, undefined, 2); + let hasBackup = false; + let finalKnownToExist = false; + try { await fsapi.writeFile(tmpPath, payload, 'utf8'); - await fsapi.rename(tmpPath, finalPath); - } catch (err) { + try { + await fsapi.rename(tmpPath, finalPath); + finalKnownToExist = true; + return; + } catch (err) { + const code = (err as NodeJS.ErrnoException | undefined)?.code; + if (!['EPERM', 'EEXIST', 'EBUSY'].includes(code ?? '')) { + throw err; + } + } + + try { + await fsapi.rename(finalPath, backupPath); + hasBackup = true; + } catch (err) { + if (!isFileNotFoundError(err)) { + throw err; + } + } + + try { + await fsapi.rename(tmpPath, finalPath); + finalKnownToExist = true; + } catch (replaceError) { + if (hasBackup) { + try { + await fsapi.rename(backupPath, finalPath); + finalKnownToExist = true; + } catch { + // Keep the backup: it is the only known copy when + // restoration cannot prove the final sidecar exists. + } + } + throw replaceError; + } + } finally { await fsapi.remove(tmpPath).catch(() => undefined); - throw err; + if (hasBackup && finalKnownToExist) { + await fsapi.remove(backupPath).catch(() => undefined); + } } } +export function hashSourceMetadataIdentity(identity: string): string { + return crypto.createHash('sha256').update(identity, 'utf8').digest('hex'); +} + +export function mergeSourceMetadataIdentityHashes( + existing: readonly string[] | undefined, + current: string | undefined, +): readonly string[] | undefined { + const ordered = [...(existing ?? [])]; + if (current && !ordered.includes(current)) { + ordered.push(current); + } + if (ordered.length === 0) { + return undefined; + } + return Object.freeze(ordered.slice(-MAX_SOURCE_METADATA_IDENTITY_HASHES)); +} + /** * Pure selector: returns the env-dir paths whose age exceeds `ttlMs`. */ @@ -290,11 +439,17 @@ function isNonEmptyTrimmedString(value: unknown): value is string { return typeof value === 'string' && value.length > 0 && value.trim() === value; } -function validateMeta(value: unknown): InlineScriptEnvMeta | undefined { +function validateMeta(value: unknown): InlineScriptEnvMeta | 'unsupported' | undefined { if (typeof value !== 'object' || value === null || Array.isArray(value)) { return undefined; } const obj = value as Record; + if (typeof obj.schemaVersion !== 'number') { + return undefined; + } + if (obj.schemaVersion > META_SCHEMA_VERSION) { + return 'unsupported'; + } if (obj.schemaVersion !== META_SCHEMA_VERSION) { return undefined; } @@ -307,15 +462,44 @@ function validateMeta(value: unknown): InlineScriptEnvMeta | undefined { if (!isCanonicalIsoTimestamp(obj.lastUsedAt)) { return undefined; } + const sourceMetadataIdentityHashes = validateSourceMetadataIdentityHashes(obj.sourceMetadataIdentityHashes); + if (obj.sourceMetadataIdentityHashes !== undefined && sourceMetadataIdentityHashes === undefined) { + return undefined; + } return { schemaVersion: META_SCHEMA_VERSION, baseInterpreterPath: obj.baseInterpreterPath, baseInterpreterVersion: obj.baseInterpreterVersion, lastUsedAt: obj.lastUsedAt, + ...(sourceMetadataIdentityHashes ? { sourceMetadataIdentityHashes } : {}), }; } +function validateSourceMetadataIdentityHashes(value: unknown): readonly string[] | undefined { + if (value === undefined) { + return undefined; + } + if (!Array.isArray(value) || value.length === 0 || value.length > MAX_SOURCE_METADATA_IDENTITY_HASHES) { + return undefined; + } + const hashes: string[] = []; + const seen = new Set(); + for (const item of value) { + if ( + typeof item !== 'string' || + item.length !== SOURCE_METADATA_IDENTITY_HASH_HEX_LENGTH || + !/^[0-9a-f]+$/.test(item) || + seen.has(item) + ) { + return undefined; + } + seen.add(item); + hashes.push(item); + } + return Object.freeze(hashes); +} + function isCanonicalIsoTimestamp(value: unknown): value is string { if (typeof value !== 'string') { return false; diff --git a/src/common/inlineScript/metadata.ts b/src/common/inlineScript/metadata.ts index 88a159861..45f6d34b9 100644 --- a/src/common/inlineScript/metadata.ts +++ b/src/common/inlineScript/metadata.ts @@ -27,6 +27,15 @@ export interface InlineScriptMetadata { * newline (or end of string if there is no trailing newline). */ readonly range: { readonly start: number; readonly end: number }; + /** + * Character offsets of the same metadata block in the original source + * text. Unlike {@link range}, these include a leading BOM and preserve + * CRLF, so they can be compared with TextDocument change offsets. + * + * Optional to keep manually constructed metadata compatible; parser + * results always supply it. + */ + readonly sourceRange?: { readonly start: number; readonly end: number }; } /** @@ -79,7 +88,9 @@ export function readInlineScriptMetadata(scriptText: string): InlineScriptMetada // "UTF-8 with BOM" on Windows have this; without stripping it the // first line becomes "\uFEFF# /// script" and the regex fails to // match. - let text = scriptText.charCodeAt(0) === 0xfeff ? scriptText.slice(1) : scriptText; + const bomOffset = scriptText.charCodeAt(0) === 0xfeff ? 1 : 0; + const sourceText = scriptText.slice(bomOffset); + let text = sourceText; // Normalize CRLF and lone CR to LF so the canonical regex (which // was authored assuming `.` matches `\r`, true in Python's re but @@ -215,9 +226,27 @@ export function readInlineScriptMetadata(scriptText: string): InlineScriptMetada dependencies, tool, range: { start: matchStart, end }, + sourceRange: { + start: bomOffset + sourceOffsetForNormalizedOffset(sourceText, matchStart), + end: bomOffset + sourceOffsetForNormalizedOffset(sourceText, end), + }, }; } +function sourceOffsetForNormalizedOffset(sourceText: string, normalizedOffset: number): number { + let sourceOffset = 0; + let currentNormalizedOffset = 0; + while (currentNormalizedOffset < normalizedOffset && sourceOffset < sourceText.length) { + if (sourceText.charCodeAt(sourceOffset) === 0x0d) { + sourceOffset += sourceText.charCodeAt(sourceOffset + 1) === 0x0a ? 2 : 1; + } else { + sourceOffset += 1; + } + currentNormalizedOffset += 1; + } + return sourceOffset; +} + /** * Read PEP 723 metadata from a file. Reads only the first * `MAX_HEADER_BYTES` bytes of the file — PEP 723 blocks live at the diff --git a/src/common/inlineScript/routingRegistry.ts b/src/common/inlineScript/routingRegistry.ts new file mode 100644 index 000000000..74dd16462 --- /dev/null +++ b/src/common/inlineScript/routingRegistry.ts @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as path from 'path'; +import { Disposable, Event, EventEmitter, Uri } from 'vscode'; +import { normalizeDependency } from './cacheKey'; +import { InlineScriptMetadata } from './metadata'; +import { normalizePath } from '../utils/pathUtils'; + +export interface InlineScriptRouteabilityChangeEvent { + readonly uri: Uri; + readonly previousRouteable: boolean; + readonly routeable: boolean; +} + +export interface InlineScriptMetadataChangeEvent { + readonly uri: Uri; + readonly metadata: InlineScriptMetadata | undefined; + readonly metadataIdentity: string | undefined; + readonly metadataRevision: number; +} + +interface ScriptRoutingState { + readonly uri?: Uri; + readonly metadata?: InlineScriptMetadata; + readonly metadataIdentity?: string; + readonly metadataRevision: number; + readonly validatedAssociation: boolean; +} + +export class InlineScriptRoutingRegistry implements Disposable { + private readonly states = new Map(); + private readonly metadataRevisions = new Map(); + private readonly _onDidChangeRouteability = new EventEmitter(); + private readonly _onDidChangeMetadata = new EventEmitter(); + + public readonly onDidChangeRouteability: Event = + this._onDidChangeRouteability.event; + + public readonly onDidChangeMetadata: Event = this._onDidChangeMetadata.event; + + public setMetadata(uri: Uri, metadata: InlineScriptMetadata | undefined): void { + const scriptPath = getInlineScriptRoutingKey(uri); + if (!scriptPath) { + return; + } + const metadataIdentity = getInlineScriptMetadataRoutingIdentity(metadata); + const metadataRevision = this.nextMetadataRevision(scriptPath); + this.update( + scriptPath, + (state) => { + return { + ...state, + uri, + metadata, + metadataIdentity, + metadataRevision, + }; + }, + true, + ); + } + + public clearMetadata(uri: Uri): void { + const scriptPath = getInlineScriptRoutingKey(uri); + if (!scriptPath) { + return; + } + const metadataRevision = this.nextMetadataRevision(scriptPath); + this.update( + scriptPath, + (state) => { + return { + ...state, + uri, + metadata: undefined, + metadataIdentity: undefined, + metadataRevision, + }; + }, + true, + ); + } + + public getMetadata(script: Uri | string): InlineScriptMetadata | undefined { + const scriptPath = getInlineScriptRoutingKey(script); + return scriptPath ? this.states.get(scriptPath)?.metadata : undefined; + } + + public getMetadataIdentity(script: Uri | string): string | undefined { + const scriptPath = getInlineScriptRoutingKey(script); + return scriptPath ? this.states.get(scriptPath)?.metadataIdentity : undefined; + } + + public getMetadataRevision(script: Uri | string): number { + const scriptPath = getInlineScriptRoutingKey(script); + return scriptPath ? (this.metadataRevisions.get(scriptPath) ?? 0) : 0; + } + + public getUri(script: Uri | string): Uri | undefined { + const scriptPath = getInlineScriptRoutingKey(script); + return scriptPath ? this.states.get(scriptPath)?.uri : undefined; + } + + public setValidatedAssociation(script: Uri | string, validatedAssociation: boolean): void { + const scriptPath = getInlineScriptRoutingKey(script); + if (!scriptPath) { + return; + } + this.update(scriptPath, (state) => ({ + ...state, + uri: script instanceof Uri ? script : state.uri, + validatedAssociation, + })); + } + + public hasValidatedAssociation(script: Uri | string): boolean { + const scriptPath = getInlineScriptRoutingKey(script); + return scriptPath ? this.states.get(scriptPath)?.validatedAssociation === true : false; + } + + public shouldRoute(uri: Uri): boolean { + const scriptPath = getInlineScriptRoutingKey(uri); + return scriptPath ? this.isRouteable(this.states.get(scriptPath)) : false; + } + + public dispose(): void { + this.states.clear(); + this.metadataRevisions.clear(); + this._onDidChangeMetadata.dispose(); + this._onDidChangeRouteability.dispose(); + } + + private update( + scriptPath: string, + updater: (state: ScriptRoutingState) => ScriptRoutingState, + fireMetadataChange: boolean = false, + ): void { + const previous = this.states.get(scriptPath) ?? { + metadataRevision: this.metadataRevisions.get(scriptPath) ?? 0, + validatedAssociation: false, + }; + const previousRouteable = this.isRouteable(previous); + const next = updater(previous); + + if (!next.metadata && !next.validatedAssociation) { + this.states.delete(scriptPath); + } else { + this.states.set(scriptPath, next); + } + + if (fireMetadataChange && next.uri) { + this._onDidChangeMetadata.fire({ + uri: next.uri, + metadata: next.metadata, + metadataIdentity: next.metadataIdentity, + metadataRevision: next.metadataRevision, + }); + } + + const routeable = this.isRouteable(next); + if (previousRouteable !== routeable && next.uri) { + this._onDidChangeRouteability.fire({ + uri: next.uri, + previousRouteable, + routeable, + }); + } + } + + private isRouteable(state: ScriptRoutingState | undefined): boolean { + return !!state?.metadata && state.validatedAssociation; + } + + private nextMetadataRevision(scriptPath: string): number { + const revision = (this.metadataRevisions.get(scriptPath) ?? 0) + 1; + this.metadataRevisions.set(scriptPath, revision); + return revision; + } +} + +export function getInlineScriptRoutingKey(script: Uri | string): string | undefined { + if (typeof script === 'string') { + return normalizePath(script); + } + if (script.scheme !== 'file') { + return undefined; + } + if (path.extname(script.fsPath).toLowerCase() !== '.py') { + return undefined; + } + return normalizePath(script.fsPath); +} + +export function getInlineScriptMetadataRoutingIdentity(metadata: InlineScriptMetadata | undefined): string | undefined { + if (!metadata) { + return undefined; + } + const normalizedDependencies = Array.from( + new Set((metadata.dependencies ?? []).map((dependency) => normalizeDependency(dependency)).filter(Boolean)), + ).sort(); + return JSON.stringify({ + requiresPython: metadata.requiresPython?.trim() ?? '', + dependencies: normalizedDependencies, + }); +} diff --git a/src/extension.ts b/src/extension.ts index f45d46aa2..c1be4885e 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -66,6 +66,7 @@ import { } from './features/envCommands'; import { PythonEnvironmentManagers } from './features/envManagers'; import { EnvVarManager, PythonEnvVariableManager } from './features/execution/envVariableManager'; +import { latchInlineScriptFeatureActivation } from './features/inlineScript/activation'; import { InlineScriptLazyDetector } from './features/inlineScript/lazyDetector'; import { applyInitialEnvironmentSelection, @@ -187,10 +188,16 @@ export async function activate(context: ExtensionContext): Promise = new Map(); private _packageManagers: Map = new Map(); + private readonly subscriptions: Disposable[] = []; /** * The last environment announced as "active" for each scope. @@ -64,6 +69,7 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { * Only mutated by setEnvironment() / setEnvironments() / refreshEnvironment(). */ private readonly _activeSelection = new Map(); + private readonly _inlineRoutingOverrides = new Map(); private readonly _selectionRevisions = new Map(); private readonly _selectionOperationCounters = new Map(); @@ -92,7 +98,20 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { public onDidChangeActiveEnvironment: Event = this._onDidChangeActiveEnvironment.event; - constructor(private readonly pm: PythonProjectManager) {} + constructor( + private readonly pm: PythonProjectManager, + private readonly inlineScriptRouting?: InlineScriptRoutingRegistry, + ) { + if (this.inlineScriptRouting) { + this.subscriptions.push( + this.inlineScriptRouting.onDidChangeRouteability((e) => { + void this.handleInlineScriptRouteabilityChange(e).catch((error) => + traceError('Failed to refresh inline-script routing:', error), + ); + }), + ); + } + } public registerEnvironmentManager(manager: EnvironmentManager, options?: { extensionId?: string }): Disposable { const registrationStopWatch = new StopWatch(); @@ -185,6 +204,8 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { public dispose() { this._environmentManagers.clear(); this._packageManagers.clear(); + this._inlineRoutingOverrides.clear(); + this.subscriptions.forEach((subscription) => subscription.dispose()); this._onDidChangeEnvironmentManager.dispose(); this._onDidChangePackageManager.dispose(); this._onDidChangeEnvironments.dispose(); @@ -198,10 +219,11 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { * * Priority: * 1. Use an exact per-script project setting. - * 2. Use a cached per-script inline selection. - * 3. Use the containing project or default setting. - * 4. Fall back to the cached project/global environment's manager. - * 5. If context is a string or PythonEnvironment, return its manager directly. + * 2. Use an explicit in-session per-script override. + * 3. Use a recognized per-script inline association. + * 4. Use the containing project or default setting. + * 5. Fall back to the cached project/global environment's manager. + * 6. If context is a string or PythonEnvironment, return its manager directly. */ public getEnvironmentManager(context: EnvironmentManagerScope): InternalEnvironmentManager | undefined { if (this._environmentManagers.size === 0) { @@ -211,47 +233,33 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { if (context === undefined || context instanceof Uri) { const project = context ? this.pm.get(context) : undefined; - if ( - context instanceof Uri && - project && - normalizePath(project.uri.fsPath) === normalizePath(context.fsPath) - ) { - const exactManagerId = getProjectEnvironmentManagerSetting(this.pm, context); - const exactManager = exactManagerId - ? this._environmentManagers.get(exactManagerId) - : undefined; - if (exactManager) { - return exactManager; - } + const exactManager = + context instanceof Uri ? this.getExactProjectEnvironmentManager(context, project) : undefined; + if (exactManager) { + return exactManager; } if (context instanceof Uri) { - const inlineEnv = this._activeSelection.get(this.getInlineScriptSelectionKey(context)); - if (inlineEnv?.envId.managerId === INLINE_SCRIPT_MANAGER_ID) { + if (this.inlineScriptRouting) { + const overrideManager = this.getInlineRoutingOverrideManager(context); + if (overrideManager) { + return overrideManager; + } const inlineManager = this._environmentManagers.get(INLINE_SCRIPT_MANAGER_ID); - if (inlineManager) { + if (inlineManager && this.inlineScriptRouting.shouldRoute(context)) { return inlineManager; } + } else { + const inlineEnv = this._activeSelection.get(this.getInlineScriptSelectionKey(context)); + if (inlineEnv?.envId.managerId === INLINE_SCRIPT_MANAGER_ID) { + const inlineManager = this._environmentManagers.get(INLINE_SCRIPT_MANAGER_ID); + if (inlineManager) { + return inlineManager; + } + } } } - - const defaultEnvManagerId = getDefaultEnvManagerSetting(this.pm, context); - if (defaultEnvManagerId !== undefined) { - const settingsManager = this._environmentManagers.get(defaultEnvManagerId); - if (settingsManager) { - return settingsManager; - } - } - - const cachedEnv = this._activeSelection.get(project ? project.uri.toString() : 'global'); - if (cachedEnv) { - const cachedManager = this._environmentManagers.get(cachedEnv.envId.managerId); - if (cachedManager) { - return cachedManager; - } - } - - return undefined; + return this.getConfiguredOrCachedEnvironmentManager(context, project); } if (typeof context === 'string') { @@ -368,6 +376,8 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { const project = scope ? this.pm.get(scope) : undefined; const key = this.getActiveSelectionKey(scope, manager, project); const operation = this.beginSelectionOperation(key); + const publishInlineSelection = + !(scope instanceof Uri) || this.shouldPublishInlineSelectionImmediately(scope, manager); const inlineClearOperation = scope instanceof Uri && manager.id !== INLINE_SCRIPT_MANAGER_ID ? this.beginSelectionOperation(this.getInlineScriptSelectionKey(scope)) @@ -400,8 +410,12 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { } if (scope instanceof Uri) { + this.updateInlineRoutingOverride(scope, manager, environment); this.clearInlineActiveSelection(scope, manager, inlineClearOperation); } + if (!publishInlineSelection) { + return; + } if (!this.commitSelectionOperation(key, operation)) { return; } @@ -475,7 +489,11 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { await setAllManagerSettings(settings); } selections.forEach((selection) => { + this.updateInlineRoutingOverride(selection.scope, manager, environment); this.clearInlineActiveSelection(selection.scope, manager, selection.inlineClearOperation); + if (!selection.publishInlineSelection) { + return; + } if (!this.commitSelectionOperation(selection.key, selection.operation)) { return; } @@ -540,6 +558,7 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { await manager.set(uris); await Promise.all( selections.map(async (selection) => { + this.clearInlineRoutingOverride(selection.scope); const newEnv = await manager.get(selection.scope); if (!this.commitSelectionOperation(selection.key, selection.operation)) { return; @@ -697,6 +716,123 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { return `inline-script:${normalizePath(scope.fsPath)}`; } + private getExactProjectEnvironmentManager( + scope: Uri, + project: PythonProject | undefined, + ): InternalEnvironmentManager | undefined { + if (!project || normalizePath(project.uri.fsPath) !== normalizePath(scope.fsPath)) { + return undefined; + } + const exactManagerId = getProjectEnvironmentManagerSetting(this.pm, scope); + return exactManagerId ? this._environmentManagers.get(exactManagerId) : undefined; + } + + private getConfiguredOrCachedEnvironmentManager( + context: Uri | undefined, + project: PythonProject | undefined, + ): InternalEnvironmentManager | undefined { + const defaultEnvManagerId = getDefaultEnvManagerSetting(this.pm, context); + if (defaultEnvManagerId !== undefined) { + const settingsManager = this._environmentManagers.get(defaultEnvManagerId); + if (settingsManager) { + return settingsManager; + } + } + + const cachedEnv = this._activeSelection.get(this.getProjectSelectionKey(project)); + if (cachedEnv) { + const cachedManager = this._environmentManagers.get(cachedEnv.envId.managerId); + if (cachedManager) { + return cachedManager; + } + } + + return undefined; + } + + private getProjectSelectionKey(project: PythonProject | undefined): string { + return project ? project.uri.toString() : 'global'; + } + + private getInlineRoutingOverrideManager(scope: Uri): InternalEnvironmentManager | undefined { + const managerId = this._inlineRoutingOverrides.get(this.getInlineScriptSelectionKey(scope)); + return managerId ? this._environmentManagers.get(managerId) : undefined; + } + + private updateInlineRoutingOverride( + scope: Uri, + manager: InternalEnvironmentManager, + environment: PythonEnvironment | undefined, + ): void { + if (!this.inlineScriptRouting) { + return; + } + const key = this.getInlineScriptSelectionKey(scope); + if (!environment || manager.id === INLINE_SCRIPT_MANAGER_ID) { + this._inlineRoutingOverrides.delete(key); + return; + } + this._inlineRoutingOverrides.set(key, manager.id); + } + + private clearInlineRoutingOverride(scope: Uri): void { + if (!this.inlineScriptRouting) { + return; + } + this._inlineRoutingOverrides.delete(this.getInlineScriptSelectionKey(scope)); + } + + private async handleInlineScriptRouteabilityChange( + event: InlineScriptRouteabilityChangeEvent, + ): Promise { + const { uri, previousRouteable } = event; + const project = this.pm.get(uri); + const exactManager = this.getExactProjectEnvironmentManager(uri, project); + if (exactManager) { + if (exactManager.id === INLINE_SCRIPT_MANAGER_ID) { + await this.refreshEnvironment(uri); + } + return; + } + + if (this.getInlineRoutingOverrideManager(uri)) { + return; + } + + const manager = this.getEnvironmentManager(uri); + if (!manager) { + return; + } + + const refreshedProject = this.pm.get(uri); + const key = this.getActiveSelectionKey(uri, manager, refreshedProject); + const operation = this.beginSelectionOperation(key); + const newEnv = await manager.get(uri); + const latestProject = this.pm.get(uri); + if (this.getEnvironmentManager(uri) !== manager || !this.commitSelectionOperation(key, operation)) { + return; + } + + const inlineKey = this.getInlineScriptSelectionKey(uri); + const oldEnv = previousRouteable + ? this._activeSelection.get(inlineKey) + : this._activeSelection.get(this.getProjectSelectionKey(latestProject)); + + if (manager.id !== INLINE_SCRIPT_MANAGER_ID) { + this._activeSelection.delete(inlineKey); + } + this._activeSelection.set(key, newEnv); + if (!this.isSameEnvironment(oldEnv, newEnv)) { + await this.fireActiveEnvironmentEvents([ + { + uri: this.getActiveSelectionUri(uri, manager, latestProject), + old: oldEnv, + new: newEnv, + }, + ]); + } + } + private beginPendingSelection(scope: Uri, manager: InternalEnvironmentManager): PendingEnvironmentSelection { const project = this.pm.get(scope); const key = this.getActiveSelectionKey(scope, manager, project); @@ -705,6 +841,7 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { project, key, operation: this.beginSelectionOperation(key), + publishInlineSelection: this.shouldPublishInlineSelectionImmediately(scope, manager), inlineClearOperation: manager.id === INLINE_SCRIPT_MANAGER_ID ? undefined @@ -712,6 +849,10 @@ export class PythonEnvironmentManagers implements EnvironmentManagers { }; } + private shouldPublishInlineSelectionImmediately(scope: Uri, manager: InternalEnvironmentManager): boolean { + return !this.inlineScriptRouting || manager.id !== INLINE_SCRIPT_MANAGER_ID || this.inlineScriptRouting.shouldRoute(scope); + } + private clearInlineActiveSelection( scope: Uri, manager: InternalEnvironmentManager, @@ -801,5 +942,6 @@ interface PendingEnvironmentSelection { readonly project: PythonProject | undefined; readonly key: string; readonly operation: number; + readonly publishInlineSelection: boolean; readonly inlineClearOperation: number | undefined; } diff --git a/src/features/inlineScript/activation.ts b/src/features/inlineScript/activation.ts new file mode 100644 index 000000000..74a9f3b64 --- /dev/null +++ b/src/features/inlineScript/activation.ts @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { InlineScriptRoutingRegistry } from '../../common/inlineScript/routingRegistry'; +import { isInlineScriptsFeatureEnabled } from '../../helpers'; + +export interface InlineScriptFeatureActivation { + readonly enabled: boolean; + readonly routingRegistry: InlineScriptRoutingRegistry | undefined; +} + +/** + * Latch the inline-script feature flag once during activation. + * The setting requires a window reload, so later config changes + * in the same activation must not change the chosen mode. + */ +export function latchInlineScriptFeatureActivation(): InlineScriptFeatureActivation { + const enabled = isInlineScriptsFeatureEnabled(); + return { + enabled, + routingRegistry: enabled ? new InlineScriptRoutingRegistry() : undefined, + }; +} diff --git a/src/features/inlineScript/lazyDetector.ts b/src/features/inlineScript/lazyDetector.ts index fb9756e1f..a9fbca979 100644 --- a/src/features/inlineScript/lazyDetector.ts +++ b/src/features/inlineScript/lazyDetector.ts @@ -2,16 +2,19 @@ // Licensed under the MIT License. import * as path from 'path'; -import { Disposable, TextDocument, TextDocumentChangeEvent, Uri } from 'vscode'; +import { Disposable, TextDocument, TextDocumentChangeEvent, TextDocumentContentChangeEvent, Uri } from 'vscode'; import { readInlineScriptMetadataFromFile } from '../../common/inlineScript/metadata'; +import { getInlineScriptRoutingKey, InlineScriptRoutingRegistry } from '../../common/inlineScript/routingRegistry'; import { traceVerbose, traceWarn } from '../../common/logging'; import { EventNames } from '../../common/telemetry/constants'; import { sendTelemetryEvent } from '../../common/telemetry/sender'; import { getOpenTextDocuments, getWorkspaceFolder, + onDidDeleteFiles, onDidChangeTextDocument, onDidOpenTextDocument, + onDidRenameFiles, onDidSaveTextDocument, } from '../../common/workspace.apis'; @@ -37,9 +40,13 @@ import { */ export class InlineScriptLazyDetector implements Disposable { private readonly subscriptions: Disposable[] = []; - // In-flight reads keyed by `uri.toString()` so rapid open+save - // doesn't double-process the same file. + // In-flight reads keyed by `uri.toString()` so rapid open events + // don't double-process the same file. private readonly inFlight = new Map>(); + // Routing reads have a generation per URI. A save that arrives while + // an older read is in flight advances its generation and queues a + // post-save read, preventing the stale read from publishing metadata. + private readonly routingReadGenerations = new Map(); // URIs (as `uri.toString()`) for which we have already emitted // `inlineScript.detected` in this session. Used to dedup the detection // event across repeat opens/saves and to gate `inlineScript.edited` so @@ -58,6 +65,8 @@ export class InlineScriptLazyDetector implements Disposable { // already torn down. private disposed = false; + constructor(private readonly routingRegistry?: InlineScriptRoutingRegistry) {} + /** * Subscribe to workspace text-document events. Safe to call once * during extension activation. @@ -84,6 +93,12 @@ export class InlineScriptLazyDetector implements Disposable { onDidSaveTextDocument((doc) => this.handleDocument(doc, 'save')), onDidChangeTextDocument((e) => this.handleChange(e)), ); + if (this.routingRegistry) { + this.subscriptions.push( + onDidDeleteFiles((e) => e.files.forEach((uri) => this.clearRouteability(uri))), + onDidRenameFiles((e) => e.files.forEach((file) => this.clearRouteability(file.oldUri))), + ); + } // Defer the catch-up pass so we observe `workspace.textDocuments` // AFTER VS Code finishes registering the document that triggered // our activation. Running the loop synchronously here can race @@ -99,19 +114,14 @@ export class InlineScriptLazyDetector implements Disposable { * `handleDocument` keeps this safe to call repeatedly. */ private replayOpenDocuments(source: 'activate'): void { - // Restrict the replay to documents that the per-event handler - // would actually look at. This keeps the activation log - // proportional to the work the detector will do — on an - // editor with many tabs open we would otherwise dump every - // URI just to throw most of them away inside - // `handleDocument`. - const openDocs = getOpenTextDocuments().filter((d) => shouldHandleUri(d.uri)); + const openDocs = getOpenTextDocuments().filter((d) => this.shouldTrackUri(d.uri)); + const candidateDescription = this.routingRegistry ? 'candidate local .py' : 'candidate .py'; if (openDocs.length === 0) { - traceVerbose(`inlineScriptLazyDetector: ${source} replay found no candidate .py documents`); + traceVerbose(`inlineScriptLazyDetector: ${source} replay found no ${candidateDescription} documents`); return; } traceVerbose( - `inlineScriptLazyDetector: ${source} replay over ${openDocs.length} candidate .py document(s): ` + + `inlineScriptLazyDetector: ${source} replay over ${openDocs.length} ${candidateDescription} document(s): ` + openDocs.map((d) => d.uri.fsPath).join(', '), ); for (const doc of openDocs) { @@ -124,6 +134,7 @@ export class InlineScriptLazyDetector implements Disposable { this.subscriptions.forEach((s) => s.dispose()); this.subscriptions.length = 0; this.inFlight.clear(); + this.routingReadGenerations.clear(); } private async handleDocument(doc: TextDocument, trigger: 'open' | 'save'): Promise { @@ -134,7 +145,7 @@ export class InlineScriptLazyDetector implements Disposable { // the `Trace` log level — to avoid flooding the default // `Info` channel. traceVerbose(`inlineScriptLazyDetector: event received (${trigger}) ${uri.toString()}`); - if (!shouldHandleUri(uri)) { + if (!this.shouldTrackUri(uri)) { traceVerbose( `inlineScriptLazyDetector: skipped (${trigger}) ${uri.toString()} ` + `(scheme='${uri.scheme}', extname='${path.extname(uri.fsPath).toLowerCase()}', ` + @@ -142,30 +153,53 @@ export class InlineScriptLazyDetector implements Disposable { ); return; } + if (this.routingRegistry && trigger === 'open' && doc.isDirty) { + traceVerbose(`inlineScriptLazyDetector: withholding dirty document metadata for ${uri.toString()}`); + this.clearRouteability(uri); + return; + } const key = uri.toString(); const existing = this.inFlight.get(key); if (existing) { - // Coalesce repeated open/save events for the same URI. - // We only parse for observation (telemetry), so the most - // recent in-flight read is good enough; there is no - // cached state downstream that could go stale. + if (this.routingRegistry && trigger === 'save') { + const routingGeneration = this.advanceRoutingReadGeneration(key); + const work = existing.then(() => + this.processOnce(uri, trigger, shouldHandleUri(uri), routingGeneration), + ); + this.trackInFlight(key, work, routingGeneration); + await work; + return; + } + // Coalesce repeated open events, and all events in telemetry-only + // mode, where there is no routing state to become stale. await existing; return; } - const work = this.processOnce(uri, trigger).finally(() => { - this.inFlight.delete(key); - }); - this.inFlight.set(key, work); + const routingGeneration = this.routingRegistry ? this.currentRoutingReadGeneration(key) : undefined; + const work = this.processOnce(uri, trigger, shouldHandleUri(uri), routingGeneration); + this.trackInFlight(key, work, routingGeneration); await work; } - private async processOnce(uri: Uri, trigger: 'open' | 'save'): Promise { + private async processOnce( + uri: Uri, + trigger: 'open' | 'save', + shouldEmitTelemetry: boolean, + routingGeneration?: number, + ): Promise { try { const metadata = await readInlineScriptMetadataFromFile(uri); if (this.disposed) { return; } - if (metadata === undefined) { + if (this.routingRegistry) { + if (this.routingReadGenerations.get(uri.toString()) === routingGeneration) { + this.routingRegistry.setMetadata(uri, metadata); + } + if (!shouldEmitTelemetry || metadata === undefined) { + return; + } + } else if (metadata === undefined) { return; } const key = uri.toString(); @@ -209,6 +243,20 @@ export class InlineScriptLazyDetector implements Disposable { if (e.contentChanges.length === 0) { return; } + if (this.routingRegistry) { + const key = e.document.uri.toString(); + const metadata = this.routingRegistry.getMetadata(e.document.uri); + if ( + (metadata && + this.contentChangesMayAffectMetadata( + e.contentChanges, + metadata.sourceRange?.end ?? metadata.range.end, + )) || + (!metadata && this.inFlight.has(key)) + ) { + this.clearRouteability(e.document.uri); + } + } const key = e.document.uri.toString(); if (!this.detectedUris.has(key)) { return; @@ -224,6 +272,62 @@ export class InlineScriptLazyDetector implements Disposable { ); sendTelemetryEvent(EventNames.INLINE_SCRIPT_EDITED, duration); } + + private contentChangesMayAffectMetadata( + changes: readonly TextDocumentContentChangeEvent[], + metadataEnd: number, + ): boolean { + return changes.some((change) => change.rangeOffset < metadataEnd); + } + + private clearRouteability(uri: Uri): void { + if (!this.routingRegistry || !shouldTrackRoutingUri(uri)) { + return; + } + const key = uri.toString(); + if (this.inFlight.has(key)) { + this.advanceRoutingReadGeneration(key); + } + this.routingRegistry.clearMetadata(uri); + this.routingRegistry.setValidatedAssociation(uri, false); + } + + private currentRoutingReadGeneration(key: string): number { + const current = this.routingReadGenerations.get(key); + if (current !== undefined) { + return current; + } + this.routingReadGenerations.set(key, 0); + return 0; + } + + private advanceRoutingReadGeneration(key: string): number { + const next = this.currentRoutingReadGeneration(key) + 1; + this.routingReadGenerations.set(key, next); + return next; + } + + private trackInFlight(key: string, work: Promise, routingGeneration: number | undefined): void { + this.inFlight.set(key, work); + void work.then( + () => this.clearInFlight(key, work, routingGeneration), + () => this.clearInFlight(key, work, routingGeneration), + ); + } + + private clearInFlight(key: string, work: Promise, routingGeneration: number | undefined): void { + if (this.inFlight.get(key) !== work) { + return; + } + this.inFlight.delete(key); + if (routingGeneration !== undefined) { + this.routingReadGenerations.delete(key); + } + } + + private shouldTrackUri(uri: Uri): boolean { + return this.routingRegistry ? shouldTrackRoutingUri(uri) : shouldHandleUri(uri); + } } /** @@ -244,3 +348,7 @@ export function shouldHandleUri(uri: Uri): boolean { } return true; } + +function shouldTrackRoutingUri(uri: Uri): boolean { + return getInlineScriptRoutingKey(uri) !== undefined; +} diff --git a/src/managers/builtin/inlineScript/envManager.ts b/src/managers/builtin/inlineScript/envManager.ts index 9ab163ed1..5889de526 100644 --- a/src/managers/builtin/inlineScript/envManager.ts +++ b/src/managers/builtin/inlineScript/envManager.ts @@ -27,17 +27,26 @@ import { computeCacheKey, normalizeDependency } from '../../../common/inlineScri import { CacheEnvironmentInspection, INLINE_SCRIPT_CACHE_DIR_NAME, + InlineScriptEnvMeta, + hashSourceMetadataIdentity, + mergeSourceMetadataIdentityHashes, META_SCHEMA_VERSION, getBaseInterpreterStatus, getScriptEnvCacheRoot, getScriptEnvDir, inspectOwnedCacheEntry, inspectMetaJson, + restoreMetaJsonBackupUnderLock, resolveCacheEntryPath, writeMetaJson, } from '../../../common/inlineScript/cacheLayout'; import { extractLowerBoundVersion, pickCompatibleInterpreter } from '../../../common/inlineScript/interpreter'; import { InlineScriptMetadata, readInlineScriptMetadataFromFile } from '../../../common/inlineScript/metadata'; +import { + getInlineScriptMetadataRoutingIdentity, + InlineScriptMetadataChangeEvent, + InlineScriptRoutingRegistry, +} from '../../../common/inlineScript/routingRegistry'; import { CONDA_MANAGER_ID, ENVS_EXTENSION_ID, @@ -61,6 +70,7 @@ import { isFileNotFoundError } from '../../../common/utils/filesystem'; import { normalizePath } from '../../../common/utils/pathUtils'; import { compareReleaseSegments, parseReleaseSegments } from '../../../common/utils/pep440Release'; import { getVenvPythonPath } from '../../../common/utils/virtualEnvironment'; +import { getOpenTextDocuments, onDidDeleteFiles, onDidRenameFiles } from '../../../common/workspace.apis'; import { NativePythonFinder } from '../../common/nativePythonFinder'; import { sortEnvironments } from '../../common/utils'; import { resolveSystemPythonEnvironmentPath } from '../utils'; @@ -84,6 +94,7 @@ const CACHED_ASSOCIATION_VALIDATION_INTERVAL_MS = 5_000; const DISCOVERY_RETRY_DELAYS_MS = [1_000, 5_000, 30_000] as const; /** Workspace-state key for PEP 723 script path to environment executable associations. */ export const INLINE_SCRIPT_ENVS_KEY = `${ENVS_EXTENSION_ID}:inline-script:SCRIPT_ENVIRONMENTS`; +const PERSISTED_ASSOCIATION_SCHEMA_VERSION = 1 as const; interface SelectedBaseInterpreter { readonly environment: PythonEnvironment; @@ -95,6 +106,7 @@ interface CreateOrReuseEnvironmentOptions { readonly packages: ReadonlyArray; readonly metadata: InlineScriptMetadata; readonly selectedBase: SelectedBaseInterpreter; + readonly pendingCreation: PendingCreationContext; } interface BuildCacheEntryResult { @@ -123,27 +135,67 @@ interface DiscoveryRefreshPass { readonly checksForSnapshotChanges: boolean; } +interface PendingCreationContext { + promise: Promise; + sourceMetadataIdentityHashes?: readonly string[]; + hasStartedRecordingSourceMetadataIdentityHashes: boolean; + recordedSourceMetadataIdentityHashes?: readonly string[]; +} + +interface MergeCacheEntrySourceMetadataIdentityHashResult { + readonly success: boolean; + readonly sourceMetadataIdentityHashes?: readonly string[]; +} + type CacheEntryInspection = | { readonly kind: 'absent' | 'stale' | 'uncertain' } | { readonly kind: 'reusable'; readonly environment: PythonEnvironment }; +interface PendingAssociationValidation { + readonly metadataIdentity: string; + readonly associationRevision: number; + readonly promise: Promise; +} + +interface PendingMetadataRefresh { + readonly metadataIdentity: string; + readonly metadataRevision: number; + readonly associationRevision: number; + readonly promise: Promise; +} + +interface ParsedPersistedAssociations { + readonly rawEntries: Record; + readonly records: PersistedInlineScriptEnvironments; + readonly invalidKeys: Set; +} + +interface SavedMetadataSnapshot { + readonly metadata?: InlineScriptMetadata; + readonly identity?: string; +} + /** Manages extension-owned PEP 723 script environments. */ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { private readonly pendingSetups = new Map>(); - private readonly pendingCreations = new Map>(); + private readonly pendingCreations = new Map(); private readonly directlyResolvedBaseInterpreters = new Map(); private baseInterpreterInstallationQueue: Promise = Promise.resolve(); private collection: PythonEnvironment[] = []; - private readonly pendingRehydrations = new Map>(); + private readonly pendingRehydrations = new Map(); + private readonly pendingMetadataRefreshes = new Map(); private readonly fsPathToEnv = new Map(); - private readonly fsPathToPersistedEnvPath = new Map(); + private readonly fsPathToPersistedAssociation = new Map(); private readonly cachedAssociationValidatedAt = new Map(); + private readonly lastValidatedMetadataIdentities = new Map(); + private readonly lastValidatedMetadataIdentityProofs = new Map(); private readonly associationRevisions = new Map(); private pendingRefresh: DiscoveryRefreshPass | undefined; private pendingSnapshotRefresh: Promise | undefined; private activationDiscoveryActive = false; private discoveryRetryAttempt = 0; private discoveryRetryTimer: ReturnType | undefined; + private readonly subscriptions: Disposable[] = []; private persistenceQueue: Promise = Promise.resolve(); private selectionQueue: Promise = Promise.resolve(); private cacheMaintenanceQueue: Promise = Promise.resolve(); @@ -175,7 +227,33 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { private readonly baseManager: EnvironmentManager, private readonly globalStorageUri: Uri, public readonly log: LogOutputChannel, - ) {} + private readonly routingRegistry: InlineScriptRoutingRegistry = new InlineScriptRoutingRegistry(), + ) { + this.subscriptions.push( + this.routingRegistry.onDidChangeMetadata((event) => { + void this.handleSavedMetadataChange(event).catch((error) => { + this.log.warn(`Failed to refresh inline-script routing state: ${getErrorMessage(error)}`); + }); + }), + onDidDeleteFiles((event) => { + void this.clearAssociationsForScripts(event.files).catch((error) => { + this.log.warn(`Failed to clear inline-script associations for deleted files: ${getErrorMessage(error)}`); + }); + }), + onDidRenameFiles((event) => { + void this.clearAssociationsForScripts(event.files.map((file) => file.oldUri)).catch((error) => { + this.log.warn(`Failed to clear inline-script associations for renamed files: ${getErrorMessage(error)}`); + }); + }), + ); + queueMicrotask(() => { + void this.initializePersistedAssociations().catch((error) => { + this.log.warn( + `Failed to prime inline-script environment associations: ${getErrorMessage(error)}`, + ); + }); + }); + } async create( scope: CreateEnvironmentScope, @@ -252,22 +330,47 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { dependencies: packages, interpreterPath: selectedBase.canonicalPath, }); + const metadataIdentity = getInlineScriptMetadataRoutingIdentity(metadata); + const sourceMetadataIdentityHash = metadataIdentity ? hashSourceMetadataIdentity(metadataIdentity) : undefined; const pending = this.pendingCreations.get(cacheKey); if (pending) { - return await pending; + const joinedAfterPendingCreationStartedRecordingSourceMetadataIdentityHashes = + pending.hasStartedRecordingSourceMetadataIdentityHashes; + this.addPendingCreationSourceMetadataIdentityHash(pending, sourceMetadataIdentityHash); + const environment = await pending.promise; + return await this.finalizeCreateForScript( + cacheKey, + environment, + sourceMetadataIdentityHash, + pending, + joinedAfterPendingCreationStartedRecordingSourceMetadataIdentityHashes, + ); } - + const pendingCreation: PendingCreationContext = { + promise: Promise.resolve(undefined), + sourceMetadataIdentityHashes: mergeSourceMetadataIdentityHashes(undefined, sourceMetadataIdentityHash), + hasStartedRecordingSourceMetadataIdentityHashes: false, + }; const creation = this.createOrReuseEnvironment({ cacheKey, packages, metadata, selectedBase, + pendingCreation, }); - this.pendingCreations.set(cacheKey, creation); + pendingCreation.promise = creation; + this.pendingCreations.set(cacheKey, pendingCreation); try { - return await creation; + const environment = await creation; + return await this.finalizeCreateForScript( + cacheKey, + environment, + sourceMetadataIdentityHash, + pendingCreation, + false, + ); } finally { - if (this.pendingCreations.get(cacheKey) === creation) { + if (this.pendingCreations.get(cacheKey) === pendingCreation) { this.pendingCreations.delete(cacheKey); } } @@ -304,6 +407,45 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { return this.installAndSelectBaseInterpreter(metadata, selection.discoveryFailed); } + private addPendingCreationSourceMetadataIdentityHash( + pendingCreation: PendingCreationContext, + sourceMetadataIdentityHash: string | undefined, + ): void { + pendingCreation.sourceMetadataIdentityHashes = mergeSourceMetadataIdentityHashes( + pendingCreation.sourceMetadataIdentityHashes, + sourceMetadataIdentityHash, + ); + } + + private async finalizeCreateForScript( + cacheKey: string, + environment: PythonEnvironment | undefined, + sourceMetadataIdentityHash: string | undefined, + pendingCreation: PendingCreationContext, + joinedAfterPendingCreationStartedRecordingSourceMetadataIdentityHashes: boolean, + ): Promise { + if (!environment || !sourceMetadataIdentityHash) { + return environment; + } + if ( + pendingCreation.recordedSourceMetadataIdentityHashes?.includes(sourceMetadataIdentityHash) !== true && + joinedAfterPendingCreationStartedRecordingSourceMetadataIdentityHashes + ) { + const mergeResult = await this.mergeCacheEntrySourceMetadataIdentityHash( + cacheKey, + sourceMetadataIdentityHash, + ); + if (!mergeResult.success) { + this.log.warn( + `Failed to durably record inline-script cache provenance for ${cacheKey}; returning no environment to the caller.`, + ); + return undefined; + } + pendingCreation.recordedSourceMetadataIdentityHashes = mergeResult.sourceMetadataIdentityHashes; + } + return environment; + } + async refresh(_scope: RefreshEnvironmentsScope): Promise { if (this.disposed) { return; @@ -750,15 +892,22 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { const updates: PendingScriptUpdate[] = []; for (const script of scripts) { const before = await this.getAssociationForMutation(script.scriptPath); - const hadPersistedAssociation = this.fsPathToPersistedEnvPath.has(script.scriptPath); - const hasSamePersistedEnvironment = - environmentPath !== undefined && - normalizePath(this.fsPathToPersistedEnvPath.get(script.scriptPath) ?? '') === - normalizePath(environmentPath); - const needsPersistence = environment ? !hasSamePersistedEnvironment : hadPersistedAssociation; + const persistedAssociation = this.getPersistedAssociationFromMemory(script.scriptPath); + const savedMetadata = environment ? await this.getSavedMetadataForPersistence(script.uri) : undefined; + const sourceMetadataIdentity = + environment && savedMetadata + ? await this.resolveVerifiedSourceMetadataIdentity(script, environment, savedMetadata) + : undefined; + const nextPersistedAssociation = environmentPath + ? this.createPersistedAssociationRecord(environmentPath, sourceMetadataIdentity, savedMetadata?.identity) + : undefined; + const needsPersistence = nextPersistedAssociation + ? !this.isSamePersistedAssociation(persistedAssociation, nextPersistedAssociation) + : persistedAssociation !== undefined; const shouldNotify = - (!this.isSameEnvironment(before, environment) && !hasSamePersistedEnvironment) || - (!environment && hadPersistedAssociation); + (!this.isSameEnvironment(before, environment) && + !this.isSamePersistedAssociation(persistedAssociation, nextPersistedAssociation)) || + (!environment && persistedAssociation !== undefined); const hasPendingRehydration = this.pendingRehydrations.has(script.scriptPath); const cached = this.fsPathToEnv.get(script.scriptPath); const needsMemoryUpdate = environment ? cached !== environment : cached !== undefined; @@ -766,6 +915,7 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { updates.push({ ...script, before, + persistedAssociation: nextPersistedAssociation, needsPersistence, shouldNotify, }); @@ -781,7 +931,7 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { await this.updatePersistedAssociations( persistenceUpdates.map((update) => ({ scriptPath: update.scriptPath, - environmentPath, + persistedAssociation: update.persistedAssociation, })), ); } @@ -793,14 +943,15 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { for (const update of updates) { this.bumpAssociationRevision(update.scriptPath); this.pendingRehydrations.delete(update.scriptPath); + this.pendingMetadataRefreshes.delete(update.scriptPath); if (environment) { this.fsPathToEnv.set(update.scriptPath, environment); - this.fsPathToPersistedEnvPath.set(update.scriptPath, environmentPath!); - this.cachedAssociationValidatedAt.set(update.scriptPath, Date.now()); + this.fsPathToPersistedAssociation.set(update.scriptPath, update.persistedAssociation!); + this.invalidateCachedAssociationValidation(update.scriptPath); } else { this.fsPathToEnv.delete(update.scriptPath); - this.fsPathToPersistedEnvPath.delete(update.scriptPath); - this.cachedAssociationValidatedAt.delete(update.scriptPath); + this.fsPathToPersistedAssociation.delete(update.scriptPath); + this.invalidateCachedAssociationValidation(update.scriptPath); } if (update.shouldNotify) { this._onDidChangeEnvironment.fire({ @@ -810,6 +961,16 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { }); } } + + await Promise.all( + updates.map(async (update) => { + if (!environment) { + this.clearValidatedRouteableState(update.uri); + return; + } + await this.updateValidatedStateForSelection(update); + }), + ); } private async getInternal(scope: GetEnvironmentScope): Promise { @@ -824,15 +985,11 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { return undefined; } - const environment = await this.getAssociation(normalizePath(scope.fsPath), scope); - if (!environment) { - return undefined; - } - - const requiresPython = metadata.requiresPython?.trim(); - return requiresPython && !this.matchesInstallConstraint(requiresPython, environment.version) - ? undefined - : environment; + return this.getAssociationForMetadata( + normalizePath(scope.fsPath), + scope, + metadata, + ); } private getScriptUris(scope: SetEnvironmentScope): ScriptReference[] { @@ -857,39 +1014,72 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { return scripts; } - private async getAssociation(scriptPath: string, scriptUri: Uri): Promise { + private async getAssociationForMetadata( + scriptPath: string, + scriptUri: Uri, + metadata: InlineScriptMetadata, + ): Promise { const pending = this.pendingRehydrations.get(scriptPath); - if (pending) { - return pending; - } - const cached = this.fsPathToEnv.get(scriptPath); const revision = this.associationRevisions.get(scriptPath) ?? 0; + const metadataIdentity = getInlineScriptMetadataRoutingIdentity(metadata)!; + const forceFreshValidation = + this.fsPathToPersistedAssociation.get(scriptPath)?.metadataBinding.kind === 'pending'; + if ( + pending && + pending.metadataIdentity === metadataIdentity && + pending.associationRevision === revision + ) { + return pending.promise; + } if (cached) { const validatedAt = this.cachedAssociationValidatedAt.get(scriptPath); if ( + !forceFreshValidation && validatedAt !== undefined && + this.lastValidatedMetadataIdentities.get(scriptPath) === metadataIdentity && Date.now() - validatedAt < CACHED_ASSOCIATION_VALIDATION_INTERVAL_MS ) { return cached; } - const validation = this.validateCachedAssociation(scriptPath, scriptUri, cached, revision); - this.pendingRehydrations.set(scriptPath, validation); + const validation = this.validateCachedAssociation( + scriptPath, + scriptUri, + cached, + revision, + metadataIdentity, + metadata, + ); + this.pendingRehydrations.set(scriptPath, { + metadataIdentity, + associationRevision: revision, + promise: validation, + }); try { return await validation; } finally { - if (this.pendingRehydrations.get(scriptPath) === validation) { + if (this.pendingRehydrations.get(scriptPath)?.promise === validation) { this.pendingRehydrations.delete(scriptPath); } } } - const rehydration = this.rehydrateAssociation(scriptPath, scriptUri, revision); - this.pendingRehydrations.set(scriptPath, rehydration); + const rehydration = this.rehydrateAssociation( + scriptPath, + scriptUri, + revision, + metadataIdentity, + metadata, + ); + this.pendingRehydrations.set(scriptPath, { + metadataIdentity, + associationRevision: revision, + promise: rehydration, + }); try { return await rehydration; } finally { - if (this.pendingRehydrations.get(scriptPath) === rehydration) { + if (this.pendingRehydrations.get(scriptPath)?.promise === rehydration) { this.pendingRehydrations.delete(scriptPath); } } @@ -909,8 +1099,11 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { scriptUri: Uri, cached: PythonEnvironment, revision: number, + metadataIdentity: string, + metadata: InlineScriptMetadata, ): Promise { const environmentPath = cached.environmentPath.fsPath; + const expectedPersistedAssociation = this.fsPathToPersistedAssociation.get(scriptPath); const envDirPath = path.dirname(path.dirname(environmentPath)); const busy = await this.isCacheEntryBusy(envDirPath); if (!this.isCurrentAssociationRevision(scriptPath, revision)) { @@ -948,13 +1141,35 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { environmentPath, revision, scriptUri, + expectedPersistedAssociation, ); return undefined; } if (ownership !== 'expected') { return undefined; } + const metadataMatch = this.inspectAssociationMetadata(scriptPath, metadataIdentity, true); + if (!this.isCurrentAssociationRevision(scriptPath, revision)) { + return this.fsPathToEnv.get(scriptPath); + } + if (metadataMatch === 'mismatched') { + return undefined; + } + const metadataIdentityProven = await this.currentCacheEntryProvesSourceMetadataIdentity( + resolved, + metadataIdentity, + metadata, + ); + if (!this.isCurrentAssociationRevision(scriptPath, revision)) { + return this.fsPathToEnv.get(scriptPath); + } + const current = this.fsPathToEnv.get(scriptPath); this.cachedAssociationValidatedAt.set(scriptPath, Date.now()); + this.lastValidatedMetadataIdentities.set(scriptPath, metadataIdentity); + this.lastValidatedMetadataIdentityProofs.set(scriptPath, metadataIdentityProven); + if (current && this.isSameEnvironment(current, resolved)) { + return current; + } if (cached.version === resolved.version) { return cached; } @@ -972,6 +1187,7 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { environmentPath, revision, scriptUri, + expectedPersistedAssociation, ); } } catch (error) { @@ -989,6 +1205,7 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { environmentPath, revision, scriptUri, + expectedPersistedAssociation, ); } } else { @@ -1004,14 +1221,17 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { scriptPath: string, scriptUri: Uri, revision: number, + metadataIdentity: string, + metadata: InlineScriptMetadata, ): Promise { - let environmentPath: string | undefined; + let persistedAssociation: PersistedAssociationRecord | undefined; try { - environmentPath = await this.getPersistedAssociation(scriptPath); + persistedAssociation = await this.getPersistedAssociation(scriptPath); } catch (error) { this.log.warn(`Failed to read inline-script environment association: ${getErrorMessage(error)}`); return undefined; } + const environmentPath = persistedAssociation?.environmentPath; if (!environmentPath) { return undefined; } @@ -1019,7 +1239,13 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { return this.fsPathToEnv.get(scriptPath); } if (!path.isAbsolute(environmentPath)) { - await this.removeStalePersistedAssociation(scriptPath, environmentPath, revision, scriptUri); + await this.removeStalePersistedAssociation( + scriptPath, + environmentPath, + revision, + scriptUri, + persistedAssociation, + ); return undefined; } const envDirPath = path.dirname(path.dirname(environmentPath)); @@ -1031,14 +1257,26 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { const stat = await fs.stat(environmentPath); if (!stat.isFile()) { if (!(await this.isCacheEntryBusy(envDirPath))) { - await this.removeStalePersistedAssociation(scriptPath, environmentPath, revision, scriptUri); + await this.removeStalePersistedAssociation( + scriptPath, + environmentPath, + revision, + scriptUri, + persistedAssociation, + ); } return undefined; } } catch (error) { if (this.isDefinitivelyStalePathError(error)) { if (!(await this.isCacheEntryBusy(envDirPath))) { - await this.removeStalePersistedAssociation(scriptPath, environmentPath, revision, scriptUri); + await this.removeStalePersistedAssociation( + scriptPath, + environmentPath, + revision, + scriptUri, + persistedAssociation, + ); } } else { this.log.warn( @@ -1081,22 +1319,66 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { return undefined; } if (ownership === 'stale') { - await this.removeStalePersistedAssociation(scriptPath, environmentPath, revision, scriptUri); + await this.removeStalePersistedAssociation( + scriptPath, + environmentPath, + revision, + scriptUri, + persistedAssociation, + ); return undefined; } if (ownership !== 'expected') { return undefined; } + const metadataMatch = this.inspectAssociationMetadata(scriptPath, metadataIdentity, true); + if (metadataMatch === 'mismatched') { + return undefined; + } + const metadataIdentityProven = await this.currentCacheEntryProvesSourceMetadataIdentity( + resolved, + metadataIdentity, + metadata, + ); + if (!this.isCurrentAssociationRevision(scriptPath, revision)) { + return this.fsPathToEnv.get(scriptPath); + } + const current = this.fsPathToEnv.get(scriptPath); + this.cachedAssociationValidatedAt.set(scriptPath, Date.now()); + this.lastValidatedMetadataIdentities.set(scriptPath, metadataIdentity); + this.lastValidatedMetadataIdentityProofs.set(scriptPath, metadataIdentityProven); + if (current && this.isSameEnvironment(current, resolved)) { + return current; + } if (!this.isCurrentAssociationRevision(scriptPath, revision) || this.fsPathToEnv.has(scriptPath)) { return this.fsPathToEnv.get(scriptPath); } this.fsPathToEnv.set(scriptPath, resolved); - this.cachedAssociationValidatedAt.set(scriptPath, Date.now()); this._onDidChangeEnvironment.fire({ uri: scriptUri, old: undefined, new: resolved }); return resolved; } + private inspectAssociationMetadata( + scriptPath: string, + metadataIdentity: string, + allowUnboundAssociation: boolean, + ): 'matched' | 'pending' | 'legacy' | 'mismatched' { + const persistedAssociation = this.fsPathToPersistedAssociation.get(scriptPath); + if (!persistedAssociation) { + return 'mismatched'; + } + if (persistedAssociation.metadataBinding.kind === 'matched') { + return persistedAssociation.metadataBinding.sourceIdentity === metadataIdentity ? 'matched' : 'mismatched'; + } + if (persistedAssociation.metadataBinding.kind === 'pending') { + return persistedAssociation.metadataBinding.sourceIdentity === metadataIdentity && allowUnboundAssociation + ? 'pending' + : 'mismatched'; + } + return allowUnboundAssociation ? 'legacy' : 'mismatched'; + } + private async inspectAssociationOwnership(environment: PythonEnvironment): Promise { if (environment.envId.managerId !== INLINE_SCRIPT_MANAGER_ID || !path.isAbsolute(environment.sysPrefix)) { return 'uncertain'; @@ -1117,33 +1399,445 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { ); } - private async getPersistedAssociation(scriptPath: string): Promise { + private async handleSavedMetadataChange(event: InlineScriptMetadataChangeEvent): Promise { + if (event.metadata === undefined) { + this.clearValidatedRouteableState(event.uri); + return; + } + await this.refreshValidatedAssociationForMetadata( + event.uri, + event.metadata, + event.metadataIdentity ?? getInlineScriptMetadataRoutingIdentity(event.metadata)!, + event.metadataRevision, + ); + } + + private async refreshValidatedAssociationForMetadata( + uri: Uri, + metadata: InlineScriptMetadata, + metadataIdentity: string, + metadataRevision: number, + ): Promise { + const scriptPath = normalizePath(uri.fsPath); + const associationRevision = this.associationRevisions.get(scriptPath) ?? 0; + const pendingRefresh = this.pendingMetadataRefreshes.get(scriptPath); + if ( + pendingRefresh && + pendingRefresh.metadataIdentity === metadataIdentity && + pendingRefresh.metadataRevision === metadataRevision && + pendingRefresh.associationRevision === associationRevision + ) { + return pendingRefresh.promise; + } + const refresh = this.refreshValidatedAssociationForMetadataInternal( + scriptPath, + uri, + metadata, + metadataIdentity, + metadataRevision, + associationRevision, + ); + this.pendingMetadataRefreshes.set(scriptPath, { + metadataIdentity, + metadataRevision, + associationRevision, + promise: refresh, + }); + try { + await refresh; + } finally { + if (this.pendingMetadataRefreshes.get(scriptPath)?.promise === refresh) { + this.pendingMetadataRefreshes.delete(scriptPath); + } + } + } + + private async refreshValidatedAssociationForMetadataInternal( + scriptPath: string, + uri: Uri, + metadata: InlineScriptMetadata, + metadataIdentity: string, + metadataRevision: number, + associationRevision: number, + ): Promise { + const environment = await this.getAssociationForMetadata(scriptPath, uri, metadata); + if (!this.isCurrentMetadataRefreshTask(uri, metadataIdentity, metadataRevision, scriptPath, associationRevision)) { + return; + } + if (!environment) { + this.clearValidatedRouteableState(uri); + return; + } + let metadataIdentityProven = this.lastValidatedMetadataIdentityProofs.get(scriptPath); + if ( + this.lastValidatedMetadataIdentities.get(scriptPath) !== metadataIdentity || + metadataIdentityProven === undefined + ) { + metadataIdentityProven = await this.currentCacheEntryProvesSourceMetadataIdentity( + environment, + metadataIdentity, + metadata, + ); + if ( + !this.isCurrentMetadataRefreshTask( + uri, + metadataIdentity, + metadataRevision, + scriptPath, + associationRevision, + ) + ) { + return; + } + this.cachedAssociationValidatedAt.set(scriptPath, Date.now()); + this.lastValidatedMetadataIdentities.set(scriptPath, metadataIdentity); + this.lastValidatedMetadataIdentityProofs.set(scriptPath, metadataIdentityProven); + } + if (metadataIdentityProven !== true) { + this.clearValidatedRouteableState(uri); + return; + } + const metadataMatch = this.inspectAssociationMetadata(scriptPath, metadataIdentity, true); + if (metadataMatch === 'pending') { + let bindResult = await this.bindPendingMetadataIdentity( + scriptPath, + environment.environmentPath.fsPath, + metadataIdentity, + metadataRevision, + associationRevision, + uri, + ); + if (!this.isCurrentRoutingMetadata(uri, metadataIdentity, metadataRevision)) { + return; + } + if ( + bindResult === 'stale' && + !this.isCurrentAssociationRevision(scriptPath, associationRevision) + ) { + const currentAssociation = this.fsPathToPersistedAssociation.get(scriptPath); + const currentAssociationRevision = this.associationRevisions.get(scriptPath) ?? 0; + if ( + currentAssociation?.metadataBinding.kind === 'pending' && + currentAssociation.metadataBinding.sourceIdentity === metadataIdentity && + normalizePath(currentAssociation.environmentPath) === + normalizePath(environment.environmentPath.fsPath) + ) { + bindResult = await this.bindPendingMetadataIdentity( + scriptPath, + environment.environmentPath.fsPath, + metadataIdentity, + metadataRevision, + currentAssociationRevision, + uri, + ); + if ( + !this.isCurrentMetadataRefreshTask( + uri, + metadataIdentity, + metadataRevision, + scriptPath, + currentAssociationRevision, + ) + ) { + return; + } + } + } else if (!this.isCurrentAssociationRevision(scriptPath, associationRevision)) { + return; + } + if (bindResult !== 'bound') { + const currentAssociation = this.fsPathToPersistedAssociation.get(scriptPath); + if ( + currentAssociation?.metadataBinding.kind === 'pending' && + currentAssociation.metadataBinding.sourceIdentity === metadataIdentity && + normalizePath(currentAssociation.environmentPath) === + normalizePath(environment.environmentPath.fsPath) + ) { + this.invalidateCachedAssociationValidation(scriptPath); + } + return; + } + } else if (metadataMatch !== 'matched') { + this.clearValidatedRouteableState(uri); + return; + } + this.routingRegistry.setValidatedAssociation(uri, true); + } + + private async updateValidatedStateForSelection(script: ScriptReference): Promise { + const savedMetadata = await this.getSavedMetadataForPersistence(script.uri); + if (!savedMetadata.identity) { + this.clearValidatedRouteableState(script.uri); + return; + } + if (this.inspectAssociationMetadata(script.scriptPath, savedMetadata.identity, false) !== 'matched') { + this.clearValidatedRouteableState(script.uri); + return; + } + this.cachedAssociationValidatedAt.set(script.scriptPath, Date.now()); + this.lastValidatedMetadataIdentities.set(script.scriptPath, savedMetadata.identity); + this.routingRegistry.setValidatedAssociation( + script.uri, + this.routingRegistry.getMetadataIdentity(script.uri) === savedMetadata.identity, + ); + } + + private async getSavedMetadataForPersistence(uri: Uri): Promise { + for (const document of getOpenTextDocuments()) { + if (document.uri.toString() === uri.toString() && document.isDirty) { + return {}; + } + } + return this.readSavedMetadataSnapshot(uri); + } + + private async readSavedMetadataSnapshot(uri: Uri): Promise { + const metadata = await readInlineScriptMetadataFromFile(uri); + return { + metadata, + identity: getInlineScriptMetadataRoutingIdentity(metadata), + }; + } + + private async currentCacheEntryProvesSourceMetadataIdentity( + environment: PythonEnvironment, + metadataIdentity: string, + metadata: InlineScriptMetadata, + ): Promise { + const sidecar = await this.readCurrentCacheEntrySidecar(environment); + return !!sidecar && this.cacheEntryProvesSourceMetadataIdentity(sidecar, environment, metadataIdentity, metadata); + } + + private async readCurrentCacheEntrySidecar(environment: PythonEnvironment): Promise { + let sidecarResult; + try { + sidecarResult = await inspectMetaJson(Uri.file(environment.sysPrefix)); + } catch { + return undefined; + } + return sidecarResult.kind === 'valid' ? sidecarResult.metadata : undefined; + } + + private cacheEntryProvesSourceMetadataIdentity( + sidecar: InlineScriptEnvMeta, + environment: PythonEnvironment, + metadataIdentity: string, + metadata: InlineScriptMetadata, + ): boolean { + return ( + this.sidecarProvesSourceMetadataIdentity(sidecar, metadataIdentity) || + this.isMetadataOnlyCacheEntryForMetadata(sidecar, environment, metadata) + ); + } + + private async resolveVerifiedSourceMetadataIdentity( + script: ScriptReference, + environment: PythonEnvironment, + savedMetadata: SavedMetadataSnapshot, + ): Promise { + if (savedMetadata.identity) { + return savedMetadata.metadata && + (await this.currentCacheEntryProvesSourceMetadataIdentity( + environment, + savedMetadata.identity, + savedMetadata.metadata, + )) + ? savedMetadata.identity + : undefined; + } + + const persistedSourceMetadataIdentity = this.getPersistedSourceMetadataIdentity( + script.scriptPath, + environment.environmentPath.fsPath, + ); + if (persistedSourceMetadataIdentity) { + const sidecar = await this.readCurrentCacheEntrySidecar(environment); + if (sidecar && this.sidecarProvesSourceMetadataIdentity(sidecar, persistedSourceMetadataIdentity)) { + return persistedSourceMetadataIdentity; + } + } + + const savedSourceMetadata = await this.readSavedMetadataSnapshot(script.uri); + if (!savedSourceMetadata.identity || !savedSourceMetadata.metadata) { + return undefined; + } + return (await this.currentCacheEntryProvesSourceMetadataIdentity( + environment, + savedSourceMetadata.identity, + savedSourceMetadata.metadata, + )) + ? savedSourceMetadata.identity + : undefined; + } + + private sidecarProvesSourceMetadataIdentity( + sidecar: InlineScriptEnvMeta, + metadataIdentity: string, + ): boolean { + if (sidecar.sourceMetadataIdentityHashes === undefined) { + return false; + } + const expectedHash = hashSourceMetadataIdentity(metadataIdentity); + return sidecar.sourceMetadataIdentityHashes.includes(expectedHash); + } + + private isMetadataOnlyCacheEntryForMetadata( + sidecar: InlineScriptEnvMeta, + environment: PythonEnvironment, + metadata: InlineScriptMetadata, + ): boolean { + if (sidecar.sourceMetadataIdentityHashes !== undefined) { + return false; + } + const expectedCacheKey = computeCacheKey({ + dependencies: metadata.dependencies ?? [], + interpreterPath: sidecar.baseInterpreterPath, + }); + if ( + normalizePath(getScriptEnvDir(this.globalStorageUri, expectedCacheKey).fsPath) !== + normalizePath(environment.sysPrefix) + ) { + return false; + } + const requiresPython = metadata.requiresPython?.trim(); + return !requiresPython || this.matchesInstallConstraint(requiresPython, environment.version); + } + + private getPersistedSourceMetadataIdentity(scriptPath: string, environmentPath: string): string | undefined { + const persistedAssociation = this.fsPathToPersistedAssociation.get(scriptPath); + return persistedAssociation && + normalizePath(persistedAssociation.environmentPath) === normalizePath(environmentPath) && + (persistedAssociation.metadataBinding.kind === 'matched' || + persistedAssociation.metadataBinding.kind === 'pending') + ? persistedAssociation.metadataBinding.sourceIdentity + : undefined; + } + + private async bindPendingMetadataIdentity( + scriptPath: string, + environmentPath: string, + metadataIdentity: string, + metadataRevision: number, + associationRevision: number, + uri: Uri, + ): Promise<'bound' | 'stale' | 'failed'> { + return this.enqueueSelection(async () => { + if ( + !this.isCurrentAssociationRevision(scriptPath, associationRevision) || + !this.isCurrentRoutingMetadata(uri, metadataIdentity, metadataRevision) + ) { + return 'stale'; + } + const expectedAssociation: PersistedAssociationRecord = { + environmentPath, + metadataBinding: { kind: 'pending', sourceIdentity: metadataIdentity }, + }; + const matchedAssociation: PersistedAssociationRecord = { + environmentPath, + metadataBinding: { kind: 'matched', sourceIdentity: metadataIdentity }, + }; + if (!this.isSamePersistedAssociation(this.fsPathToPersistedAssociation.get(scriptPath), expectedAssociation)) { + return 'stale'; + } + try { + await this.updatePersistedAssociations([ + { + scriptPath, + persistedAssociation: matchedAssociation, + expectedPersistedAssociation: expectedAssociation, + }, + ]); + } catch (error) { + this.log.warn(`Failed to bind inline-script metadata identity: ${getErrorMessage(error)}`); + return 'failed'; + } + if ( + !this.isCurrentAssociationRevision(scriptPath, associationRevision) || + !this.isCurrentRoutingMetadata(uri, metadataIdentity, metadataRevision) + ) { + return 'stale'; + } + return this.isSamePersistedAssociation(this.fsPathToPersistedAssociation.get(scriptPath), matchedAssociation) + ? 'bound' + : 'stale'; + }); + } + + private isCurrentMetadataRefreshTask( + uri: Uri, + metadataIdentity: string, + metadataRevision: number, + scriptPath: string, + associationRevision: number, + ): boolean { + return ( + this.isCurrentRoutingMetadata(uri, metadataIdentity, metadataRevision) && + this.isCurrentAssociationRevision(scriptPath, associationRevision) + ); + } + + private isCurrentRoutingMetadata(uri: Uri, metadataIdentity: string, metadataRevision: number): boolean { + return ( + this.routingRegistry.getMetadataIdentity(uri) === metadataIdentity && + this.routingRegistry.getMetadataRevision(uri) === metadataRevision + ); + } + + private clearValidatedRouteableState(script: Uri | string): void { + const scriptPath = typeof script === 'string' ? script : normalizePath(script.fsPath); + this.invalidateCachedAssociationValidation(scriptPath); + this.routingRegistry.setValidatedAssociation(script, false); + } + + private invalidateCachedAssociationValidation(scriptPath: string): void { + this.cachedAssociationValidatedAt.delete(scriptPath); + this.lastValidatedMetadataIdentities.delete(scriptPath); + this.lastValidatedMetadataIdentityProofs.delete(scriptPath); + } + + private initializePersistedAssociations(): Promise { + return this.enqueuePersistence(async (state) => { + const rawAssociations = await state.get(INLINE_SCRIPT_ENVS_KEY); + const parsed = this.parsePersistedAssociations(rawAssociations); + this.applyPersistedAssociations(parsed?.records ?? {}); + }).then(async () => { + await Promise.all( + [...this.fsPathToPersistedAssociation.keys()].map(async (scriptPath) => { + const uri = this.routingRegistry.getUri(scriptPath); + const metadata = this.routingRegistry.getMetadata(scriptPath); + if (uri && metadata) { + await this.refreshValidatedAssociationForMetadata( + uri, + metadata, + getInlineScriptMetadataRoutingIdentity(metadata)!, + this.routingRegistry.getMetadataRevision(uri), + ); + } + }), + ); + }); + } + + private async getPersistedAssociation(scriptPath: string): Promise { await this.persistenceQueue; const state = await getWorkspacePersistentState(); - const raw = await state.get(INLINE_SCRIPT_ENVS_KEY); - if (raw === undefined) { - this.fsPathToPersistedEnvPath.delete(scriptPath); + const rawAssociations = await state.get(INLINE_SCRIPT_ENVS_KEY); + if (rawAssociations === undefined) { + this.applyPersistedAssociations({}); return undefined; } - const associations = this.asPersistedAssociations(raw); - if (!associations) { + const parsed = this.parsePersistedAssociations(rawAssociations); + if (!parsed) { await this.removeInvalidPersistedAssociation(scriptPath); - this.fsPathToPersistedEnvPath.delete(scriptPath); - return undefined; + return this.getPersistedAssociationFromMemory(scriptPath); } - const rawValue = (raw as Record)[scriptPath]; - if (rawValue !== undefined && (typeof rawValue !== 'string' || rawValue.length === 0)) { + const rawValue = (rawAssociations as Record)[scriptPath]; + if (rawValue !== undefined && this.parsePersistedAssociationValue(rawValue).kind === 'invalid') { await this.removeInvalidPersistedAssociation(scriptPath); - this.fsPathToPersistedEnvPath.delete(scriptPath); - return undefined; + return this.getPersistedAssociationFromMemory(scriptPath); } - const environmentPath = associations[scriptPath]; - if (environmentPath) { - this.fsPathToPersistedEnvPath.set(scriptPath, environmentPath); - } else { - this.fsPathToPersistedEnvPath.delete(scriptPath); - } - return environmentPath; + this.applyPersistedAssociations(parsed.records); + return this.getPersistedAssociationFromMemory(scriptPath); } private async removeStalePersistedAssociation( @@ -1151,23 +1845,31 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { expectedEnvironmentPath: string, revision: number, scriptUri?: Uri, + expectedPersistedAssociation?: PersistedAssociationRecord, ): Promise { await this.enqueueSelection(async () => { if (!this.isCurrentAssociationRevision(scriptPath, revision)) { return; } try { - await this.updatePersistedAssociations([{ scriptPath, expectedEnvironmentPath }]); + const persistedPathBeforeUpdate = this.fsPathToPersistedAssociation.get(scriptPath)?.environmentPath; + await this.updatePersistedAssociations([ + { + scriptPath, + expectedEnvironmentPath, + expectedPersistedAssociation, + }, + ]); if ( - normalizePath(this.fsPathToPersistedEnvPath.get(scriptPath) ?? '') === - normalizePath(expectedEnvironmentPath) && + normalizePath(persistedPathBeforeUpdate ?? '') === normalizePath(expectedEnvironmentPath) && + !this.fsPathToPersistedAssociation.has(scriptPath) && this.isCurrentAssociationRevision(scriptPath, revision) ) { const old = this.fsPathToEnv.get(scriptPath); this.bumpAssociationRevision(scriptPath); this.fsPathToEnv.delete(scriptPath); - this.fsPathToPersistedEnvPath.delete(scriptPath); - this.cachedAssociationValidatedAt.delete(scriptPath); + this.fsPathToPersistedAssociation.delete(scriptPath); + this.clearValidatedRouteableState(scriptPath); if (old && scriptUri) { this._onDidChangeEnvironment.fire({ uri: scriptUri, old, new: undefined }); } @@ -1182,54 +1884,218 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { private removeInvalidPersistedAssociation(scriptPath: string): Promise { return this.enqueuePersistence(async (state) => { - const raw = await state.get(INLINE_SCRIPT_ENVS_KEY); - if (raw === undefined) { + const rawAssociations = await state.get(INLINE_SCRIPT_ENVS_KEY); + if (rawAssociations === undefined) { + this.applyPersistedAssociations({}); return; } - const associations = this.asPersistedAssociations(raw); - if (!associations) { + const parsed = this.parsePersistedAssociations(rawAssociations); + if (!parsed) { await state.set(INLINE_SCRIPT_ENVS_KEY, {}); + this.applyPersistedAssociations({}); return; } - const rawValue = (raw as Record)[scriptPath]; - if (rawValue !== undefined && (typeof rawValue !== 'string' || rawValue.length === 0)) { - delete associations[scriptPath]; - await state.set(INLINE_SCRIPT_ENVS_KEY, associations); + if (parsed.invalidKeys.has(scriptPath)) { + delete parsed.rawEntries[scriptPath]; + delete parsed.records[scriptPath]; + parsed.invalidKeys.delete(scriptPath); + await state.set(INLINE_SCRIPT_ENVS_KEY, parsed.rawEntries); } + this.applyPersistedAssociations(parsed.records); }); } private updatePersistedAssociations(changes: readonly PersistedAssociationChange[]): Promise { return this.enqueuePersistence(async (state) => { - const raw = await state.get(INLINE_SCRIPT_ENVS_KEY); - const associations = { ...(this.asPersistedAssociations(raw) ?? {}) }; + const rawAssociations = await state.get(INLINE_SCRIPT_ENVS_KEY); + const parsed = this.parsePersistedAssociations(rawAssociations); + const rawEntries = { ...(parsed?.rawEntries ?? {}) }; + const associations = { ...(parsed?.records ?? {}) }; for (const change of changes) { const current = associations[change.scriptPath]; - if (change.environmentPath) { - associations[change.scriptPath] = change.environmentPath; + if (change.persistedAssociation) { + if ( + change.expectedPersistedAssociation && + !this.isSamePersistedAssociation(current, change.expectedPersistedAssociation) + ) { + continue; + } + associations[change.scriptPath] = change.persistedAssociation; + rawEntries[change.scriptPath] = this.serializePersistedAssociation(change.persistedAssociation); } else if ( - change.expectedEnvironmentPath === undefined || - (current !== undefined && - normalizePath(current) === normalizePath(change.expectedEnvironmentPath)) + (change.expectedPersistedAssociation && + this.isSamePersistedAssociation(current, change.expectedPersistedAssociation)) || + (change.expectedPersistedAssociation === undefined && + (change.expectedEnvironmentPath === undefined || + (current !== undefined && + normalizePath(current.environmentPath) === normalizePath(change.expectedEnvironmentPath)))) ) { delete associations[change.scriptPath]; + delete rawEntries[change.scriptPath]; } } - await state.set(INLINE_SCRIPT_ENVS_KEY, associations); + await state.set(INLINE_SCRIPT_ENVS_KEY, rawEntries); + this.applyPersistedAssociations(associations); }); } - private asPersistedAssociations(value: unknown): PersistedInlineScriptEnvironments | undefined { + private parsePersistedAssociations(value: unknown): ParsedPersistedAssociations | undefined { + if (value === undefined) { + return { + rawEntries: {}, + records: {}, + invalidKeys: new Set(), + }; + } if (!value || typeof value !== 'object' || Array.isArray(value)) { return undefined; } - const associations: PersistedInlineScriptEnvironments = {}; - for (const [scriptPath, environmentPath] of Object.entries(value)) { - if (typeof environmentPath === 'string' && environmentPath.length > 0) { - associations[scriptPath] = environmentPath; + const rawEntries = { ...(value as Record) }; + const records: PersistedInlineScriptEnvironments = {}; + const invalidKeys = new Set(); + for (const [scriptPath, association] of Object.entries(rawEntries)) { + const parsed = this.parsePersistedAssociationValue(association); + if (parsed.kind === 'valid') { + records[scriptPath] = parsed.record; + } else if (parsed.kind === 'invalid') { + invalidKeys.add(scriptPath); } } - return associations; + return { rawEntries, records, invalidKeys }; + } + + private getPersistedAssociationFromMemory(scriptPath: string): PersistedAssociationRecord | undefined { + return this.fsPathToPersistedAssociation.get(scriptPath); + } + + private createPersistedAssociationRecord( + environmentPath: string, + sourceMetadataIdentity: string | undefined, + currentMetadataIdentity: string | undefined, + ): PersistedAssociationRecord { + if (!sourceMetadataIdentity) { + return { + environmentPath, + metadataBinding: { kind: 'legacy' }, + }; + } + return { + environmentPath, + metadataBinding: + currentMetadataIdentity === sourceMetadataIdentity + ? { kind: 'matched', sourceIdentity: sourceMetadataIdentity } + : { kind: 'pending', sourceIdentity: sourceMetadataIdentity }, + }; + } + + private isSamePersistedAssociation( + first: PersistedAssociationRecord | undefined, + second: PersistedAssociationRecord | undefined, + ): boolean { + if (first === second) { + return true; + } + if (!first || !second) { + return false; + } + if (normalizePath(first.environmentPath) !== normalizePath(second.environmentPath)) { + return false; + } + if (first.metadataBinding.kind !== second.metadataBinding.kind) { + return false; + } + if (first.metadataBinding.kind === 'matched' && second.metadataBinding.kind === 'matched') { + return first.metadataBinding.sourceIdentity === second.metadataBinding.sourceIdentity; + } + if (first.metadataBinding.kind === 'pending' && second.metadataBinding.kind === 'pending') { + return first.metadataBinding.sourceIdentity === second.metadataBinding.sourceIdentity; + } + return true; + } + + private parsePersistedAssociationValue(value: unknown): + | { readonly kind: 'valid'; readonly record: PersistedAssociationRecord } + | { readonly kind: 'future' } + | { readonly kind: 'invalid' } { + if (typeof value === 'string' && value.length > 0) { + return { + kind: 'valid', + record: { + environmentPath: value, + metadataBinding: { kind: 'legacy' }, + }, + }; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return { kind: 'invalid' }; + } + const association = value as Record; + const schemaVersion = association.schemaVersion; + if (typeof schemaVersion !== 'number') { + return { kind: 'invalid' }; + } + if (schemaVersion !== PERSISTED_ASSOCIATION_SCHEMA_VERSION) { + return { kind: 'future' }; + } + const environmentPath = association.environmentPath; + const metadataBinding = association.metadataBinding; + if (typeof environmentPath !== 'string' || environmentPath.length === 0) { + return { kind: 'invalid' }; + } + if (!metadataBinding || typeof metadataBinding !== 'object' || Array.isArray(metadataBinding)) { + return { kind: 'invalid' }; + } + const binding = metadataBinding as Record; + if (binding.kind === 'pending') { + if (typeof binding.sourceIdentity === 'string' && binding.sourceIdentity.trim().length > 0) { + return { + kind: 'valid', + record: { + environmentPath, + metadataBinding: { kind: 'pending', sourceIdentity: binding.sourceIdentity }, + }, + }; + } + return { kind: 'invalid' }; + } + if (binding.kind === 'legacy') { + return { + kind: 'valid', + record: { environmentPath, metadataBinding: { kind: 'legacy' } }, + }; + } + if ( + binding.kind === 'matched' && + typeof binding.sourceIdentity === 'string' && + binding.sourceIdentity.trim().length > 0 + ) { + return { + kind: 'valid', + record: { + environmentPath, + metadataBinding: { + kind: 'matched', + sourceIdentity: binding.sourceIdentity, + }, + }, + }; + } + return { kind: 'invalid' }; + } + + private serializePersistedAssociation( + association: PersistedAssociationRecord, + ): PersistedInlineScriptAssociationValue { + return { + schemaVersion: PERSISTED_ASSOCIATION_SCHEMA_VERSION, + environmentPath: association.environmentPath, + metadataBinding: + association.metadataBinding.kind === 'matched' + ? { kind: 'matched', sourceIdentity: association.metadataBinding.sourceIdentity } + : association.metadataBinding.kind === 'pending' + ? { kind: 'pending', sourceIdentity: association.metadataBinding.sourceIdentity } + : { kind: association.metadataBinding.kind }, + }; } private enqueuePersistence(operation: (state: PersistentState) => Promise): Promise { @@ -1274,6 +2140,37 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { return run; } + private clearAssociationsForScripts(scripts: readonly Uri[]): Promise { + return this.enqueueSelection(async () => { + const changes = scripts + .filter((uri) => uri.scheme === 'file') + .map((uri) => ({ + uri, + scriptPath: normalizePath(uri.fsPath), + })) + .filter((script, index, all) => all.findIndex((candidate) => candidate.scriptPath === script.scriptPath) === index) + .filter( + (script) => + this.fsPathToEnv.has(script.scriptPath) || + this.fsPathToPersistedAssociation.has(script.scriptPath), + ); + + if (changes.length === 0) { + return; + } + + await this.updatePersistedAssociations(changes.map(({ scriptPath }) => ({ scriptPath }))); + for (const change of changes) { + this.bumpAssociationRevision(change.scriptPath); + this.pendingRehydrations.delete(change.scriptPath); + this.pendingMetadataRefreshes.delete(change.scriptPath); + this.fsPathToEnv.delete(change.scriptPath); + this.fsPathToPersistedAssociation.delete(change.scriptPath); + this.clearValidatedRouteableState(change.uri); + } + }); + } + private async isCacheEntryBusy(envDirPath: string): Promise { if (this.pendingCreations.has(path.basename(envDirPath))) { return true; @@ -1306,7 +2203,8 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { } return ( first.envId.managerId === second.envId.managerId && - normalizePath(first.environmentPath.fsPath) === normalizePath(second.environmentPath.fsPath) + normalizePath(first.environmentPath.fsPath) === normalizePath(second.environmentPath.fsPath) && + first.version === second.version ); } @@ -1589,74 +2487,147 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { return { kind: 'installed', installedPath: promptResult.pythonPath }; } + /** + * Serializes cache-entry mutation across extension-host processes. + * Every `writeMetaJson` call in this manager is either directly inside + * this callback or reachable only through `createOrReuseEnvironment`, + * which invokes it under this lock. + */ + private async withCacheEntryLock( + envDir: Uri, + action: (lock: AcquiredFileLock) => Promise, + ): Promise { + const lock = await acquireFileLock(envDir.fsPath, { + timeoutMs: CACHE_LOCK_TIMEOUT_MS, + retryIntervalMs: CACHE_LOCK_RETRY_MS, + }); + try { + return await action(lock); + } finally { + try { + await lock.release(); + } catch (error) { + this.log.warn(`Failed to release inline-script cache lock: ${getErrorMessage(error)}`); + } + } + } + + private mergePendingCreationSourceMetadataIdentityHashes( + existing: readonly string[] | undefined, + pendingCreation: PendingCreationContext, + ): readonly string[] | undefined { + let merged = existing; + for (const sourceMetadataIdentityHash of pendingCreation.sourceMetadataIdentityHashes ?? []) { + merged = mergeSourceMetadataIdentityHashes(merged, sourceMetadataIdentityHash); + } + return merged; + } + + private async mergeCacheEntrySourceMetadataIdentityHash( + cacheKey: string, + sourceMetadataIdentityHash: string, + ): Promise { + const envDir = getScriptEnvDir(this.globalStorageUri, cacheKey); + try { + return await this.withCacheEntryLock(envDir, async () => { + const sidecarResult = await inspectMetaJson(envDir); + if (sidecarResult.kind !== 'valid') { + return { success: false }; + } + if (sidecarResult.metadata.sourceMetadataIdentityHashes?.includes(sourceMetadataIdentityHash)) { + return { + success: true, + sourceMetadataIdentityHashes: sidecarResult.metadata.sourceMetadataIdentityHashes, + }; + } + const sourceMetadataIdentityHashes = mergeSourceMetadataIdentityHashes( + sidecarResult.metadata.sourceMetadataIdentityHashes, + sourceMetadataIdentityHash, + ); + await writeMetaJson(envDir, { + ...sidecarResult.metadata, + ...(sourceMetadataIdentityHashes ? { sourceMetadataIdentityHashes } : {}), + }); + return { + success: true, + sourceMetadataIdentityHashes, + }; + }); + } catch (error) { + this.log.warn(`Failed to update inline-script cache provenance: ${getErrorMessage(error)}`); + return { success: false }; + } + } + private async createOrReuseEnvironment({ cacheKey, packages, metadata, selectedBase, + pendingCreation, }: CreateOrReuseEnvironmentOptions): Promise { const dependencyCount = this.getTelemetryDependencyCount(packages); const cacheRoot = getScriptEnvCacheRoot(this.globalStorageUri); const envDir = getScriptEnvDir(this.globalStorageUri, cacheKey); - await fs.ensureDir(cacheRoot.fsPath); - let lock: AcquiredFileLock | undefined; try { - lock = await acquireFileLock(envDir.fsPath, { - timeoutMs: CACHE_LOCK_TIMEOUT_MS, - retryIntervalMs: CACHE_LOCK_RETRY_MS, - }); - - const cached = await this.inspectCacheEntry(cacheRoot, envDir, metadata, selectedBase); - if (cached.kind === 'reusable') { - this.sendInlineScriptEnvReuseHitTelemetry(dependencyCount); - return cached.environment; - } - if (cached.kind === 'uncertain') { - this.log.warn( - `Preserving an inline-script cache entry that could not be safely inspected: ${envDir.fsPath}`, + await fs.ensureDir(cacheRoot.fsPath); + return await this.withCacheEntryLock(envDir, async (lock) => { + const cached = await this.inspectCacheEntry( + cacheRoot, + envDir, + metadata, + selectedBase, + pendingCreation, ); - this.sendInlineScriptEnvErrorTelemetry('setup-failure'); - return undefined; - } - if (cached.kind === 'stale') { - if (!(await this.removeCacheEntry(envDir))) { + if (cached.kind === 'reusable') { + this.sendInlineScriptEnvReuseHitTelemetry(dependencyCount); + return cached.environment; + } + if (cached.kind === 'uncertain') { + this.log.warn( + `Preserving an inline-script cache entry that could not be safely inspected: ${envDir.fsPath}`, + ); this.sendInlineScriptEnvErrorTelemetry('setup-failure'); return undefined; } - } + if (cached.kind === 'stale') { + if (!(await this.removeCacheEntry(envDir))) { + this.sendInlineScriptEnvErrorTelemetry('setup-failure'); + return undefined; + } + } - const buildStartAtMs = Date.now(); - const build = await this.buildCacheEntry(envDir, cacheRoot, packages, selectedBase); - if (build.retainLock) { - try { - await lock.retain(); - } catch (error) { - this.log.error( - `Failed to mark the inline-script cache lock as retained: ${getErrorMessage(error)}`, - ); + const buildStartAtMs = Date.now(); + const build = await this.buildCacheEntry( + envDir, + cacheRoot, + packages, + selectedBase, + pendingCreation, + ); + if (build.retainLock) { + try { + await lock.retain(); + } catch (error) { + this.log.error( + `Failed to mark the inline-script cache lock as retained: ${getErrorMessage(error)}`, + ); + } } - } - if (build.environment) { - this.sendInlineScriptEnvCreatedTelemetry(buildStartAtMs, dependencyCount); - return build.environment; - } - if (build.errorCategory) { - this.sendInlineScriptEnvErrorTelemetry(build.errorCategory); - } - return undefined; + if (build.environment) { + this.sendInlineScriptEnvCreatedTelemetry(buildStartAtMs, dependencyCount); + return build.environment; + } + if (build.errorCategory) { + this.sendInlineScriptEnvErrorTelemetry(build.errorCategory); + } + return undefined; + }); } catch (error) { this.sendInlineScriptEnvErrorTelemetry(this.getCreateOrReuseErrorCategory(error)); this.log.error(`Failed to create or reuse inline-script cache entry: ${getErrorMessage(error)}`); return undefined; - } finally { - if (lock) { - try { - await lock.release(); - } catch (error) { - this.log.warn(`Failed to release inline-script cache lock: ${getErrorMessage(error)}`); - } - } } } @@ -1665,6 +2636,7 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { envDir: Uri, metadata: InlineScriptMetadata, selectedBase: SelectedBaseInterpreter, + pendingCreation: PendingCreationContext, ): Promise { try { const stat = await fs.lstat(envDir.fsPath); @@ -1689,17 +2661,24 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { let sidecarResult; try { sidecarResult = await inspectMetaJson(envDir); + if (sidecarResult.kind === 'missing') { + sidecarResult = await restoreMetaJsonBackupUnderLock(envDir, (candidate) => + this.matchesSelectedBase(candidate, selectedBase), + ); + } } catch { return { kind: 'uncertain' }; } if (sidecarResult.kind !== 'valid') { - return { kind: sidecarResult.kind === 'unavailable' ? 'uncertain' : 'stale' }; + return { + kind: + sidecarResult.kind === 'unavailable' || sidecarResult.kind === 'unsupported' + ? 'uncertain' + : 'stale', + }; } const sidecar = sidecarResult.metadata; - if ( - normalizePath(sidecar.baseInterpreterPath) !== normalizePath(selectedBase.canonicalPath) || - sidecar.baseInterpreterVersion !== selectedBase.environment.version - ) { + if (!this.matchesSelectedBase(sidecar, selectedBase)) { return { kind: 'stale' }; } @@ -1729,20 +2708,37 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { if (requiresPython && !this.matchesInstallConstraint(requiresPython, environment.version)) { return { kind: 'stale' }; } - try { - await writeMetaJson(envDir, { ...sidecar, lastUsedAt: new Date().toISOString() }); + pendingCreation.hasStartedRecordingSourceMetadataIdentityHashes = true; + const sourceMetadataIdentityHashes = this.mergePendingCreationSourceMetadataIdentityHashes( + sidecar.sourceMetadataIdentityHashes, + pendingCreation, + ); + await writeMetaJson(envDir, { + ...sidecar, + lastUsedAt: new Date().toISOString(), + ...(sourceMetadataIdentityHashes ? { sourceMetadataIdentityHashes } : {}), + }); + pendingCreation.recordedSourceMetadataIdentityHashes = sourceMetadataIdentityHashes; } catch (error) { this.log.warn(`Failed to update inline-script cache metadata: ${getErrorMessage(error)}`); } return { kind: 'reusable', environment }; } + private matchesSelectedBase(sidecar: InlineScriptEnvMeta, selectedBase: SelectedBaseInterpreter): boolean { + return ( + normalizePath(sidecar.baseInterpreterPath) === normalizePath(selectedBase.canonicalPath) && + sidecar.baseInterpreterVersion === selectedBase.environment.version + ); + } + private async buildCacheEntry( envDir: Uri, cacheRoot: Uri, packages: ReadonlyArray, selectedBase: SelectedBaseInterpreter, + pendingCreation: PendingCreationContext, ): Promise { let result; try { @@ -1784,14 +2780,20 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { await this.removeCacheEntry(envDir); return { errorCategory: 'setup-failure' }; } - try { + pendingCreation.hasStartedRecordingSourceMetadataIdentityHashes = true; + const sourceMetadataIdentityHashes = this.mergePendingCreationSourceMetadataIdentityHashes( + undefined, + pendingCreation, + ); await writeMetaJson(envDir, { schemaVersion: META_SCHEMA_VERSION, baseInterpreterPath: selectedBase.canonicalPath, baseInterpreterVersion: selectedBase.environment.version, lastUsedAt: new Date().toISOString(), + ...(sourceMetadataIdentityHashes ? { sourceMetadataIdentityHashes } : {}), }); + pendingCreation.recordedSourceMetadataIdentityHashes = sourceMetadataIdentityHashes; } catch (error) { this.log.error(`Failed to record inline-script cache metadata: ${getErrorMessage(error)}`); await this.removeCacheEntry(envDir); @@ -1817,9 +2819,12 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { ...Object.keys(persistedAssociations), ...this.associationRevisions.keys(), ...this.cachedAssociationValidatedAt.keys(), + ...this.lastValidatedMetadataIdentities.keys(), + ...this.lastValidatedMetadataIdentityProofs.keys(), ...this.fsPathToEnv.keys(), - ...this.fsPathToPersistedEnvPath.keys(), + ...this.fsPathToPersistedAssociation.keys(), ...this.pendingRehydrations.keys(), + ...this.pendingMetadataRefreshes.keys(), ]); const priorSelections = new Map(); scriptPaths.forEach((scriptPath) => { @@ -2112,8 +3117,8 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { const invalidatedScriptPaths = new Set(); for (const scriptPath of scriptPaths) { const environmentPaths = [ - persistedAssociations[scriptPath], - this.fsPathToPersistedEnvPath.get(scriptPath), + persistedAssociations[scriptPath]?.environmentPath, + this.fsPathToPersistedAssociation.get(scriptPath)?.environmentPath, this.fsPathToEnv.get(scriptPath)?.environmentPath.fsPath, ].filter((value): value is string => value !== undefined); const states = await Promise.all( @@ -2175,7 +3180,8 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { await this.updatePersistedAssociations( persistedPathsToClear.map((scriptPath) => ({ scriptPath, - expectedEnvironmentPath: persistedAssociations[scriptPath], + expectedEnvironmentPath: persistedAssociations[scriptPath].environmentPath, + expectedPersistedAssociation: persistedAssociations[scriptPath], })), ); } @@ -2187,9 +3193,10 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { for (const scriptPath of invalidatedScriptPaths) { this.bumpAssociationRevision(scriptPath); this.pendingRehydrations.delete(scriptPath); + this.pendingMetadataRefreshes.delete(scriptPath); this.fsPathToEnv.delete(scriptPath); - this.fsPathToPersistedEnvPath.delete(scriptPath); - this.cachedAssociationValidatedAt.delete(scriptPath); + this.fsPathToPersistedAssociation.delete(scriptPath); + this.clearValidatedRouteableState(scriptPath); const environment = priorSelections.get(scriptPath); if (environment) { @@ -2206,7 +3213,7 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { private async getPersistedAssociationSnapshot(): Promise { await this.persistenceQueue; const state = await getWorkspacePersistentState(); - return this.asPersistedAssociations(await state.get(INLINE_SCRIPT_ENVS_KEY)) ?? {}; + return this.parsePersistedAssociations(await state.get(INLINE_SCRIPT_ENVS_KEY))?.records ?? {}; } private async removeCacheEntry(envDir: Uri): Promise { @@ -2274,16 +3281,49 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable { dispose(): void { this.disposed = true; this.stopActivationDiscovery(); + this.pendingMetadataRefreshes.clear(); + this.subscriptions.forEach((subscription) => subscription.dispose()); this._onDidChangeEnvironments.dispose(); this._onDidChangeEnvironment.dispose(); } + + private applyPersistedAssociations(associations: PersistedInlineScriptEnvironments): void { + const nextPaths = new Set(Object.keys(associations)); + for (const scriptPath of this.fsPathToPersistedAssociation.keys()) { + if (!nextPaths.has(scriptPath)) { + this.fsPathToPersistedAssociation.delete(scriptPath); + this.clearValidatedRouteableState(scriptPath); + } + } + for (const [scriptPath, association] of Object.entries(associations)) { + this.fsPathToPersistedAssociation.set(scriptPath, association); + } + } } -type PersistedInlineScriptEnvironments = Record; +type PersistedInlineScriptEnvironments = Record; +type PersistedInlineScriptAssociationValue = string | PersistedInlineScriptAssociationObject; + +type PersistedMetadataBinding = + | { readonly kind: 'legacy' } + | { readonly kind: 'pending'; readonly sourceIdentity: string } + | { readonly kind: 'matched'; readonly sourceIdentity: string }; + +interface PersistedInlineScriptAssociationObject { + readonly schemaVersion: typeof PERSISTED_ASSOCIATION_SCHEMA_VERSION; + readonly environmentPath: string; + readonly metadataBinding: PersistedMetadataBinding; +} + +interface PersistedAssociationRecord { + readonly environmentPath: string; + readonly metadataBinding: PersistedMetadataBinding; +} interface PersistedAssociationChange { readonly scriptPath: string; - readonly environmentPath?: string; + readonly persistedAssociation?: PersistedAssociationRecord; + readonly expectedPersistedAssociation?: PersistedAssociationRecord; readonly expectedEnvironmentPath?: string; } @@ -2294,6 +3334,7 @@ interface ScriptReference { interface PendingScriptUpdate extends ScriptReference { readonly before: PythonEnvironment | undefined; + readonly persistedAssociation?: PersistedAssociationRecord; readonly needsPersistence: boolean; readonly shouldNotify: boolean; } diff --git a/src/managers/builtin/inlineScript/main.ts b/src/managers/builtin/inlineScript/main.ts index 46760ff72..daf9ad4d6 100644 --- a/src/managers/builtin/inlineScript/main.ts +++ b/src/managers/builtin/inlineScript/main.ts @@ -4,15 +4,14 @@ import { Disposable, LogOutputChannel, Uri } from 'vscode'; import { EnvironmentManager, PythonEnvironmentApi } from '../../../api'; import { traceInfo, traceVerbose } from '../../../common/logging'; +import { InlineScriptFeatureActivation } from '../../../features/inlineScript/activation'; import { getPythonApi } from '../../../features/pythonApi'; -import { isInlineScriptsFeatureEnabled } from '../../../helpers'; import { NativePythonFinder } from '../../common/nativePythonFinder'; import { InlineScriptEnvManager } from './envManager'; /** - * Register the inline-script env manager when the internal - * `python-envs.inlineScripts.enabled` flag is true. The flag is - * undeclared in `package.json`, so default users see nothing. + * Register the inline-script env manager when the activation-latched + * `python-envs.inlineScripts.enabled` flag is true. */ export async function registerInlineScriptFeatures( nativeFinder: NativePythonFinder, @@ -20,14 +19,19 @@ export async function registerInlineScriptFeatures( log: LogOutputChannel, baseManager: EnvironmentManager, globalStorageUri: Uri, + activation: InlineScriptFeatureActivation, ): Promise { - if (!isInlineScriptsFeatureEnabled()) { + if (!activation.enabled) { traceVerbose('Inline-script env manager: skipping registration (internal flag is off)'); return; } + const { routingRegistry } = activation; + if (!routingRegistry) { + throw new Error('Inline-script env manager requires a routing registry when the feature flag is on'); + } const api: PythonEnvironmentApi = await getPythonApi(); - const mgr = new InlineScriptEnvManager(nativeFinder, api, baseManager, globalStorageUri, log); + const mgr = new InlineScriptEnvManager(nativeFinder, api, baseManager, globalStorageUri, log, routingRegistry); disposables.push(mgr, api.registerEnvironmentManager(mgr)); setImmediate(() => mgr.startActivationDiscovery()); traceInfo('Inline-script env manager: registered (internal flag is on)'); diff --git a/src/test/common/inlineScript/cacheLayout.unit.test.ts b/src/test/common/inlineScript/cacheLayout.unit.test.ts index ce63d1c49..2f0765bb6 100644 --- a/src/test/common/inlineScript/cacheLayout.unit.test.ts +++ b/src/test/common/inlineScript/cacheLayout.unit.test.ts @@ -11,23 +11,29 @@ import { Uri } from 'vscode'; import { PythonEnvironment } from '../../../api'; import { CacheEntrySummary, + MAX_SOURCE_METADATA_IDENTITY_HASHES, + SOURCE_METADATA_IDENTITY_HASH_HEX_LENGTH, INLINE_SCRIPT_CACHE_DIR_NAME, InlineScriptEnvMeta, META_JSON_FILENAME, META_SCHEMA_VERSION, getBaseInterpreterStatus, + hashSourceMetadataIdentity, getMetaJsonPath, getScriptEnvCacheRoot, getScriptEnvDir, inspectOwnedCacheEntry, inspectMetaJson, + mergeSourceMetadataIdentityHashes, readMetaJson, + restoreMetaJsonBackupUnderLock, resolveCacheEntryPath, selectStaleEntries, verifyBaseInterpreterExists, writeMetaJson, } from '../../../common/inlineScript/cacheLayout'; import * as logging from '../../../common/logging'; +import { createDeferred } from '../../../common/utils/deferred'; import * as platformUtils from '../../../common/utils/platformUtils'; import { getVenvPythonPath } from '../../../common/utils/virtualEnvironment'; @@ -89,7 +95,9 @@ suite('inlineScriptCacheLayout', () => { }); test('writeMetaJson then readMetaJson returns the same object', async () => { - const meta = makeMeta(); + const meta = makeMeta({ + sourceMetadataIdentityHashes: [hashSourceMetadataIdentity('{"requiresPython":">=3.11","dependencies":["requests"]}')], + }); await writeMetaJson(envDir, meta); const read = await readMetaJson(envDir); assert.deepStrictEqual(read, meta); @@ -122,24 +130,131 @@ suite('inlineScriptCacheLayout', () => { ); }); - test('concurrent writeMetaJson calls leave one valid sidecar, never a missing one', async () => { - await writeMetaJson(envDir, makeMeta({ lastUsedAt: '2020-01-01T00:00:00.000Z' })); + test('restores an existing sidecar when both replacement attempts fail', async () => { + const existing = makeMeta({ lastUsedAt: '2020-01-01T00:00:00.000Z' }); + await writeMetaJson(envDir, existing); + const replacementError = Object.assign(new Error('sharing violation'), { code: 'EPERM' }); + const originalRename = fsExtra.rename; + const renameStub = sinon.stub(fsExtra, 'rename') as unknown as sinon.SinonStub; + renameStub.callsFake(async (source: string, destination: string) => { + switch (renameStub.callCount) { + case 1: + case 3: + throw replacementError; + default: + return originalRename(source, destination); + } + }); + + await assert.rejects( + writeMetaJson(envDir, makeMeta({ lastUsedAt: '2030-01-01T00:00:00.000Z' })), + (error) => error === replacementError, + ); + + assert.deepStrictEqual(await readMetaJson(envDir), existing); + assert.strictEqual(renameStub.callCount, 4, 'retryable replacement failure should restore after retry'); + const entries = await fs.readdir(envDir.fsPath); + assert.deepStrictEqual( + entries.filter((name) => name.includes('.tmp-') || name.includes('.backup-')), + [], + 'failed replacement must clean up temporary and backup files', + ); + }); + + test('retains the backup when restoration cannot prove the final sidecar exists', async () => { + const existing = makeMeta({ lastUsedAt: '2020-01-01T00:00:00.000Z' }); + await writeMetaJson(envDir, existing); + const replacementError = Object.assign(new Error('sharing violation'), { code: 'EPERM' }); + const restoreError = Object.assign(new Error('restore failed'), { code: 'EIO' }); + const originalRename = fsExtra.rename; + const renameStub = sinon.stub(fsExtra, 'rename') as unknown as sinon.SinonStub; + renameStub.callsFake(async (source: string, destination: string) => { + switch (renameStub.callCount) { + case 1: + case 3: + throw replacementError; + case 2: + return originalRename(source, destination); + default: + throw restoreError; + } + }); + + await assert.rejects( + writeMetaJson(envDir, makeMeta({ lastUsedAt: '2030-01-01T00:00:00.000Z' })), + (error) => error === replacementError, + ); + + assert.strictEqual(await readMetaJson(envDir), undefined); + const entries = await fs.readdir(envDir.fsPath); + const backups = entries.filter((name) => name.includes('.backup-')); + assert.strictEqual(backups.length, 1, 'failed restoration must retain the only known good copy'); + assert.deepStrictEqual( + JSON.parse(await fs.readFile(path.join(envDir.fsPath, backups[0]), 'utf8')), + existing, + ); + assert.deepStrictEqual(entries.filter((name) => name.includes('.tmp-')), []); + }); + + test('serializes concurrent writes for the same sidecar and retains the latest metadata', async () => { const a = makeMeta({ lastUsedAt: '2025-01-01T00:00:00.000Z' }); const b = makeMeta({ lastUsedAt: '2030-01-01T00:00:00.000Z' }); - await Promise.all([writeMetaJson(envDir, a), writeMetaJson(envDir, b)]); + const firstRenameStarted = createDeferred(); + const allowFirstRename = createDeferred(); + const originalRename = fsExtra.rename; + const renameStub = sinon.stub(fsExtra, 'rename') as unknown as sinon.SinonStub; + renameStub.callsFake(async (source: string, destination: string) => { + if (renameStub.callCount === 1) { + firstRenameStarted.resolve(); + await allowFirstRename.promise; + } + return originalRename(source, destination); + }); + + const first = writeMetaJson(envDir, a); + await firstRenameStarted.promise; + const second = writeMetaJson(envDir, b); + assert.strictEqual(renameStub.callCount, 1, 'the second same-path write must wait for the first'); + allowFirstRename.resolve(); + await Promise.all([first, second]); + const read = await readMetaJson(envDir); assert.ok(read, 'sidecar must exist after concurrent writes'); - assert.ok( - read.lastUsedAt === a.lastUsedAt || read.lastUsedAt === b.lastUsedAt, - `final write must be one of the concurrent payloads, got ${read.lastUsedAt}`, - ); + assert.deepStrictEqual(read, b); const entries = await fs.readdir(envDir.fsPath); assert.deepStrictEqual( - entries.filter((name) => name.includes('.tmp-')), + entries.filter((name) => name.includes('.tmp-') || name.includes('.backup-')), [], ); }); + test('does not serialize writes for different sidecars', async () => { + const otherEnvDir = Uri.file(path.join(tmpDir, 'other-env')); + const firstFinalPath = getMetaJsonPath(envDir).fsPath; + const firstRenameStarted = createDeferred(); + const allowFirstRename = createDeferred(); + const originalRename = fsExtra.rename; + const renameStub = sinon.stub(fsExtra, 'rename') as unknown as sinon.SinonStub; + let delayedFirstRename = false; + renameStub.callsFake(async (source: string, destination: string) => { + if (!delayedFirstRename && destination === firstFinalPath) { + delayedFirstRename = true; + firstRenameStarted.resolve(); + await allowFirstRename.promise; + } + return originalRename(source, destination); + }); + + const first = writeMetaJson(envDir, makeMeta({ lastUsedAt: '2025-01-01T00:00:00.000Z' })); + await firstRenameStarted.promise; + const other = makeMeta({ lastUsedAt: '2030-01-01T00:00:00.000Z' }); + await writeMetaJson(otherEnvDir, other); + + assert.deepStrictEqual(await readMetaJson(otherEnvDir), other); + allowFirstRename.resolve(); + await first; + }); + test('writeMetaJson only serializes environment-level metadata', async () => { const meta = makeMeta(); await writeMetaJson(envDir, meta); @@ -156,6 +271,116 @@ suite('inlineScriptCacheLayout', () => { }); }); + suite('restoreMetaJsonBackupUnderLock', () => { + let tmpDir: string; + let envDir: Uri; + + setup(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'isclayout-backup-recovery-')); + envDir = Uri.file(path.join(tmpDir, 'env')); + await fs.ensureDir(envDir.fsPath); + }); + + teardown(async () => { + await fs.remove(tmpDir); + }); + + function backupPath(suffix: string): string { + return `${getMetaJsonPath(envDir).fsPath}.backup-${suffix}`; + } + + async function writeBackup(suffix: string, content: string | Buffer): Promise { + await fs.writeFile(backupPath(suffix), content); + } + + test('leaves valid backups untouched for unlocked readers, then restores one under the entry lock', async () => { + const metadata = makeMeta(); + await writeBackup('abcdef123456', JSON.stringify(metadata)); + + assert.deepStrictEqual(await inspectMetaJson(envDir), { kind: 'missing' }); + assert.strictEqual(await readMetaJson(envDir), undefined); + assert.strictEqual(await fs.pathExists(getMetaJsonPath(envDir).fsPath), false); + + assert.deepStrictEqual(await restoreMetaJsonBackupUnderLock(envDir), { kind: 'valid', metadata }); + assert.deepStrictEqual(await readMetaJson(envDir), metadata); + assert.strictEqual(await fs.pathExists(backupPath('abcdef123456')), false); + }); + + test('selects the newest valid backup and uses its path as a stable tie-breaker', async () => { + const older = makeMeta({ lastUsedAt: '2020-01-01T00:00:00.000Z' }); + const sameTimeLaterPath = makeMeta({ lastUsedAt: '2030-01-01T00:00:00.000Z' }); + const sameTimeEarlierPath = makeMeta({ lastUsedAt: '2030-01-01T00:00:00.000Z', baseInterpreterVersion: '3.13.0' }); + await writeBackup('ffffffffffff', JSON.stringify(older)); + await writeBackup('eeeeeeeeeeee', JSON.stringify(sameTimeLaterPath)); + await writeBackup('000000000000', JSON.stringify(sameTimeEarlierPath)); + + assert.deepStrictEqual(await restoreMetaJsonBackupUnderLock(envDir), { + kind: 'valid', + metadata: sameTimeEarlierPath, + }); + assert.deepStrictEqual(await readMetaJson(envDir), sameTimeEarlierPath); + assert.strictEqual(await fs.pathExists(backupPath('000000000000')), false); + }); + + test('leaves valid backups in place when none meet the compatibility predicate', async () => { + await writeBackup('abcdef123456', JSON.stringify(makeMeta())); + + assert.deepStrictEqual(await restoreMetaJsonBackupUnderLock(envDir, () => false), { kind: 'missing' }); + assert.strictEqual(await fs.pathExists(backupPath('abcdef123456')), true); + assert.strictEqual(await fs.pathExists(getMetaJsonPath(envDir).fsPath), false); + }); + + test('rejects temp, malformed, unsupported, and oversized artifacts without restoring them', async () => { + const finalPath = getMetaJsonPath(envDir).fsPath; + await fs.writeFile(`${finalPath}.tmp-abcdef123456`, JSON.stringify(makeMeta())); + await fs.writeFile(`${finalPath}.backup-ABCDEF123456`, JSON.stringify(makeMeta())); + await writeBackup('111111111111', 'not json'); + await writeBackup('222222222222', JSON.stringify({ ...makeMeta(), schemaVersion: 99 })); + await writeBackup('333333333333', Buffer.alloc(1024 * 1024 + 1, 0x20)); + + assert.deepStrictEqual(await restoreMetaJsonBackupUnderLock(envDir), { kind: 'missing' }); + assert.strictEqual(await fs.pathExists(finalPath), false); + }); + + test('rejects a symlink backup without restoring it', async function () { + const externalPath = path.join(tmpDir, 'external-meta.json'); + await fs.writeFile(externalPath, JSON.stringify(makeMeta())); + try { + await fs.symlink(externalPath, backupPath('abcdef123456'), 'file'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'EPERM' || code === 'EACCES') { + this.skip(); + return; + } + throw error; + } + + assert.deepStrictEqual(await restoreMetaJsonBackupUnderLock(envDir), { kind: 'missing' }); + assert.strictEqual(await fs.pathExists(getMetaJsonPath(envDir).fsPath), false); + }); + + test('preserves the entry when backup scanning is uncertain', async () => { + const backup = backupPath('abcdef123456'); + await writeBackup('abcdef123456', JSON.stringify(makeMeta())); + sinon.stub(fsExtra, 'readdir').rejects(Object.assign(new Error('I/O error'), { code: 'EIO' })); + + assert.deepStrictEqual(await restoreMetaJsonBackupUnderLock(envDir), { kind: 'unavailable' }); + assert.strictEqual(await fs.pathExists(backup), true); + assert.strictEqual(await fs.pathExists(getMetaJsonPath(envDir).fsPath), false); + }); + + test('preserves the backup when restoration is uncertain', async () => { + const backup = backupPath('abcdef123456'); + await writeBackup('abcdef123456', JSON.stringify(makeMeta())); + sinon.stub(fsExtra, 'rename').rejects(Object.assign(new Error('I/O error'), { code: 'EIO' })); + + assert.deepStrictEqual(await restoreMetaJsonBackupUnderLock(envDir), { kind: 'unavailable' }); + assert.strictEqual(await fs.pathExists(backup), true); + assert.strictEqual(await fs.pathExists(getMetaJsonPath(envDir).fsPath), false); + }); + }); + suite('readMetaJson rejection paths', () => { let tmpDir: string; let envDir: Uri; @@ -190,6 +415,11 @@ suite('inlineScriptCacheLayout', () => { assert.deepStrictEqual(await inspectMetaJson(envDir), { kind: 'valid', metadata }); }); + test('classifies a newer schema as unsupported without treating it as malformed', async () => { + await writeRaw(JSON.stringify({ ...makeMeta(), schemaVersion: 99 })); + assert.deepStrictEqual(await inspectMetaJson(envDir), { kind: 'unsupported' }); + }); + test('classifies non-ENOENT sidecar stat failures as unavailable', async () => { sinon.stub(fsExtra, 'lstat').rejects(Object.assign(new Error('permission denied'), { code: 'EACCES' })); assert.deepStrictEqual(await inspectMetaJson(envDir), { kind: 'unavailable' }); @@ -233,11 +463,10 @@ suite('inlineScriptCacheLayout', () => { ); }); - test('returns undefined for an unknown schemaVersion', async () => { + test('classifies a newer schemaVersion as unsupported', async () => { await writeRaw(JSON.stringify({ ...makeMeta(), schemaVersion: 99 })); - const result = await readMetaJson(envDir); - assert.strictEqual(result, undefined); - assert.ok(traceWarnStub.called); + const result = await inspectMetaJson(envDir); + assert.deepStrictEqual(result, { kind: 'unsupported' }); }); test('returns undefined when baseInterpreterPath is missing', async () => { @@ -273,6 +502,31 @@ suite('inlineScriptCacheLayout', () => { assert.strictEqual(await readMetaJson(envDir), undefined); }); + test('returns undefined for malformed sourceMetadataIdentityHashes', async () => { + await writeRaw(JSON.stringify({ ...makeMeta(), sourceMetadataIdentityHashes: 'not-an-array' })); + assert.strictEqual(await readMetaJson(envDir), undefined); + await writeRaw(JSON.stringify({ ...makeMeta(), sourceMetadataIdentityHashes: [] })); + assert.strictEqual(await readMetaJson(envDir), undefined); + await writeRaw(JSON.stringify({ ...makeMeta(), sourceMetadataIdentityHashes: ['bad-hash'] })); + assert.strictEqual(await readMetaJson(envDir), undefined); + }); + + test('returns undefined for duplicate or oversized sourceMetadataIdentityHashes', async () => { + const hash = hashSourceMetadataIdentity('same'); + await writeRaw(JSON.stringify({ ...makeMeta(), sourceMetadataIdentityHashes: [hash, hash] })); + assert.strictEqual(await readMetaJson(envDir), undefined); + await writeRaw( + JSON.stringify({ + ...makeMeta(), + sourceMetadataIdentityHashes: Array.from( + { length: MAX_SOURCE_METADATA_IDENTITY_HASHES + 1 }, + (_, index) => hashSourceMetadataIdentity(`id-${index}`), + ), + }), + ); + assert.strictEqual(await readMetaJson(envDir), undefined); + }); + test('returns undefined when lastUsedAt is not parseable', async () => { await writeRaw(JSON.stringify({ ...makeMeta(), lastUsedAt: 'not-a-date' })); const result = await readMetaJson(envDir); @@ -334,6 +588,13 @@ suite('inlineScriptCacheLayout', () => { assert.strictEqual('_internal' in result, false); }); + test('old sidecars without sourceMetadataIdentityHashes remain valid', async () => { + const result = await inspectMetaJson(envDir); + assert.deepStrictEqual(result, { kind: 'missing' }); + await writeRaw(JSON.stringify(makeMeta())); + assert.ok(await readMetaJson(envDir)); + }); + test('returns undefined when the sidecar path is a directory rather than a file', async () => { await fs.remove(getMetaJsonPath(envDir).fsPath).catch(() => undefined); await fs.ensureDir(getMetaJsonPath(envDir).fsPath); @@ -341,6 +602,24 @@ suite('inlineScriptCacheLayout', () => { assert.ok(traceWarnStub.called); }); + suite('source metadata hash helpers', () => { + test('hashSourceMetadataIdentity returns fixed-size lowercase hex', () => { + const hash = hashSourceMetadataIdentity('metadata-identity'); + assert.strictEqual(hash.length, SOURCE_METADATA_IDENTITY_HASH_HEX_LENGTH); + assert.ok(/^[0-9a-f]+$/.test(hash)); + }); + + test('mergeSourceMetadataIdentityHashes dedupes and caps the newest hashes', () => { + const hashes = Array.from({ length: MAX_SOURCE_METADATA_IDENTITY_HASHES }, (_, index) => + hashSourceMetadataIdentity(`id-${index}`), + ); + const merged = mergeSourceMetadataIdentityHashes(hashes, hashSourceMetadataIdentity('latest')); + assert.ok(merged); + assert.strictEqual(merged.length, MAX_SOURCE_METADATA_IDENTITY_HASHES); + assert.strictEqual(merged[merged.length - 1], hashSourceMetadataIdentity('latest')); + }); + }); + test('returns undefined when the sidecar exceeds the size cap (1 MiB)', async () => { const big = Buffer.alloc(1024 * 1024 + 1, 0x20); await fs.writeFile(getMetaJsonPath(envDir).fsPath, big); diff --git a/src/test/common/inlineScript/metadata.unit.test.ts b/src/test/common/inlineScript/metadata.unit.test.ts index 1426c0eac..87dd12575 100644 --- a/src/test/common/inlineScript/metadata.unit.test.ts +++ b/src/test/common/inlineScript/metadata.unit.test.ts @@ -131,6 +131,8 @@ suite('inlineScriptMetadata', () => { const md = readInlineScriptMetadata(text); assert.ok(md); assert.deepStrictEqual([...(md.dependencies ?? [])], ['a']); + assert.deepStrictEqual(md.sourceRange, { start: 0, end: text.length }); + assert.strictEqual(md.range.end, text.replace(/\r\n/g, '\n').length); }); test('lone-CR line endings parse identically to LF', () => { @@ -155,6 +157,8 @@ suite('inlineScriptMetadata', () => { const md = readInlineScriptMetadata(text); assert.ok(md); assert.deepStrictEqual([...(md.dependencies ?? [])], ['a']); + assert.strictEqual(md.range.start, 0, 'normalized parser offsets continue to exclude the BOM'); + assert.deepStrictEqual(md.sourceRange, { start: 1, end: text.length }); }); test('shebang before block does not block detection', () => { diff --git a/src/test/common/inlineScript/routingRegistry.unit.test.ts b/src/test/common/inlineScript/routingRegistry.unit.test.ts new file mode 100644 index 000000000..2807e6877 --- /dev/null +++ b/src/test/common/inlineScript/routingRegistry.unit.test.ts @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import assert from 'assert'; +import { Uri } from 'vscode'; +import { + InlineScriptRoutingRegistry, + getInlineScriptMetadataRoutingIdentity, +} from '../../../common/inlineScript/routingRegistry'; + +const METADATA = { + requiresPython: '>=3.11', + dependencies: ['requests'], + range: { start: 0, end: 40 }, +}; + +suite('InlineScriptRoutingRegistry', () => { + test('keeps metadata revisions monotonic after an empty state is removed', () => { + const registry = new InlineScriptRoutingRegistry(); + const uri = Uri.file('/workspace/script.py'); + + registry.setMetadata(uri, METADATA); + const firstRevision = registry.getMetadataRevision(uri); + registry.clearMetadata(uri); + const clearedRevision = registry.getMetadataRevision(uri); + registry.setMetadata(uri, METADATA); + const restoredRevision = registry.getMetadataRevision(uri); + + assert.strictEqual(firstRevision, 1); + assert.strictEqual(clearedRevision, 2); + assert.strictEqual(restoredRevision, 3); + assert.strictEqual(registry.getMetadataIdentity(uri), getInlineScriptMetadataRoutingIdentity(METADATA)); + registry.dispose(); + }); +}); diff --git a/src/test/features/envManagers.lastKnown.unit.test.ts b/src/test/features/envManagers.lastKnown.unit.test.ts index 589a1a25c..c7c369ddf 100644 --- a/src/test/features/envManagers.lastKnown.unit.test.ts +++ b/src/test/features/envManagers.lastKnown.unit.test.ts @@ -23,6 +23,8 @@ import { PythonProject, } from '../../api'; import * as extensionApis from '../../common/extension.apis'; +import { InlineScriptMetadata } from '../../common/inlineScript/metadata'; +import { InlineScriptRoutingRegistry } from '../../common/inlineScript/routingRegistry'; import { PythonEnvironmentManagers } from '../../features/envManagers'; import * as settingHelpers from '../../features/settings/settingHelpers'; import { InternalPackageManager, PythonProjectManager } from '../../internal.api'; @@ -34,6 +36,13 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { let projectsByUri: Map; let defaultManagerId: string; let exactManagerSettings: Map; + let routingRegistry: InlineScriptRoutingRegistry; + + const INLINE_METADATA: InlineScriptMetadata = { + requiresPython: '>=3.11', + dependencies: ['requests'], + range: { start: 0, end: 40 }, + }; function makeEnv(id: string): PythonEnvironment { const envId: PythonEnvironmentId = { id, managerId: 'test-manager' }; @@ -64,11 +73,12 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { setupNonThenable(projectManager); projectsByUri = new Map(); exactManagerSettings = new Map(); + routingRegistry = new InlineScriptRoutingRegistry(); projectManager .setup((pm) => pm.get(typeMoq.It.isAny())) .returns((uri) => projectsByUri.get(uri.toString())); - envManagers = new PythonEnvironmentManagers(projectManager.object); + envManagers = new PythonEnvironmentManagers(projectManager.object, routingRegistry); sinon.stub(settingHelpers, 'getDefaultEnvManagerSetting').callsFake(() => defaultManagerId); sinon .stub(settingHelpers, 'getProjectEnvironmentManagerSetting') @@ -114,6 +124,16 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { sinon.stub(envManagers, 'getPackageManager').returns(packageManager.object); } + function markInlineScript(uri: Uri, associated: boolean = true, metadata: InlineScriptMetadata = INLINE_METADATA): void { + routingRegistry.setMetadata(uri, metadata); + routingRegistry.setValidatedAssociation(uri, associated); + } + + function recreateEnvManagersWithoutRouting(): void { + envManagers.dispose(); + envManagers = new PythonEnvironmentManagers(projectManager.object); + } + test('returns undefined before any environment has been resolved', () => { registerManager(async () => makeEnv('env1')); assert.strictEqual(envManagers.getLastKnownEnvironment(undefined), undefined); @@ -267,6 +287,7 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { settings.onSecondCall().resolves(); const events: DidChangeEnvironmentEventArgs[] = []; envManagers.onDidChangeActiveEnvironment((event) => events.push(event)); + markInlineScript(scope); const olderSelection = envManagers.setEnvironment(scope, first); await firstWriteStarted; @@ -293,6 +314,7 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { }; const events: DidChangeEnvironmentEventArgs[] = []; envManagers.onDidChangeActiveEnvironment((event) => events.push(event)); + markInlineScript(scope); await envManagers.setEnvironment(scope, first, false); await envManagers.setEnvironment(scope, second, false); @@ -322,6 +344,7 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { }; const events: DidChangeEnvironmentEventArgs[] = []; envManagers.onDidChangeActiveEnvironment((event) => events.push(event)); + markInlineScript(scope); await envManagers.setEnvironment(scope, first, false); await envManagers.setEnvironment(scope, regenerated, false); @@ -369,6 +392,8 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { const managerId = registerManager(async () => undefined, async () => undefined, 'inline-script'); const first = { ...makeEnv('first'), envId: { id: 'first', managerId } }; const second = { ...makeEnv('second'), envId: { id: 'second', managerId } }; + markInlineScript(firstUri); + markInlineScript(secondUri); await envManagers.setEnvironment(firstUri, first, false); await envManagers.setEnvironment(secondUri, second, false); @@ -377,6 +402,62 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { assert.strictEqual(envManagers.getLastKnownEnvironment(secondUri), second); }); + test('does not route inline metadata without an associated environment', () => { + const script = Uri.file('/workspace/project/script.py'); + projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); + const defaultId = registerManager(async () => makeEnv('default'), async () => undefined, 'venv'); + registerManager(async () => makeEnv('inline'), async () => undefined, 'inline-script'); + defaultManagerId = defaultId; + routingRegistry.setMetadata(script, INLINE_METADATA); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, defaultId); + }); + + test('does not route an associated inline environment without known metadata', () => { + const script = Uri.file('/workspace/project/script.py'); + projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); + const defaultId = registerManager(async () => makeEnv('default'), async () => undefined, 'venv'); + registerManager(async () => makeEnv('inline'), async () => undefined, 'inline-script'); + defaultManagerId = defaultId; + routingRegistry.setValidatedAssociation(script, true); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, defaultId); + }); + + test('without a routing registry, does not route metadata-only inline associations', () => { + recreateEnvManagersWithoutRouting(); + const shouldRouteSpy = sinon.spy(InlineScriptRoutingRegistry.prototype, 'shouldRoute'); + const script = Uri.file('/workspace/project/script.py'); + projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); + const defaultId = registerManager(async () => makeEnv('default'), async () => undefined, 'venv'); + registerManager(async () => makeEnv('inline'), async () => undefined, 'inline-script'); + defaultManagerId = defaultId; + routingRegistry.setMetadata(script, INLINE_METADATA); + routingRegistry.setValidatedAssociation(script, true); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, defaultId); + assert.strictEqual(shouldRouteSpy.called, false, 'off-mode should not consult inline routeability'); + }); + + test('without a routing registry, keeps baseline cached inline selections without routeability checks', async () => { + recreateEnvManagersWithoutRouting(); + const shouldRouteSpy = sinon.spy(InlineScriptRoutingRegistry.prototype, 'shouldRoute'); + const script = Uri.file('/workspace/project/script.py'); + projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); + const defaultId = registerManager(async () => makeEnv('default'), async () => undefined, 'venv'); + let inlineEnvironment: PythonEnvironment; + const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); + inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; + defaultManagerId = defaultId; + + await envManagers.setEnvironment(script, inlineEnvironment, false); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, inlineId); + assert.strictEqual(await envManagers.getEnvironment(script), inlineEnvironment); + assert.strictEqual(envManagers.getLastKnownEnvironment(script), inlineEnvironment); + assert.strictEqual(shouldRouteSpy.called, false, 'off-mode should not consult inline routeability'); + }); + test('routes an active inline-script selection before the containing project default', async () => { const script = Uri.file('/workspace/project/script.py'); projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); @@ -385,6 +466,7 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; defaultManagerId = defaultId; + markInlineScript(script); await envManagers.setEnvironment(script, inlineEnvironment, false); @@ -400,6 +482,7 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; defaultManagerId = selectedId; + markInlineScript(script); await envManagers.setEnvironment(script, inlineEnvironment, false); exactManagerSettings.set(script.toString(), selectedId); @@ -417,6 +500,7 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { selectedEnvironment = { ...makeEnv('selected'), envId: { id: 'selected', managerId: selectedId } }; inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; defaultManagerId = selectedId; + markInlineScript(script); await envManagers.setEnvironment(script, inlineEnvironment, false); await envManagers.setEnvironment(script, selectedEnvironment, false); @@ -424,6 +508,33 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, selectedId); }); + test('ignores routeability changes while an explicit non-inline override wins', async () => { + const script = Uri.file('/workspace/project/script.py'); + projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); + let selectedEnvironment: PythonEnvironment; + const selectedId = registerManager(async () => selectedEnvironment, async () => undefined, 'venv'); + let inlineEnvironment: PythonEnvironment; + const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); + selectedEnvironment = { ...makeEnv('selected'), envId: { id: 'selected', managerId: selectedId } }; + inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; + defaultManagerId = selectedId; + markInlineScript(script); + + await envManagers.setEnvironment(script, inlineEnvironment, false); + await envManagers.setEnvironment(script, selectedEnvironment, false); + await new Promise((resolve) => setImmediate(resolve)); + + const events: DidChangeEnvironmentEventArgs[] = []; + envManagers.onDidChangeActiveEnvironment((event) => events.push(event)); + routingRegistry.clearMetadata(script); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + assert.deepStrictEqual(events, []); + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, selectedId); + assert.strictEqual(envManagers.getLastKnownEnvironment(script), selectedEnvironment); + }); + test('clears inline routing after a no-op inline refresh during settings persistence', async () => { const script = Uri.file('/workspace/project/script.py'); projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); @@ -434,6 +545,7 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { selectedEnvironment = { ...makeEnv('selected'), envId: { id: 'selected', managerId: selectedId } }; inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; defaultManagerId = selectedId; + markInlineScript(script); await envManagers.setEnvironment(script, inlineEnvironment, false); stubPackageManager(); let releaseSettings: (() => void) | undefined; @@ -459,6 +571,133 @@ suite('PythonEnvironmentManagers getLastKnownEnvironment', () => { assert.strictEqual(envManagers.getLastKnownEnvironment(script), selectedEnvironment); }); + test('refreshes to the inline manager when a persisted association becomes routeable', async () => { + const script = Uri.file('/workspace/project/script.py'); + projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); + const defaultEnvironment = makeEnv('default'); + const defaultId = registerManager(async () => defaultEnvironment, async () => undefined, 'venv'); + let inlineEnvironment: PythonEnvironment; + const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); + inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; + defaultManagerId = defaultId; + routingRegistry.setMetadata(script, INLINE_METADATA); + + await envManagers.refreshEnvironment(script); + routingRegistry.setValidatedAssociation(script, true); + await new Promise((resolve) => setImmediate(resolve)); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, inlineId); + assert.strictEqual(envManagers.getLastKnownEnvironment(script), inlineEnvironment); + }); + + test('does not publish an inline selection while routeability is false, then publishes once when it validates', async () => { + const script = Uri.file('/workspace/project/script.py'); + const project = { name: 'project', uri: Uri.file('/workspace/project') }; + projectsByUri.set(script.toString(), project); + const defaultEnvironment = makeEnv('default'); + const defaultId = registerManager(async () => defaultEnvironment, async () => undefined, 'venv'); + let inlineEnvironment: PythonEnvironment; + const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); + inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; + defaultManagerId = defaultId; + + await envManagers.refreshEnvironment(script); + await new Promise((resolve) => setImmediate(resolve)); + const events: DidChangeEnvironmentEventArgs[] = []; + envManagers.onDidChangeActiveEnvironment((event) => events.push(event)); + + await envManagers.setEnvironment(script, inlineEnvironment, false); + await new Promise((resolve) => setImmediate(resolve)); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, defaultId); + assert.strictEqual(envManagers.getLastKnownEnvironment(script), defaultEnvironment); + assert.deepStrictEqual(events, []); + + routingRegistry.setMetadata(script, INLINE_METADATA); + routingRegistry.setValidatedAssociation(script, true); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, inlineId); + assert.strictEqual(envManagers.getLastKnownEnvironment(script), inlineEnvironment); + assert.deepStrictEqual(events, [{ uri: script, old: defaultEnvironment, new: inlineEnvironment }]); + }); + + test('does not publish batch inline selections until each script becomes routeable', async () => { + const first = Uri.file('/workspace/project/first.py'); + const second = Uri.file('/workspace/project/second.py'); + const project = { name: 'project', uri: Uri.file('/workspace/project') }; + projectsByUri.set(first.toString(), project); + projectsByUri.set(second.toString(), project); + const defaultEnvironment = makeEnv('default'); + const defaultId = registerManager(async () => defaultEnvironment, async () => undefined, 'venv'); + let inlineEnvironment: PythonEnvironment; + const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); + inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; + defaultManagerId = defaultId; + + await envManagers.refreshEnvironment(first); + await envManagers.refreshEnvironment(second); + await new Promise((resolve) => setImmediate(resolve)); + const events: DidChangeEnvironmentEventArgs[] = []; + envManagers.onDidChangeActiveEnvironment((event) => events.push(event)); + + await envManagers.setEnvironments([first, second], inlineEnvironment, false); + await new Promise((resolve) => setImmediate(resolve)); + + assert.strictEqual(envManagers.getLastKnownEnvironment(first), defaultEnvironment); + assert.strictEqual(envManagers.getLastKnownEnvironment(second), defaultEnvironment); + assert.deepStrictEqual(events, []); + + routingRegistry.setMetadata(first, INLINE_METADATA); + routingRegistry.setValidatedAssociation(first, true); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + assert.strictEqual(envManagers.getLastKnownEnvironment(first), inlineEnvironment); + assert.strictEqual(envManagers.getLastKnownEnvironment(second), defaultEnvironment); + assert.deepStrictEqual(events, [{ uri: first, old: defaultEnvironment, new: inlineEnvironment }]); + }); + + test('falls back when inline-script metadata is invalidated after routing', async () => { + const script = Uri.file('/workspace/project/script.py'); + const project = { name: 'project', uri: Uri.file('/workspace/project') }; + projectsByUri.set(script.toString(), project); + const defaultEnvironment = makeEnv('default'); + const defaultId = registerManager(async () => defaultEnvironment, async () => undefined, 'venv'); + let inlineEnvironment: PythonEnvironment; + const inlineId = registerManager(async () => inlineEnvironment, async () => undefined, 'inline-script'); + inlineEnvironment = { ...makeEnv('inline'), envId: { id: 'inline', managerId: inlineId } }; + defaultManagerId = defaultId; + await envManagers.refreshEnvironment(script); + markInlineScript(script); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const events: DidChangeEnvironmentEventArgs[] = []; + envManagers.onDidChangeActiveEnvironment((event) => events.push(event)); + routingRegistry.clearMetadata(script); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, defaultId); + assert.strictEqual(envManagers.getLastKnownEnvironment(script), defaultEnvironment); + assert.deepStrictEqual(events[events.length - 1], { + uri: project.uri, + old: inlineEnvironment, + new: defaultEnvironment, + }); + }); + + test('ignores routeable inline state when the inline manager is not registered', () => { + const script = Uri.file('/workspace/project/script.py'); + projectsByUri.set(script.toString(), { name: 'project', uri: Uri.file('/workspace/project') }); + const defaultId = registerManager(async () => makeEnv('default'), async () => undefined, 'venv'); + defaultManagerId = defaultId; + markInlineScript(script); + + assert.strictEqual(envManagers.getEnvironmentManager(script)?.id, defaultId); + }); + test('does not persist an inline-script manager for the containing project', async () => { const script = Uri.file('/workspace/project/script.py'); const containingProject = { name: 'project', uri: Uri.file('/workspace/project') }; diff --git a/src/test/features/inlineScript/lazyDetector.unit.test.ts b/src/test/features/inlineScript/lazyDetector.unit.test.ts index f8262e146..9abf10bc1 100644 --- a/src/test/features/inlineScript/lazyDetector.unit.test.ts +++ b/src/test/features/inlineScript/lazyDetector.unit.test.ts @@ -6,25 +6,31 @@ import * as path from 'path'; import * as sinon from 'sinon'; import { Disposable, TextDocument, TextDocumentChangeEvent, TextDocumentContentChangeEvent, Uri } from 'vscode'; import * as ism from '../../../common/inlineScript/metadata'; +import { InlineScriptRoutingRegistry } from '../../../common/inlineScript/routingRegistry'; import { EventNames } from '../../../common/telemetry/constants'; import * as telemetrySender from '../../../common/telemetry/sender'; +import { createDeferred } from '../../../common/utils/deferred'; import * as wapi from '../../../common/workspace.apis'; import { InlineScriptLazyDetector, shouldHandleUri } from '../../../features/inlineScript/lazyDetector'; -// Build a minimal TextDocument stub. Only the `uri` field is read by -// the detector; the rest exists to satisfy the type. +let docDirtyByUri = new Map(); + function makeDoc(uri: Uri): TextDocument { - return { uri } as TextDocument; + return { + uri, + getText: () => '', + isDirty: docDirtyByUri.get(uri.toString()) ?? false, + } as TextDocument; } -// A non-empty change event payload. The actual content of the -// changes is not inspected by the detector; only `contentChanges.length` -// matters. const NON_EMPTY_CHANGES: readonly TextDocumentContentChangeEvent[] = [ { range: undefined as never, rangeOffset: 0, rangeLength: 0, text: 'x' }, ]; -function makeChange(uri: Uri, changes: readonly TextDocumentContentChangeEvent[] = NON_EMPTY_CHANGES): TextDocumentChangeEvent { +function makeChange( + uri: Uri, + changes: readonly TextDocumentContentChangeEvent[] = NON_EMPTY_CHANGES, +): TextDocumentChangeEvent { return { document: makeDoc(uri), contentChanges: changes, @@ -43,18 +49,27 @@ suite('InlineScriptLazyDetector', () => { let onDidOpenStub: sinon.SinonStub; let onDidSaveStub: sinon.SinonStub; let onDidChangeStub: sinon.SinonStub; + let onDidDeleteStub: sinon.SinonStub; + let onDidRenameStub: sinon.SinonStub; let getOpenTextDocumentsStub: sinon.SinonStub; let getWorkspaceFolderStub: sinon.SinonStub; let readMetadataStub: sinon.SinonStub; let sendTelemetryStub: sinon.SinonStub; + let routingRegistry: InlineScriptRoutingRegistry; let openListener: ((doc: TextDocument) => unknown) | undefined; let saveListener: ((doc: TextDocument) => unknown) | undefined; let changeListener: ((e: TextDocumentChangeEvent) => unknown) | undefined; + let deleteListener: ((e: { files: readonly Uri[] }) => unknown) | undefined; + let renameListener: ((e: { files: readonly { oldUri: Uri; newUri: Uri }[] }) => unknown) | undefined; setup(() => { openListener = undefined; saveListener = undefined; changeListener = undefined; + deleteListener = undefined; + renameListener = undefined; + docDirtyByUri = new Map(); + routingRegistry = new InlineScriptRoutingRegistry(); onDidOpenStub = sinon.stub(wapi, 'onDidOpenTextDocument'); onDidOpenStub.callsFake((listener: (doc: TextDocument) => unknown) => { @@ -80,15 +95,26 @@ suite('InlineScriptLazyDetector', () => { }); }); - // Default to an empty list of open documents. Tests that - // exercise the catch-up replay override this. + onDidDeleteStub = sinon.stub(wapi, 'onDidDeleteFiles'); + onDidDeleteStub.callsFake((listener: (e: { files: readonly Uri[] }) => unknown) => { + deleteListener = listener; + return new Disposable(() => { + deleteListener = undefined; + }); + }); + + onDidRenameStub = sinon.stub(wapi, 'onDidRenameFiles'); + onDidRenameStub.callsFake((listener: (e: { files: readonly { oldUri: Uri; newUri: Uri }[] }) => unknown) => { + renameListener = listener; + return new Disposable(() => { + renameListener = undefined; + }); + }); + getOpenTextDocumentsStub = sinon.stub(wapi, 'getOpenTextDocuments'); getOpenTextDocumentsStub.returns([]); getWorkspaceFolderStub = sinon.stub(wapi, 'getWorkspaceFolder'); - // By default, every URI is treated as being inside a workspace - // folder. Tests that want to exercise the "not in workspace" - // branch override this. getWorkspaceFolderStub.callsFake((uri: Uri) => ({ uri: Uri.file(path.dirname(uri.fsPath)), name: 'mockWorkspace', @@ -105,6 +131,18 @@ suite('InlineScriptLazyDetector', () => { sinon.restore(); }); + function createDetector(): InlineScriptLazyDetector { + const detector = new InlineScriptLazyDetector(routingRegistry); + detector.activate(); + return detector; + } + + function createDetectorWithoutRouting(): InlineScriptLazyDetector { + const detector = new InlineScriptLazyDetector(); + detector.activate(); + return detector; + } + async function fireOpen(uri: Uri): Promise { assert.ok(openListener, 'open listener should be registered after activate()'); await openListener!(makeDoc(uri)); @@ -120,50 +158,97 @@ suite('InlineScriptLazyDetector', () => { changeListener!(makeChange(uri, changes)); } - // Filter `sendTelemetryStub.getCalls()` to a single inline script event name. + function makeContentChanges(rangeOffset: number): readonly TextDocumentContentChangeEvent[] { + return [{ range: undefined as never, rangeOffset, rangeLength: 0, text: 'x' }]; + } + + function setDocDirty(uri: Uri, isDirty: boolean): void { + docDirtyByUri.set(uri.toString(), isDirty); + } + + function fireDelete(...uris: Uri[]): void { + assert.ok(deleteListener, 'delete listener should be registered after activate()'); + deleteListener!({ files: uris }); + } + + function fireRename(oldUri: Uri, newUri: Uri): void { + assert.ok(renameListener, 'rename listener should be registered after activate()'); + renameListener!({ files: [{ oldUri, newUri }] }); + } + function callsFor(name: EventNames): sinon.SinonSpyCall[] { return sendTelemetryStub.getCalls().filter((c) => c.args[0] === name); } - test('activate() subscribes to onDidOpen, onDidSave, and onDidChange', () => { - const detector = new InlineScriptLazyDetector(); - detector.activate(); + function flushImmediate(): Promise { + return new Promise((resolve) => setImmediate(resolve)); + } + + test('activate() subscribes to document and file events', () => { + const detector = createDetector(); assert.ok(onDidOpenStub.calledOnce, 'should subscribe to onDidOpenTextDocument'); assert.ok(onDidSaveStub.calledOnce, 'should subscribe to onDidSaveTextDocument'); assert.ok(onDidChangeStub.calledOnce, 'should subscribe to onDidChangeTextDocument'); + assert.ok(onDidDeleteStub.calledOnce, 'should subscribe to onDidDeleteFiles'); + assert.ok(onDidRenameStub.calledOnce, 'should subscribe to onDidRenameFiles'); + detector.dispose(); + }); + + test('activate() without routing subscribes only to document events', () => { + const detector = createDetectorWithoutRouting(); + assert.ok(onDidOpenStub.calledOnce, 'should subscribe to onDidOpenTextDocument'); + assert.ok(onDidSaveStub.calledOnce, 'should subscribe to onDidSaveTextDocument'); + assert.ok(onDidChangeStub.calledOnce, 'should subscribe to onDidChangeTextDocument'); + assert.ok(onDidDeleteStub.notCalled, 'should not subscribe to onDidDeleteFiles'); + assert.ok(onDidRenameStub.notCalled, 'should not subscribe to onDidRenameFiles'); detector.dispose(); }); test('skips non-file URI schemes', async () => { - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(Uri.parse('untitled:foo.py')); assert.ok(readMetadataStub.notCalled, 'should not read metadata for non-file URI'); detector.dispose(); }); test('skips non-.py files', async () => { - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(Uri.file(path.resolve('/ws/foo.txt'))); assert.ok(readMetadataStub.notCalled, 'should not read metadata for non-.py files'); detector.dispose(); }); - test('skips files outside any workspace folder', async () => { + test('skips telemetry for files outside any workspace folder but still refreshes saved routing metadata', async () => { getWorkspaceFolderStub.returns(undefined); - const detector = new InlineScriptLazyDetector(); - detector.activate(); - await fireOpen(Uri.file(path.resolve('/elsewhere/foo.py'))); - assert.ok(readMetadataStub.notCalled, 'should not read metadata for out-of-workspace files'); + readMetadataStub.resolves(VALID_METADATA); + routingRegistry.setValidatedAssociation(Uri.file(path.resolve('/elsewhere/foo.py')), true); + const detector = createDetector(); + const uri = Uri.file(path.resolve('/elsewhere/foo.py')); + await fireOpen(uri); + assert.ok(readMetadataStub.calledOnceWithExactly(uri), 'should still read saved metadata for routing'); + assert.strictEqual(callsFor(EventNames.INLINE_SCRIPT_DETECTED).length, 0, 'should not emit telemetry'); + detector.dispose(); + }); + + test('without routing skips files outside any workspace folder', async () => { + const uri = Uri.file(path.resolve('/elsewhere/foo.py')); + const setMetadataSpy = sinon.spy(InlineScriptRoutingRegistry.prototype, 'setMetadata'); + getWorkspaceFolderStub.returns(undefined); + readMetadataStub.resolves(VALID_METADATA); + const detector = createDetectorWithoutRouting(); + + await fireOpen(uri); + + assert.ok(readMetadataStub.notCalled, 'should not read files outside the workspace'); + assert.strictEqual(callsFor(EventNames.INLINE_SCRIPT_DETECTED).length, 0, 'should not emit telemetry'); + assert.ok(setMetadataSpy.notCalled, 'off-mode should not update routing metadata'); detector.dispose(); }); test('reads metadata for an in-workspace .py file on open', async () => { const uri = Uri.file(path.resolve('/ws/foo.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); assert.strictEqual(readMetadataStub.callCount, 1, 'open should trigger exactly one read'); assert.strictEqual((readMetadataStub.firstCall.args[0] as Uri).toString(), uri.toString()); @@ -173,33 +258,121 @@ suite('InlineScriptLazyDetector', () => { test('reads metadata for an in-workspace .py file on save', async () => { const uri = Uri.file(path.resolve('/ws/bar.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireSave(uri); assert.strictEqual(readMetadataStub.callCount, 1, 'save should trigger exactly one read'); detector.dispose(); }); + test('withholds routeability and skips disk reads for dirty documents on open', async () => { + const uri = Uri.file(path.resolve('/ws/dirty.py')); + setDocDirty(uri, true); + routingRegistry.setMetadata(uri, VALID_METADATA); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + + await fireOpen(uri); + + assert.ok(readMetadataStub.notCalled, 'dirty open should not read saved metadata'); + assert.strictEqual(routingRegistry.getMetadata(uri), undefined); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + detector.dispose(); + }); + + test('without routing replays dirty workspace documents from saved disk and preserves edited duration gating', async () => { + const uri = Uri.file(path.resolve('/ws/restoredDirty.py')); + const setMetadataSpy = sinon.spy(InlineScriptRoutingRegistry.prototype, 'setMetadata'); + const clearMetadataSpy = sinon.spy(InlineScriptRoutingRegistry.prototype, 'clearMetadata'); + const validateAssociationSpy = sinon.spy(InlineScriptRoutingRegistry.prototype, 'setValidatedAssociation'); + sinon.stub(Date, 'now').onFirstCall().returns(1_000).onSecondCall().returns(1_250); + setDocDirty(uri, true); + getOpenTextDocumentsStub.returns([makeDoc(uri)]); + readMetadataStub.resolves(VALID_METADATA); + const detector = createDetectorWithoutRouting(); + + await flushImmediate(); + await flushImmediate(); + fireChange(uri); + + assert.ok(readMetadataStub.calledOnceWithExactly(uri), 'dirty replay should still read saved metadata'); + const detectedCalls = callsFor(EventNames.INLINE_SCRIPT_DETECTED); + assert.strictEqual(detectedCalls.length, 1, 'dirty replay should still emit detection telemetry'); + assert.strictEqual(detectedCalls[0].args[2].trigger, 'open'); + const editedCalls = callsFor(EventNames.INLINE_SCRIPT_EDITED); + assert.strictEqual(editedCalls.length, 1, 'first edit after dirty replay should still emit telemetry'); + assert.strictEqual(editedCalls[0].args[1], 250, 'edited duration should still be based on the detection time'); + assert.ok(setMetadataSpy.notCalled, 'off-mode should not write routing metadata'); + assert.ok(clearMetadataSpy.notCalled, 'off-mode should not clear routing metadata'); + assert.ok(validateAssociationSpy.notCalled, 'off-mode should not change routing associations'); + detector.dispose(); + }); + test('concurrent open + open coalesces to a single read', async () => { const uri = Uri.file(path.resolve('/ws/dedup.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await Promise.all([fireOpen(uri), fireOpen(uri)]); assert.strictEqual(readMetadataStub.callCount, 1, 'open+open should coalesce to a single read'); detector.dispose(); }); - test('concurrent open + save coalesces to a single read', async () => { + test('an in-flight open followed by save reads fresh metadata and cannot publish stale routing data', async () => { const uri = Uri.file(path.resolve('/ws/race.py')); + const staleRead = createDeferred(); + const savedMetadata = { + ...VALID_METADATA, + dependencies: ['saved'], + } satisfies ism.InlineScriptMetadata; + const savedRead = createDeferred(); + readMetadataStub.onFirstCall().returns(staleRead.promise); + readMetadataStub.onSecondCall().returns(savedRead.promise); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + const open = openListener!(makeDoc(uri)) as Promise; + const save = saveListener!(makeDoc(uri)) as Promise; + + assert.strictEqual(readMetadataStub.callCount, 1, 'save must wait for the older open read'); + staleRead.resolve(VALID_METADATA); + await open; + await flushImmediate(); + + assert.strictEqual(readMetadataStub.callCount, 2, 'save must trigger a fresh post-save read'); + assert.strictEqual(routingRegistry.getMetadata(uri), undefined, 'stale open data must not be published'); + savedRead.resolve(savedMetadata); + await save; + + assert.deepStrictEqual(routingRegistry.getMetadata(uri), savedMetadata); + assert.strictEqual(routingRegistry.shouldRoute(uri), true); + detector.dispose(); + }); + + test('a metadata edit invalidates an in-flight open read before metadata is registered', async () => { + const uri = Uri.file(path.resolve('/ws/edit-race.py')); + const staleRead = createDeferred(); + readMetadataStub.returns(staleRead.promise); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + + const open = openListener!(makeDoc(uri)) as Promise; + fireChange(uri, makeContentChanges(0)); + staleRead.resolve(VALID_METADATA); + await open; + + assert.strictEqual(routingRegistry.getMetadata(uri), undefined); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + assert.strictEqual( + (detector as unknown as { routingReadGenerations: Map }).routingReadGenerations.size, + 0, + ); + detector.dispose(); + }); + + test('telemetry-only concurrent open + save still coalesces to a single read', async () => { + const uri = Uri.file(path.resolve('/ws/telemetry-race.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetectorWithoutRouting(); await Promise.all([fireOpen(uri), fireSave(uri)]); - // The slim observer has no cached state to keep fresh, so - // simple URI-level dedup is sufficient: a save concurrent - // with an in-flight open coalesces with it. - assert.strictEqual(readMetadataStub.callCount, 1, 'concurrent open+save should coalesce to a single read'); + assert.strictEqual(readMetadataStub.callCount, 1, 'telemetry-only mode should retain the original coalescing'); detector.dispose(); }); @@ -212,28 +385,214 @@ suite('InlineScriptLazyDetector', () => { }), ); - const detector = new InlineScriptLazyDetector(); - detector.activate(); - // Kick off the open without awaiting it; the read is parked - // on our manual resolver above. + const detector = createDetector(); const inFlight = openListener!(makeDoc(uri)) as Promise | undefined; - // Tear the detector down BEFORE the read settles. detector.dispose(); - // Now let the in-flight read complete with metadata. The - // `disposed` guard inside processOnce must prevent any - // further work — including the detection telemetry event. resolveRead!(VALID_METADATA); await assert.doesNotReject(inFlight ?? Promise.resolve()); assert.strictEqual(callsFor(EventNames.INLINE_SCRIPT_DETECTED).length, 0, 'no detection event after dispose'); }); - // ---------- catch-up replay over `getOpenTextDocuments` ---------- + test('tracks loose local .py files for routing even when telemetry skips them', async () => { + const uri = Uri.file(path.resolve('/elsewhere/loose.py')); + readMetadataStub.resolves(VALID_METADATA); + routingRegistry.setValidatedAssociation(uri.fsPath, true); + getWorkspaceFolderStub.returns(undefined); + const detector = createDetector(); - // Drain the microtask queue and the next `setImmediate` slot so - // the deferred catch-up replay can run before assertions. - function flushImmediate(): Promise { - return new Promise((resolve) => setImmediate(resolve)); - } + await fireOpen(uri); + + assert.strictEqual(routingRegistry.shouldRoute(uri), true); + assert.ok(readMetadataStub.calledOnceWithExactly(uri), 'loose files should still refresh saved metadata'); + detector.dispose(); + }); + + test('replays already-open loose .py documents for routing on activation', async () => { + const uri = Uri.file(path.resolve('/elsewhere/replayed.py')); + readMetadataStub.resolves(VALID_METADATA); + routingRegistry.setValidatedAssociation(uri.fsPath, true); + getWorkspaceFolderStub.returns(undefined); + getOpenTextDocumentsStub.returns([makeDoc(uri)]); + + const detector = createDetector(); + await flushImmediate(); + + assert.strictEqual(routingRegistry.shouldRoute(uri), true); + assert.ok(readMetadataStub.calledOnceWithExactly(uri), 'loose replay should refresh saved metadata'); + detector.dispose(); + }); + + test('clears routeability on header edits and refreshes saved metadata on the next save', async () => { + const uri = Uri.file(path.resolve('/elsewhere/edited.py')); + routingRegistry.setValidatedAssociation(uri, true); + readMetadataStub.onFirstCall().resolves(VALID_METADATA); + readMetadataStub.onSecondCall().resolves(VALID_METADATA); + const detector = createDetector(); + + await fireOpen(uri); + assert.strictEqual(routingRegistry.shouldRoute(uri), true); + + fireChange(uri, makeContentChanges(0)); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + assert.strictEqual( + (detector as unknown as { routingReadGenerations: Map }).routingReadGenerations.size, + 0, + ); + + await fireSave(uri); + assert.deepStrictEqual(routingRegistry.getMetadata(uri), VALID_METADATA); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + detector.dispose(); + }); + + test('preserves routing when edits are after the metadata block', async () => { + const uri = Uri.file(path.resolve('/elsewhere/bodyEdit.py')); + readMetadataStub.resolves(VALID_METADATA); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + + await fireOpen(uri); + const metadata = routingRegistry.getMetadata(uri); + assert.ok(metadata, 'expected routing metadata after open'); + + fireChange(uri, makeContentChanges(metadata!.range.end + 5)); + + assert.strictEqual(routingRegistry.shouldRoute(uri), true); + detector.dispose(); + }); + + test('invalidates routing for a CRLF dependency edit using source offsets', async () => { + const uri = Uri.file(path.resolve('/elsewhere/crlf.py')); + const source = [ + '# /// script', + ...Array.from({ length: 40 }, () => '#'), + '# dependencies = ["requests"]', + '# ///', + 'print("body")', + ].join('\r\n'); + const metadata = ism.readInlineScriptMetadata(source); + assert.ok(metadata?.sourceRange, 'parsed metadata should include source offsets'); + const dependencyOffset = source.indexOf('# dependencies'); + assert.ok( + dependencyOffset >= metadata.range.end, + 'test requires a raw CRLF dependency offset beyond the normalized range', + ); + assert.ok(dependencyOffset < metadata.sourceRange.end); + readMetadataStub.resolves(metadata); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + + await fireOpen(uri); + fireChange(uri, makeContentChanges(dependencyOffset)); + + assert.strictEqual(routingRegistry.getMetadata(uri), undefined); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + detector.dispose(); + }); + + test('invalidates routing for an edit at the end of a BOM-prefixed metadata block', async () => { + const uri = Uri.file(path.resolve('/elsewhere/bom.py')); + const source = + '\uFEFF# /// script\r\n# dependencies = ["requests"]\r\n# ///\r\nprint("hello")\r\n'; + const metadata = ism.readInlineScriptMetadata(source); + assert.ok(metadata?.sourceRange); + assert.deepStrictEqual(metadata.sourceRange, { + start: 1, + end: source.indexOf('print'), + }); + readMetadataStub.resolves(metadata); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + + await fireOpen(uri); + fireChange(uri, makeContentChanges(metadata.sourceRange.end - 1)); + + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + detector.dispose(); + }); + + test('save rehydrates routing from saved file metadata rather than the live buffer', async () => { + const uri = Uri.file(path.resolve('/elsewhere/savedState.py')); + readMetadataStub.resolves(VALID_METADATA); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + + await fireSave(uri); + + assert.strictEqual(routingRegistry.shouldRoute(uri), true); + detector.dispose(); + }); + + test('restored dirty open with removed metadata stays non-routeable until save', async () => { + const uri = Uri.file(path.resolve('/elsewhere/restoredDirtyRemoved.py')); + setDocDirty(uri, true); + routingRegistry.setMetadata(uri, VALID_METADATA); + routingRegistry.setValidatedAssociation(uri, true); + readMetadataStub.resolves(undefined); + const detector = createDetector(); + + await fireOpen(uri); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + assert.ok(readMetadataStub.notCalled); + + setDocDirty(uri, false); + await fireSave(uri); + assert.strictEqual(routingRegistry.getMetadata(uri), undefined); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + detector.dispose(); + }); + + test('restored dirty open with changed metadata stays non-routeable until save', async () => { + const uri = Uri.file(path.resolve('/elsewhere/restoredDirtyChanged.py')); + const changedMetadata = { + ...VALID_METADATA, + dependencies: ['urllib3'], + } satisfies ism.InlineScriptMetadata; + setDocDirty(uri, true); + routingRegistry.setMetadata(uri, VALID_METADATA); + routingRegistry.setValidatedAssociation(uri, true); + readMetadataStub.resolves(changedMetadata); + const detector = createDetector(); + + await fireOpen(uri); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + assert.ok(readMetadataStub.notCalled); + + setDocDirty(uri, false); + await fireSave(uri); + assert.deepStrictEqual(routingRegistry.getMetadata(uri), changedMetadata); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + detector.dispose(); + }); + + test('clears routing metadata and validation when a file is deleted', async () => { + const uri = Uri.file(path.resolve('/elsewhere/deleted.py')); + readMetadataStub.resolves(VALID_METADATA); + routingRegistry.setValidatedAssociation(uri, true); + const detector = createDetector(); + await fireOpen(uri); + + fireDelete(uri); + + assert.strictEqual(routingRegistry.getMetadata(uri), undefined); + assert.strictEqual(routingRegistry.shouldRoute(uri), false); + detector.dispose(); + }); + + test('clears routing metadata and validation for the old path when a file is renamed', async () => { + const oldUri = Uri.file(path.resolve('/elsewhere/old.py')); + const newUri = Uri.file(path.resolve('/elsewhere/new.py')); + readMetadataStub.resolves(VALID_METADATA); + routingRegistry.setValidatedAssociation(oldUri, true); + const detector = createDetector(); + await fireOpen(oldUri); + + fireRename(oldUri, newUri); + + assert.strictEqual(routingRegistry.getMetadata(oldUri), undefined); + assert.strictEqual(routingRegistry.shouldRoute(oldUri), false); + detector.dispose(); + }); test('activate() replays already-open .py documents via setImmediate', async () => { const uriWithMeta = Uri.file(path.resolve('/ws/withMeta.py')); @@ -244,15 +603,10 @@ suite('InlineScriptLazyDetector', () => { ); getOpenTextDocumentsStub.returns([makeDoc(uriWithMeta), makeDoc(uriPlain), makeDoc(uriNonPy)]); - const detector = new InlineScriptLazyDetector(); - detector.activate(); - // Wait for the deferred catch-up. + const detector = createDetector(); await flushImmediate(); - // Then await any in-flight reads kicked off by the replay. await flushImmediate(); - // The non-`.py` URI must be filtered out by `shouldHandleUri` - // BEFORE the read is attempted. assert.strictEqual(readMetadataStub.callCount, 2, 'should read each candidate .py document exactly once'); const readUris = readMetadataStub.getCalls().map((c) => (c.args[0] as Uri).toString()); assert.ok(readUris.includes(uriWithMeta.toString())); @@ -263,21 +617,16 @@ suite('InlineScriptLazyDetector', () => { test('dispose() cancels the pending catch-up replay', async () => { getOpenTextDocumentsStub.returns([makeDoc(Uri.file(path.resolve('/ws/never.py')))]); - const detector = new InlineScriptLazyDetector(); - detector.activate(); - // Tear down BEFORE the `setImmediate` slot fires. + const detector = createDetector(); detector.dispose(); await flushImmediate(); assert.ok(readMetadataStub.notCalled, 'dispose() must clear the pending setImmediate handle'); }); - // ---------- inlineScript.detected telemetry ---------- - test('inlineScript.detected fires once with trigger=open + dependencyCount + hasRequiresPython', async () => { const uri = Uri.file(path.resolve('/ws/detect.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); const detectedCalls = callsFor(EventNames.INLINE_SCRIPT_DETECTED); @@ -291,8 +640,7 @@ suite('InlineScriptLazyDetector', () => { test('inlineScript.detected fires with trigger=save when surfaced by a save event', async () => { const uri = Uri.file(path.resolve('/ws/detectOnSave.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireSave(uri); const detectedCalls = callsFor(EventNames.INLINE_SCRIPT_DETECTED); @@ -304,8 +652,7 @@ suite('InlineScriptLazyDetector', () => { test('inlineScript.detected does not fire when the file has no metadata block', async () => { const uri = Uri.file(path.resolve('/ws/plain.py')); readMetadataStub.resolves(undefined); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); assert.strictEqual(callsFor(EventNames.INLINE_SCRIPT_DETECTED).length, 0); detector.dispose(); @@ -314,8 +661,7 @@ suite('InlineScriptLazyDetector', () => { test('inlineScript.detected is deduplicated across repeated opens and saves of the same URI', async () => { const uri = Uri.file(path.resolve('/ws/repeat.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); await fireSave(uri); await fireSave(uri); @@ -332,8 +678,7 @@ suite('InlineScriptLazyDetector', () => { tool: undefined, range: { start: 0, end: 20 }, } satisfies ism.InlineScriptMetadata); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); const [, measures, properties] = callsFor(EventNames.INLINE_SCRIPT_DETECTED)[0].args; @@ -342,19 +687,15 @@ suite('InlineScriptLazyDetector', () => { detector.dispose(); }); - // ---------- inlineScript.edited telemetry ---------- - test('inlineScript.edited fires once on first content change after detection', async () => { const uri = Uri.file(path.resolve('/ws/edit.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); fireChange(uri); const editedCalls = callsFor(EventNames.INLINE_SCRIPT_EDITED); assert.strictEqual(editedCalls.length, 1, 'edited event should fire exactly once'); - // Second arg is the measure (number → { duration }); accept either form. const measureArg = editedCalls[0].args[1]; assert.strictEqual(typeof measureArg, 'number', 'measure should be a number (latency ms)'); assert.ok((measureArg as number) >= 0, 'duration should be non-negative'); @@ -364,8 +705,7 @@ suite('InlineScriptLazyDetector', () => { test('inlineScript.edited is deduplicated across repeated edits of the same URI', async () => { const uri = Uri.file(path.resolve('/ws/multiEdit.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); fireChange(uri); fireChange(uri); @@ -377,8 +717,7 @@ suite('InlineScriptLazyDetector', () => { test('inlineScript.edited does not fire for changes on a URI that was never detected', async () => { const uri = Uri.file(path.resolve('/ws/notDetected.py')); readMetadataStub.resolves(undefined); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); fireChange(uri); assert.strictEqual(callsFor(EventNames.INLINE_SCRIPT_EDITED).length, 0); @@ -388,15 +727,10 @@ suite('InlineScriptLazyDetector', () => { test('inlineScript.edited ignores change events with no content changes', async () => { const uri = Uri.file(path.resolve('/ws/noOpChange.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); - // VS Code can fire a change event with an empty contentChanges - // array for things like dirty-state toggles; that's not a user - // edit and must not count. fireChange(uri, []); assert.strictEqual(callsFor(EventNames.INLINE_SCRIPT_EDITED).length, 0); - // A real edit still counts after the no-op was ignored. fireChange(uri); assert.strictEqual(callsFor(EventNames.INLINE_SCRIPT_EDITED).length, 1); detector.dispose(); @@ -405,8 +739,7 @@ suite('InlineScriptLazyDetector', () => { test('inlineScript.edited is suppressed after dispose()', async () => { const uri = Uri.file(path.resolve('/ws/disposedEdit.py')); readMetadataStub.resolves(VALID_METADATA); - const detector = new InlineScriptLazyDetector(); - detector.activate(); + const detector = createDetector(); await fireOpen(uri); const grabbedChangeListener = changeListener!; detector.dispose(); diff --git a/src/test/features/pythonApi.unit.test.ts b/src/test/features/pythonApi.unit.test.ts index 0287828e5..bd464b4b0 100644 --- a/src/test/features/pythonApi.unit.test.ts +++ b/src/test/features/pythonApi.unit.test.ts @@ -1,65 +1,119 @@ import * as assert from 'assert'; +import * as sinon from 'sinon'; import { EventEmitter, Uri } from 'vscode'; -import { PythonProject } from '../../api'; +import { PythonEnvironment, PythonProject } from '../../api'; +import * as managerReady from '../../features/common/managerReady'; import { PythonEnvironmentApiImpl } from '../../features/pythonApi'; import { PythonProjectManager } from '../../internal.api'; suite('PythonEnvironmentApiImpl - onDidChangePythonProjects', () => { - test('Fires event with correct added and removed projects', async () => { - // 1. Create a mock EventEmitter to simulate the internal project manager + test('fires event with correct added and removed projects', () => { const onDidChangeProjectsEmitter = new EventEmitter(); - - // 2. Mock the PythonProjectManager let currentProjects: PythonProject[] = []; const mockProjectManager = { getProjects: () => currentProjects, onDidChangeProjects: onDidChangeProjectsEmitter.event, } as unknown as PythonProjectManager; - // 3. Mock the other required constructor arguments using ConstructorParameters type ApiArgs = ConstructorParameters; - const mockEnvManagers = { onDidChangeActiveEnvironment: new EventEmitter().event } as unknown as ApiArgs[0]; const mockProjectCreators = {} as unknown as ApiArgs[2]; const mockTerminalManager = {} as unknown as ApiArgs[3]; const mockEnvVarManager = { onDidChangeEnvironmentVariables: new EventEmitter().event } as unknown as ApiArgs[4]; - // 4. Initialize the API instance const api = new PythonEnvironmentApiImpl( mockEnvManagers, mockProjectManager, mockProjectCreators, mockTerminalManager, - mockEnvVarManager + mockEnvVarManager, ); - // 5. Listen to the public event we are testing let firedEventPayload: unknown = null; - api.onDidChangePythonProjects((e: unknown) => { - firedEventPayload = e; + api.onDidChangePythonProjects((event: unknown) => { + firedEventPayload = event; }); - // 6. Simulate adding a project const newProject = { uri: Uri.joinPath(Uri.file(process.cwd()), 'fake', 'path') } as unknown as PythonProject; - currentProjects = [newProject]; // Update the mock's state - - // Fire the internal event + currentProjects = [newProject]; onDidChangeProjectsEmitter.fire(); - // 7. Assert the public event fired with the correct delta assert.ok(firedEventPayload, 'Event should have fired'); - assert.strictEqual((firedEventPayload as { added: PythonProject[] }).added.length, 1, 'Should have 1 added project'); + assert.strictEqual((firedEventPayload as { added: PythonProject[] }).added.length, 1); assert.strictEqual((firedEventPayload as { added: PythonProject[] }).added[0].uri.fsPath, newProject.uri.fsPath); - assert.strictEqual((firedEventPayload as { removed: PythonProject[] }).removed.length, 0, 'Should have 0 removed projects'); + assert.strictEqual((firedEventPayload as { removed: PythonProject[] }).removed.length, 0); - // 8. Simulate removing the project firedEventPayload = null; currentProjects = []; onDidChangeProjectsEmitter.fire(); assert.ok(firedEventPayload, 'Event should have fired'); - assert.strictEqual((firedEventPayload as { added: PythonProject[] }).added.length, 0, 'Should have 0 added projects'); - assert.strictEqual((firedEventPayload as { removed: PythonProject[] }).removed.length, 1, 'Should have 1 removed project'); - assert.strictEqual((firedEventPayload as { removed: PythonProject[] }).removed[0].uri.fsPath, newProject.uri.fsPath); + assert.strictEqual((firedEventPayload as { added: PythonProject[] }).added.length, 0); + assert.strictEqual((firedEventPayload as { removed: PythonProject[] }).removed.length, 1); + assert.strictEqual( + (firedEventPayload as { removed: PythonProject[] }).removed[0].uri.fsPath, + newProject.uri.fsPath, + ); + }); +}); + +suite('PythonEnvironmentApiImpl - getEnvironment timeout fallback', () => { + let clock: sinon.SinonFakeTimers; + + setup(() => { + clock = sinon.useFakeTimers(); + sinon.stub(managerReady, 'waitForEnvManager').resolves(); + }); + + teardown(() => { + sinon.restore(); + }); + + test('returns the last-known environment while a slower lookup continues in the background', async () => { + const scope = Uri.file('/workspace/script.py'); + const lastKnown: PythonEnvironment = { + envId: { id: 'default', managerId: 'ms-python.python:venv' }, + name: 'default', + displayName: 'default', + displayPath: '/env/default', + version: '3.11.0', + environmentPath: Uri.file('/env/default'), + execInfo: { run: { executable: '/env/default/python', args: [] } }, + sysPrefix: '/env/default', + }; + let resolveEnvironment: ((value: PythonEnvironment | undefined) => void) | undefined; + + const mockProjectManager = { + getProjects: () => [], + onDidChangeProjects: new EventEmitter().event, + } as unknown as PythonProjectManager; + + type ApiArgs = ConstructorParameters; + const mockEnvManagers = { + onDidChangeActiveEnvironment: new EventEmitter().event, + getEnvironment: sinon.stub().returns( + new Promise((resolve) => { + resolveEnvironment = resolve; + }), + ), + getLastKnownEnvironment: sinon.stub().withArgs(scope).returns(lastKnown), + } as unknown as ApiArgs[0]; + const mockProjectCreators = {} as unknown as ApiArgs[2]; + const mockTerminalManager = {} as unknown as ApiArgs[3]; + const mockEnvVarManager = { onDidChangeEnvironmentVariables: new EventEmitter().event } as unknown as ApiArgs[4]; + + const api = new PythonEnvironmentApiImpl( + mockEnvManagers, + mockProjectManager, + mockProjectCreators, + mockTerminalManager, + mockEnvVarManager, + ); + + const pending = api.getEnvironment(scope); + await clock.tickAsync(1_000); + + assert.strictEqual(await pending, lastKnown); + resolveEnvironment?.(undefined); }); -}); \ No newline at end of file +}); diff --git a/src/test/managers/builtin/inlineScript/envManager.unit.test.ts b/src/test/managers/builtin/inlineScript/envManager.unit.test.ts index cce4d22f9..63e47c709 100644 --- a/src/test/managers/builtin/inlineScript/envManager.unit.test.ts +++ b/src/test/managers/builtin/inlineScript/envManager.unit.test.ts @@ -7,11 +7,17 @@ import * as fs from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; import * as sinon from 'sinon'; -import { LogOutputChannel, Uri } from 'vscode'; -import { EnvironmentChangeKind, EnvironmentManager, PythonEnvironment, PythonEnvironmentApi } from '../../../../api'; +import { Disposable, LogOutputChannel, TextDocument, Uri } from 'vscode'; +import { + EnvironmentChangeKind, + EnvironmentManager, + PythonEnvironment, + PythonEnvironmentApi, +} from '../../../../api'; import * as cacheKey from '../../../../common/inlineScript/cacheKey'; import * as cacheLayout from '../../../../common/inlineScript/cacheLayout'; import * as metadataReader from '../../../../common/inlineScript/metadata'; +import { InlineScriptRoutingRegistry } from '../../../../common/inlineScript/routingRegistry'; import * as lockfileApis from '../../../../common/lockfile.apis'; import * as persistentState from '../../../../common/persistentState'; import { EventNames } from '../../../../common/telemetry/constants'; @@ -19,6 +25,7 @@ import * as telemetrySender from '../../../../common/telemetry/sender'; import { isWindows } from '../../../../common/utils/platformUtils'; import { normalizePath } from '../../../../common/utils/pathUtils'; import { getVenvPythonPath } from '../../../../common/utils/virtualEnvironment'; +import * as workspaceApis from '../../../../common/workspace.apis'; import { InlineScriptEnvManager, INLINE_SCRIPT_ENVS_KEY, @@ -35,6 +42,10 @@ const VALID_METADATA: metadataReader.InlineScriptMetadata = { dependencies: ['requests'], range: { start: 0, end: 40 }, }; +const VALID_METADATA_IDENTITY = JSON.stringify({ + requiresPython: '>=3.11', + dependencies: ['requests'], +}); function makeFakeLog(): LogOutputChannel { return { @@ -113,9 +124,15 @@ suite('InlineScriptEnvManager', () => { let releaseLockStub: sinon.SinonStub; let resolveSystemPythonStub: sinon.SinonStub; let resolveVenvStub: sinon.SinonStub; + let routingRegistry: InlineScriptRoutingRegistry; + let sidecarsByEnvDir: Map; + let environmentsByExecutablePath: Map; + let cacheKeysByInputs: Map; let tempRoot: string; let baseInterpreterStatusStub: sinon.SinonStub; let writeMetaStub: sinon.SinonStub; + let deleteFilesListener: ((e: { files: readonly Uri[] }) => unknown) | undefined; + let renameFilesListener: ((e: { files: readonly { oldUri: Uri; newUri: Uri }[] }) => unknown) | undefined; let workspaceState: { get: sinon.SinonStub; set: sinon.SinonStub; @@ -137,6 +154,12 @@ suite('InlineScriptEnvManager', () => { refreshEnvironments: apiRefreshEnvironmentsStub, } as unknown as PythonEnvironmentApi; nativeFinder = {} as NativePythonFinder; + routingRegistry = new InlineScriptRoutingRegistry(); + sidecarsByEnvDir = new Map(); + environmentsByExecutablePath = new Map(); + cacheKeysByInputs = new Map(); + deleteFilesListener = undefined; + renameFilesListener = undefined; baseManager = {} as EnvironmentManager; persistedAssociations = undefined; workspaceState = { @@ -157,7 +180,10 @@ suite('InlineScriptEnvManager', () => { sinon.stub(persistentState, 'getWorkspacePersistentState').resolves(workspaceState); readMetadataStub = sinon.stub(metadataReader, 'readInlineScriptMetadataFromFile').resolves(VALID_METADATA); - computeCacheKeyStub = sinon.stub(cacheKey, 'computeCacheKey').returns(CACHE_KEY); + computeCacheKeyStub = sinon.stub(cacheKey, 'computeCacheKey').callsFake((inputs) => { + return cacheKeysByInputs.get(getCacheKeyInputKey(inputs.dependencies, inputs.interpreterPath)) ?? CACHE_KEY; + }); + registerCacheKey(CACHE_KEY, VALID_METADATA.dependencies ?? [], baseExecutable); getAvailablePythonVersionsStub = sinon.stub(uvPythonInstaller, 'getAvailablePythonVersions').resolves([]); ensureUvForVersionLookupStub = sinon .stub(uvPythonInstaller, 'ensureUvForInlineScriptVersionLookupDetailed') @@ -166,32 +192,66 @@ suite('InlineScriptEnvManager', () => { .stub(uvPythonInstaller, 'promptInstallPythonViaUvDetailed') .resolves({ kind: 'declined' }); sendTelemetryStub = sinon.stub(telemetrySender, 'sendTelemetryEvent'); - inspectMetaStub = sinon.stub(cacheLayout, 'inspectMetaJson').resolves({ kind: 'missing' }); + inspectMetaStub = sinon.stub(cacheLayout, 'inspectMetaJson').callsFake(async (envDir: Uri) => { + const result = sidecarsByEnvDir.get(normalizePath(envDir.fsPath)) ?? 'missing'; + if (result === 'missing' || result === 'invalid' || result === 'unavailable') { + return { kind: result }; + } + return { kind: 'valid', metadata: result }; + }); baseInterpreterStatusStub = sinon.stub(cacheLayout, 'getBaseInterpreterStatus').resolves('available'); - writeMetaStub = sinon.stub(cacheLayout, 'writeMetaJson').resolves(); + writeMetaStub = sinon.stub(cacheLayout, 'writeMetaJson').callsFake(async (envDir: Uri, meta: cacheLayout.InlineScriptEnvMeta) => { + sidecarsByEnvDir.set(normalizePath(envDir.fsPath), meta); + }); retainLockStub = sinon.stub().resolves(); releaseLockStub = sinon.stub().resolves(); lockStub = sinon .stub(lockfileApis, 'acquireFileLock') .resolves({ release: releaseLockStub, retain: retainLockStub }); resolveSystemPythonStub = sinon.stub(builtinUtils, 'resolveSystemPythonEnvironmentPath').resolves(undefined); - resolveVenvStub = sinon.stub(venvUtils, 'resolveVenvPythonEnvironmentPath').resolves(undefined); + resolveVenvStub = sinon.stub(venvUtils, 'resolveVenvPythonEnvironmentPath').callsFake(async (environmentPath: string) => { + return environmentsByExecutablePath.get(normalizePath(environmentPath)); + }); + sinon.stub(workspaceApis, 'onDidDeleteFiles').callsFake((listener: (e: { files: readonly Uri[] }) => unknown) => { + deleteFilesListener = listener; + return new Disposable(() => { + deleteFilesListener = undefined; + }); + }); + sinon + .stub(workspaceApis, 'onDidRenameFiles') + .callsFake((listener: (e: { files: readonly { oldUri: Uri; newUri: Uri }[] }) => unknown) => { + renameFilesListener = listener; + return new Disposable(() => { + renameFilesListener = undefined; + }); + }); + sinon.stub(workspaceApis, 'getOpenTextDocuments').returns([]); createWithProgressStub = sinon.stub(venvUtils, 'createWithProgress').callsFake(async (...args: unknown[]) => { const envDir = args[6] as string; const selectedBase = args[4] as PythonEnvironment; await fs.outputFile(getVenvPythonPath(envDir), ''); + const environment = makeEnvironment( + 'ms-python.python:inline-script', + selectedBase.version, + getVenvPythonPath(envDir), + envDir, + ); + environmentsByExecutablePath.set(normalizePath(environment.environmentPath.fsPath), environment); return { - environment: makeEnvironment( - 'ms-python.python:inline-script', - selectedBase.version, - getVenvPythonPath(envDir), - envDir, - ), + environment, }; }); clock = sinon.useFakeTimers({ now: NOW, toFake: ['Date'] }); - manager = new InlineScriptEnvManager(nativeFinder, api, baseManager, globalStorageUri, makeFakeLog()); + manager = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + routingRegistry, + ); }); teardown(async () => { @@ -208,8 +268,21 @@ suite('InlineScriptEnvManager', () => { return cacheLayout.getScriptEnvDir(globalStorageUri, CACHE_KEY); } - function setSidecar(metadata: cacheLayout.InlineScriptEnvMeta): void { - inspectMetaStub.resolves({ kind: 'valid', metadata }); + function getCacheKeyInputKey(dependencies: readonly string[], interpreterPath: string): string { + return JSON.stringify({ + dependencies: Array.from( + new Set(dependencies.map((dependency) => cacheKey.normalizeDependency(dependency)).filter(Boolean)), + ).sort(), + interpreterPath: normalizePath(interpreterPath), + }); + } + + function registerCacheKey(cacheKeyValue: string, dependencies: readonly string[], interpreterPath: string): void { + cacheKeysByInputs.set(getCacheKeyInputKey(dependencies, interpreterPath), cacheKeyValue); + } + + function setSidecar(metadata: cacheLayout.InlineScriptEnvMeta, targetEnvDir: Uri = envDir()): void { + sidecarsByEnvDir.set(normalizePath(targetEnvDir.fsPath), metadata); } async function makeSidecar( @@ -239,11 +312,25 @@ suite('InlineScriptEnvManager', () => { ): Promise { const location = cacheLayout.getScriptEnvDir(globalStorageUri, cacheKey).fsPath; const executable = getVenvPythonPath(location); + const baseInterpreterPath = + cacheKey === CACHE_KEY + ? baseExecutable + : path.join(tempRoot, `base-python-${cacheKey}`, isWindows() ? 'python.exe' : 'python'); + await fs.outputFile(baseInterpreterPath, ''); await fs.outputFile(executable, ''); - return { + registerCacheKey(cacheKey, VALID_METADATA.dependencies ?? [], baseInterpreterPath); + setSidecar({ + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath, + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: NOW.toISOString(), + }, Uri.file(location)); + const environment = { ...makeEnvironment('ms-python.python:inline-script', '3.12.4', executable, location), envId: { managerId: 'ms-python.python:inline-script', id: envId }, }; + environmentsByExecutablePath.set(normalizePath(executable), environment); + return environment; } async function waitForStubCall(stub: sinon.SinonStub): Promise { @@ -256,14 +343,14 @@ suite('InlineScriptEnvManager', () => { assert.fail('Expected the stub to be called'); } - async function waitForStubCallCount(stub: sinon.SinonStub, expectedCallCount: number): Promise { + async function waitForStubCallCount(stub: { callCount: number }, expectedCallCount: number): Promise { for (let attempt = 0; attempt < 20; attempt += 1) { if (stub.callCount >= expectedCallCount) { return; } await new Promise((resolve) => setTimeout(resolve, 5)); } - assert.fail(`Expected the stub to be called ${expectedCallCount} times`); + assert.fail(`Expected the stub to be called at least ${expectedCallCount} times`); } async function waitForCondition( @@ -283,6 +370,119 @@ suite('InlineScriptEnvManager', () => { return new Promise((resolve) => setImmediate(resolve)); } + function fireDelete(...files: Uri[]): void { + assert.ok(deleteFilesListener, 'delete listener should be registered'); + deleteFilesListener!({ files }); + } + + function fireRename(oldUri: Uri, newUri: Uri): void { + assert.ok(renameFilesListener, 'rename listener should be registered'); + renameFilesListener!({ files: [{ oldUri, newUri }] }); + } + + function workspaceStateSetCalls(key: string): readonly sinon.SinonSpyCall[] { + return workspaceState.set.getCalls().filter((call) => call.args[0] === key); + } + + function matchedAssociationRecord(environmentPath: string, metadataIdentity: string = VALID_METADATA_IDENTITY): unknown { + return { + schemaVersion: 1, + environmentPath, + metadataBinding: { + kind: 'matched', + sourceIdentity: metadataIdentity, + }, + }; + } + + function pendingAssociationRecord(environmentPath: string, metadataIdentity: string = VALID_METADATA_IDENTITY): unknown { + return { + schemaVersion: 1, + environmentPath, + metadataBinding: { + kind: 'pending', + sourceIdentity: metadataIdentity, + }, + }; + } + + function futureAssociationRecord(environmentPath: string): unknown { + return { + schemaVersion: 2, + environmentPath, + metadataBinding: { + kind: 'matched', + sourceIdentity: 'future', + }, + }; + } + + async function triggerSavedMetadataChange( + registry: InlineScriptRoutingRegistry, + managerInstance: InlineScriptEnvManager, + uri: Uri, + metadata: metadataReader.InlineScriptMetadata = VALID_METADATA, + ): Promise { + registry.setMetadata(uri, metadata); + await ( + managerInstance as unknown as { + handleSavedMetadataChange(event: { + uri: Uri; + metadata: metadataReader.InlineScriptMetadata; + metadataIdentity: string | undefined; + metadataRevision: number; + }): Promise; + } + ).handleSavedMetadataChange({ + uri, + metadata, + metadataIdentity: registry.getMetadataIdentity(uri), + metadataRevision: registry.getMetadataRevision(uri), + }); + } + + function asMetadataRefreshManager(managerInstance: InlineScriptEnvManager): { + refreshValidatedAssociationForMetadataInternal( + scriptPath: string, + uri: Uri, + metadata: metadataReader.InlineScriptMetadata, + metadataIdentity: string, + metadataRevision: number, + associationRevision: number, + ): Promise; + currentCacheEntryProvesSourceMetadataIdentity( + candidate: PythonEnvironment, + metadataIdentity: string, + metadata: metadataReader.InlineScriptMetadata, + ): Promise; + cachedAssociationValidatedAt: Map; + lastValidatedMetadataIdentities: Map; + lastValidatedMetadataIdentityProofs: Map; + associationRevisions: Map; + subscriptions: Disposable[]; + } { + return managerInstance as unknown as { + refreshValidatedAssociationForMetadataInternal( + scriptPath: string, + uri: Uri, + metadata: metadataReader.InlineScriptMetadata, + metadataIdentity: string, + metadataRevision: number, + associationRevision: number, + ): Promise; + currentCacheEntryProvesSourceMetadataIdentity( + candidate: PythonEnvironment, + metadataIdentity: string, + metadata: metadataReader.InlineScriptMetadata, + ): Promise; + cachedAssociationValidatedAt: Map; + lastValidatedMetadataIdentities: Map; + lastValidatedMetadataIdentityProofs: Map; + associationRevisions: Map; + subscriptions: Disposable[]; + }; + } + suite('static metadata and deferred methods', () => { test('exposes creation but leaves later-phase methods empty', async () => { const asInterface: EnvironmentManager = manager; @@ -1016,6 +1216,9 @@ suite('InlineScriptEnvManager', () => { baseInterpreterPath: baseExecutable, baseInterpreterVersion: baseEnvironment.version, lastUsedAt: NOW.toISOString(), + sourceMetadataIdentityHashes: [ + cacheLayout.hashSourceMetadataIdentity(VALID_METADATA_IDENTITY), + ], }, ]); assert.strictEqual( @@ -1035,6 +1238,61 @@ suite('InlineScriptEnvManager', () => { assert.ok(options.retryIntervalMs > 0); }); + test('reuses a restart cache entry from an older backup matching the selected base', async () => { + const directory = envDir(); + const executable = venvPythonPath(directory.fsPath); + const sidecar = { + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: baseExecutable, + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: NOW.toISOString(), + } satisfies cacheLayout.InlineScriptEnvMeta; + const newerIncompatibleSidecar = { + ...sidecar, + baseInterpreterPath: path.join(tempRoot, 'other-base-python'), + baseInterpreterVersion: '3.13.0', + lastUsedAt: '2030-01-01T00:00:00.000Z', + } satisfies cacheLayout.InlineScriptEnvMeta; + const environment = makeEnvironment( + 'ms-python.python:inline-script', + baseEnvironment.version, + executable, + directory.fsPath, + ); + await fs.outputFile(executable, ''); + await fs.writeFile( + `${cacheLayout.getMetaJsonPath(directory).fsPath}.backup-abcdef123456`, + JSON.stringify(sidecar), + ); + await fs.writeFile( + `${cacheLayout.getMetaJsonPath(directory).fsPath}.backup-ffffffffffff`, + JSON.stringify(newerIncompatibleSidecar), + ); + environmentsByExecutablePath.set(normalizePath(executable), environment); + inspectMetaStub.restore(); + + const result = await manager.create(scriptUri()); + + assert.strictEqual(result, environment); + assert.strictEqual(createWithProgressStub.callCount, 0, 'recovered cache entry must not rebuild'); + assert.deepStrictEqual(await cacheLayout.readMetaJson(directory), sidecar); + assert.strictEqual( + await fs.pathExists(`${cacheLayout.getMetaJsonPath(directory).fsPath}.backup-abcdef123456`), + false, + ); + }); + + test('preserves a restart cache entry when backup recovery is uncertain', async () => { + const markerPath = path.join(envDir().fsPath, 'keep.txt'); + await fs.outputFile(markerPath, 'keep'); + inspectMetaStub.resolves({ kind: 'missing' }); + sinon.stub(cacheLayout, 'restoreMetaJsonBackupUnderLock').resolves({ kind: 'unavailable' }); + + assert.strictEqual(await manager.create(scriptUri()), undefined); + assert.strictEqual(await fs.readFile(markerPath, 'utf8'), 'keep'); + assert.strictEqual(createWithProgressStub.callCount, 0); + }); + test('coalesces simultaneous same-key creation within one extension host', async () => { let continueCreation: (() => void) | undefined; let creationStarted: (() => void) | undefined; @@ -1081,6 +1339,482 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(createWithProgressStub.callCount, 1); }); + test('records every successful same-key coalesced caller provenance for later set and restart routing', async () => { + const cacheKeyValue = 'fedcba9876543210'; + const firstUri = scriptUri('a.py'); + const secondUri = scriptUri('b.py'); + const secondMetadata = { + ...VALID_METADATA, + requiresPython: '>=3.12', + } satisfies metadataReader.InlineScriptMetadata; + const firstIdentity = VALID_METADATA_IDENTITY; + const secondIdentity = JSON.stringify({ + requiresPython: secondMetadata.requiresPython, + dependencies: secondMetadata.dependencies, + }); + const metadataByScript = new Map([ + [normalizePath(firstUri.fsPath), VALID_METADATA], + [normalizePath(secondUri.fsPath), secondMetadata], + ]); + readMetadataStub.callsFake(async (uri: Uri) => metadataByScript.get(normalizePath(uri.fsPath))); + routingRegistry.setMetadata(firstUri, VALID_METADATA); + routingRegistry.setMetadata(secondUri, secondMetadata); + registerCacheKey(cacheKeyValue, ['requests', 'pytest'], baseExecutable); + + let continueCreation: (() => void) | undefined; + let creationStarted: (() => void) | undefined; + let secondCallHashed: (() => void) | undefined; + const started = new Promise((resolve) => { + creationStarted = resolve; + }); + const secondHashed = new Promise((resolve) => { + secondCallHashed = resolve; + }); + const gate = new Promise((resolve) => { + continueCreation = resolve; + }); + computeCacheKeyStub.callsFake((inputs: cacheKey.CacheKeyInputs) => { + if (computeCacheKeyStub.callCount === 2) { + secondCallHashed!(); + } + return cacheKeysByInputs.get(getCacheKeyInputKey(inputs.dependencies, inputs.interpreterPath)) ?? CACHE_KEY; + }); + createWithProgressStub.callsFake(async (...args: unknown[]) => { + const target = args[6] as string; + await fs.outputFile(venvPythonPath(target), ''); + const environment = makeEnvironment( + 'ms-python.python:inline-script', + '3.12.4', + venvPythonPath(target), + target, + ); + environmentsByExecutablePath.set(normalizePath(environment.environmentPath.fsPath), environment); + creationStarted!(); + await gate; + return { environment }; + }); + + const first = manager.create(firstUri, { additionalPackages: ['pytest'] }); + await started; + const second = manager.create(secondUri, { additionalPackages: ['pytest'] }); + await secondHashed; + continueCreation!(); + const [firstEnvironment, secondEnvironment] = await Promise.all([first, second]); + + assert.ok(firstEnvironment); + assert.strictEqual(firstEnvironment, secondEnvironment); + assert.strictEqual(lockStub.callCount, 1); + assert.strictEqual(createWithProgressStub.callCount, 1); + assert.deepStrictEqual( + ( + sidecarsByEnvDir.get( + normalizePath(cacheLayout.getScriptEnvDir(globalStorageUri, cacheKeyValue).fsPath), + ) as cacheLayout.InlineScriptEnvMeta + ).sourceMetadataIdentityHashes, + [ + cacheLayout.hashSourceMetadataIdentity(firstIdentity), + cacheLayout.hashSourceMetadataIdentity(secondIdentity), + ], + ); + + await manager.set(firstUri, firstEnvironment); + await manager.set(secondUri, secondEnvironment); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(firstUri.fsPath)]: matchedAssociationRecord(firstEnvironment.environmentPath.fsPath, firstIdentity), + [normalizePath(secondUri.fsPath)]: matchedAssociationRecord(secondEnvironment!.environmentPath.fsPath, secondIdentity), + }); + assert.strictEqual(routingRegistry.hasValidatedAssociation(firstUri), true); + assert.strictEqual(routingRegistry.hasValidatedAssociation(secondUri), true); + + persistedAssociations = {}; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + restartRoutingRegistry.setMetadata(firstUri, VALID_METADATA); + restartRoutingRegistry.setMetadata(secondUri, secondMetadata); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + + await restarted.set(firstUri, firstEnvironment); + await restarted.set(secondUri, secondEnvironment); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(firstUri.fsPath)]: matchedAssociationRecord(firstEnvironment.environmentPath.fsPath, firstIdentity), + [normalizePath(secondUri.fsPath)]: matchedAssociationRecord(secondEnvironment!.environmentPath.fsPath, secondIdentity), + }); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(firstUri), true); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(secondUri), true); + restarted.dispose(); + }); + + test('merges a late same-key caller that arrives while the initial sidecar write is in flight', async () => { + const cacheKeyValue = 'fedcba9876543210'; + const firstUri = scriptUri('a.py'); + const secondUri = scriptUri('b.py'); + const secondMetadata = { + ...VALID_METADATA, + requiresPython: '>=3.12', + } satisfies metadataReader.InlineScriptMetadata; + const secondIdentity = JSON.stringify({ + requiresPython: secondMetadata.requiresPython, + dependencies: secondMetadata.dependencies, + }); + const firstHash = cacheLayout.hashSourceMetadataIdentity(VALID_METADATA_IDENTITY); + const secondHash = cacheLayout.hashSourceMetadataIdentity(secondIdentity); + const metadataByScript = new Map([ + [normalizePath(firstUri.fsPath), VALID_METADATA], + [normalizePath(secondUri.fsPath), secondMetadata], + ]); + readMetadataStub.callsFake(async (uri: Uri) => metadataByScript.get(normalizePath(uri.fsPath))); + registerCacheKey(cacheKeyValue, ['requests', 'pytest'], baseExecutable); + let secondCallHashed: (() => void) | undefined; + const secondHashed = new Promise((resolve) => { + secondCallHashed = resolve; + }); + computeCacheKeyStub.callsFake((inputs: cacheKey.CacheKeyInputs) => { + if (computeCacheKeyStub.callCount === 2) { + secondCallHashed!(); + } + return cacheKeysByInputs.get(getCacheKeyInputKey(inputs.dependencies, inputs.interpreterPath)) ?? CACHE_KEY; + }); + + let releaseFirstWrite: (() => void) | undefined; + let firstWriteStarted: (() => void) | undefined; + const firstWriteGate = new Promise((resolve) => { + releaseFirstWrite = resolve; + }); + const firstWritePending = new Promise((resolve) => { + firstWriteStarted = resolve; + }); + let firstWrittenHashes: readonly string[] | undefined; + writeMetaStub.callsFake(async (envDir: Uri, meta: cacheLayout.InlineScriptEnvMeta) => { + if (writeMetaStub.callCount === 1) { + firstWrittenHashes = meta.sourceMetadataIdentityHashes; + firstWriteStarted!(); + await firstWriteGate; + } + sidecarsByEnvDir.set(normalizePath(envDir.fsPath), meta); + }); + createWithProgressStub.callsFake(async (...args: unknown[]) => { + const target = args[6] as string; + await fs.outputFile(venvPythonPath(target), ''); + const environment = makeEnvironment( + 'ms-python.python:inline-script', + '3.12.4', + venvPythonPath(target), + target, + ); + environmentsByExecutablePath.set(normalizePath(environment.environmentPath.fsPath), environment); + return { environment }; + }); + + const first = manager.create(firstUri, { additionalPackages: ['pytest'] }); + await firstWritePending; + const second = manager.create(secondUri, { additionalPackages: ['pytest'] }); + await secondHashed; + const pendingCreations = ( + manager as unknown as { + pendingCreations: Map; + } + ).pendingCreations; + for (let attempt = 0; attempt < 20; attempt += 1) { + if (pendingCreations.get(cacheKeyValue)?.sourceMetadataIdentityHashes?.includes(secondHash)) { + break; + } + await nextTurn(); + } + + assert.deepStrictEqual(firstWrittenHashes, [firstHash]); + assert.strictEqual( + pendingCreations.get(cacheKeyValue)?.sourceMetadataIdentityHashes?.includes(secondHash), + true, + ); + + releaseFirstWrite!(); + const [firstEnvironment, secondEnvironment] = await Promise.all([first, second]); + + assert.ok(firstEnvironment); + assert.strictEqual(firstEnvironment, secondEnvironment); + assert.strictEqual(createWithProgressStub.callCount, 1); + assert.strictEqual(lockStub.callCount, 2); + assert.deepStrictEqual( + ( + sidecarsByEnvDir.get( + normalizePath(cacheLayout.getScriptEnvDir(globalStorageUri, cacheKeyValue).fsPath), + ) as cacheLayout.InlineScriptEnvMeta + ).sourceMetadataIdentityHashes, + [firstHash, secondHash], + ); + }); + + for (const failureMode of ['lock', 'read', 'write'] as const) { + test(`late same-key caller returns undefined when durable provenance merge ${failureMode} fails, but first caller and retry succeed`, async () => { + const cacheKeyValue = 'fedcba9876543210'; + const firstUri = scriptUri('a.py'); + const secondUri = scriptUri('b.py'); + const secondMetadata = { + ...VALID_METADATA, + requiresPython: '>=3.12', + } satisfies metadataReader.InlineScriptMetadata; + const secondIdentity = JSON.stringify({ + requiresPython: secondMetadata.requiresPython, + dependencies: secondMetadata.dependencies, + }); + const firstHash = cacheLayout.hashSourceMetadataIdentity(VALID_METADATA_IDENTITY); + const secondHash = cacheLayout.hashSourceMetadataIdentity(secondIdentity); + const metadataByScript = new Map([ + [normalizePath(firstUri.fsPath), VALID_METADATA], + [normalizePath(secondUri.fsPath), secondMetadata], + ]); + readMetadataStub.callsFake(async (uri: Uri) => metadataByScript.get(normalizePath(uri.fsPath))); + registerCacheKey(cacheKeyValue, ['requests', 'pytest'], baseExecutable); + let secondCallHashed: (() => void) | undefined; + const secondHashed = new Promise((resolve) => { + secondCallHashed = resolve; + }); + computeCacheKeyStub.callsFake((inputs: cacheKey.CacheKeyInputs) => { + if (computeCacheKeyStub.callCount === 2) { + secondCallHashed!(); + } + return cacheKeysByInputs.get(getCacheKeyInputKey(inputs.dependencies, inputs.interpreterPath)) ?? CACHE_KEY; + }); + + let releaseFirstWrite: (() => void) | undefined; + let firstWriteStarted: (() => void) | undefined; + const firstWriteGate = new Promise((resolve) => { + releaseFirstWrite = resolve; + }); + const firstWritePending = new Promise((resolve) => { + firstWriteStarted = resolve; + }); + writeMetaStub.callsFake(async (envDir: Uri, meta: cacheLayout.InlineScriptEnvMeta) => { + if (writeMetaStub.callCount === 1) { + firstWriteStarted!(); + await firstWriteGate; + } + sidecarsByEnvDir.set(normalizePath(envDir.fsPath), meta); + }); + if (failureMode === 'lock') { + lockStub.onSecondCall().rejects(new Error('merge lock failed')); + } else if (failureMode === 'read') { + inspectMetaStub.onFirstCall().rejects(new Error('merge read failed')); + } else { + writeMetaStub.onSecondCall().rejects(new Error('merge write failed')); + } + createWithProgressStub.callsFake(async (...args: unknown[]) => { + const target = args[6] as string; + await fs.outputFile(venvPythonPath(target), ''); + const environment = makeEnvironment( + 'ms-python.python:inline-script', + '3.12.4', + venvPythonPath(target), + target, + ); + environmentsByExecutablePath.set(normalizePath(environment.environmentPath.fsPath), environment); + return { environment }; + }); + + const first = manager.create(firstUri, { additionalPackages: ['pytest'] }); + await firstWritePending; + const second = manager.create(secondUri, { additionalPackages: ['pytest'] }); + await secondHashed; + releaseFirstWrite!(); + const [firstEnvironment, secondEnvironment] = await Promise.all([first, second]); + + assert.ok(firstEnvironment); + assert.strictEqual(secondEnvironment, undefined); + assert.strictEqual(createWithProgressStub.callCount, 1); + assert.deepStrictEqual( + ( + sidecarsByEnvDir.get( + normalizePath(cacheLayout.getScriptEnvDir(globalStorageUri, cacheKeyValue).fsPath), + ) as cacheLayout.InlineScriptEnvMeta + ).sourceMetadataIdentityHashes, + [firstHash], + ); + + const retried = await manager.create(secondUri, { additionalPackages: ['pytest'] }); + + assert.ok(retried); + assert.strictEqual(normalizePath(retried!.environmentPath.fsPath), normalizePath(firstEnvironment.environmentPath.fsPath)); + assert.deepStrictEqual( + ( + sidecarsByEnvDir.get( + normalizePath(cacheLayout.getScriptEnvDir(globalStorageUri, cacheKeyValue).fsPath), + ) as cacheLayout.InlineScriptEnvMeta + ).sourceMetadataIdentityHashes, + [firstHash, secondHash], + ); + assert.strictEqual(createWithProgressStub.callCount, 1); + }); + } + + test('does not record provenance when a shared same-key creation fails', async () => { + const firstUri = scriptUri('a.py'); + const secondUri = scriptUri('b.py'); + const secondMetadata = { + ...VALID_METADATA, + requiresPython: '>=3.12', + } satisfies metadataReader.InlineScriptMetadata; + const metadataByScript = new Map([ + [normalizePath(firstUri.fsPath), VALID_METADATA], + [normalizePath(secondUri.fsPath), secondMetadata], + ]); + readMetadataStub.callsFake(async (uri: Uri) => metadataByScript.get(normalizePath(uri.fsPath))); + registerCacheKey(CACHE_KEY, ['requests', 'pytest'], baseExecutable); + + let continueCreation: (() => void) | undefined; + let creationStarted: (() => void) | undefined; + let secondCallHashed: (() => void) | undefined; + const started = new Promise((resolve) => { + creationStarted = resolve; + }); + const secondHashed = new Promise((resolve) => { + secondCallHashed = resolve; + }); + const gate = new Promise((resolve) => { + continueCreation = resolve; + }); + computeCacheKeyStub.callsFake((inputs: cacheKey.CacheKeyInputs) => { + if (computeCacheKeyStub.callCount === 2) { + secondCallHashed!(); + } + return cacheKeysByInputs.get(getCacheKeyInputKey(inputs.dependencies, inputs.interpreterPath)) ?? CACHE_KEY; + }); + createWithProgressStub.callsFake(async () => { + creationStarted!(); + await gate; + return { envCreationErr: 'boom' }; + }); + + const first = manager.create(firstUri, { additionalPackages: ['pytest'] }); + await started; + const second = manager.create(secondUri, { additionalPackages: ['pytest'] }); + await secondHashed; + continueCreation!(); + + assert.deepStrictEqual(await Promise.all([first, second]), [undefined, undefined]); + assert.strictEqual(writeMetaStub.callCount, 0); + assert.strictEqual(sidecarsByEnvDir.size, 0); + }); + + test('dedupes coalesced same-key provenance hashes before the first sidecar write', async () => { + const cacheKeyValue = 'fedcba9876543210'; + const scriptSpecs = [ + ['script-0.py', '>=3.0'], + ['script-1.py', '>=3.1'], + ['script-2.py', '>=3.2'], + ['script-3.py', '>=3.3'], + ['script-4.py', '>=3.4'], + ['script-5.py', '>=3.5'], + ['script-6.py', '>=3.6'], + ['script-7.py', '>=3.7'], + ['script-8.py', '>=3.8'], + ['script-9.py', '>=3.8'], + ] as const; + const metadataByScript = new Map( + scriptSpecs.map(([name, requiresPython]) => [ + normalizePath(scriptUri(name).fsPath), + { + ...VALID_METADATA, + requiresPython, + }, + ]), + ); + let expectedHashes: readonly string[] | undefined; + for (const [, requiresPython] of scriptSpecs) { + expectedHashes = cacheLayout.mergeSourceMetadataIdentityHashes( + expectedHashes, + cacheLayout.hashSourceMetadataIdentity( + JSON.stringify({ + requiresPython, + dependencies: ['requests'], + }), + ), + ); + } + readMetadataStub.callsFake(async (uri: Uri) => metadataByScript.get(normalizePath(uri.fsPath))); + registerCacheKey(cacheKeyValue, ['requests', 'pytest'], baseExecutable); + + let continueCreation: (() => void) | undefined; + let creationStarted: (() => void) | undefined; + const started = new Promise((resolve) => { + creationStarted = resolve; + }); + const gate = new Promise((resolve) => { + continueCreation = resolve; + }); + createWithProgressStub.callsFake(async (...args: unknown[]) => { + const target = args[6] as string; + await fs.outputFile(venvPythonPath(target), ''); + const environment = makeEnvironment( + 'ms-python.python:inline-script', + '3.12.4', + venvPythonPath(target), + target, + ); + environmentsByExecutablePath.set(normalizePath(environment.environmentPath.fsPath), environment); + creationStarted!(); + await gate; + return { environment }; + }); + + const pendingCreates = [manager.create(scriptUri(scriptSpecs[0][0]), { additionalPackages: ['pytest'] })]; + await started; + const pendingCreations = ( + manager as unknown as { + pendingCreations: Map; + } + ).pendingCreations; + const addPendingCreationSourceMetadataIdentityHashStub = sinon + .stub( + manager as unknown as { + addPendingCreationSourceMetadataIdentityHash( + pendingCreation: { sourceMetadataIdentityHashes?: readonly string[] }, + sourceMetadataIdentityHash: string | undefined, + ): void; + }, + 'addPendingCreationSourceMetadataIdentityHash', + ) + .callThrough(); + for (const [name, requiresPython] of scriptSpecs.slice(1)) { + const hash = cacheLayout.hashSourceMetadataIdentity( + JSON.stringify({ + requiresPython, + dependencies: ['requests'], + }), + ); + pendingCreates.push(manager.create(scriptUri(name), { additionalPackages: ['pytest'] })); + await waitForStubCallCount(addPendingCreationSourceMetadataIdentityHashStub, pendingCreates.length - 1); + assert.strictEqual( + pendingCreations.get(cacheKeyValue)?.sourceMetadataIdentityHashes?.includes(hash), + true, + ); + } + assert.deepStrictEqual( + [...(pendingCreations.get(cacheKeyValue)?.sourceMetadataIdentityHashes ?? [])].sort(), + [...(expectedHashes ?? [])].sort(), + ); + continueCreation!(); + const environments = await Promise.all(pendingCreates); + + assert.ok(environments[0]); + assert.ok(environments.every((environment) => environment === environments[0])); + assert.strictEqual(lockStub.callCount, 1); + const sourceMetadataIdentityHashes = ( + sidecarsByEnvDir.get( + normalizePath(cacheLayout.getScriptEnvDir(globalStorageUri, cacheKeyValue).fsPath), + ) as cacheLayout.InlineScriptEnvMeta + ).sourceMetadataIdentityHashes; + assert.deepStrictEqual([...(sourceMetadataIdentityHashes ?? [])].sort(), [...(expectedHashes ?? [])].sort()); + assert.strictEqual(sourceMetadataIdentityHashes?.length, expectedHashes?.length); + assert.strictEqual(sourceMetadataIdentityHashes ? new Set(sourceMetadataIdentityHashes).size : 0, sourceMetadataIdentityHashes?.length); + }); + test('returns undefined without building when the cache lock cannot be acquired', async () => { lockStub.rejects(Object.assign(new Error('already locked'), { code: 'ELOCKED' })); assert.strictEqual(await manager.create(scriptUri()), undefined); @@ -1115,11 +1849,91 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(resolveVenvStub.firstCall.args[0], venvPythonPath(envDir().fsPath)); assert.deepStrictEqual(writeMetaStub.firstCall.args, [ envDir(), - { ...sidecar, lastUsedAt: NOW.toISOString() }, + { + ...sidecar, + lastUsedAt: NOW.toISOString(), + sourceMetadataIdentityHashes: [ + cacheLayout.hashSourceMetadataIdentity(VALID_METADATA_IDENTITY), + ], + }, ]); }); - test('returns a valid hit even when the last-used timestamp cannot be updated', async () => { + test('merges the current metadata identity hash into a reused cache sidecar', async () => { + await fs.ensureDir(envDir().fsPath); + setSidecar({ + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: baseExecutable, + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: '2026-07-01T00:00:00.000Z', + sourceMetadataIdentityHashes: [cacheLayout.hashSourceMetadataIdentity('{"requiresPython":">=3.12","dependencies":["rich"]}')], + }); + const cached = makeEnvironment( + 'ms-python.python:inline-script', + '3.12.4', + venvPythonPath(envDir().fsPath), + envDir().fsPath, + ); + await fs.outputFile(venvPythonPath(envDir().fsPath), ''); + resolveVenvStub.resolves(cached); + + await manager.create(scriptUri()); + + assert.deepStrictEqual(writeMetaStub.firstCall.args[1], { + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: baseExecutable, + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: NOW.toISOString(), + sourceMetadataIdentityHashes: [ + cacheLayout.hashSourceMetadataIdentity('{"requiresPython":">=3.12","dependencies":["rich"]}'), + cacheLayout.hashSourceMetadataIdentity(VALID_METADATA_IDENTITY), + ], + }); + }); + + test('dedupes and caps reused cache provenance hashes', async () => { + await fs.ensureDir(envDir().fsPath); + const currentHash = cacheLayout.hashSourceMetadataIdentity(VALID_METADATA_IDENTITY); + const hashes = [ + currentHash, + ...Array.from({ length: cacheLayout.MAX_SOURCE_METADATA_IDENTITY_HASHES - 1 }, (_, index) => + cacheLayout.hashSourceMetadataIdentity(`identity-${index}`), + ), + ]; + setSidecar({ + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: baseExecutable, + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: '2026-07-01T00:00:00.000Z', + sourceMetadataIdentityHashes: hashes, + }); + const cached = makeEnvironment( + 'ms-python.python:inline-script', + '3.12.4', + venvPythonPath(envDir().fsPath), + envDir().fsPath, + ); + await fs.outputFile(venvPythonPath(envDir().fsPath), ''); + resolveVenvStub.resolves(cached); + + await manager.create(scriptUri()); + + assert.strictEqual((writeMetaStub.firstCall.args[1] as cacheLayout.InlineScriptEnvMeta).sourceMetadataIdentityHashes?.length, cacheLayout.MAX_SOURCE_METADATA_IDENTITY_HASHES); + }); + + test('preserves a cache entry with a future sidecar schema version', async () => { + await fs.ensureDir(envDir().fsPath); + await fs.outputFile(venvPythonPath(envDir().fsPath), ''); + const markerPath = path.join(envDir().fsPath, 'keep.txt'); + await fs.outputFile(markerPath, 'keep'); + sidecarsByEnvDir.set(normalizePath(envDir().fsPath), 'unavailable'); + inspectMetaStub.callsFake(async () => ({ kind: 'unsupported' } as cacheLayout.InlineScriptMetaReadResult)); + + assert.strictEqual(await manager.create(scriptUri()), undefined); + assert.strictEqual(await fs.pathExists(markerPath), true); + }); + + test('returns a valid hit even when the last-used timestamp cannot be updated', async () => { await fs.ensureDir(envDir().fsPath); setSidecar({ schemaVersion: cacheLayout.META_SCHEMA_VERSION, @@ -2360,6 +3174,47 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(telemetryCalls(EventNames.INLINE_SCRIPT_ENV_ERROR).length, 0); }); + test('emits one setup-failure when coalesced cache-root creation fails', async () => { + readMetadataStub.onSecondCall().resolves({ ...VALID_METADATA, requiresPython: '>=3.12' }); + const cacheRootPath = cacheLayout.getScriptEnvCacheRoot(globalStorageUri).fsPath; + const originalEnsureDir = fsExtra.ensureDir; + let rejectCacheRoot: ((error: Error) => void) | undefined; + let signalCacheRoot: (() => void) | undefined; + const cacheRootStarted = new Promise((resolve) => { + signalCacheRoot = resolve; + }); + const cacheRootGate = new Promise((_resolve, reject) => { + rejectCacheRoot = reject; + }); + sinon.stub(fsExtra, 'ensureDir').callsFake(async (target: string) => { + if (normalizePath(target) === normalizePath(cacheRootPath)) { + signalCacheRoot!(); + return cacheRootGate; + } + return originalEnsureDir(target); + }); + + const first = manager.create(scriptUri('a.py')); + await cacheRootStarted; + const second = manager.create(scriptUri('b.py')); + const pendingManager = manager as unknown as { + pendingCreations: Map; + }; + await waitForCondition( + () => [...pendingManager.pendingCreations.values()][0]?.sourceMetadataIdentityHashes?.length === 2, + 'Expected the second request to join the pending cache creation', + ); + rejectCacheRoot!(new Error('global storage unavailable')); + + assert.deepStrictEqual(await Promise.all([first, second]), [undefined, undefined]); + assert.deepStrictEqual( + telemetryCalls(EventNames.INLINE_SCRIPT_ENV_ERROR).map((call) => call.args), + [[EventNames.INLINE_SCRIPT_ENV_ERROR, undefined, { category: 'setup-failure' }]], + ); + assert.strictEqual(lockStub.callCount, 0); + assert.strictEqual(createWithProgressStub.callCount, 0); + }); + test('excludes lock and cache inspection time from envCreated duration', async () => { await fs.ensureDir(envDir().fsPath); lockStub.callsFake(async () => { @@ -2494,7 +3349,7 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(await manager.get(uri), environment); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), }); assert.strictEqual(workspaceState.set.firstCall.args[0], INLINE_SCRIPT_ENVS_KEY); assert.strictEqual(listener.callCount, 1); @@ -2509,172 +3364,1385 @@ suite('InlineScriptEnvManager', () => { assert.deepStrictEqual(listener.secondCall.args[0], { uri, old: environment, new: undefined }); }); - test('persists a batch atomically and reports each distinct script URI exactly once', async () => { - const first = scriptUri('first.py'); - const second = scriptUri('second.py'); + test('updates validated routing state when selections are set and unset', async () => { + const uri = scriptUri(); const environment = await createOwnedEnvironment(); - const listener = sinon.spy(); - manager.onDidChangeEnvironment(listener); + routingRegistry.setMetadata(uri, VALID_METADATA); - await manager.set([first, second, first], environment); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); - assert.deepStrictEqual(persistedAssociations, { - [normalizePath(first.fsPath)]: environment.environmentPath.fsPath, - [normalizePath(second.fsPath)]: environment.environmentPath.fsPath, - }); - assert.strictEqual(workspaceState.set.callCount, 1); - assert.strictEqual(listener.callCount, 2); - assert.strictEqual(listener.firstCall.args[0].uri, first); - assert.strictEqual(listener.secondCall.args[0].uri, second); - assert.strictEqual(await manager.get(first), environment); - assert.strictEqual(await manager.get(second), environment); + await manager.set(uri, environment); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), true); + + await manager.set(uri, undefined); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); }); - test('serializes concurrent selections so neither persisted association is lost', async () => { - const firstUri = scriptUri('first.py'); - const secondUri = scriptUri('second.py'); - const firstEnvironment = await createOwnedEnvironment(); - const secondEnvironment = await createOwnedEnvironment('fedcba9876543210'); + test('persists the saved metadata identity separately from the environment path', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); - await Promise.all([ - manager.set(firstUri, firstEnvironment), - manager.set(secondUri, secondEnvironment), - ]); + await manager.set(uri, environment); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(firstUri.fsPath)]: firstEnvironment.environmentPath.fsPath, - [normalizePath(secondUri.fsPath)]: secondEnvironment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), }); - assert.strictEqual(await manager.get(firstUri), firstEnvironment); - assert.strictEqual(await manager.get(secondUri), secondEnvironment); }); - test('rehydrates a persisted owned association on demand after restart', async () => { + test('routes an environment created with additional packages by saved metadata identity', async () => { const uri = scriptUri(); - const persistedEnvironment = await createOwnedEnvironment(); - persistedAssociations = { [normalizePath(uri.fsPath)]: persistedEnvironment.environmentPath.fsPath }; - const rehydrated = { ...persistedEnvironment, envId: { ...persistedEnvironment.envId, id: 'rehydrated' } }; - resolveVenvStub.resolves(rehydrated); - const restarted = new InlineScriptEnvManager(nativeFinder, api, baseManager, globalStorageUri, makeFakeLog()); + routingRegistry.setMetadata(uri, VALID_METADATA); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + + await manager.set(uri, environment!); + + assert.strictEqual(await manager.get(uri), environment); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), true); + }); + + test('reselecting the same matched additional-packages environment after restart preserves matched provenance', async () => { + const uri = scriptUri(); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + await manager.set(uri, environment!); + + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + + await restarted.set(uri, environment!); - assert.strictEqual(await restarted.get(uri), rehydrated); - assert.strictEqual(resolveVenvStub.callCount, 1); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: persistedEnvironment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment!.environmentPath.fsPath), }); + restarted.dispose(); + }); - const listener = sinon.spy(); - restarted.onDidChangeEnvironment(listener); - await restarted.set(uri, persistedEnvironment); - assert.strictEqual(listener.callCount, 0, 'different generated IDs for the same executable are not a change'); + test('create with additional packages can route after reload before the first set via sidecar provenance', async () => { + const uri = scriptUri(); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + persistedAssociations = {}; + + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + + await restarted.set(uri, environment!); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment!.environmentPath.fsPath), + }); restarted.dispose(); }); - test('preserves and retries a cold association when resolution rejects', async () => { + test('does not reuse matched provenance after the same cache path is rebuilt for a different generation', async () => { const uri = scriptUri(); - const environment = await createOwnedEnvironment(); - persistedAssociations = { [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath }; - resolveVenvStub.onFirstCall().rejects(new Error('resolver unavailable')); - resolveVenvStub.onSecondCall().resolves(environment); + const cacheKeyValue = 'fedcba9876543210'; + routingRegistry.setMetadata(uri, VALID_METADATA); + const environment = await createOwnedEnvironment(cacheKeyValue); + setSidecar( + { + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: path.join( + tempRoot, + `base-python-${cacheKeyValue}`, + isWindows() ? 'python.exe' : 'python', + ), + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: NOW.toISOString(), + sourceMetadataIdentityHashes: [ + cacheLayout.hashSourceMetadataIdentity(VALID_METADATA_IDENTITY), + ], + }, + Uri.file(path.dirname(path.dirname(environment.environmentPath.fsPath))), + ); - assert.strictEqual(await manager.get(uri), undefined); + await manager.set(uri, environment); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment!.environmentPath.fsPath), }); - assert.strictEqual(await manager.get(uri), environment); + + const rebuiltMetadata = { + ...VALID_METADATA, + requiresPython: '>=3.12', + } satisfies metadataReader.InlineScriptMetadata; + const rebuiltBaseExecutable = path.join(tempRoot, 'rebuilt-base', isWindows() ? 'python.exe' : 'python'); + await fs.outputFile(rebuiltBaseExecutable, ''); + setSidecar( + { + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: rebuiltBaseExecutable, + baseInterpreterVersion: '3.12.9', + lastUsedAt: NOW.toISOString(), + sourceMetadataIdentityHashes: [ + cacheLayout.hashSourceMetadataIdentity( + JSON.stringify({ + requiresPython: rebuiltMetadata.requiresPython, + dependencies: rebuiltMetadata.dependencies, + }), + ), + ], + }, + Uri.file(path.dirname(path.dirname(environment!.environmentPath.fsPath))), + ); + + await manager.set(uri, environment); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: { + schemaVersion: 1, + environmentPath: environment!.environmentPath.fsPath, + metadataBinding: { kind: 'legacy' }, + }, + }); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); }); - test('preserves and retries a cold association when ownership inspection rejects', async () => { - const uri = scriptUri(); - const environment = await createOwnedEnvironment(); - persistedAssociations = { [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath }; - resolveVenvStub.resolves(environment); - const inspectionManager = manager as unknown as { - inspectAssociationOwnership( - candidate: PythonEnvironment, - ): Promise<'expected' | 'stale' | 'uncertain'>; - }; - const ownershipStub = sinon.stub(inspectionManager, 'inspectAssociationOwnership').callThrough(); - ownershipStub.onFirstCall().rejects(new Error('filesystem unavailable')); + test('does not infer matched provenance when the sidecar source identity hash does not match', async () => { + const sourceUri = scriptUri('source.py'); + const targetUri = scriptUri('target.py'); + const sourceMetadata = { + ...VALID_METADATA, + dependencies: ['rich'], + } satisfies metadataReader.InlineScriptMetadata; + routingRegistry.setMetadata(targetUri, VALID_METADATA); + registerCacheKey('fedcba9876543210', ['rich', 'pytest'], baseExecutable); + readMetadataStub.resolves(sourceMetadata); + const environment = await manager.create(sourceUri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + readMetadataStub.resolves(VALID_METADATA); + + await manager.set(targetUri, environment!); - assert.strictEqual(await manager.get(uri), undefined); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(targetUri.fsPath)]: { + schemaVersion: 1, + environmentPath: environment!.environmentPath.fsPath, + metadataBinding: { kind: 'legacy' }, + }, }); - assert.strictEqual(await manager.get(uri), environment); + assert.strictEqual(routingRegistry.hasValidatedAssociation(targetUri), false); }); - test('notifies when a slow persisted association finishes rehydrating', async () => { + test('reselecting a different owned env after restart does not inherit matched provenance', async () => { const uri = scriptUri(); - const environment = await createOwnedEnvironment(); - persistedAssociations = { [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath }; - let resolveRehydration: ((value: PythonEnvironment) => void) | undefined; - resolveVenvStub.callsFake( - () => - new Promise((resolve) => { - resolveRehydration = resolve; - }), + const otherUri = scriptUri('other.py'); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const matchedEnvironment = await manager.create(uri, { additionalPackages: ['pytest'] }); + const otherMetadata = { + ...VALID_METADATA, + dependencies: ['urllib3'], + } satisfies metadataReader.InlineScriptMetadata; + registerCacheKey('0011223344556677', ['urllib3', 'pytest', 'rich'], baseExecutable); + readMetadataStub.resolves(otherMetadata); + const differentOwnedEnvironment = await manager.create(otherUri, { additionalPackages: ['pytest', 'rich'] }); + readMetadataStub.resolves(VALID_METADATA); + assert.ok(matchedEnvironment); + assert.ok(differentOwnedEnvironment); + await manager.set(uri, matchedEnvironment!); + + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, ); - const listener = sinon.spy(); - manager.onDidChangeEnvironment(listener); - const pending = manager.get(uri); - await waitForStubCall(resolveVenvStub); - assert.strictEqual(listener.callCount, 0); - resolveRehydration!(environment); + await restarted.set(uri, differentOwnedEnvironment!); - assert.strictEqual(await pending, environment); - sinon.assert.calledOnceWithExactly(listener, { uri, old: undefined, new: environment }); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: { + schemaVersion: 1, + environmentPath: differentOwnedEnvironment!.environmentPath.fsPath, + metadataBinding: { kind: 'legacy' }, + }, + }); + restarted.dispose(); }); - test('does not rewrite or notify when a restart reselects the same persisted executable', async () => { + test('old sidecars without provenance keep additional-packages envs conservative on reload', async () => { const uri = scriptUri(); - const environment = await createOwnedEnvironment(); - persistedAssociations = { [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath }; - const restarted = new InlineScriptEnvManager(nativeFinder, api, baseManager, globalStorageUri, makeFakeLog()); - const listener = sinon.spy(); - restarted.onDidChangeEnvironment(listener); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + const envDirPath = path.dirname(path.dirname(environment!.environmentPath.fsPath)); + setSidecar({ + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: baseExecutable, + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: NOW.toISOString(), + }, Uri.file(envDirPath)); + persistedAssociations = {}; + + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); await restarted.set(uri, environment); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: { + schemaVersion: 1, + environmentPath: environment!.environmentPath.fsPath, + metadataBinding: { kind: 'legacy' }, + }, }); - assert.strictEqual(workspaceState.set.callCount, 0); - assert.strictEqual(listener.callCount, 0); - assert.strictEqual(resolveVenvStub.callCount, 0); - restarted.dispose(); }); - test('does not return a retained association when current metadata no longer accepts its Python version', async () => { + test('stores a pending verified binding for a dirty selection and promotes it on matching save', async () => { const uri = scriptUri(); - const environment = await createOwnedEnvironment(); - await manager.set(uri, environment); - readMetadataStub.resolves({ ...VALID_METADATA, requiresPython: '==3.11.*' }); + const openDocumentsStub = workspaceApis.getOpenTextDocuments as unknown as sinon.SinonStub; + openDocumentsStub.returns([{ uri, isDirty: true } as unknown as TextDocument]); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); - assert.strictEqual(await manager.get(uri), undefined); + await manager.set(uri, environment!); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment!.environmentPath.fsPath), }); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); - readMetadataStub.resolves(VALID_METADATA); - assert.strictEqual(await manager.get(uri), environment); + openDocumentsStub.returns([]); + await triggerSavedMetadataChange(routingRegistry, manager, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment!.environmentPath.fsPath), + }); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), true); }); - test('uses full PEP 440 semantics when validating a retained association', async () => { + test('dirty pending binding for the same path after restart keeps pending until saved metadata is consistent', async () => { const uri = scriptUri(); - const environment = { - ...(await createOwnedEnvironment()), - version: '3.15.0', + const environment = await createOwnedEnvironment('fedcba9876543210'); + persistedAssociations = { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), }; - await manager.set(uri, environment); - readMetadataStub.resolves({ ...VALID_METADATA, requiresPython: '!=3.15.0rc2' }); + const openDocumentsStub = workspaceApis.getOpenTextDocuments as unknown as sinon.SinonStub; + openDocumentsStub.returns([{ uri, isDirty: true } as unknown as TextDocument]); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + + await restarted.set(uri, environment); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('keeps a dirty pending binding non-routeable when the saved metadata identity no longer matches', async () => { + const uri = scriptUri(); + const changedMetadata = { + ...VALID_METADATA, + dependencies: ['urllib3'], + } satisfies metadataReader.InlineScriptMetadata; + const openDocumentsStub = workspaceApis.getOpenTextDocuments as unknown as sinon.SinonStub; + openDocumentsStub.returns([{ uri, isDirty: true } as unknown as TextDocument]); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + + await manager.set(uri, environment!); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment!.environmentPath.fsPath), + }); + + openDocumentsStub.returns([]); + routingRegistry.setMetadata(uri, changedMetadata); + await triggerSavedMetadataChange(routingRegistry, manager, uri, changedMetadata); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment!.environmentPath.fsPath), + }); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + }); + + test('failed pending bind invalidates warm validation before a retry within 5s', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment('fedcba9876543210'); + persistedAssociations = { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), + }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + resolveVenvStub.resolves(environment); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + workspaceState.set.onFirstCall().rejects(new Error('Memento unavailable')); + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + await fs.remove(environment.environmentPath.fsPath); + clock.tick(5_000 - 1); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.deepStrictEqual(persistedAssociations, {}); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('removes a dirty pending binding when the environment was deleted before save validation', async () => { + const uri = scriptUri(); + const openDocumentsStub = workspaceApis.getOpenTextDocuments as unknown as sinon.SinonStub; + openDocumentsStub.returns([{ uri, isDirty: true } as unknown as TextDocument]); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + + await manager.set(uri, environment!); + await fs.remove(environment!.environmentPath.fsPath); + + openDocumentsStub.returns([]); + routingRegistry.setMetadata(uri, VALID_METADATA); + await triggerSavedMetadataChange(routingRegistry, manager, uri); + + assert.deepStrictEqual(persistedAssociations, {}); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + }); + + test('keeps a dirty pending binding non-routeable when validation is transiently unavailable on save', async () => { + const uri = scriptUri(); + const openDocumentsStub = workspaceApis.getOpenTextDocuments as unknown as sinon.SinonStub; + openDocumentsStub.returns([{ uri, isDirty: true } as unknown as TextDocument]); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + + await manager.set(uri, environment!); + resolveVenvStub.resolves(undefined); + openDocumentsStub.returns([]); + routingRegistry.setMetadata(uri, VALID_METADATA); + await triggerSavedMetadataChange(routingRegistry, manager, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment!.environmentPath.fsPath), + }); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + }); + + test('keeps a dirty pending binding non-routeable when ownership validation changes on save', async () => { + const uri = scriptUri(); + const openDocumentsStub = workspaceApis.getOpenTextDocuments as unknown as sinon.SinonStub; + openDocumentsStub.returns([{ uri, isDirty: true } as unknown as TextDocument]); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + + await manager.set(uri, environment!); + resolveVenvStub.resolves({ + ...environment!, + envId: { ...environment!.envId, managerId: 'ms-python.python:system' }, + }); + openDocumentsStub.returns([]); + routingRegistry.setMetadata(uri, VALID_METADATA); + await triggerSavedMetadataChange(routingRegistry, manager, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment!.environmentPath.fsPath), + }); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + }); + + test('removes only the requested malformed entry while preserving valid and legacy records', async () => { + const invalidUri = scriptUri('invalid.py'); + const validUri = scriptUri('valid.py'); + const legacyUri = scriptUri('legacy.py'); + const validEnvironment = await createOwnedEnvironment('fedcba9876543210'); + const legacyEnvironment = await createOwnedEnvironment('0011223344556677'); + persistedAssociations = { + [normalizePath(invalidUri.fsPath)]: { schemaVersion: 1, environmentPath: '', metadataBinding: { kind: 'pending' } }, + [normalizePath(validUri.fsPath)]: matchedAssociationRecord(validEnvironment.environmentPath.fsPath), + [normalizePath(legacyUri.fsPath)]: legacyEnvironment.environmentPath.fsPath, + }; + resolveVenvStub.callsFake(async (environmentPath: string) => { + const normalized = normalizePath(environmentPath); + if (normalized === normalizePath(validEnvironment.environmentPath.fsPath)) { + return validEnvironment; + } + if (normalized === normalizePath(legacyEnvironment.environmentPath.fsPath)) { + return legacyEnvironment; + } + return undefined; + }); + + assert.strictEqual(await manager.get(invalidUri), undefined); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(validUri.fsPath)]: matchedAssociationRecord(validEnvironment.environmentPath.fsPath), + [normalizePath(legacyUri.fsPath)]: legacyEnvironment.environmentPath.fsPath, + }); + assert.strictEqual(await manager.get(validUri), validEnvironment); + assert.strictEqual(await manager.get(legacyUri), legacyEnvironment); + }); + + test('preserves unknown future-version entries when repairing a malformed requested entry', async () => { + const invalidUri = scriptUri('invalid.py'); + const futureUri = scriptUri('future.py'); + const futureEnvironment = await createOwnedEnvironment('8899aabbccddeeff'); + persistedAssociations = { + [normalizePath(invalidUri.fsPath)]: { schemaVersion: 1, environmentPath: '', metadataBinding: { kind: 'pending' } }, + [normalizePath(futureUri.fsPath)]: futureAssociationRecord(futureEnvironment.environmentPath.fsPath), + }; + + assert.strictEqual(await manager.get(invalidUri), undefined); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(futureUri.fsPath)]: futureAssociationRecord(futureEnvironment.environmentPath.fsPath), + }); + assert.strictEqual(await manager.get(futureUri), undefined); + }); + + test('removes a requested record with an unknown current binding kind without affecting unrelated entries', async () => { + const invalidUri = scriptUri('invalid.py'); + const validUri = scriptUri('valid.py'); + const validEnvironment = await createOwnedEnvironment('fedcba9876543210'); + persistedAssociations = { + [normalizePath(invalidUri.fsPath)]: { + schemaVersion: 1, + environmentPath: validEnvironment.environmentPath.fsPath, + metadataBinding: { kind: 'mystery' }, + }, + [normalizePath(validUri.fsPath)]: matchedAssociationRecord(validEnvironment.environmentPath.fsPath), + }; + resolveVenvStub.resolves(validEnvironment); + + assert.strictEqual(await manager.get(invalidUri), undefined); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(validUri.fsPath)]: matchedAssociationRecord(validEnvironment.environmentPath.fsPath), + }); + assert.strictEqual(await manager.get(validUri), validEnvironment); + }); + + test('persists a batch atomically and reports each distinct script URI exactly once', async () => { + const first = scriptUri('first.py'); + const second = scriptUri('second.py'); + const environment = await createOwnedEnvironment(); + const listener = sinon.spy(); + manager.onDidChangeEnvironment(listener); + + await manager.set([first, second, first], environment); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(first.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + [normalizePath(second.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(workspaceStateSetCalls(INLINE_SCRIPT_ENVS_KEY).length, 1); + assert.strictEqual(listener.callCount, 2); + assert.strictEqual(listener.firstCall.args[0].uri, first); + assert.strictEqual(listener.secondCall.args[0].uri, second); + assert.strictEqual(await manager.get(first), environment); + assert.strictEqual(await manager.get(second), environment); + }); + + test('serializes concurrent selections so neither persisted association is lost', async () => { + const firstUri = scriptUri('first.py'); + const secondUri = scriptUri('second.py'); + const firstEnvironment = await createOwnedEnvironment(); + const secondEnvironment = await createOwnedEnvironment('fedcba9876543210'); + + await Promise.all([ + manager.set(firstUri, firstEnvironment), + manager.set(secondUri, secondEnvironment), + ]); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(firstUri.fsPath)]: matchedAssociationRecord(firstEnvironment.environmentPath.fsPath), + [normalizePath(secondUri.fsPath)]: matchedAssociationRecord(secondEnvironment.environmentPath.fsPath), + }); + assert.strictEqual(await manager.get(firstUri), firstEnvironment); + assert.strictEqual(await manager.get(secondUri), secondEnvironment); + }); + + test('does not let pending binding overwrite a newer unset', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), + }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + let resolvePending: ((value: PythonEnvironment | undefined) => void) | undefined; + resolveVenvStub.callsFake( + () => + new Promise((resolve) => { + resolvePending = resolve; + }), + ); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + const pendingBind = triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + await waitForStubCall(resolveVenvStub); + await restarted.set(uri, undefined); + resolvePending!(environment); + await pendingBind; + + assert.deepStrictEqual(persistedAssociations, {}); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('does not let pending binding overwrite a newer matched selection', async () => { + const uri = scriptUri(); + const oldEnvironment = await createOwnedEnvironment(); + const newEnvironment = await createOwnedEnvironment('fedcba9876543210'); + persistedAssociations = { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(oldEnvironment.environmentPath.fsPath), + }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + let resolvePending: ((value: PythonEnvironment | undefined) => void) | undefined; + resolveVenvStub.callsFake( + () => + new Promise((resolve) => { + resolvePending = resolve; + }), + ); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + const pendingBind = triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + await waitForStubCall(resolveVenvStub); + await restarted.set(uri, newEnvironment); + resolvePending!(oldEnvironment); + await pendingBind; + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(newEnvironment.environmentPath.fsPath), + }); + assert.strictEqual(await restarted.get(uri), newEnvironment); + restarted.dispose(); + }); + + test('preserves a concurrent valid set while repairing an unrelated malformed entry', async () => { + const invalidUri = scriptUri('invalid.py'); + const validUri = scriptUri('valid.py'); + const validEnvironment = await createOwnedEnvironment('fedcba9876543210'); + persistedAssociations = { + [normalizePath(invalidUri.fsPath)]: { schemaVersion: 1, environmentPath: '', metadataBinding: { kind: 'pending' } }, + }; + + await Promise.all([manager.get(invalidUri), manager.set(validUri, validEnvironment)]); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(validUri.fsPath)]: matchedAssociationRecord(validEnvironment.environmentPath.fsPath), + }); + assert.strictEqual(await manager.get(validUri), validEnvironment); + }); + + test('leaves a pending binding non-routeable when persistence fails', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), + }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + resolveVenvStub.resolves(environment); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + ((restarted as unknown as { subscriptions: Disposable[] }).subscriptions[0]).dispose(); + workspaceState.set.onFirstCall().rejects(new Error('Memento unavailable')); + workspaceState.set.onSecondCall().rejects(new Error('Memento unavailable')); + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('does not publish routeability from raw persisted associations after startup', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('legacy string associations stay non-routeable after restart but remain retrievable', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + assert.strictEqual(await restarted.get(uri), environment); + restarted.dispose(); + }); + + test('routes a persisted matched additional-packages association on restart when the current sidecar hash matches', async () => { + const uri = scriptUri(); + routingRegistry.setMetadata(uri, VALID_METADATA); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + persistedAssociations = { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), true); + restarted.dispose(); + }); + + test('does not route a persisted matched association on restart when the same cache path was rebuilt for another identity', async () => { + const uri = scriptUri(); + const cacheKeyValue = 'fedcba9876543210'; + const environment = await createOwnedEnvironment(cacheKeyValue); + persistedAssociations = { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }; + const rebuiltMetadata = { + ...VALID_METADATA, + requiresPython: '>=3.12', + } satisfies metadataReader.InlineScriptMetadata; + const rebuiltBaseExecutable = path.join(tempRoot, 'rebuilt-base-restart', isWindows() ? 'python.exe' : 'python'); + await fs.outputFile(rebuiltBaseExecutable, ''); + setSidecar( + { + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: rebuiltBaseExecutable, + baseInterpreterVersion: '3.12.9', + lastUsedAt: NOW.toISOString(), + sourceMetadataIdentityHashes: [ + cacheLayout.hashSourceMetadataIdentity( + JSON.stringify({ + requiresPython: rebuiltMetadata.requiresPython, + dependencies: rebuiltMetadata.dependencies, + }), + ), + ], + }, + Uri.file(path.dirname(path.dirname(environment.environmentPath.fsPath))), + ); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + assert.strictEqual(await restarted.get(uri), environment); + restarted.dispose(); + }); + + test('does not promote a pending association when the current sidecar hash does not prove its source identity', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment('fedcba9876543210'); + persistedAssociations = { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), + }; + setSidecar( + { + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: path.join( + tempRoot, + 'base-python-fedcba9876543210', + isWindows() ? 'python.exe' : 'python', + ), + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: NOW.toISOString(), + sourceMetadataIdentityHashes: [ + cacheLayout.hashSourceMetadataIdentity('{"requiresPython":">=3.12","dependencies":["requests"]}'), + ], + }, + Uri.file(path.dirname(path.dirname(environment.environmentPath.fsPath))), + ); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: pendingAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('preserves a persisted matched association with a future sidecar but leaves it non-routeable', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + const markerPath = path.join(environment.sysPrefix, 'keep.txt'); + await fs.outputFile(markerPath, 'keep'); + persistedAssociations = { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }; + inspectMetaStub.callsFake(async (envDir: Uri) => + normalizePath(envDir.fsPath) === normalizePath(environment.sysPrefix) + ? ({ kind: 'unsupported' } as cacheLayout.InlineScriptMetaReadResult) + : ({ kind: 'missing' } as cacheLayout.InlineScriptMetaReadResult), + ); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + assert.strictEqual(await restarted.get(uri), environment); + assert.strictEqual(await fs.pathExists(markerPath), true); + restarted.dispose(); + }); + + test('keeps a persisted matched additional-packages association non-routeable on restart when only an old sidecar remains', async () => { + const uri = scriptUri(); + routingRegistry.setMetadata(uri, VALID_METADATA); + registerCacheKey('fedcba9876543210', ['requests', 'pytest'], baseExecutable); + const environment = await manager.create(uri, { additionalPackages: ['pytest'] }); + assert.ok(environment); + setSidecar( + { + schemaVersion: cacheLayout.META_SCHEMA_VERSION, + baseInterpreterPath: baseExecutable, + baseInterpreterVersion: baseEnvironment.version, + lastUsedAt: NOW.toISOString(), + }, + Uri.file(path.dirname(path.dirname(environment.environmentPath.fsPath))), + ); + persistedAssociations = { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('enables routeability only after persisted validation succeeds on restart', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + resolveVenvStub.resolves(environment); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + const pending = triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + await waitForStubCall(resolveVenvStub); + await pending; + + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), true); + restarted.dispose(); + }); + + test('keeps routeability disabled while persisted restart validation is still in flight', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + let resolveRehydration: ((value: PythonEnvironment) => void) | undefined; + resolveVenvStub.callsFake( + () => + new Promise((resolve) => { + resolveRehydration = resolve; + }), + ); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + const pending = triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + await waitForStubCall(resolveVenvStub); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + resolveRehydration!(environment); + await pending; + + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), true); + restarted.dispose(); + }); + + test('ignores a stale saved-metadata refresh when metadata changes while sidecar proof awaits', async () => { + const uri = scriptUri(); + const scriptPath = normalizePath(uri.fsPath); + const environment = await createOwnedEnvironment(); + await manager.set(uri, environment); + const refreshManager = asMetadataRefreshManager(manager); + refreshManager.subscriptions[0].dispose(); + const validatedAtBefore = refreshManager.cachedAssociationValidatedAt.get(scriptPath); + assert.ok(validatedAtBefore !== undefined); + const routeabilityListener = sinon.spy(); + routingRegistry.onDidChangeRouteability(routeabilityListener); + clock.tick(1); + routingRegistry.setMetadata(uri, VALID_METADATA); + const metadataIdentity = routingRegistry.getMetadataIdentity(uri)!; + const metadataRevision = routingRegistry.getMetadataRevision(uri); + let resolveProof: ((value: boolean) => void) | undefined; + const proofStub = sinon.stub(refreshManager, 'currentCacheEntryProvesSourceMetadataIdentity').callThrough(); + proofStub.onFirstCall().returns( + new Promise((resolve) => { + resolveProof = resolve; + }), + ); + + const pendingRefresh = refreshManager.refreshValidatedAssociationForMetadataInternal( + scriptPath, + uri, + VALID_METADATA, + metadataIdentity, + metadataRevision, + refreshManager.associationRevisions.get(scriptPath) ?? 0, + ); + await waitForStubCall(proofStub); + routingRegistry.setMetadata(uri, { + ...VALID_METADATA, + requiresPython: '>=3.12', + }); + resolveProof!(true); + await pendingRefresh; + + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + assert.strictEqual(routeabilityListener.callCount, 0); + assert.strictEqual(refreshManager.cachedAssociationValidatedAt.get(scriptPath), validatedAtBefore); + assert.strictEqual(refreshManager.lastValidatedMetadataIdentities.get(scriptPath), VALID_METADATA_IDENTITY); + assert.strictEqual(refreshManager.lastValidatedMetadataIdentityProofs.has(scriptPath), false); + }); + + test('ignores a stale refresh when the same metadata returns after routeability is cleared', async () => { + const uri = scriptUri(); + const scriptPath = normalizePath(uri.fsPath); + const environment = await createOwnedEnvironment(); + await manager.set(uri, environment); + const refreshManager = asMetadataRefreshManager(manager); + refreshManager.subscriptions[0].dispose(); + routingRegistry.setMetadata(uri, VALID_METADATA); + const metadataIdentity = routingRegistry.getMetadataIdentity(uri)!; + const staleRevision = routingRegistry.getMetadataRevision(uri); + let resolveProof: ((value: boolean) => void) | undefined; + const proofStub = sinon.stub(refreshManager, 'currentCacheEntryProvesSourceMetadataIdentity').returns( + new Promise((resolve) => { + resolveProof = resolve; + }), + ); + + const pendingRefresh = refreshManager.refreshValidatedAssociationForMetadataInternal( + scriptPath, + uri, + VALID_METADATA, + metadataIdentity, + staleRevision, + refreshManager.associationRevisions.get(scriptPath) ?? 0, + ); + await waitForStubCall(proofStub); + routingRegistry.clearMetadata(uri); + routingRegistry.setMetadata(uri, VALID_METADATA); + assert.ok(routingRegistry.getMetadataRevision(uri) > staleRevision); + resolveProof!(true); + await pendingRefresh; + + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + }); + + test('ignores a stale saved-metadata refresh when an unset wins while sidecar proof awaits', async () => { + const uri = scriptUri(); + const scriptPath = normalizePath(uri.fsPath); + const environment = await createOwnedEnvironment(); + const refreshManager = asMetadataRefreshManager(manager); + refreshManager.subscriptions[0].dispose(); + routingRegistry.setMetadata(uri, VALID_METADATA); + await manager.set(uri, environment); + const routeabilityListener = sinon.spy(); + routingRegistry.onDidChangeRouteability(routeabilityListener); + clock.tick(1); + let resolveProof: ((value: boolean) => void) | undefined; + const proofStub = sinon.stub(refreshManager, 'currentCacheEntryProvesSourceMetadataIdentity').callThrough(); + proofStub.onFirstCall().returns( + new Promise((resolve) => { + resolveProof = resolve; + }), + ); + + const pendingRefresh = refreshManager.refreshValidatedAssociationForMetadataInternal( + scriptPath, + uri, + VALID_METADATA, + routingRegistry.getMetadataIdentity(uri)!, + routingRegistry.getMetadataRevision(uri), + refreshManager.associationRevisions.get(scriptPath) ?? 0, + ); + await waitForStubCall(proofStub); + await manager.set(uri, undefined); + resolveProof!(true); + await pendingRefresh; + + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + sinon.assert.calledOnceWithExactly(routeabilityListener, { + uri, + previousRouteable: true, + routeable: false, + }); + assert.strictEqual(refreshManager.cachedAssociationValidatedAt.has(scriptPath), false); + assert.strictEqual(refreshManager.lastValidatedMetadataIdentities.has(scriptPath), false); + assert.strictEqual(refreshManager.lastValidatedMetadataIdentityProofs.has(scriptPath), false); + assert.deepStrictEqual(persistedAssociations, {}); + }); + + test('ignores a stale saved-metadata refresh when a replacement wins while sidecar proof awaits', async () => { + const uri = scriptUri(); + const scriptPath = normalizePath(uri.fsPath); + const oldEnvironment = await createOwnedEnvironment(); + const replacementEnvironment = await createOwnedEnvironment('fedcba9876543210'); + const refreshManager = asMetadataRefreshManager(manager); + refreshManager.subscriptions[0].dispose(); + routingRegistry.setMetadata(uri, VALID_METADATA); + await manager.set(uri, oldEnvironment); + const routeabilityListener = sinon.spy(); + routingRegistry.onDidChangeRouteability(routeabilityListener); + clock.tick(1); + let resolveProof: ((value: boolean) => void) | undefined; + const proofStub = sinon.stub(refreshManager, 'currentCacheEntryProvesSourceMetadataIdentity').callThrough(); + proofStub.onFirstCall().returns( + new Promise((resolve) => { + resolveProof = resolve; + }), + ); + + const pendingRefresh = refreshManager.refreshValidatedAssociationForMetadataInternal( + scriptPath, + uri, + VALID_METADATA, + routingRegistry.getMetadataIdentity(uri)!, + routingRegistry.getMetadataRevision(uri), + refreshManager.associationRevisions.get(scriptPath) ?? 0, + ); + await waitForStubCall(proofStub); + await manager.set(uri, replacementEnvironment); + const validatedAtAfterReplacement = refreshManager.cachedAssociationValidatedAt.get(scriptPath); + resolveProof!(false); + await pendingRefresh; + + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), true); + assert.strictEqual(routeabilityListener.callCount, 0); + assert.strictEqual( + refreshManager.cachedAssociationValidatedAt.get(scriptPath), + validatedAtAfterReplacement, + ); + assert.strictEqual(refreshManager.lastValidatedMetadataIdentities.get(scriptPath), VALID_METADATA_IDENTITY); + assert.strictEqual(refreshManager.lastValidatedMetadataIdentityProofs.has(scriptPath), false); + assert.deepStrictEqual(persistedAssociations, { + [scriptPath]: matchedAssociationRecord(replacementEnvironment.environmentPath.fsPath), + }); + }); + + test('preserves a persisted restart candidate after transient validation failure and retries later', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + resolveVenvStub.onFirstCall().rejects(new Error('resolver unavailable')); + resolveVenvStub.onSecondCall().resolves(environment); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + await waitForStubCall(resolveVenvStub); + + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), true); + restarted.dispose(); + }); + + test('clears a stale persisted restart candidate instead of routing it', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + await fs.remove(environment.environmentPath.fsPath); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + + assert.deepStrictEqual(persistedAssociations, {}); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + restarted.dispose(); + }); + + test('rehydrates a persisted owned association on demand after restart', async () => { + const uri = scriptUri(); + const persistedEnvironment = await createOwnedEnvironment(); + persistedAssociations = { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(persistedEnvironment.environmentPath.fsPath), + }; + const rehydrated = { ...persistedEnvironment, envId: { ...persistedEnvironment.envId, id: 'rehydrated' } }; + resolveVenvStub.resolves(rehydrated); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + + assert.strictEqual(await restarted.get(uri), rehydrated); + assert.strictEqual(resolveVenvStub.callCount, 1); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(persistedEnvironment.environmentPath.fsPath), + }); + + const listener = sinon.spy(); + restarted.onDidChangeEnvironment(listener); + await restarted.set(uri, persistedEnvironment); + assert.strictEqual(listener.callCount, 0, 'different generated IDs for the same executable are not a change'); + + restarted.dispose(); + }); + + test('preserves and retries a cold association when resolution rejects', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + resolveVenvStub.onFirstCall().rejects(new Error('resolver unavailable')); + resolveVenvStub.onSecondCall().resolves(environment); + + assert.strictEqual(await manager.get(uri), undefined); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(await manager.get(uri), environment); + }); + + test('preserves and retries a cold association when ownership inspection rejects', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + resolveVenvStub.resolves(environment); + const inspectionManager = manager as unknown as { + inspectAssociationOwnership( + candidate: PythonEnvironment, + ): Promise<'expected' | 'stale' | 'uncertain'>; + }; + const ownershipStub = sinon.stub(inspectionManager, 'inspectAssociationOwnership').callThrough(); + ownershipStub.onFirstCall().rejects(new Error('filesystem unavailable')); + + assert.strictEqual(await manager.get(uri), undefined); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(await manager.get(uri), environment); + }); + + test('notifies when a slow persisted association finishes rehydrating', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + let resolveRehydration: ((value: PythonEnvironment) => void) | undefined; + resolveVenvStub.callsFake( + () => + new Promise((resolve) => { + resolveRehydration = resolve; + }), + ); + const listener = sinon.spy(); + manager.onDidChangeEnvironment(listener); + + const pending = manager.get(uri); + await waitForStubCall(resolveVenvStub); + assert.strictEqual(listener.callCount, 0); + resolveRehydration!(environment); + + assert.strictEqual(await pending, environment); + sinon.assert.calledOnceWithExactly(listener, { uri, old: undefined, new: environment }); + }); + + test('coalesces repeated saved-metadata validation for the same identity', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + let resolveRehydration: ((value: PythonEnvironment | undefined) => void) | undefined; + resolveVenvStub.callsFake( + () => + new Promise((resolve) => { + resolveRehydration = resolve; + }), + ); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + const listener = sinon.spy(); + restarted.onDidChangeEnvironment(listener); + await nextTurn(); + + const first = triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + await waitForStubCall(resolveVenvStub); + const second = triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + assert.strictEqual(resolveVenvStub.callCount, 1); + + resolveRehydration!(environment); + await Promise.all([first, second]); + + assert.strictEqual(listener.callCount, 1); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), true); + restarted.dispose(); + }); + + test('does not rewrite or notify when a restart reselects the same persisted executable', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + const listener = sinon.spy(); + restarted.onDidChangeEnvironment(listener); + + await restarted.set(uri, environment); + + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + assert.strictEqual(workspaceState.set.callCount, 0); + assert.strictEqual(listener.callCount, 0); + assert.strictEqual(resolveVenvStub.callCount, 0); + + restarted.dispose(); + }); + + test('does not return a retained association when current metadata no longer accepts its Python version', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + await manager.set(uri, environment); + readMetadataStub.resolves({ ...VALID_METADATA, requiresPython: '==3.11.*' }); + + assert.strictEqual(await manager.get(uri), undefined); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + + readMetadataStub.resolves(VALID_METADATA); + assert.strictEqual(await manager.get(uri), environment); + }); + + test('does not return a retained association when current metadata dependencies changed', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + await manager.set(uri, environment); + computeCacheKeyStub + .withArgs( + sinon.match((inputs: cacheKey.CacheKeyInputs) => inputs.dependencies.length === 1 && inputs.dependencies[0] === 'urllib3'), + ) + .returns('different-cache-key'); + readMetadataStub.resolves({ ...VALID_METADATA, dependencies: ['urllib3'] }); + + assert.strictEqual(await manager.get(uri), undefined); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), + }); + readMetadataStub.resolves(VALID_METADATA); assert.strictEqual(await manager.get(uri), environment); }); + test('does not return a retained association when current requires-python identity changed, even if compatible', async () => { + const uri = scriptUri(); + const environment = { + ...(await createOwnedEnvironment()), + version: '3.15.0', + }; + await manager.set(uri, environment); + resolveVenvStub.resolves(environment); + readMetadataStub.resolves({ ...VALID_METADATA, requiresPython: '!=3.15.0rc2' }); + + assert.strictEqual(await manager.get(uri), undefined); + }); + test('does not resolve or discard an association when metadata is absent or unreadable', async () => { const uri = scriptUri(); const environment = await createOwnedEnvironment(); @@ -2703,6 +4771,62 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(resolveVenvStub.callCount, 0); }); + test('clears the routing registry when a stale persisted association is removed', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + persistedAssociations = { [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + resolveVenvStub.resolves(environment); + const restartRoutingRegistry = new InlineScriptRoutingRegistry(); + + const restarted = new InlineScriptEnvManager( + nativeFinder, + api, + baseManager, + globalStorageUri, + makeFakeLog(), + restartRoutingRegistry, + ); + await nextTurn(); + await triggerSavedMetadataChange(restartRoutingRegistry, restarted, uri); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), true); + + await fs.remove(environment.environmentPath.fsPath); + clock.tick(5_000); + assert.strictEqual(await restarted.get(uri), undefined); + assert.strictEqual(restartRoutingRegistry.hasValidatedAssociation(uri), false); + + restarted.dispose(); + }); + + test('clears persisted association state when the script path is deleted', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + await manager.set(uri, environment); + + fireDelete(uri); + await nextTurn(); + await nextTurn(); + + assert.deepStrictEqual(persistedAssociations, {}); + assert.strictEqual(await manager.get(uri), undefined); + assert.strictEqual(routingRegistry.hasValidatedAssociation(uri), false); + }); + + test('clears persisted association state for the old path when a script is renamed', async () => { + const oldUri = scriptUri('old.py'); + const newUri = scriptUri('new.py'); + const environment = await createOwnedEnvironment(); + await manager.set(oldUri, environment); + + fireRename(oldUri, newUri); + await nextTurn(); + await nextTurn(); + + assert.deepStrictEqual(persistedAssociations, {}); + assert.strictEqual(await manager.get(oldUri), undefined); + assert.strictEqual(routingRegistry.hasValidatedAssociation(oldUri), false); + }); + test('removes and notifies for a warm association whose executable was deleted', async () => { const uri = scriptUri(); const environment = await createOwnedEnvironment(); @@ -2753,7 +4877,7 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(await manager.get(uri), undefined); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), }); assert.strictEqual(listener.callCount, 0); }); @@ -2775,7 +4899,7 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(await manager.get(uri), undefined); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), }); assert.strictEqual(listener.callCount, 0); }); @@ -2814,6 +4938,94 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(listener.callCount, 0); }); + test('refreshes warm validation timestamps when validation keeps the same environment', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + await manager.set(uri, environment); + resolveVenvStub.resolves({ + ...environment, + envId: { ...environment.envId, id: 'new-generated-id' }, + }); + clock.tick(5_000); + + assert.strictEqual(await manager.get(uri), environment); + assert.strictEqual(resolveVenvStub.callCount, 1); + assert.strictEqual(await manager.get(uri), environment); + assert.strictEqual(resolveVenvStub.callCount, 1); + }); + + test('lets an unset win while warm validation awaits sidecar proof', async () => { + const uri = scriptUri(); + const environment = await createOwnedEnvironment(); + await manager.set(uri, environment); + const validationManager = manager as unknown as { + currentCacheEntryProvesSourceMetadataIdentity( + candidate: PythonEnvironment, + metadataIdentity: string, + metadata: metadataReader.InlineScriptMetadata, + ): Promise; + }; + let resolveProof: ((value: boolean) => void) | undefined; + const proofStub = sinon.stub(validationManager, 'currentCacheEntryProvesSourceMetadataIdentity').callThrough(); + proofStub.onFirstCall().returns( + new Promise((resolve) => { + resolveProof = resolve; + }), + ); + const listener = sinon.spy(); + manager.onDidChangeEnvironment(listener); + clock.tick(5_000); + + const pendingGet = manager.get(uri); + await waitForStubCall(proofStub); + await manager.set(uri, undefined); + resolveProof!(true); + + assert.strictEqual(await pendingGet, undefined); + assert.strictEqual(await manager.get(uri), undefined); + sinon.assert.calledOnceWithExactly(listener, { uri, old: environment, new: undefined }); + }); + + test('lets a replacement win while warm validation awaits sidecar proof', async () => { + const uri = scriptUri(); + const oldEnvironment = await createOwnedEnvironment(); + const replacementEnvironment = await createOwnedEnvironment('fedcba9876543210'); + await manager.set(uri, oldEnvironment); + const validationManager = manager as unknown as { + currentCacheEntryProvesSourceMetadataIdentity( + candidate: PythonEnvironment, + metadataIdentity: string, + metadata: metadataReader.InlineScriptMetadata, + ): Promise; + }; + let resolveProof: ((value: boolean) => void) | undefined; + const proofStub = sinon.stub(validationManager, 'currentCacheEntryProvesSourceMetadataIdentity').callThrough(); + proofStub.onFirstCall().returns( + new Promise((resolve) => { + resolveProof = resolve; + }), + ); + const listener = sinon.spy(); + manager.onDidChangeEnvironment(listener); + clock.tick(5_000); + + const pendingGet = manager.get(uri); + await waitForStubCall(proofStub); + await manager.set(uri, replacementEnvironment); + resolveProof!(true); + + assert.strictEqual(await pendingGet, replacementEnvironment); + assert.strictEqual(await manager.get(uri), replacementEnvironment); + assert.deepStrictEqual(persistedAssociations, { + [normalizePath(uri.fsPath)]: matchedAssociationRecord(replacementEnvironment.environmentPath.fsPath), + }); + sinon.assert.calledOnceWithExactly(listener, { + uri, + old: oldEnvironment, + new: replacementEnvironment, + }); + }); + test('coalesces concurrent validation of an expired warm association', async () => { const uri = scriptUri(); const environment = await createOwnedEnvironment(); @@ -2876,7 +5088,7 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(await pendingGet, selectedEnvironment); assert.strictEqual(await manager.get(uri), selectedEnvironment); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: selectedEnvironment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(selectedEnvironment.environmentPath.fsPath), }); assert.strictEqual(resolveVenvStub.callCount, 0); sinon.assert.calledOnceWithExactly(listener, { @@ -2938,14 +5150,22 @@ suite('InlineScriptEnvManager', () => { const environment = await createOwnedEnvironment(); const scriptPath = normalizePath(uri.fsPath); persistedAssociations = { [scriptPath]: 42 }; - workspaceState.get.onSecondCall().callsFake(async () => { - persistedAssociations = { [scriptPath]: environment.environmentPath.fsPath }; - return persistedAssociations; + let envKeyReads = 0; + workspaceState.get.callsFake(async (key: string) => { + if (key === INLINE_SCRIPT_ENVS_KEY) { + envKeyReads += 1; + if (envKeyReads === 1) { + return { [scriptPath]: 42 }; + } + persistedAssociations = { [scriptPath]: matchedAssociationRecord(environment.environmentPath.fsPath) }; + return persistedAssociations; + } + return undefined; }); - assert.strictEqual(await manager.get(uri), undefined); + assert.strictEqual(await manager.get(uri), environment); assert.deepStrictEqual(persistedAssociations, { - [scriptPath]: environment.environmentPath.fsPath, + [scriptPath]: matchedAssociationRecord(environment.environmentPath.fsPath), }); assert.strictEqual(workspaceState.set.callCount, 0); }); @@ -3028,7 +5248,7 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(await manager.get(uri), first); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: first.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(first.environmentPath.fsPath), }); assert.strictEqual(listener.callCount, 1); }); @@ -3045,7 +5265,7 @@ suite('InlineScriptEnvManager', () => { await assert.rejects(manager.set(uri, undefined), /Memento unavailable/); assert.strictEqual(await manager.get(uri), environment); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), }); assert.strictEqual(listener.callCount, 1); }); @@ -3127,9 +5347,9 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(await pendingGet, environment); assert.strictEqual(await manager.get(uri), environment); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), }); - assert.strictEqual(workspaceState.set.callCount, 0); + assert.strictEqual(workspaceStateSetCalls(INLINE_SCRIPT_ENVS_KEY).length, 1); }); test('retains a pending rehydration when a competing persistence write fails', async () => { @@ -3345,7 +5565,7 @@ suite('InlineScriptEnvManager', () => { await assert.rejects(manager.clearCache(), /being created/); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(uri.fsPath)]: environment.environmentPath.fsPath, + [normalizePath(uri.fsPath)]: matchedAssociationRecord(environment.environmentPath.fsPath), }); assert.strictEqual(await manager.get(uri), environment); }); @@ -3500,7 +5720,9 @@ suite('InlineScriptEnvManager', () => { assert.strictEqual(await fs.pathExists(firstEnvironment.sysPrefix), false); assert.strictEqual(await fs.pathExists(secondEnvironment.sysPrefix), true); assert.deepStrictEqual(persistedAssociations, { - [normalizePath(secondUri.fsPath)]: secondEnvironment.environmentPath.fsPath, + [normalizePath(secondUri.fsPath)]: matchedAssociationRecord( + secondEnvironment.environmentPath.fsPath, + ), }); assert.strictEqual(await manager.get(firstUri), undefined); assert.strictEqual(await manager.get(secondUri), secondEnvironment); diff --git a/src/test/managers/builtin/inlineScript/main.unit.test.ts b/src/test/managers/builtin/inlineScript/main.unit.test.ts index 3531930fd..7dc820dca 100644 --- a/src/test/managers/builtin/inlineScript/main.unit.test.ts +++ b/src/test/managers/builtin/inlineScript/main.unit.test.ts @@ -5,6 +5,9 @@ import assert from 'assert'; import * as sinon from 'sinon'; import { Disposable, LogOutputChannel, Uri } from 'vscode'; import { EnvironmentManager, PythonEnvironmentApi } from '../../../../api'; +import { InlineScriptRoutingRegistry } from '../../../../common/inlineScript/routingRegistry'; +import * as workspaceApis from '../../../../common/workspace.apis'; +import { latchInlineScriptFeatureActivation } from '../../../../features/inlineScript/activation'; import * as pythonApi from '../../../../features/pythonApi'; import * as helpers from '../../../../helpers'; import { InlineScriptEnvManager } from '../../../../managers/builtin/inlineScript/envManager'; @@ -40,11 +43,14 @@ suite('registerInlineScriptFeatures (feature-flag gate)', () => { const nativeFinder = {} as NativePythonFinder; const baseManager = {} as EnvironmentManager; const globalStorageUri = Uri.file('inline-script-global-storage'); + const routingRegistry = new InlineScriptRoutingRegistry(); setup(() => { isEnabledStub = sinon.stub(helpers, 'isInlineScriptsFeatureEnabled'); registerEnvironmentManagerStub = sinon.stub<[unknown], Disposable>().returns({ dispose: () => undefined }); startActivationDiscoveryStub = sinon.stub(InlineScriptEnvManager.prototype, 'startActivationDiscovery'); + sinon.stub(workspaceApis, 'onDidDeleteFiles').returns(new Disposable(() => undefined)); + sinon.stub(workspaceApis, 'onDidRenameFiles').returns(new Disposable(() => undefined)); getPythonApiStub = sinon.stub(pythonApi, 'getPythonApi').resolves({ registerEnvironmentManager: registerEnvironmentManagerStub, } as unknown as PythonEnvironmentApi); @@ -55,21 +61,53 @@ suite('registerInlineScriptFeatures (feature-flag gate)', () => { }); test('when the feature flag is FALSE: does not register, does not even fetch the API', async () => { - isEnabledStub.returns(false); const disposables: Disposable[] = []; - await registerInlineScriptFeatures(nativeFinder, disposables, makeFakeLog(), baseManager, globalStorageUri); + await registerInlineScriptFeatures( + nativeFinder, + disposables, + makeFakeLog(), + baseManager, + globalStorageUri, + { enabled: false, routingRegistry: undefined }, + ); assert.strictEqual(disposables.length, 0, 'no disposables should be added when flag is off'); assert.strictEqual(getPythonApiStub.called, false, 'should not even call getPythonApi when gated off'); assert.strictEqual(registerEnvironmentManagerStub.called, false); }); + test('when the feature flag is TRUE without a routing registry: fails before touching the API', async () => { + const disposables: Disposable[] = []; + + await assert.rejects( + registerInlineScriptFeatures( + nativeFinder, + disposables, + makeFakeLog(), + baseManager, + globalStorageUri, + { enabled: true, routingRegistry: undefined }, + ), + /routing registry/i, + ); + + assert.strictEqual(disposables.length, 0, 'no disposables should be added when the registry is missing'); + assert.strictEqual(getPythonApiStub.called, false, 'should fail before getPythonApi when the registry is missing'); + assert.strictEqual(registerEnvironmentManagerStub.called, false); + }); + test('when the feature flag is TRUE: registers the manager and pushes the disposable', async () => { - isEnabledStub.returns(true); const disposables: Disposable[] = []; - await registerInlineScriptFeatures(nativeFinder, disposables, makeFakeLog(), baseManager, globalStorageUri); + await registerInlineScriptFeatures( + nativeFinder, + disposables, + makeFakeLog(), + baseManager, + globalStorageUri, + { enabled: true, routingRegistry }, + ); assert.strictEqual(getPythonApiStub.callCount, 1); assert.strictEqual(registerEnvironmentManagerStub.callCount, 1); @@ -86,10 +124,16 @@ suite('registerInlineScriptFeatures (feature-flag gate)', () => { }); test('when the feature flag is TRUE: defers activation-time discovery to the next turn', async () => { - isEnabledStub.returns(true); const disposables: Disposable[] = []; - await registerInlineScriptFeatures(nativeFinder, disposables, makeFakeLog(), baseManager, globalStorageUri); + await registerInlineScriptFeatures( + nativeFinder, + disposables, + makeFakeLog(), + baseManager, + globalStorageUri, + { enabled: true, routingRegistry }, + ); assert.strictEqual( startActivationDiscoveryStub.callCount, @@ -102,4 +146,54 @@ suite('registerInlineScriptFeatures (feature-flag gate)', () => { sinon.assert.calledOnceWithExactly(startActivationDiscoveryStub); disposables.forEach((disposable) => disposable.dispose()); }); + + test('latches FALSE through deferred registration even if the live setting flips TRUE later', async () => { + isEnabledStub.onFirstCall().returns(false); + isEnabledStub.onSecondCall().returns(true); + const activation = latchInlineScriptFeatureActivation(); + const disposables: Disposable[] = []; + + await (activation.enabled + ? registerInlineScriptFeatures( + nativeFinder, + disposables, + makeFakeLog(), + baseManager, + globalStorageUri, + activation, + ) + : Promise.resolve()); + + assert.strictEqual(activation.enabled, false); + assert.strictEqual(activation.routingRegistry, undefined); + assert.strictEqual(isEnabledStub.callCount, 1, 'activation should read the setting only once'); + assert.strictEqual(disposables.length, 0, 'disabled activation should not add disposables later'); + assert.strictEqual(getPythonApiStub.called, false, 'disabled activation should never touch the API later'); + assert.strictEqual(registerEnvironmentManagerStub.called, false); + }); + + test('latches TRUE through deferred registration even if the live setting flips FALSE later', async () => { + isEnabledStub.onFirstCall().returns(true); + isEnabledStub.onSecondCall().returns(false); + const activation = latchInlineScriptFeatureActivation(); + const disposables: Disposable[] = []; + + await (activation.enabled + ? registerInlineScriptFeatures( + nativeFinder, + disposables, + makeFakeLog(), + baseManager, + globalStorageUri, + activation, + ) + : Promise.resolve()); + + assert.strictEqual(activation.enabled, true); + assert.ok(activation.routingRegistry, 'enabled activation should latch a routing registry'); + assert.strictEqual(isEnabledStub.callCount, 1, 'deferred registration should not reread the setting'); + assert.strictEqual(getPythonApiStub.callCount, 1); + assert.strictEqual(registerEnvironmentManagerStub.callCount, 1); + assert.strictEqual(disposables.length, 2, 'enabled activation should still register later'); + }); });