diff --git a/CHANGELOG.md b/CHANGELOG.md index 0925b8e0d9b..da528e489d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,12 @@ ### Enhancements -* None. +* Add a target-aware analyzer execution mode with bounded isolated worker + processes, whole-target collecting rules, opt-in `unused_import` batches of + at most 32 requested files, deterministic reporting, and versioned execution + evidence. + [Derek Pearson](https://github.com/dpearson2699) + [#3020](https://github.com/realm/SwiftLint/issues/3020) ### Bug Fixes diff --git a/README.md b/README.md index 4bb2c15642c..9e45ceb4a2d 100644 --- a/README.md +++ b/README.md @@ -936,6 +936,57 @@ This can be obtained by Analyzer rules tend to be considerably slower than lint rules. +Analyzer work can also run in bounded, isolated SwiftLint processes. A build +system integration supplies a versioned target plan with `--target-plan`, sets +the global process limit with `--jobs`, and chooses where SwiftLint writes the +run record with `--execution-evidence`. The target plan describes each target's +complete source inventory, compilation database, configured analyzer rules, +and any requested-source batches. + +For example, a plan that divides `unused_import` work into two jobs has this +shape: + +```json +{ + "schemaIdentity": "swiftlint-analyzer-target-plan", + "schemaVersion": 1, + "compilerLogSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "workingDirectory": "/path/to/package", + "rules": ["unused_import"], + "targets": [{ + "targetId": "application", + "moduleName": "FixtureApp", + "sourceRoot": "Sources", + "compileCommandsPath": "application.compile-commands.json", + "compileCommandsSha256": "1111111111111111111111111111111111111111111111111111111111111111", + "sourceFiles": ["Sources/App.swift", "Sources/Model.swift"], + "sourceFilesSha256": "2222222222222222222222222222222222222222222222222222222222222222", + "rulePlans": [{ + "rule": "unused_import", + "mode": "batches", + "batches": [ + {"batchIndex": 0, "requestedPaths": ["Sources/App.swift"]}, + {"batchIndex": 1, "requestedPaths": ["Sources/Model.swift"]} + ] + }] + }] +} +``` + +Replace the example digests with SHA-256 values for the compiler log and +compilation database. `sourceFilesSha256` is the SHA-256 digest of the sorted +`sourceFiles` array encoded as compact JSON with a trailing newline. SwiftLint +resolves `workingDirectory` from the invocation directory, source paths from +`workingDirectory`, and compilation database paths from the target-plan file. +Every target and rule must appear in deterministic order. + +Collecting rules always receive their complete target source inventory. Other +rules may use batches only when they explicitly declare that capability; +`unused_import` initially permits batches of at most 32 requested files. Every +worker still receives the complete target compilation database. SwiftLint +cancels the remaining workers after a failure and merges successful results in +target-plan order so reporter output remains deterministic. + ## Using Multiple Configuration Files SwiftLint offers a variety of ways to include multiple configuration files. diff --git a/Source/SwiftLintBuiltInRules/Rules/Lint/UnusedImportRule.swift b/Source/SwiftLintBuiltInRules/Rules/Lint/UnusedImportRule.swift index 64bfca04471..0a96aa28264 100644 --- a/Source/SwiftLintBuiltInRules/Rules/Lint/UnusedImportRule.swift +++ b/Source/SwiftLintBuiltInRules/Rules/Lint/UnusedImportRule.swift @@ -4,9 +4,11 @@ import SourceKittenFramework private let moduleToLog = ProcessInfo.processInfo.environment["SWIFTLINT_LOG_MODULE_USAGE"] @DisabledWithoutSourceKit -struct UnusedImportRule: CorrectableRule, AnalyzerRule { +struct UnusedImportRule: CorrectableRule, AnalyzerRule, AnalyzerBatchingRule { var configuration = UnusedImportConfiguration() + let analyzerBatchSize = 32 + static let description = RuleDescription( identifier: "unused_import", name: "Unused Import", diff --git a/Source/SwiftLintCore/Protocols/Rule.swift b/Source/SwiftLintCore/Protocols/Rule.swift index d1db8b02a82..14f8592cecd 100644 --- a/Source/SwiftLintCore/Protocols/Rule.swift +++ b/Source/SwiftLintCore/Protocols/Rule.swift @@ -100,6 +100,16 @@ public protocol Rule: Sendable { func notifyRuleDisabledOnce() } +/// Opt-in capability for analyzer rules that can safely run against deterministic subsets of a target's sources. +/// +/// Absence of this capability means the rule must receive the entire target. The worker coordinator uses this +/// package-only protocol to reject unsafe target-plan batches. +package protocol AnalyzerBatchingRule: Rule { + /// The largest requested-source batch that preserves the rule's analyzer semantics when the worker also receives + /// the complete target compilation database. + var analyzerBatchSize: Int { get } +} + public extension Rule { var shouldLintEmptyFiles: Bool { false diff --git a/Source/SwiftLintFramework/AnalyzerProcessCoordinator.swift b/Source/SwiftLintFramework/AnalyzerProcessCoordinator.swift new file mode 100644 index 00000000000..30523317341 --- /dev/null +++ b/Source/SwiftLintFramework/AnalyzerProcessCoordinator.swift @@ -0,0 +1,913 @@ +import Foundation +@preconcurrency import SwiftLintCore + +// Versioned plan, worker, aggregate, and evidence schemas intentionally share one owned protocol file. +// swiftlint:disable file_length + +package struct AnalyzerRuleCapability: Equatable, Sendable { + package let isCollecting: Bool + package let maximumBatchSize: Int? + + package init(isCollecting: Bool, maximumBatchSize: Int?) { + self.isCollecting = isCollecting + self.maximumBatchSize = maximumBatchSize + } +} + +package struct AnalyzerTargetPlan: Codable, Equatable, Sendable { + package static let schemaIdentity = "swiftlint-analyzer-target-plan" + package static let currentVersion = 1 + + package struct Target: Codable, Equatable, Sendable { + package var targetId: String + package var moduleName: String + package var sourceRoot: String + package var compileCommandsPath: String + package var compileCommandsSha256: String + package var sourceFiles: [String] + package var sourceFilesSha256: String + package var rulePlans: [RulePlan] + + package init( + targetId: String, + moduleName: String, + sourceRoot: String, + compileCommandsPath: String, + compileCommandsSha256: String, + sourceFiles: [String], + sourceFilesSha256: String, + rulePlans: [RulePlan] + ) { + self.targetId = targetId + self.moduleName = moduleName + self.sourceRoot = sourceRoot + self.compileCommandsPath = compileCommandsPath + self.compileCommandsSha256 = compileCommandsSha256 + self.sourceFiles = sourceFiles + self.sourceFilesSha256 = sourceFilesSha256 + self.rulePlans = rulePlans + } + } + + package struct RulePlan: Codable, Equatable, Sendable { + // swiftlint:disable:next nesting + package enum Mode: String, Codable, Sendable { + // swiftlint:disable:next redundant_string_enum_value + case wholeTarget = "wholeTarget" + case batches + } + + package var rule: String + package var mode: Mode + package var batches: [Batch] + + package init(rule: String, mode: Mode, batches: [Batch]) { + self.rule = rule + self.mode = mode + self.batches = batches + } + } + + package struct Batch: Codable, Equatable, Sendable { + package var batchIndex: Int + package var requestedPaths: [String] + + package init(batchIndex: Int, requestedPaths: [String]) { + self.batchIndex = batchIndex + self.requestedPaths = requestedPaths + } + } + + package var schemaIdentity: String + package var schemaVersion: Int + package var compilerLogSha256: String + package var workingDirectory: String + package var rules: [String] + package var targets: [Target] + + package init( + schemaIdentity: String, + schemaVersion: Int, + compilerLogSha256: String, + workingDirectory: String, + rules: [String], + targets: [Target] + ) { + self.schemaIdentity = schemaIdentity + self.schemaVersion = schemaVersion + self.compilerLogSha256 = compilerLogSha256 + self.workingDirectory = workingDirectory + self.rules = rules + self.targets = targets + } + + package static func sourceInventoryDigest(_ paths: [String]) -> String { + var data: Data + do { + data = try AnalyzerJSON.encoder.encode(paths) + } catch { + preconditionFailure("String arrays must be JSON encodable: \(error)") + } + data.append(0x0a) + return AnalyzerSHA256.hexDigest(data) + } + + package static func sha256(_ data: Data) -> String { + AnalyzerSHA256.hexDigest(data) + } + + // swiftlint:disable:next cyclomatic_complexity function_body_length + package func validatedJobGraph(capabilities: [String: AnalyzerRuleCapability]) throws -> [AnalyzerJob] { + guard schemaIdentity == Self.schemaIdentity, schemaVersion == Self.currentVersion else { + throw AnalyzerTargetPlanError.invalidSchema + } + guard Self.isSHA256(compilerLogSha256), !workingDirectory.isEmpty else { + throw AnalyzerTargetPlanError.invalidProvenance + } + try Self.requireUniqueNonempty(rules, label: "rule") + guard Set(capabilities.keys) == Set(rules) else { + throw AnalyzerTargetPlanError.ruleCapabilityMismatch + } + try Self.requireUniqueNonempty(targets.map(\.targetId), label: "target") + + var jobs = [AnalyzerJob]() + for target in targets { + guard !target.moduleName.isEmpty, + !target.sourceRoot.isEmpty, + !target.compileCommandsPath.isEmpty, + Self.isSHA256(target.compileCommandsSha256), + target.sourceFiles.isNotEmpty, + target.sourceFiles == target.sourceFiles.sorted(), + target.sourceFilesSha256 == Self.sourceInventoryDigest(target.sourceFiles) else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + try Self.requireUniqueNonempty(target.sourceFiles, label: "source path") + guard target.rulePlans.map(\.rule) == rules else { + throw AnalyzerTargetPlanError.rulePlanMismatch(target.targetId) + } + + for rulePlan in target.rulePlans { + guard let capability = capabilities[rulePlan.rule] else { + throw AnalyzerTargetPlanError.ruleCapabilityMismatch + } + switch rulePlan.mode { + case .wholeTarget: + guard rulePlan.batches.isEmpty else { + throw AnalyzerTargetPlanError.invalidBatches(target: target.targetId, rule: rulePlan.rule) + } + jobs.append( + AnalyzerJob( + workerId: "", + jobId: "\(target.targetId)/\(rulePlan.rule)", + targetId: target.targetId, + moduleName: target.moduleName, + sourceRoot: target.sourceRoot, + ruleIdentifier: rulePlan.rule, + batchIndex: nil, + requestedPaths: target.sourceFiles, + targetSourcePaths: target.sourceFiles, + compileCommandsPath: target.compileCommandsPath, + compileCommandsSha256: target.compileCommandsSha256 + ) + ) + case .batches: + guard !capability.isCollecting, + let maximumBatchSize = capability.maximumBatchSize, + maximumBatchSize > 0, + rulePlan.batches.isNotEmpty else { + throw AnalyzerTargetPlanError.unsafeBatching(rulePlan.rule) + } + let expectedIndices = Array(rulePlan.batches.indices) + guard rulePlan.batches.map(\.batchIndex) == expectedIndices, + rulePlan.batches.allSatisfy({ + !$0.requestedPaths.isEmpty && $0.requestedPaths.count <= maximumBatchSize + }), + rulePlan.batches.flatMap(\.requestedPaths) == target.sourceFiles else { + throw AnalyzerTargetPlanError.invalidBatches(target: target.targetId, rule: rulePlan.rule) + } + for batch in rulePlan.batches { + jobs.append( + AnalyzerJob( + workerId: "", + jobId: "\(target.targetId)/\(rulePlan.rule)/batch-" + + String(format: "%03d", batch.batchIndex), + targetId: target.targetId, + moduleName: target.moduleName, + sourceRoot: target.sourceRoot, + ruleIdentifier: rulePlan.rule, + batchIndex: batch.batchIndex, + requestedPaths: batch.requestedPaths, + targetSourcePaths: target.sourceFiles, + compileCommandsPath: target.compileCommandsPath, + compileCommandsSha256: target.compileCommandsSha256 + ) + ) + } + } + } + } + + return jobs.enumerated().map { index, job in + var job = job + job.workerId = "worker-\(String(format: "%04d", index))" + return job + } + } + + private static func requireUniqueNonempty(_ values: [String], label: String) throws { + guard values.allSatisfy({ !$0.isEmpty }), Set(values).count == values.count else { + throw AnalyzerTargetPlanError.duplicateIdentity(label) + } + } + + private static func isSHA256(_ value: String) -> Bool { + value.count == 64 && value.allSatisfy(\.isHexDigit) + } +} + +package enum AnalyzerTargetPlanError: LocalizedError, Equatable { + case invalidSchema + case invalidProvenance + case duplicateIdentity(String) + case ruleCapabilityMismatch + case invalidTarget(String) + case rulePlanMismatch(String) + case unsafeBatching(String) + case invalidBatches(target: String, rule: String) + + package var errorDescription: String? { + switch self { + case .invalidSchema: + "Unsupported analyzer target-plan schema." + case .invalidProvenance: + "Analyzer target-plan provenance is invalid." + case .duplicateIdentity(let identity): + "Analyzer target plan contains an empty or duplicate \(identity)." + case .ruleCapabilityMismatch: + "Analyzer target-plan rules do not match configured analyzer capabilities." + case .invalidTarget(let target): + "Analyzer target '\(target)' has invalid source or compilation-database provenance." + case .rulePlanMismatch(let target): + "Analyzer target '\(target)' does not contain the exact configured rule order." + case .unsafeBatching(let rule): + "Analyzer rule '\(rule)' does not explicitly permit batching." + case let .invalidBatches(target, rule): + "Analyzer batches for '\(target)/\(rule)' do not exactly partition the target source inventory." + } + } +} + +package struct AnalyzerJob: Codable, Equatable, Sendable { + package var workerId: String + package var jobId: String + package let targetId: String + package let moduleName: String + package let sourceRoot: String + package let ruleIdentifier: String + package let batchIndex: Int? + package let requestedPaths: [String] + package let targetSourcePaths: [String] + package var compileCommandsPath: String + package let compileCommandsSha256: String +} + +package struct AnalyzerProcessCoordinator: Sendable { + private let jobs: Int + private let operation: @Sendable (Input) async throws -> Output + + package init(jobs: Int, operation: @escaping @Sendable (Input) async throws -> Output) { + self.jobs = jobs + self.operation = operation + } + + package func run(_ inputs: [Input]) async throws -> [Output] { + guard jobs > 0 else { + throw AnalyzerWorkerContractError.invalidJobs(jobs) + } + guard inputs.isNotEmpty else { return [] } + + return try await withThrowingTaskGroup(of: (Int, Output).self) { group in + var nextIndex = 0 + var outputs = [Int: Output]() + + func addNext() { + let index = nextIndex + nextIndex += 1 + let input = inputs[index] + group.addTask { + try Task.checkCancellation() + return (index, try await operation(input)) + } + } + + for _ in 0..= 0, + jobId.hasSuffix("/batch-\(String(format: "%03d", batchIndex))") else { + throw AnalyzerWorkerContractError.requestMismatch(jobId) + } + } else { + guard requestedPaths == targetSourcePaths else { + throw AnalyzerWorkerContractError.requestMismatch(jobId) + } + } + } +} + +package struct AnalyzerFileTiming: Codable, Equatable, Sendable { + package let path: String + package let seconds: Double +} + +package struct AnalyzerRuleTiming: Codable, Equatable, Sendable { + package let ruleIdentifier: String + package let seconds: Double +} + +package struct AnalyzerWorkerResult: Codable, Equatable, Sendable { + package static let schemaIdentity = "swiftlint-analyzer-worker-result" + package static let currentVersion = 1 + + package var schemaIdentity: String + package var schemaVersion: Int + package var workerId: String + package var jobId: String + package var targetId: String + package var ruleIdentifier: String + package var batchIndex: Int? + package var requestedPaths: [String] + package var compileCommandsSha256: String + package var violations: [StyleViolation] + package var files: [URL] + package var fileTimings: [AnalyzerFileTiming] + package var ruleTimings: [AnalyzerRuleTiming] + package var startOffsetSeconds: Double + package var durationSeconds: Double + package var exitCode: Int32 + + package init( + schemaIdentity: String, + schemaVersion: Int, + workerId: String, + jobId: String, + targetId: String, + ruleIdentifier: String, + batchIndex: Int?, + requestedPaths: [String], + compileCommandsSha256: String, + violations: [StyleViolation], + files: [URL], + fileTimings: [AnalyzerFileTiming], + ruleTimings: [AnalyzerRuleTiming], + startOffsetSeconds: Double, + durationSeconds: Double, + exitCode: Int32 + ) { + self.schemaIdentity = schemaIdentity + self.schemaVersion = schemaVersion + self.workerId = workerId + self.jobId = jobId + self.targetId = targetId + self.ruleIdentifier = ruleIdentifier + self.batchIndex = batchIndex + self.requestedPaths = requestedPaths + self.compileCommandsSha256 = compileCommandsSha256 + self.violations = violations + self.files = files + self.fileTimings = fileTimings + self.ruleTimings = ruleTimings + self.startOffsetSeconds = startOffsetSeconds + self.durationSeconds = durationSeconds + self.exitCode = exitCode + } + + // swiftlint:disable:next cyclomatic_complexity + package func validate(for job: AnalyzerJob) throws { + guard schemaIdentity == Self.schemaIdentity else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [schemaIdentity]") + } + guard schemaVersion == Self.currentVersion else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [schemaVersion]") + } + guard workerId == job.workerId else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [workerId]") + } + guard jobId == job.jobId else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [jobId]") + } + guard targetId == job.targetId else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [targetId]") + } + guard ruleIdentifier == job.ruleIdentifier else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [ruleIdentifier]") + } + guard batchIndex == job.batchIndex else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [batchIndex]") + } + guard requestedPaths == job.requestedPaths else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [requestedPaths]") + } + guard compileCommandsSha256 == job.compileCommandsSha256 else { + throw AnalyzerWorkerContractError.resultMismatch("\(job.jobId) [compileCommandsSha256]") + } + guard exitCode == 0 else { + throw AnalyzerWorkerContractError.workerFailed( + job: job.jobId, + reason: "exit \(exitCode)", + diagnostics: "Worker result reported a nonzero exit." + ) + } + guard durationSeconds.isFinite, durationSeconds >= 0, + startOffsetSeconds.isFinite, startOffsetSeconds >= 0 else { + throw AnalyzerWorkerContractError.invalidTiming(job.jobId) + } + } +} + +package struct AnalyzerWorkerAggregate: Equatable, Sendable { + package let violations: [StyleViolation] + package let files: [URL] + package let fileTimings: [AnalyzerFileTiming] + package let ruleTimings: [AnalyzerRuleTiming] + + package static func merge(graph: [AnalyzerJob], results: [AnalyzerWorkerResult]) throws -> Self { + guard graph.count == results.count else { + throw AnalyzerWorkerContractError.jobCount(expected: graph.count, actual: results.count) + } + guard Set(graph.map(\.workerId)).count == graph.count, + Set(graph.map(\.jobId)).count == graph.count, + Set(results.map(\.workerId)).count == results.count, + Set(results.map(\.jobId)).count == results.count else { + throw AnalyzerWorkerContractError.duplicateIdentity + } + + let resultByJob = Dictionary(uniqueKeysWithValues: results.map { ($0.jobId, $0) }) + let orderedResults = try graph.map { job in + guard let result = resultByJob[job.jobId] else { + throw AnalyzerWorkerContractError.missingJob(job.jobId) + } + try result.validate(for: job) + return result + } + guard Set(resultByJob.keys) == Set(graph.map(\.jobId)) else { + throw AnalyzerWorkerContractError.extraJob + } + + let rawViolations = orderedResults.flatMap(\.violations) + guard Set(rawViolations).count == rawViolations.count else { + throw AnalyzerWorkerContractError.duplicateRawFinding + } + + return Self( + violations: rawViolations.sorted(by: violationOrder), + files: orderedUnique(orderedResults.flatMap(\.files)), + fileTimings: orderedResults.flatMap(\.fileTimings), + ruleTimings: orderedResults.flatMap(\.ruleTimings) + ) + } + + private static func violationOrder(_ lhs: StyleViolation, _ rhs: StyleViolation) -> Bool { + if lhs.location != rhs.location { return lhs.location < rhs.location } + if lhs.ruleIdentifier != rhs.ruleIdentifier { return lhs.ruleIdentifier < rhs.ruleIdentifier } + if lhs.reason != rhs.reason { return lhs.reason < rhs.reason } + return lhs.severity.rawValue < rhs.severity.rawValue + } + + private static func orderedUnique(_ values: [T]) -> [T] { + var seen = Set() + return values.filter { seen.insert($0).inserted } + } +} + +private struct AnalyzerCanonicalFinding: Codable { + let file: String? + let line: Int? + let character: Int? + let severity: String + let type: String + let ruleId: String + let reason: String + + enum CodingKeys: String, CodingKey { + case file, line, character, severity, type, reason + case ruleId = "rule_id" + } +} + +package struct AnalyzerExecutionEvidence: Codable, Equatable, Sendable { + package static let schemaIdentity = "swiftlint-analyzer-execution-evidence" + package static let currentVersion = 1 + + package struct Worker: Codable, Equatable, Sendable { + package let workerId: String + package let jobId: String + package let targetId: String + package let ruleIdentifier: String + package let batchIndex: Int? + package let requestedPaths: [String] + package let compileCommandsSha256: String + package let startOffsetSeconds: Double + package let durationSeconds: Double + package let exitCode: Int32 + package let rawFindingCount: Int + } + + package struct Topology: Codable, Equatable, Sendable { + package let targetCount: Int + package let ruleCount: Int + package let wholeTargetJobCount: Int + package let batchedJobCount: Int + package let totalJobCount: Int + package let jobOrder: [String] + } + + package var schemaIdentity: String + package var schemaVersion: Int + package var targetPlanSha256: String + package var jobsRequested: Int + package var peakConcurrentJobs: Int + package var peakChildResidentMemoryBytes: UInt64 + package var elapsedSeconds: Double + package var topology: Topology + package var workers: [Worker] + package var findingsCount: Int + package var findingsSha256: String + package var reporterOutputSha256: String + + // swiftlint:disable:next function_parameter_count + package static func make( + targetPlanSha256: String, + jobsRequested: Int, + peakConcurrentJobs: Int, + peakChildResidentMemoryBytes: UInt64, + elapsedSeconds: Double, + graph: [AnalyzerJob], + results: [AnalyzerWorkerResult], + findings: [StyleViolation], + reporterOutput: Data, + workingDirectory: URL = .cwd + ) throws -> Self { + _ = try AnalyzerWorkerAggregate.merge(graph: graph, results: results) + let resultByJob = Dictionary(uniqueKeysWithValues: results.map { ($0.jobId, $0) }) + return Self( + schemaIdentity: schemaIdentity, + schemaVersion: currentVersion, + targetPlanSha256: targetPlanSha256, + jobsRequested: jobsRequested, + peakConcurrentJobs: peakConcurrentJobs, + peakChildResidentMemoryBytes: peakChildResidentMemoryBytes, + elapsedSeconds: elapsedSeconds, + topology: Topology( + targetCount: Set(graph.map(\.targetId)).count, + ruleCount: Set(graph.map(\.ruleIdentifier)).count, + wholeTargetJobCount: graph.filter({ $0.batchIndex == nil }).count, + batchedJobCount: graph.filter({ $0.batchIndex != nil }).count, + totalJobCount: graph.count, + jobOrder: graph.map(\.jobId) + ), + workers: try graph.map { job in + guard let result = resultByJob[job.jobId] else { + throw AnalyzerWorkerContractError.missingJob(job.jobId) + } + return Worker( + workerId: result.workerId, + jobId: result.jobId, + targetId: result.targetId, + ruleIdentifier: result.ruleIdentifier, + batchIndex: result.batchIndex, + requestedPaths: result.requestedPaths, + compileCommandsSha256: result.compileCommandsSha256, + startOffsetSeconds: result.startOffsetSeconds, + durationSeconds: result.durationSeconds, + exitCode: result.exitCode, + rawFindingCount: result.violations.count + ) + }, + findingsCount: findings.count, + findingsSha256: try digestFindings(findings, workingDirectory: workingDirectory), + reporterOutputSha256: AnalyzerSHA256.hexDigest(reporterOutput) + ) + } + + package func validate(graph: [AnalyzerJob], targetPlanSha256 expectedPlanDigest: String) throws { + guard schemaIdentity == Self.schemaIdentity, + schemaVersion == Self.currentVersion, + targetPlanSha256 == expectedPlanDigest, + jobsRequested > 0, + peakConcurrentJobs > 0, + peakConcurrentJobs <= jobsRequested, + elapsedSeconds.isFinite, + elapsedSeconds >= 0, + topology == Topology( + targetCount: Set(graph.map(\.targetId)).count, + ruleCount: Set(graph.map(\.ruleIdentifier)).count, + wholeTargetJobCount: graph.filter({ $0.batchIndex == nil }).count, + batchedJobCount: graph.filter({ $0.batchIndex != nil }).count, + totalJobCount: graph.count, + jobOrder: graph.map(\.jobId) + ), + workers.count == graph.count, + workers.map(\.workerId) == graph.map(\.workerId), + workers.map(\.jobId) == graph.map(\.jobId), + Set(workers.map(\.workerId)).count == workers.count, + Set(workers.map(\.jobId)).count == workers.count else { + throw AnalyzerWorkerContractError.evidenceMismatch + } + for (worker, job) in zip(workers, graph) { + guard worker.targetId == job.targetId, + worker.ruleIdentifier == job.ruleIdentifier, + worker.batchIndex == job.batchIndex, + worker.requestedPaths == job.requestedPaths, + worker.compileCommandsSha256 == job.compileCommandsSha256, + worker.exitCode == 0, + worker.startOffsetSeconds.isFinite, + worker.startOffsetSeconds >= 0, + worker.durationSeconds.isFinite, + worker.durationSeconds >= 0, + worker.rawFindingCount >= 0 else { + throw AnalyzerWorkerContractError.evidenceMismatch + } + } + } + + private static func digestFindings(_ findings: [StyleViolation], workingDirectory: URL) throws -> String { + let directoryPath = workingDirectory.standardizedFileURL.path + let prefix = directoryPath.hasSuffix("/") ? directoryPath : directoryPath + "/" + let canonical = findings.map { finding in + let absolutePath = finding.location.file?.standardizedFileURL.path + let file = absolutePath.map { path in + path.hasPrefix(prefix) ? String(path.dropFirst(prefix.count)) : path + } + return AnalyzerCanonicalFinding( + file: file, + line: finding.location.line, + character: finding.location.character, + severity: finding.severity.rawValue.capitalized, + type: finding.ruleName, + ruleId: finding.ruleIdentifier, + reason: finding.reason + ) + } + var data = try AnalyzerJSON.encoder.encode(canonical) + data.append(0x0a) + return AnalyzerSHA256.hexDigest(data) + } +} + +package enum AnalyzerWorkerContractError: LocalizedError, Equatable { + case invalidJobs(Int) + case requestMismatch(String) + case resultMismatch(String) + case duplicateIdentity + case missingJob(String) + case extraJob + case jobCount(expected: Int, actual: Int) + case duplicateRawFinding + case findingsMismatch + case evidenceMismatch + case invalidTiming(String) + case workerFailed(job: String, reason: String, diagnostics: String) + + package var errorDescription: String? { + switch self { + case .invalidJobs(let jobs): + "Analyzer jobs must be positive; received \(jobs)." + case .requestMismatch(let job): + "Analyzer worker request does not match scheduled job '\(job)'." + case .resultMismatch(let job): + "Analyzer worker result does not match scheduled job '\(job)'." + case .duplicateIdentity: + "Analyzer workers contain a duplicate job or worker identity." + case .missingJob(let job): + "Analyzer worker result is missing scheduled job '\(job)'." + case .extraJob: + "Analyzer worker results contain an unscheduled job." + case let .jobCount(expected, actual): + "Analyzer worker result count \(actual) does not match scheduled job count \(expected)." + case .duplicateRawFinding: + "Analyzer workers produced an unexpected duplicate raw finding." + case .findingsMismatch: + "Analyzer evidence findings do not match the deterministic worker aggregate." + case .evidenceMismatch: + "Analyzer execution evidence does not match the scheduled target plan." + case .invalidTiming(let job): + "Analyzer worker '\(job)' reported an invalid timing." + case let .workerFailed(job, reason, diagnostics): + "Analyzer worker '\(job)' failed (\(reason)): \(diagnostics)" + } + } +} + +private enum AnalyzerSHA256 { + static func hexDigest(_ data: Data) -> String { + hash(Array(data)).map { String(format: "%02x", $0) }.joined() + } + + // FIPS 180-4 SHA-256. Kept local so the worker protocol hashes identically on macOS and Linux. + // swiftlint:disable:next function_body_length + private static func hash(_ bytes: [UInt8]) -> [UInt8] { + let constants: [UInt32] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + ] + var message = bytes + let bitLength = UInt64(message.count) * 8 + message.append(0x80) + while message.count % 64 != 56 { message.append(0) } + message.append(contentsOf: withUnsafeBytes(of: bitLength.bigEndian, Array.init)) + + var state: [UInt32] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + ] + for offset in stride(from: 0, to: message.count, by: 64) { + var words = [UInt32](repeating: 0, count: 64) + for index in 0..<16 { + let start = offset + index * 4 + words[index] = UInt32(message[start]) << 24 + | UInt32(message[start + 1]) << 16 + | UInt32(message[start + 2]) << 8 + | UInt32(message[start + 3]) + } + for index in 16..<64 { + let value15 = words[index - 15] + let value2 = words[index - 2] + let sigma0 = value15.rotateRight(7) ^ value15.rotateRight(18) ^ (value15 >> 3) + let sigma1 = value2.rotateRight(17) ^ value2.rotateRight(19) ^ (value2 >> 10) + words[index] = words[index - 16] &+ sigma0 &+ words[index - 7] &+ sigma1 + } + + var working0 = state[0] + var working1 = state[1] + var working2 = state[2] + var working3 = state[3] + var working4 = state[4] + var working5 = state[5] + var working6 = state[6] + var working7 = state[7] + for index in 0..<64 { + let sum1 = working4.rotateRight(6) ^ working4.rotateRight(11) ^ working4.rotateRight(25) + let choose = (working4 & working5) ^ (~working4 & working6) + let temporary1 = working7 &+ sum1 &+ choose &+ constants[index] &+ words[index] + let sum0 = working0.rotateRight(2) ^ working0.rotateRight(13) ^ working0.rotateRight(22) + let majority = (working0 & working1) ^ (working0 & working2) ^ (working1 & working2) + let temporary2 = sum0 &+ majority + working7 = working6 + working6 = working5 + working5 = working4 + working4 = working3 &+ temporary1 + working3 = working2 + working2 = working1 + working1 = working0 + working0 = temporary1 &+ temporary2 + } + state[0] &+= working0 + state[1] &+= working1 + state[2] &+= working2 + state[3] &+= working3 + state[4] &+= working4 + state[5] &+= working5 + state[6] &+= working6 + state[7] &+= working7 + } + return state.flatMap { word in withUnsafeBytes(of: word.bigEndian, Array.init) } + } +} + +private extension UInt32 { + func rotateRight(_ amount: UInt32) -> UInt32 { + self >> amount | self << (32 - amount) + } +} diff --git a/Source/SwiftLintFramework/AnalyzerWorkerProcessRunner.swift b/Source/SwiftLintFramework/AnalyzerWorkerProcessRunner.swift new file mode 100644 index 00000000000..3f572a3234d --- /dev/null +++ b/Source/SwiftLintFramework/AnalyzerWorkerProcessRunner.swift @@ -0,0 +1,559 @@ +import Dispatch +import Foundation + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +#if canImport(Darwin) +private typealias AnalyzerSpawnFileActions = posix_spawn_file_actions_t? +private typealias AnalyzerSpawnAttributes = posix_spawnattr_t? +#elseif canImport(Glibc) +private typealias AnalyzerSpawnFileActions = posix_spawn_file_actions_t +private typealias AnalyzerSpawnAttributes = posix_spawnattr_t +#endif + +// Process-group creation, signal forwarding, metrics, and quiescence are intentionally colocated. +// swiftlint:disable file_length + +package enum AnalyzerWorkerEnvironment { + package static let requestPath = "SWIFTLINT_ANALYZER_WORKER_REQUEST_V1" +} + +package struct AnalyzerWorkerInvocation: Sendable { + package let jobId: String + package let requestURL: URL + package let resultURL: URL + package let standardOutputURL: URL + package let standardErrorURL: URL + package let arguments: [String] + package let environment: [String: String] + + package init( + jobId: String, + requestURL: URL, + resultURL: URL, + standardOutputURL: URL, + standardErrorURL: URL, + arguments: [String], + environment: [String: String] + ) { + self.jobId = jobId + self.requestURL = requestURL + self.resultURL = resultURL + self.standardOutputURL = standardOutputURL + self.standardErrorURL = standardErrorURL + self.arguments = arguments + self.environment = environment + } +} + +package struct AnalyzerWorkerProcessOutput: Sendable { + package let jobId: String + package let resultData: Data + package let diagnostics: String +} + +package struct AnalyzerWorkerProcessMetrics: Equatable, Sendable { + package let peakConcurrentJobs: Int + package let peakAggregateResidentMemoryBytes: UInt64 +} + +package final class AnalyzerWorkerProcessRunner: @unchecked Sendable { + private let executableURL: URL + private let currentDirectoryURL: URL + private let environment: [String: String] + private let registry = AnalyzerProcessRegistry() + + package init(executableURL: URL, currentDirectoryURL: URL, environment: [String: String]) { + self.executableURL = executableURL + self.currentDirectoryURL = currentDirectoryURL + self.environment = environment + } + + package func prepare( + requests: [AnalyzerWorkerRequest], + graph: [AnalyzerJob], + in directory: URL + ) throws -> [AnalyzerWorkerInvocation] { + guard requests.count == graph.count else { + throw AnalyzerWorkerContractError.jobCount(expected: graph.count, actual: requests.count) + } + return try zip(requests, graph).enumerated().map { index, pair in + let (request, job) = pair + try request.validate(for: job) + let requestURL = directory.appending(path: "request-\(index).json", directoryHint: .notDirectory) + let standardOutputURL = directory.appending(path: "stdout-\(index).txt", directoryHint: .notDirectory) + let standardErrorURL = directory.appending(path: "stderr-\(index).txt", directoryHint: .notDirectory) + try AnalyzerJSON.encoder.encode(request).write(to: requestURL, options: .atomic) + guard FileManager.default.createFile(atPath: standardOutputURL.path, contents: nil), + FileManager.default.createFile(atPath: standardErrorURL.path, contents: nil) else { + throw AnalyzerProcessRunnerError.cannotCreateDiagnostics(job.jobId) + } + var childEnvironment = environment + childEnvironment[AnalyzerWorkerEnvironment.requestPath] = requestURL.path + childEnvironment.removeValue(forKey: "BUILD_WORKSPACE_DIRECTORY") + return AnalyzerWorkerInvocation( + jobId: job.jobId, + requestURL: requestURL, + resultURL: request.resultURL, + standardOutputURL: standardOutputURL, + standardErrorURL: standardErrorURL, + arguments: ["analyze"], + environment: childEnvironment + ) + } + } + + package func run(_ invocation: AnalyzerWorkerInvocation) async throws -> AnalyzerWorkerProcessOutput { + try await withTaskCancellationHandler { + let processIdentifier = try registry.spawn( + executableURL: executableURL, + arguments: invocation.arguments, + environment: invocation.environment, + currentDirectoryURL: currentDirectoryURL, + standardOutputURL: invocation.standardOutputURL, + standardErrorURL: invocation.standardErrorURL + ) + let sampler = Task.detached(priority: .utility) { [registry] in + while !Task.isCancelled { + registry.sampleResidentMemory() + try? await Task.sleep(for: .milliseconds(250)) + } + } + let status = await Task.detached(priority: .userInitiated) { + AnalyzerProcessRegistry.wait(for: processIdentifier) + }.value + sampler.cancel() + registry.sampleResidentMemory() + let wasCancelled = registry.finished(processIdentifier) + let groupWasQuiescent = registry.ensureGroupQuiescence(processIdentifier) + if wasCancelled { + throw CancellationError() + } + guard groupWasQuiescent else { + throw AnalyzerWorkerContractError.workerFailed( + job: invocation.jobId, + reason: "descendant cleanup failed", + diagnostics: "The worker process group remained live after its leader exited." + ) + } + return try Self.processOutput(for: invocation, status: status) + } onCancel: { + registry.terminateAll(forwarding: SIGTERM) + } + } + + package func cancelAll() { + registry.terminateAll(forwarding: SIGTERM) + } + + package func forward(signal signalNumber: Int32) { + registry.terminateAll(forwarding: signalNumber) + } + + package var metrics: AnalyzerWorkerProcessMetrics { + registry.metrics + } + + private static func processOutput( + for invocation: AnalyzerWorkerInvocation, + status: Int32 + ) throws -> AnalyzerWorkerProcessOutput { + let diagnostics = try String(contentsOf: invocation.standardErrorURL, encoding: .utf8) + let output = try String(contentsOf: invocation.standardOutputURL, encoding: .utf8) + guard output.isEmpty else { + throw AnalyzerWorkerContractError.workerFailed( + job: invocation.jobId, + reason: "unexpected standard output", + diagnostics: output + ) + } + guard let exitCode = Self.exitCode(status), exitCode == 0 else { + let reason = Self.terminationDescription(status) + throw AnalyzerWorkerContractError.workerFailed( + job: invocation.jobId, + reason: reason, + diagnostics: diagnostics + ) + } + guard diagnostics.isEmpty else { + throw AnalyzerWorkerContractError.workerFailed( + job: invocation.jobId, + reason: "unexpected standard error", + diagnostics: diagnostics + ) + } + guard FileManager.default.fileExists(atPath: invocation.resultURL.path) else { + throw AnalyzerWorkerContractError.workerFailed( + job: invocation.jobId, + reason: "missing result", + diagnostics: "The worker exited successfully without writing its versioned result." + ) + } + return AnalyzerWorkerProcessOutput( + jobId: invocation.jobId, + resultData: try Data(contentsOf: invocation.resultURL), + diagnostics: diagnostics + ) + } + + private static func exitCode(_ status: Int32) -> Int32? { + status & 0x7f == 0 ? (status >> 8) & 0xff : nil + } + + private static func terminationDescription(_ status: Int32) -> String { + if let exitCode = exitCode(status) { + return "exit \(exitCode)" + } + return "signal \(status & 0x7f)" + } +} + +package final class AnalyzerSignalForwarder: @unchecked Sendable { + private struct Registration { + let signalNumber: Int32 + let previousHandler: sig_t? + let source: any DispatchSourceSignal + } + + private let registrations: [Registration] + + package init(runner: AnalyzerWorkerProcessRunner) { + registrations = [SIGINT, SIGTERM].map { signalNumber in + let previousHandler = signal(signalNumber, SIG_IGN) + let source = DispatchSource.makeSignalSource( + signal: signalNumber, + queue: DispatchQueue.global(qos: .userInitiated) + ) + source.setEventHandler { + runner.forward(signal: signalNumber) + _ = signal(signalNumber, SIG_DFL) + _ = kill(getpid(), signalNumber) + } + source.activate() + return Registration( + signalNumber: signalNumber, + previousHandler: previousHandler, + source: source + ) + } + } + + deinit { + for registration in registrations { + registration.source.cancel() + if let previousHandler = registration.previousHandler { + _ = signal(registration.signalNumber, previousHandler) + } + } + } +} + +package enum AnalyzerProcessRunnerError: LocalizedError { + case cannotCreateDiagnostics(String) + case invalidExecutable(String) + case spawnFailed(String, Int32) + case stringAllocation + + package var errorDescription: String? { + switch self { + case .cannotCreateDiagnostics(let job): + "Could not create analyzer worker diagnostics for '\(job)'." + case .invalidExecutable(let path): + "Analyzer worker executable is not executable: '\(path)'." + case let .spawnFailed(path, code): + "Could not spawn analyzer worker '\(path)': \(String(cString: strerror(code)))." + case .stringAllocation: + "Could not allocate analyzer worker process arguments." + } + } +} + +// All mutable state is protected by `lock`; process groups are registered atomically with `posix_spawn`. +private final class AnalyzerProcessRegistry: @unchecked Sendable { + private let lock = NSLock() + private var processGroups = Set() + private var cancelledProcessGroups = Set() + private var cancellationRequested = false + private var maximumActiveJobs = 0 + private var maximumResidentMemoryBytes: UInt64 = 0 + + var metrics: AnalyzerWorkerProcessMetrics { + lock.withLock { + AnalyzerWorkerProcessMetrics( + peakConcurrentJobs: maximumActiveJobs, + peakAggregateResidentMemoryBytes: maximumResidentMemoryBytes + ) + } + } + + // swiftlint:disable:next function_parameter_count + func spawn( + executableURL: URL, + arguments: [String], + environment: [String: String], + currentDirectoryURL: URL, + standardOutputURL: URL, + standardErrorURL: URL + ) throws -> pid_t { + guard FileManager.default.isExecutableFile(atPath: executableURL.path) else { + throw AnalyzerProcessRunnerError.invalidExecutable(executableURL.path) + } + + return try lock.withLock { + guard !cancellationRequested else { throw CancellationError() } + + var fileActions = Self.makeSpawnFileActions() + var attributes = Self.makeSpawnAttributes() + guard posix_spawn_file_actions_init(&fileActions) == 0, + posix_spawnattr_init(&attributes) == 0 else { + throw AnalyzerProcessRunnerError.spawnFailed(executableURL.path, errno) + } + defer { + posix_spawn_file_actions_destroy(&fileActions) + posix_spawnattr_destroy(&attributes) + } + + try Self.configureFileActions( + &fileActions, + currentDirectoryURL: currentDirectoryURL, + standardOutputURL: standardOutputURL, + standardErrorURL: standardErrorURL + ) + let flags = Int16(POSIX_SPAWN_SETPGROUP) + let flagsResult = posix_spawnattr_setflags(&attributes, flags) + let groupResult = posix_spawnattr_setpgroup(&attributes, 0) + guard flagsResult == 0, groupResult == 0 else { + throw AnalyzerProcessRunnerError.spawnFailed( + executableURL.path, + flagsResult == 0 ? groupResult : flagsResult + ) + } + + var processIdentifier: pid_t = 0 + let environmentStrings = environment.keys.sorted().compactMap { key in + environment[key].map { "\(key)=\($0)" } + } + let spawnResult = try Self.withCStringArray([executableURL.path] + arguments) { argumentPointer in + try Self.withCStringArray(environmentStrings) { environmentPointer in + posix_spawn( + &processIdentifier, + executableURL.path, + &fileActions, + &attributes, + argumentPointer, + environmentPointer + ) + } + } + guard spawnResult == 0 else { + throw AnalyzerProcessRunnerError.spawnFailed(executableURL.path, spawnResult) + } + processGroups.insert(processIdentifier) + maximumActiveJobs = max(maximumActiveJobs, processGroups.count) + return processIdentifier + } + } + + func finished(_ processGroup: pid_t) -> Bool { + lock.withLock { + processGroups.remove(processGroup) + return cancelledProcessGroups.remove(processGroup) != nil + } + } + + func terminateAll(forwarding signalNumber: Int32) { + let groups = lock.withLock { () -> [pid_t] in + cancellationRequested = true + cancelledProcessGroups.formUnion(processGroups) + return Array(processGroups) + } + Self.signal(groups: groups, signalNumber: signalNumber) + let deadline = ContinuousClock.now.advanced(by: .seconds(2)) + while ContinuousClock.now < deadline, groups.contains(where: Self.groupExists) { + usleep(20_000) + } + let survivingGroups = groups.filter(Self.groupExists) + Self.signal(groups: survivingGroups, signalNumber: SIGKILL) + } + + func ensureGroupQuiescence(_ processGroup: pid_t) -> Bool { + guard Self.groupExists(processGroup) else { return true } + Self.signal(groups: [processGroup], signalNumber: SIGTERM) + let gracefulDeadline = ContinuousClock.now.advanced(by: .seconds(2)) + while ContinuousClock.now < gracefulDeadline, Self.groupExists(processGroup) { + usleep(20_000) + } + if Self.groupExists(processGroup) { + Self.signal(groups: [processGroup], signalNumber: SIGKILL) + } + let killDeadline = ContinuousClock.now.advanced(by: .seconds(2)) + while ContinuousClock.now < killDeadline, Self.groupExists(processGroup) { + usleep(20_000) + } + return !Self.groupExists(processGroup) + } + + func sampleResidentMemory() { + let groups = lock.withLock { Array(processGroups) } + let residentMemory = groups.reduce(UInt64(0)) { partial, processIdentifier in + partial + Self.residentMemoryBytes(processIdentifier) + } + lock.withLock { + maximumResidentMemoryBytes = max(maximumResidentMemoryBytes, residentMemory) + } + } + + static func wait(for processIdentifier: pid_t) -> Int32 { + var status: Int32 = 0 + while waitpid(processIdentifier, &status, 0) == -1, errno == EINTR { + // Retry after an interrupted system call. + } + return status + } + + private static func makeSpawnFileActions() -> AnalyzerSpawnFileActions { + #if canImport(Darwin) + nil + #else + AnalyzerSpawnFileActions() + #endif + } + + private static func makeSpawnAttributes() -> AnalyzerSpawnAttributes { + #if canImport(Darwin) + nil + #else + AnalyzerSpawnAttributes() + #endif + } + + private static func configureFileActions( + _ actions: inout AnalyzerSpawnFileActions, + currentDirectoryURL: URL, + standardOutputURL: URL, + standardErrorURL: URL + ) throws { + let changeDirectoryResult: Int32 + #if canImport(Darwin) + if #available(macOS 26.0, *) { + changeDirectoryResult = posix_spawn_file_actions_addchdir(&actions, currentDirectoryURL.path) + } else { + changeDirectoryResult = posix_spawn_file_actions_addchdir_np(&actions, currentDirectoryURL.path) + } + #else + changeDirectoryResult = posix_spawn_file_actions_addchdir_np(&actions, currentDirectoryURL.path) + #endif + let mode = mode_t(S_IRUSR | S_IWUSR) + let stdoutResult = posix_spawn_file_actions_addopen( + &actions, + STDOUT_FILENO, + standardOutputURL.path, + O_WRONLY | O_CREAT | O_TRUNC, + mode + ) + let stderrResult = posix_spawn_file_actions_addopen( + &actions, + STDERR_FILENO, + standardErrorURL.path, + O_WRONLY | O_CREAT | O_TRUNC, + mode + ) + guard changeDirectoryResult == 0 else { + throw AnalyzerProcessRunnerError.spawnFailed(currentDirectoryURL.path, changeDirectoryResult) + } + guard stdoutResult == 0 else { + throw AnalyzerProcessRunnerError.spawnFailed(standardOutputURL.path, stdoutResult) + } + guard stderrResult == 0 else { + throw AnalyzerProcessRunnerError.spawnFailed(standardErrorURL.path, stderrResult) + } + } + + private static func withCStringArray( + _ values: [String], + body: (UnsafeMutablePointer?>) throws -> Result + ) throws -> Result { + var pointers = [UnsafeMutablePointer]() + defer { pointers.forEach { free(UnsafeMutableRawPointer($0)) } } + for value in values { + guard let pointer = strdup(value) else { + throw AnalyzerProcessRunnerError.stringAllocation + } + pointers.append(pointer) + } + var nullablePointers = pointers.map(Optional.some) + nullablePointers.append(nil) + return try nullablePointers.withUnsafeMutableBufferPointer { buffer in + guard let baseAddress = buffer.baseAddress else { + throw AnalyzerProcessRunnerError.stringAllocation + } + return try body(baseAddress) + } + } + + private static func signal(groups: [pid_t], signalNumber: Int32) { + for group in groups where group > 0 { + if kill(-group, signalNumber) == -1, errno != ESRCH { + // The wait path reports the authoritative worker outcome. + } + } + } + + private static func groupExists(_ group: pid_t) -> Bool { + guard group > 0 else { return false } + if kill(-group, 0) == -1 { + return errno == EPERM + } + #if canImport(Glibc) + guard let entries = try? FileManager.default.contentsOfDirectory(atPath: "/proc") else { + return true + } + var foundGroupMember = false + for entry in entries where pid_t(entry) != nil { + guard let stat = try? String(contentsOfFile: "/proc/\(entry)/stat", encoding: .utf8), + let commandEnd = stat.range(of: ")", options: .backwards)?.upperBound else { + continue + } + let fields = stat[commandEnd...].split(whereSeparator: \.isWhitespace) + guard fields.count > 2, pid_t(fields[2]) == group else { continue } + foundGroupMember = true + if fields[0] != "Z", fields[0] != "X" { + return true + } + } + return !foundGroupMember && kill(-group, 0) == 0 + #else + return true + #endif + } + + private static func residentMemoryBytes(_ processIdentifier: pid_t) -> UInt64 { + #if canImport(Darwin) + var info = proc_taskinfo() + let size = Int32(MemoryLayout.size) + guard proc_pidinfo(processIdentifier, PROC_PIDTASKINFO, 0, &info, size) == size else { return 0 } + return info.pti_resident_size + #elseif canImport(Glibc) + guard let contents = try? String(contentsOfFile: "/proc/\(processIdentifier)/status", encoding: .utf8), + let line = contents.split(separator: "\n").first(where: { $0.hasPrefix("VmRSS:") }), + let kilobytes = UInt64(line.split(whereSeparator: \.isWhitespace).dropFirst().first ?? "") else { + return 0 + } + return kilobytes * 1024 + #else + return 0 + #endif + } +} + +package enum AnalyzerJSON { + package static var encoder: JSONEncoder { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + return encoder + } +} diff --git a/Source/SwiftLintFramework/LintOrAnalyzeCommand+AnalyzerWorker.swift b/Source/SwiftLintFramework/LintOrAnalyzeCommand+AnalyzerWorker.swift new file mode 100644 index 00000000000..a2a4053b9b2 --- /dev/null +++ b/Source/SwiftLintFramework/LintOrAnalyzeCommand+AnalyzerWorker.swift @@ -0,0 +1,543 @@ +import Foundation +@preconcurrency import SwiftLintCore + +// The worker protocol is kept in one file so its schemas, validators, and command boundary evolve atomically. +// swiftlint:disable file_length + +package struct AnalyzerPreparedTargetPlan: Sendable { + package let plan: AnalyzerTargetPlan + package let planDirectory: URL + package let workingDirectory: URL + package let digest: String +} + +package struct AnalyzerNativeRun: Sendable { + package let aggregate: AnalyzerWorkerAggregate + package let graph: [AnalyzerJob] + package let results: [AnalyzerWorkerResult] + package let metrics: AnalyzerWorkerProcessMetrics + package let elapsedSeconds: Double + package let targetPlanSha256: String + package let jobsRequested: Int + package let workingDirectory: URL +} + +// All mutable state is private, protected by `lock`, and no mutable reference escapes. +private final class AnalyzerWorkerResultCollector: @unchecked Sendable { + private let lock = NSLock() + private var violations = [StyleViolation]() + private var fileTimings = [AnalyzerFileTiming]() + private var ruleTimings = [AnalyzerRuleTiming]() + + func record( + violations: [StyleViolation], + fileTiming: AnalyzerFileTiming, + ruleTimings: [AnalyzerRuleTiming] + ) { + lock.withLock { + self.violations += violations + fileTimings.append(fileTiming) + self.ruleTimings += ruleTimings + } + } + + func result( + request: AnalyzerWorkerRequest, + files: [URL], + durationSeconds: Double + ) -> AnalyzerWorkerResult { + lock.withLock { + AnalyzerWorkerResult( + schemaIdentity: AnalyzerWorkerResult.schemaIdentity, + schemaVersion: AnalyzerWorkerResult.currentVersion, + workerId: request.workerId, + jobId: request.jobId, + targetId: request.targetId, + ruleIdentifier: request.ruleIdentifier, + batchIndex: request.batchIndex, + requestedPaths: request.requestedPaths, + compileCommandsSha256: request.compileCommandsSha256, + violations: violations.sorted(by: analyzerViolationPrecedes), + files: files.sorted(by: { $0.path < $1.path }), + fileTimings: fileTimings.sorted(by: analyzerFileTimingPrecedes), + ruleTimings: ruleTimings.sorted(by: analyzerRuleTimingPrecedes), + startOffsetSeconds: 0, + durationSeconds: durationSeconds, + exitCode: 0 + ) + } + } +} + +// swiftlint:disable:next type_body_length +package extension LintOrAnalyzeCommand { + static func analyzerExecutableURL( + argument: String = CommandLine.arguments[0], + environment: [String: String] = ProcessInfo.processInfo.environment + ) throws -> URL { + let fileManager = FileManager.default + if argument.contains("/") { + let url = URL(filePath: argument).standardizedFileURL + guard fileManager.isExecutableFile(atPath: url.path) else { + throw SwiftLintError.usageError( + description: "Could not locate the SwiftLint executable at '\(url.path)'." + ) + } + return url + } + + for directory in environment["PATH", default: ""].split(separator: ":", omittingEmptySubsequences: false) { + let baseURL = directory.isEmpty + ? URL(filePath: fileManager.currentDirectoryPath, directoryHint: .isDirectory) + : URL(filePath: String(directory), directoryHint: .isDirectory) + let candidate = baseURL.appending(path: argument, directoryHint: .notDirectory) + if fileManager.isExecutableFile(atPath: candidate.path) { + return candidate.standardizedFileURL + } + } + throw SwiftLintError.usageError( + description: "Could not locate the SwiftLint executable '\(argument)' in PATH." + ) + } + + static func validateTargetAwareAnalyzeOptions( + targetPlanPath: String, + executionEvidencePath: String, + jobs: Int, + autocorrect: Bool + ) throws { + guard !targetPlanPath.isEmpty, !executionEvidencePath.isEmpty else { + throw SwiftLintError.usageError( + description: "Target-aware analyze requires both --target-plan and --execution-evidence." + ) + } + guard jobs > 0 else { + throw SwiftLintError.usageError(description: "--jobs must be a positive integer.") + } + guard !autocorrect || jobs == 1 else { + throw SwiftLintError.usageError( + description: "Analyzer autocorrection is serial; use --jobs 1 with --fix." + ) + } + } + + static func prepareTargetAwareAnalyze(_ options: LintOrAnalyzeOptions) throws -> AnalyzerPreparedTargetPlan { + guard let targetPlanURL = options.targetPlan else { + throw SwiftLintError.usageError(description: "Target-aware analyze is missing --target-plan.") + } + let planData = try Data(contentsOf: targetPlanURL) + try validateTargetPlanJSONShape(planData) + let plan = try JSONDecoder().decode(AnalyzerTargetPlan.self, from: planData) + guard plan.schemaIdentity == AnalyzerTargetPlan.schemaIdentity, + plan.schemaVersion == AnalyzerTargetPlan.currentVersion else { + throw AnalyzerTargetPlanError.invalidSchema + } + let invocationDirectory = URL( + filePath: FileManager.default.currentDirectoryPath, + directoryHint: .isDirectory + ) + let workingDirectory = resolve(plan.workingDirectory, relativeTo: invocationDirectory) + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: workingDirectory.path, isDirectory: &isDirectory), + isDirectory.boolValue else { + throw SwiftLintError.usageError( + description: "Analyzer target-plan working directory does not exist: '\(workingDirectory.path)'." + ) + } + return AnalyzerPreparedTargetPlan( + plan: plan, + planDirectory: targetPlanURL.deletingLastPathComponent().standardizedFileURL, + workingDirectory: workingDirectory, + digest: AnalyzerTargetPlan.sha256(planData) + ) + } + + // swiftlint:disable:next function_body_length + static func executeTargetAwareAnalyze( + _ options: LintOrAnalyzeOptions, + preparedPlan: AnalyzerPreparedTargetPlan + ) async throws -> AnalyzerNativeRun { + let configuration = Configuration(options: options) + let analyzerRules = configuration.rules.filter { $0 is any AnalyzerRule } + var capabilities = [String: AnalyzerRuleCapability]() + for rule in analyzerRules { + let identifier = type(of: rule).identifier + guard capabilities[identifier] == nil else { + throw AnalyzerTargetPlanError.duplicateIdentity("configured analyzer rule") + } + capabilities[identifier] = AnalyzerRuleCapability( + isCollecting: rule is any AnyCollectingRule, + maximumBatchSize: (rule as? any AnalyzerBatchingRule)?.analyzerBatchSize + ) + } + var resolvedGraph = try preparedPlan.plan.validatedJobGraph(capabilities: capabilities) + try validateTargetInputs(preparedPlan, graph: &resolvedGraph) + let graph = resolvedGraph + + let temporaryDirectory = FileManager.default.temporaryDirectory + .appending(path: "swiftlint-analyzer-\(UUID().uuidString)", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: temporaryDirectory) } + + let requests = zip(graph.indices, graph).map { index, job in + AnalyzerWorkerRequest( + job: job, + options: options, + resultURL: temporaryDirectory.appending( + path: "result-\(index).json", + directoryHint: .notDirectory + ) + ) + } + let runner = AnalyzerWorkerProcessRunner( + executableURL: try analyzerExecutableURL(), + currentDirectoryURL: preparedPlan.workingDirectory, + environment: ProcessInfo.processInfo.environment + ) + let invocations = try runner.prepare(requests: requests, graph: graph, in: temporaryDirectory) + let signalForwarder = AnalyzerSignalForwarder(runner: runner) + _ = signalForwarder + + let clock = ContinuousClock() + let runStarted = clock.now + let coordinator = AnalyzerProcessCoordinator( + jobs: options.analyzerJobs + ) { invocation in + let workerStarted = clock.now + let output = try await runner.run(invocation) + try validateWorkerResultJSONShape(output.resultData) + var result = try JSONDecoder().decode(AnalyzerWorkerResult.self, from: output.resultData) + guard let graphIndex = graph.firstIndex(where: { $0.jobId == invocation.jobId }) else { + throw AnalyzerWorkerContractError.extraJob + } + let job = graph[graphIndex] + try result.validate(for: job) + result.startOffsetSeconds = runStarted.duration(to: workerStarted).analyzerSeconds + result.durationSeconds = workerStarted.duration(to: clock.now).analyzerSeconds + return result + } + + do { + let results = try await coordinator.run(invocations) + let aggregate = try AnalyzerWorkerAggregate.merge(graph: graph, results: results) + return AnalyzerNativeRun( + aggregate: aggregate, + graph: graph, + results: results, + metrics: runner.metrics, + elapsedSeconds: runStarted.duration(to: clock.now).analyzerSeconds, + targetPlanSha256: preparedPlan.digest, + jobsRequested: options.analyzerJobs, + workingDirectory: preparedPlan.workingDirectory + ) + } catch { + runner.cancelAll() + throw error + } + } + + static func writeAnalyzerExecutionEvidence( + _ run: AnalyzerNativeRun, + findings: [StyleViolation], + reporterOutput: Data, + to path: String + ) throws { + let evidence = try AnalyzerExecutionEvidence.make( + targetPlanSha256: run.targetPlanSha256, + jobsRequested: run.jobsRequested, + peakConcurrentJobs: run.metrics.peakConcurrentJobs, + peakChildResidentMemoryBytes: run.metrics.peakAggregateResidentMemoryBytes, + elapsedSeconds: run.elapsedSeconds, + graph: run.graph, + results: run.results, + findings: findings, + reporterOutput: reporterOutput, + workingDirectory: run.workingDirectory + ) + var data = try AnalyzerJSON.encoder.encode(evidence) + data.append(0x0a) + try data.write(to: URL(filePath: path), options: .atomic) + } + + // swiftlint:disable:next function_body_length + static func runAnalyzerWorker(requestAt requestURL: URL) async throws { + let requestData = try Data(contentsOf: requestURL) + try validateWorkerRequestJSONShape(requestData) + let request = try JSONDecoder().decode(AnalyzerWorkerRequest.self, from: requestData) + try request.validate() + let compileCommandsData = try Data(contentsOf: URL(filePath: request.compileCommandsPath)) + guard AnalyzerTargetPlan.sha256(compileCommandsData) == request.compileCommandsSha256 else { + throw AnalyzerWorkerContractError.requestMismatch(request.jobId) + } + + let workerDirectory = URL( + filePath: FileManager.default.currentDirectoryPath, + directoryHint: .isDirectory + ) + let requestedURLs = request.requestedPaths.map { resolve($0, relativeTo: workerDirectory) } + let workerOptions = LintOrAnalyzeOptions( + mode: .analyze, + paths: requestedURLs, + useSTDIN: false, + configurationFiles: request.options.configurationFiles, + strict: false, + lenient: false, + forceExclude: request.options.forceExclude, + useExcludingByPrefix: request.options.useExcludingByPrefix, + useScriptInputFiles: false, + useScriptInputFileLists: false, + benchmark: true, + reporter: nil, + baseline: nil, + writeBaseline: nil, + workingDirectory: nil, + quiet: true, + output: nil, + progress: false, + cachePath: nil, + ignoreCache: true, + enableAllRules: false, + onlyRule: [request.ruleIdentifier], + autocorrect: request.options.autocorrect, + format: false, + disableSourceKit: request.options.disableSourceKit, + compilerLogPath: nil, + compileCommands: request.compileCommandsPath, + checkForUpdates: false + ) + let configuration = Configuration(options: workerOptions) + let configuredAnalyzerRules = configuration.rules + .filter { $0 is any AnalyzerRule } + .map { type(of: $0).identifier } + guard configuredAnalyzerRules == [request.ruleIdentifier] else { + throw AnalyzerWorkerContractError.requestMismatch(request.jobId) + } + + let storage = RuleStorage() + let collector = AnalyzerWorkerResultCollector() + let clock = ContinuousClock() + let started = clock.now + let files = try await configuration.visitLintableFiles( + options: workerOptions, + cache: nil, + storage: storage + ) { linter in + let fileStarted = clock.now + if request.options.autocorrect { + _ = linter.correct(using: storage) + collector.record( + violations: [], + fileTiming: AnalyzerFileTiming( + path: linter.file.path?.path ?? "", + seconds: fileStarted.duration(to: clock.now).analyzerSeconds + ), + ruleTimings: [] + ) + } else { + let (rawViolations, rawRuleTimes) = linter.styleViolationsAndRuleTimes(using: storage) + let violations = rawViolations.filter { $0.ruleIdentifier == request.ruleIdentifier } + collector.record( + violations: violations, + fileTiming: AnalyzerFileTiming( + path: linter.file.path?.path ?? "", + seconds: fileStarted.duration(to: clock.now).analyzerSeconds + ), + ruleTimings: rawRuleTimes + .filter { $0.id == request.ruleIdentifier } + .map { AnalyzerRuleTiming(ruleIdentifier: $0.id, seconds: $0.time) } + ) + } + } + let fileURLs = try files.map { file in + guard let path = file.path else { + throw AnalyzerWorkerContractError.resultMismatch(request.jobId) + } + return path.standardizedFileURL + } + let expectedURLs = requestedURLs + guard fileURLs.sorted(by: { $0.path < $1.path }) == expectedURLs.sorted(by: { $0.path < $1.path }) else { + throw AnalyzerWorkerContractError.resultMismatch( + "\(request.jobId) [file inventory expected=\(expectedURLs.map(\.path)) actual=\(fileURLs.map(\.path))]" + ) + } + let result = collector.result( + request: request, + files: fileURLs, + durationSeconds: started.duration(to: clock.now).analyzerSeconds + ) + try AnalyzerJSON.encoder.encode(result).write(to: request.resultURL, options: .atomic) + } + + private static func validateTargetInputs( + _ preparedPlan: AnalyzerPreparedTargetPlan, + graph: inout [AnalyzerJob] + ) throws { + let targetsByID = Dictionary(uniqueKeysWithValues: preparedPlan.plan.targets.map { ($0.targetId, $0) }) + for target in preparedPlan.plan.targets { + let root = resolve(target.sourceRoot, relativeTo: preparedPlan.workingDirectory) + guard isContained(root, in: preparedPlan.workingDirectory) else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + let sourceURLs = target.sourceFiles.map { resolve($0, relativeTo: preparedPlan.workingDirectory) } + guard sourceURLs.allSatisfy({ isContained($0, in: root) }), + sourceURLs.allSatisfy({ FileManager.default.fileExists(atPath: $0.path) }) else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + + let compileCommandsURL = resolve(target.compileCommandsPath, relativeTo: preparedPlan.planDirectory) + let compileCommandsData = try Data(contentsOf: compileCommandsURL) + guard AnalyzerTargetPlan.sha256(compileCommandsData) == target.compileCommandsSha256 else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + try validateCompilationDatabase( + compileCommandsData, + target: target, + workingDirectory: preparedPlan.workingDirectory + ) + } + + for index in graph.indices { + guard let target = targetsByID[graph[index].targetId] else { + throw AnalyzerTargetPlanError.invalidTarget(graph[index].targetId) + } + graph[index].compileCommandsPath = resolve( + target.compileCommandsPath, + relativeTo: preparedPlan.planDirectory + ).path + } + } + + private static func validateCompilationDatabase( + _ data: Data, + target: AnalyzerTargetPlan.Target, + workingDirectory: URL + ) throws { + guard let entries = try JSONSerialization.jsonObject(with: data) as? [[String: Any]], + entries.count == target.sourceFiles.count else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + let expectedFiles = Set(target.sourceFiles.map { resolve($0, relativeTo: workingDirectory).path }) + var actualFiles = Set() + for entry in entries { + guard let file = entry["file"] as? String, + let arguments = entry["arguments"] as? [String] else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + let absoluteFile = resolve(file, relativeTo: workingDirectory).path + guard actualFiles.insert(absoluteFile).inserted, + expectedFiles.contains(absoluteFile), + arguments.contains(absoluteFile), + expectedFiles.allSatisfy(arguments.contains), + arguments.adjacentPairs().contains(where: { + $0.0 == "-module-name" && $0.1 == target.moduleName + }) else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + } + guard actualFiles == expectedFiles else { + throw AnalyzerTargetPlanError.invalidTarget(target.targetId) + } + } + + private static func validateTargetPlanJSONShape(_ data: Data) throws { + guard let root = try JSONSerialization.jsonObject(with: data) as? [String: Any], + Set(root.keys) == [ + "schemaIdentity", "schemaVersion", "compilerLogSha256", "workingDirectory", "rules", "targets", + ], + let targets = root["targets"] as? [[String: Any]] else { + throw AnalyzerTargetPlanError.invalidSchema + } + for target in targets { + guard Set(target.keys) == [ + "targetId", "moduleName", "sourceRoot", "compileCommandsPath", "compileCommandsSha256", + "sourceFiles", "sourceFilesSha256", "rulePlans", + ], let rulePlans = target["rulePlans"] as? [[String: Any]] else { + throw AnalyzerTargetPlanError.invalidSchema + } + for rulePlan in rulePlans { + guard Set(rulePlan.keys) == ["rule", "mode", "batches"], + let batches = rulePlan["batches"] as? [[String: Any]], + batches.allSatisfy({ Set($0.keys) == ["batchIndex", "requestedPaths"] }) else { + throw AnalyzerTargetPlanError.invalidSchema + } + } + } + } + + private static func validateWorkerRequestJSONShape(_ data: Data) throws { + let requiredRootKeys: Set = [ + "schemaIdentity", "schemaVersion", "workerId", "jobId", "targetId", "moduleName", "sourceRoot", + "ruleIdentifier", "requestedPaths", "targetSourcePaths", "compileCommandsPath", + "compileCommandsSha256", "options", "resultURL", + ] + let allowedRootKeys = requiredRootKeys.union(["batchIndex"]) + let requiredOptionKeys: Set = [ + "configurationFiles", "strict", "lenient", "forceExclude", "useExcludingByPrefix", + "benchmark", "quiet", "disableSourceKit", "autocorrect", + ] + let allowedOptionKeys = requiredOptionKeys.union(["reporter"]) + guard let root = try JSONSerialization.jsonObject(with: data) as? [String: Any], + requiredRootKeys.isSubset(of: root.keys), + Set(root.keys).isSubset(of: allowedRootKeys), + let options = root["options"] as? [String: Any], + requiredOptionKeys.isSubset(of: options.keys), + Set(options.keys).isSubset(of: allowedOptionKeys) else { + throw AnalyzerWorkerContractError.requestMismatch("unknown") + } + } + + private static func validateWorkerResultJSONShape(_ data: Data) throws { + let requiredKeys: Set = [ + "schemaIdentity", "schemaVersion", "workerId", "jobId", "targetId", "ruleIdentifier", + "requestedPaths", "compileCommandsSha256", "violations", "files", "fileTimings", "ruleTimings", + "startOffsetSeconds", "durationSeconds", "exitCode", + ] + let allowedKeys = requiredKeys.union(["batchIndex"]) + guard let root = try JSONSerialization.jsonObject(with: data) as? [String: Any], + requiredKeys.isSubset(of: root.keys), + Set(root.keys).isSubset(of: allowedKeys) else { + throw AnalyzerWorkerContractError.resultMismatch("unknown") + } + } + + private static func resolve(_ path: String, relativeTo base: URL) -> URL { + let url = URL(filePath: path) + return (url.path.hasPrefix("/") ? url : base.appending(path: path)).standardizedFileURL + } + + private static func isContained(_ url: URL, in directory: URL) -> Bool { + let path = url.standardizedFileURL.path + let root = directory.standardizedFileURL.path + return path == root || path.hasPrefix(root.hasSuffix("/") ? root : root + "/") + } +} + +private func analyzerViolationPrecedes(_ lhs: StyleViolation, _ rhs: StyleViolation) -> Bool { + if lhs.location != rhs.location { return lhs.location < rhs.location } + if lhs.ruleIdentifier != rhs.ruleIdentifier { return lhs.ruleIdentifier < rhs.ruleIdentifier } + if lhs.reason != rhs.reason { return lhs.reason < rhs.reason } + return lhs.severity.rawValue < rhs.severity.rawValue +} + +private func analyzerFileTimingPrecedes(_ lhs: AnalyzerFileTiming, _ rhs: AnalyzerFileTiming) -> Bool { + lhs.path == rhs.path ? lhs.seconds < rhs.seconds : lhs.path < rhs.path +} + +private func analyzerRuleTimingPrecedes(_ lhs: AnalyzerRuleTiming, _ rhs: AnalyzerRuleTiming) -> Bool { + lhs.ruleIdentifier == rhs.ruleIdentifier + ? lhs.seconds < rhs.seconds + : lhs.ruleIdentifier < rhs.ruleIdentifier +} + +private extension Duration { + var analyzerSeconds: Double { + let components = components + return Double(components.seconds) + Double(components.attoseconds) / 1_000_000_000_000_000_000 + } +} + +private extension Collection { + func adjacentPairs() -> [(Element, Element)] { + zip(self, dropFirst()).map { ($0, $1) } + } +} diff --git a/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift b/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift index a6214bb8c62..a1a8735061c 100644 --- a/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift +++ b/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift @@ -58,6 +58,9 @@ package struct LintOrAnalyzeOptions { let compilerLogPath: String? let compileCommands: String? let checkForUpdates: Bool + let targetPlan: URL? + let analyzerJobs: Int + let executionEvidence: URL? package init(mode: LintOrAnalyzeMode, paths: [URL], @@ -86,7 +89,10 @@ package struct LintOrAnalyzeOptions { disableSourceKit: Bool, compilerLogPath: String?, compileCommands: String?, - checkForUpdates: Bool) { + checkForUpdates: Bool, + targetPlan: URL? = nil, + analyzerJobs: Int = 1, + executionEvidence: URL? = nil) { self.mode = mode self.paths = paths self.useSTDIN = useSTDIN @@ -115,6 +121,9 @@ package struct LintOrAnalyzeOptions { self.compilerLogPath = compilerLogPath self.compileCommands = compileCommands self.checkForUpdates = checkForUpdates + self.targetPlan = targetPlan + self.analyzerJobs = analyzerJobs + self.executionEvidence = executionEvidence } var verb: String { @@ -146,7 +155,97 @@ package struct LintOrAnalyzeCommand { } } try await Signposts.record(name: "LintOrAnalyzeCommand.run") { - try await options.autocorrect ? autocorrect(options) : lintOrAnalyze(options) + if options.mode == .analyze, options.targetPlan != nil || options.executionEvidence != nil { + try await targetAwareAnalyze(options) + } else if options.autocorrect { + try await autocorrect(options) + } else { + try await lintOrAnalyze(options) + } + } + } + + // Target-aware analysis deliberately keeps all final command semantics in this parent-owned stage. + // swiftlint:disable:next function_body_length + private static func targetAwareAnalyze(_ options: LintOrAnalyzeOptions) async throws { + let targetPlanPath = options.targetPlan?.path ?? "" + let executionEvidencePath = options.executionEvidence?.path ?? "" + try validateTargetAwareAnalyzeOptions( + targetPlanPath: targetPlanPath, + executionEvidencePath: executionEvidencePath, + jobs: options.analyzerJobs, + autocorrect: options.autocorrect + ) + + let preparedPlan = try prepareTargetAwareAnalyze(options) + let originalDirectory = FileManager.default.currentDirectoryPath + guard FileManager.default.changeCurrentDirectoryPath(preparedPlan.workingDirectory.path) else { + throw SwiftLintError.usageError( + description: "Could not enter analyzer target-plan working directory " + + "'\(preparedPlan.workingDirectory.path)'." + ) + } + defer { + if !FileManager.default.changeCurrentDirectoryPath(originalDirectory) { + queuedFatalError("Could not restore analyzer working directory '\(originalDirectory)'.") + } + } + + let nativeRun = try await executeTargetAwareAnalyze(options, preparedPlan: preparedPlan) + let lintedFiles = nativeRun.aggregate.files.compactMap { SwiftLintFile(path: $0) } + guard lintedFiles.count == nativeRun.aggregate.files.count else { + throw AnalyzerWorkerContractError.findingsMismatch + } + if options.autocorrect { + try writeAnalyzerExecutionEvidence( + nativeRun, + findings: [], + reporterOutput: Data(), + to: executionEvidencePath + ) + if !options.quiet { + printStatus(violations: [], files: lintedFiles, serious: 0, verb: options.verb) + } + return + } + + let builder = LintOrAnalyzeResultBuilder(options) + let violationsBeforeBaseline = applyLeniency( + options: options, + strict: builder.configuration.strict, + lenient: builder.configuration.lenient, + violations: nativeRun.aggregate.violations + ) + let baseline = try baseline(options, builder.configuration) + let groupedViolations = Dictionary(grouping: violationsBeforeBaseline) { + $0.location.file?.path ?? "" + } + let filteredViolations = groupedViolations.keys.sorted().flatMap { path in + let violations = groupedViolations[path, default: []] + return baseline?.filter(violations) ?? violations + } + builder.unfilteredViolations = violationsBeforeBaseline + builder.violations = filteredViolations + builder.report(violations: filteredViolations, realtimeCondition: true) + + if let baselineOutputPath = options.writeBaseline ?? builder.configuration.writeBaseline { + try Baseline(violations: builder.unfilteredViolations).write(toPath: baselineOutputPath) + } + let numberOfSeriousViolations = try Signposts.record(name: "LintOrAnalyzeCommand.PostProcessViolations") { + try postProcessViolations(files: lintedFiles, builder: builder) + } + let reporterOutput = options.output.flatMap { try? Data(contentsOf: $0) } ?? Data() + try writeAnalyzerExecutionEvidence( + nativeRun, + findings: builder.violations, + reporterOutput: reporterOutput, + to: executionEvidencePath + ) + if options.checkForUpdates || builder.configuration.checkForUpdates { + await UpdateChecker.checkForUpdates() + } + if numberOfSeriousViolations > 0 { + exit(2) } } diff --git a/Source/swiftlint/Commands/Analyze.swift b/Source/swiftlint/Commands/Analyze.swift index 0e81aa88248..7871aac337e 100644 --- a/Source/swiftlint/Commands/Analyze.swift +++ b/Source/swiftlint/Commands/Analyze.swift @@ -14,10 +14,27 @@ extension SwiftLint { var compilerLogPath: String? @Option(help: "The path of a compilation database to use when running AnalyzerRules.") var compileCommands: String? + @Option( + name: .customLong("target-plan"), + help: "The versioned target plan for process-isolated analysis." + ) + var targetPlan: URL? + @Option(help: "The global worker-process limit used with --target-plan.") + var jobs = 1 + @Option( + name: .customLong("execution-evidence"), + help: "The path where analyzer execution evidence should be written." + ) + var executionEvidence: URL? @Argument(help: pathsArgumentDescription(for: .analyze)) var paths = [URL]() func run() async throws { + if let requestPath = ProcessInfo.processInfo.environment[AnalyzerWorkerEnvironment.requestPath] { + try await LintOrAnalyzeCommand.runAnalyzerWorker(requestAt: URL(filePath: requestPath)) + return + } + // Analyze files in current working directory if no paths were specified. let allPaths = paths.isNotEmpty ? paths : [URL.cwd] let options = LintOrAnalyzeOptions( @@ -48,7 +65,10 @@ extension SwiftLint { disableSourceKit: false, compilerLogPath: compilerLogPath, compileCommands: compileCommands, - checkForUpdates: common.checkForUpdates + checkForUpdates: common.checkForUpdates, + targetPlan: targetPlan, + analyzerJobs: jobs, + executionEvidence: executionEvidence ) try await LintOrAnalyzeCommand.run(options) diff --git a/Tests/FrameworkTests/AnalyzerProcessCoordinatorTests.swift b/Tests/FrameworkTests/AnalyzerProcessCoordinatorTests.swift new file mode 100644 index 00000000000..fe5b7b408b2 --- /dev/null +++ b/Tests/FrameworkTests/AnalyzerProcessCoordinatorTests.swift @@ -0,0 +1,824 @@ +import Foundation +import SourceKittenFramework +import SwiftLintCore +import Testing + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +// This is the only plan-owned compiled test file for the complete local worker protocol. +// swiftlint:disable file_length + +@testable import SwiftLintBuiltInRules +@testable import SwiftLintFramework + +private let runningUnderBazel = ProcessInfo.processInfo.environment["TEST_SRCDIR"] != nil + +// swiftlint:disable type_body_length +@Suite +struct AnalyzerProcessCoordinatorTests { + @Test + func targetPlanBuildsDeterministicTargetRuleBatchOrder() throws { + let plan = targetPlan() + + let graph = try plan.validatedJobGraph(capabilities: capabilities()) + + #expect(graph.map(\.jobId) == [ + "application/capture_variable", + "application/unused_import/batch-000", + "application/unused_import/batch-001", + "unitTests/capture_variable", + "unitTests/unused_import/batch-000", + ]) + #expect(graph.map(\.workerId) == [ + "worker-0000", + "worker-0001", + "worker-0002", + "worker-0003", + "worker-0004", + ]) + } + + @Test + func coordinatorUsesOneGlobalBoundAndReturnsGraphOrder() async throws { + let probe = ConcurrencyProbe() + let coordinator = AnalyzerProcessCoordinator(jobs: 2) { value in + await probe.started() + try await Task.sleep(for: .milliseconds(value.isMultiple(of: 2) ? 40 : 10)) + await probe.finished() + return value + } + + let values = try await coordinator.run([0, 1, 2, 3, 4]) + + #expect(values == [0, 1, 2, 3, 4]) + #expect(await probe.maximumActiveJobs == 2) + } + + @Test + func coordinatorCancelsPeersAfterFirstFailure() async { + let probe = CancellationProbe() + let coordinator = AnalyzerProcessCoordinator(jobs: 2) { value in + await probe.started(value) + await probe.waitUntilBothStarted() + if value == 1 { + throw TestWorkerError.failed + } + do { + try await Task.sleep(for: .seconds(5)) + return value + } catch is CancellationError { + await probe.cancelled(value) + throw CancellationError() + } catch { + throw error + } + } + + do { + _ = try await coordinator.run([0, 1]) + Issue.record("Expected the first worker failure to reject the run") + } catch TestWorkerError.failed { + await probe.waitUntilCancelled(0) + #expect(await probe.cancelledRequests == [0]) + } catch { + Issue.record("Unexpected error: \(error)") + } + } + + @Test + func unusedImportDeclaresTheOnlyInitialBatchCapability() { + let unusedImport = UnusedImportRule() + + #expect(unusedImport.analyzerBatchSize == 32) + } + + @Test + func batchedJobKeepsFullTargetCompileCommandsContext() throws { + let graph = try targetPlan().validatedJobGraph(capabilities: capabilities()) + let batched = try #require(graph.first(where: { $0.jobId == "application/unused_import/batch-000" })) + + #expect(batched.requestedPaths == ["Sources/A.swift", "Sources/B.swift"]) + #expect(batched.targetSourcePaths == ["Sources/A.swift", "Sources/B.swift", "Sources/C.swift"]) + #expect(batched.compileCommandsPath == "application.compile-commands.json") + #expect(batched.compileCommandsSha256 == String(repeating: "a", count: 64)) + } + + @Test + func collectingRuleRemainsOneWholeTargetJob() throws { + let graph = try targetPlan().validatedJobGraph(capabilities: capabilities()) + let job = try #require(graph.first) + + #expect(job.ruleIdentifier == "capture_variable") + #expect(job.batchIndex == nil) + #expect(job.requestedPaths == ["Sources/A.swift", "Sources/B.swift", "Sources/C.swift"]) + } + + @Test(arguments: invalidPlanMutations()) + fileprivate func planRejectsIncompleteOrAmbiguousBatchCoverage(mutation: InvalidPlanMutation) { + var plan = targetPlan() + mutation.mutate(&plan) + + #expect(throws: AnalyzerTargetPlanError.self) { + try plan.validatedJobGraph(capabilities: capabilities()) + } + } + + @Test + func planRejectsBatchingWithoutAnExplicitRuleCapability() { + var plan = targetPlan() + plan.targets[0].rulePlans[0] = .init( + rule: "capture_variable", + mode: .batches, + batches: [.init(batchIndex: 0, requestedPaths: plan.targets[0].sourceFiles)] + ) + + #expect(throws: AnalyzerTargetPlanError.self) { + try plan.validatedJobGraph(capabilities: capabilities()) + } + } + + @Test + func planRejectsBatchLargerThanRuleCapability() { + let sources = (0..<33).map { "Sources/File\($0).swift" } + var plan = targetPlan() + plan.targets[0].sourceFiles = sources + plan.targets[0].sourceFilesSha256 = AnalyzerTargetPlan.sourceInventoryDigest(sources) + plan.targets[0].rulePlans[1].batches = [.init(batchIndex: 0, requestedPaths: sources)] + + #expect(throws: AnalyzerTargetPlanError.self) { + try plan.validatedJobGraph(capabilities: capabilities()) + } + } + + @Test + func planRejectsDuplicateTargetAndRuleIdentities() { + var duplicateTarget = targetPlan() + duplicateTarget.targets.append(duplicateTarget.targets[0]) + var duplicateRule = targetPlan() + duplicateRule.rules.append("unused_import") + + #expect(throws: AnalyzerTargetPlanError.self) { + try duplicateTarget.validatedJobGraph(capabilities: capabilities()) + } + #expect(throws: AnalyzerTargetPlanError.self) { + try duplicateRule.validatedJobGraph(capabilities: capabilities()) + } + } + + @Test + func workerRequestRejectsIdentityAndInventoryDrift() throws { + let job = try #require(targetPlan().validatedJobGraph(capabilities: capabilities()).first) + let request = AnalyzerWorkerRequest( + job: job, + options: analyzeOptions(), + resultURL: URL(filePath: "/tmp/result") + ) + var wrongJob = request + wrongJob.jobId = "unexpected/job" + var wrongSources = request + wrongSources.requestedPaths = ["Sources/Unexpected.swift"] + + try request.validate(for: job) + #expect(throws: AnalyzerWorkerContractError.self) { + try wrongJob.validate(for: job) + } + #expect(throws: AnalyzerWorkerContractError.self) { + try wrongSources.validate(for: job) + } + } + + @Test + func aggregateRejectsDuplicateWorkerAndJobIdentities() throws { + let graph = try targetPlan().validatedJobGraph(capabilities: capabilities()) + let first = workerResult(job: graph[0]) + var duplicateWorker = workerResult(job: graph[1]) + duplicateWorker.workerId = first.workerId + var duplicateJob = workerResult(job: graph[1]) + duplicateJob.jobId = first.jobId + + #expect(throws: AnalyzerWorkerContractError.self) { + try AnalyzerWorkerAggregate.merge(graph: Array(graph.prefix(2)), results: [first, duplicateWorker]) + } + #expect(throws: AnalyzerWorkerContractError.self) { + try AnalyzerWorkerAggregate.merge(graph: Array(graph.prefix(2)), results: [first, duplicateJob]) + } + } + + @Test + func aggregateRejectsMissingAndExtraJobs() throws { + let graph = try targetPlan().validatedJobGraph(capabilities: capabilities()) + let firstTwo = Array(graph.prefix(2)) + + #expect(throws: AnalyzerWorkerContractError.self) { + try AnalyzerWorkerAggregate.merge(graph: firstTwo, results: [workerResult(job: firstTwo[0])]) + } + #expect(throws: AnalyzerWorkerContractError.self) { + try AnalyzerWorkerAggregate.merge( + graph: [firstTwo[0]], + results: firstTwo.map(workerResult) + ) + } + } + + @Test + func aggregateRejectsUnexpectedDuplicateRawFindings() throws { + let graph = try targetPlan().validatedJobGraph(capabilities: capabilities()) + let violation = violation(rule: "unused_import", file: "Sources/A.swift", line: 1) + var first = workerResult(job: graph[1]) + first.violations = [violation] + var second = workerResult(job: graph[2]) + second.violations = [violation] + + #expect(throws: AnalyzerWorkerContractError.self) { + try AnalyzerWorkerAggregate.merge(graph: Array(graph[1...2]), results: [first, second]) + } + } + + @Test + func aggregateMergesFindingsAndReporterInputDeterministically() throws { + let graph = try targetPlan().validatedJobGraph(capabilities: capabilities()) + let firstViolation = violation(rule: "capture_variable", file: "Sources/C.swift", line: 3) + let secondViolation = violation(rule: "unused_import", file: "Sources/A.swift", line: 1) + var first = workerResult(job: graph[0]) + first.violations = [firstViolation] + var second = workerResult(job: graph[1]) + second.violations = [secondViolation] + + let forward = try AnalyzerWorkerAggregate.merge(graph: Array(graph.prefix(2)), results: [first, second]) + let reverse = try AnalyzerWorkerAggregate.merge(graph: Array(graph.prefix(2)), results: [second, first]) + + #expect(forward.violations == [secondViolation, firstViolation]) + #expect(forward == reverse) + #expect(JSONReporter.generateReport(forward.violations) == JSONReporter.generateReport(reverse.violations)) + #expect(XcodeReporter.generateReport(forward.violations) == XcodeReporter.generateReport(reverse.violations)) + } + + @Test + func executionEvidenceRejectsDuplicateIdentitiesAndWrongProvenance() throws { + let graph = try targetPlan().validatedJobGraph(capabilities: capabilities()) + let results = graph.map(workerResult) + var evidence = try AnalyzerExecutionEvidence.make( + targetPlanSha256: String(repeating: "f", count: 64), + jobsRequested: 4, + peakConcurrentJobs: 4, + peakChildResidentMemoryBytes: 1024, + elapsedSeconds: 1, + graph: graph, + results: results, + findings: [], + reporterOutput: Data("[]\n".utf8) + ) + + try evidence.validate(graph: graph, targetPlanSha256: String(repeating: "f", count: 64)) + evidence.targetPlanSha256 = String(repeating: "0", count: 64) + #expect(throws: AnalyzerWorkerContractError.self) { + try evidence.validate(graph: graph, targetPlanSha256: String(repeating: "f", count: 64)) + } + } + + @Test + func workerFailurePreservesExitAndDiagnostics() { + let error = AnalyzerWorkerContractError.workerFailed( + job: "application/unused_import/batch-000", + reason: "signal 9", + diagnostics: "SourceKit terminated" + ) + + #expect(error.localizedDescription.contains("signal 9")) + #expect(error.localizedDescription.contains("SourceKit terminated")) + } + + @Test + func targetAwareOptionsRequirePositiveJobsAndSerialAutocorrection() throws { + try LintOrAnalyzeCommand.validateTargetAwareAnalyzeOptions( + targetPlanPath: "/tmp/plan.json", + executionEvidencePath: "/tmp/evidence.json", + jobs: 1, + autocorrect: true + ) + try LintOrAnalyzeCommand.validateTargetAwareAnalyzeOptions( + targetPlanPath: "/tmp/plan.json", + executionEvidencePath: "/tmp/evidence.json", + jobs: 4, + autocorrect: false + ) + + #expect(throws: SwiftLintError.self) { + try LintOrAnalyzeCommand.validateTargetAwareAnalyzeOptions( + targetPlanPath: "/tmp/plan.json", + executionEvidencePath: "/tmp/evidence.json", + jobs: 0, + autocorrect: false + ) + } + #expect(throws: SwiftLintError.self) { + try LintOrAnalyzeCommand.validateTargetAwareAnalyzeOptions( + targetPlanPath: "/tmp/plan.json", + executionEvidencePath: "/tmp/evidence.json", + jobs: 2, + autocorrect: true + ) + } + } + + @Test + func processRunnerPreservesCrashExitAndDiagnostics() async throws { + let fixture = try ProcessFixture() + defer { fixture.remove() } + let runner = fixture.runner() + let invocation = fixture.invocation( + jobId: "crashing-worker", + script: "printf 'SourceKit terminated' >&2; exit 7" + ) + + do { + _ = try await runner.run(invocation) + Issue.record("Expected the crashing worker to fail") + } catch let error as AnalyzerWorkerContractError { + #expect(error.localizedDescription.contains("exit 7")) + #expect(error.localizedDescription.contains("SourceKit terminated")) + } + } + + @Test + func cancellationTerminatesWorkerDescendants() async throws { + let fixture = try ProcessFixture() + defer { fixture.remove() } + let descendantPID = fixture.directory.appending(path: "descendant.pid") + let runner = fixture.runner() + let invocation = fixture.invocation( + jobId: "cancelled-worker", + script: "sleep 30 & child=$!; printf '%s' \"$child\" > \"$PID_FILE\"; wait" + ) + let task = Task { try await runner.run(invocation) } + let processIdentifier = try await waitForProcessIdentifier(at: descendantPID) + + task.cancel() + await #expect(throws: CancellationError.self) { + try await task.value + } + #expect(await processExits(processIdentifier)) + } + + @Test + func forwardedSignalTerminatesWorkerDescendants() async throws { + let fixture = try ProcessFixture() + defer { fixture.remove() } + let descendantPID = fixture.directory.appending(path: "forwarded-descendant.pid") + let runner = fixture.runner() + let invocation = fixture.invocation( + jobId: "forwarded-worker", + script: "sleep 30 & child=$!; printf '%s' \"$child\" > \"$PID_FILE\"; wait" + ) + let task = Task { try await runner.run(invocation) } + let processIdentifier = try await waitForProcessIdentifier(at: descendantPID) + + runner.forward(signal: SIGTERM) + await #expect(throws: CancellationError.self) { + try await task.value + } + #expect(await processExits(processIdentifier)) + } + + @Test + func cancellationCannotMissAWorkerDuringSpawnRegistration() async throws { + for index in 0..<20 { + let fixture = try ProcessFixture(suffix: "-\(index)") + defer { fixture.remove() } + let runner = fixture.runner() + let invocation = fixture.invocation( + jobId: "spawn-race-\(index)", + script: "sleep 30" + ) + let task = Task { try await runner.run(invocation) } + + runner.cancelAll() + await #expect(throws: CancellationError.self) { + try await task.value + } + #expect(runner.metrics.peakConcurrentJobs <= 1) + } + } + + // swiftlint:disable function_body_length + @Test(.disabled(if: runningUnderBazel, "The Bazel FrameworkTests target does not stage the swiftlint executable.")) + func targetAwareAnalyzeCommandPublishesReporterAndEvidence() throws { + let fixture = try ProcessFixture() + defer { fixture.remove() } + let sourceURL = fixture.directory.appending(path: "Source.swift") + try Data("let value = 1\n".utf8).write(to: sourceURL) + + let compileCommandsURL = fixture.directory.appending(path: "compile-commands.json") + var compilerArguments = ["-module-name", "Fixture"] + compilerArguments += try platformCompilerContextArguments() + compilerArguments.append(sourceURL.path) + let compileCommands: [[String: Any]] = [ + [ + "directory": fixture.directory.path, + "file": sourceURL.path, + "arguments": compilerArguments, + ], + ] + let compileCommandsData = try JSONSerialization.data( + withJSONObject: compileCommands, + options: [.sortedKeys, .withoutEscapingSlashes] + ) + try compileCommandsData.write(to: compileCommandsURL) + + let sourceFiles = ["Source.swift"] + let plan = AnalyzerTargetPlan( + schemaIdentity: AnalyzerTargetPlan.schemaIdentity, + schemaVersion: AnalyzerTargetPlan.currentVersion, + compilerLogSha256: String(repeating: "c", count: 64), + workingDirectory: fixture.directory.path, + rules: ["unused_import"], + targets: [ + .init( + targetId: "fixture", + moduleName: "Fixture", + sourceRoot: ".", + compileCommandsPath: compileCommandsURL.lastPathComponent, + compileCommandsSha256: AnalyzerTargetPlan.sha256(compileCommandsData), + sourceFiles: sourceFiles, + sourceFilesSha256: AnalyzerTargetPlan.sourceInventoryDigest(sourceFiles), + rulePlans: [ + .init( + rule: "unused_import", + mode: .batches, + batches: [.init(batchIndex: 0, requestedPaths: sourceFiles)] + ), + ] + ), + ] + ) + let planURL = fixture.directory.appending(path: "target-plan.json") + try AnalyzerJSON.encoder.encode(plan).write(to: planURL) + let reporterURL = fixture.directory.appending(path: "reporter.json") + let evidenceURL = fixture.directory.appending(path: "execution-evidence.json") + + let process = Process() + process.executableURL = try testSwiftLintExecutable() + process.currentDirectoryURL = URL(filePath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + process.arguments = [ + "analyze", + "--target-plan", planURL.path, + "--jobs", "1", + "--execution-evidence", evidenceURL.path, + "--only-rule", "unused_import", + "--reporter", "json", + "--output", reporterURL.path, + "--quiet", + ] + let standardOutput = Pipe() + let standardError = Pipe() + process.standardOutput = standardOutput + process.standardError = standardError + try process.run() + process.waitUntilExit() + + let diagnostics = String( + data: standardError.fileHandleForReading.readDataToEndOfFile(), + encoding: .utf8 + ) ?? "" + #expect(process.terminationStatus == 0, Comment(rawValue: diagnostics)) + #expect( + try Data(contentsOf: reporterURL) + == Data((JSONReporter.generateReport([]) + "\n").utf8) + ) + let evidence = try JSONDecoder().decode( + AnalyzerExecutionEvidence.self, + from: Data(contentsOf: evidenceURL) + ) + #expect(evidence.schemaIdentity == AnalyzerExecutionEvidence.schemaIdentity) + #expect(evidence.jobsRequested == 1) + #expect(evidence.peakConcurrentJobs == 1) + #expect(evidence.workers.map(\.jobId) == ["fixture/unused_import/batch-000"]) + #expect(evidence.findingsCount == 0) + } + // swiftlint:enable function_body_length +} +// swiftlint:enable type_body_length + +struct InvalidPlanMutation: Sendable, CustomTestStringConvertible { + let name: String + let mutate: @Sendable (inout AnalyzerTargetPlan) -> Void + + var testDescription: String { name } +} + +private func invalidPlanMutations() -> [InvalidPlanMutation] { + [ + .init(name: "missing path") { plan in + plan.targets[0].rulePlans[1].batches[1].requestedPaths = [] + }, + .init(name: "duplicate path") { plan in + plan.targets[0].rulePlans[1].batches[1].requestedPaths = ["Sources/B.swift", "Sources/C.swift"] + }, + .init(name: "overlapping path") { plan in + plan.targets[0].rulePlans[1].batches[0].requestedPaths.append("Sources/C.swift") + }, + .init(name: "extra path") { plan in + plan.targets[0].rulePlans[1].batches[1].requestedPaths.append("Sources/Unexpected.swift") + }, + .init(name: "duplicate batch index") { plan in + plan.targets[0].rulePlans[1].batches[1].batchIndex = 0 + }, + ] +} + +private func targetPlan() -> AnalyzerTargetPlan { + let applicationSources = ["Sources/A.swift", "Sources/B.swift", "Sources/C.swift"] + let unitTestSources = ["Tests/A.swift"] + return AnalyzerTargetPlan( + schemaIdentity: AnalyzerTargetPlan.schemaIdentity, + schemaVersion: AnalyzerTargetPlan.currentVersion, + compilerLogSha256: String(repeating: "c", count: 64), + workingDirectory: "ios", + rules: ["capture_variable", "unused_import"], + targets: [ + .init( + targetId: "application", + moduleName: "FixtureApp", + sourceRoot: "Sources", + compileCommandsPath: "application.compile-commands.json", + compileCommandsSha256: String(repeating: "a", count: 64), + sourceFiles: applicationSources, + sourceFilesSha256: AnalyzerTargetPlan.sourceInventoryDigest(applicationSources), + rulePlans: [ + .init(rule: "capture_variable", mode: .wholeTarget, batches: []), + .init(rule: "unused_import", mode: .batches, batches: [ + .init(batchIndex: 0, requestedPaths: Array(applicationSources.prefix(2))), + .init(batchIndex: 1, requestedPaths: Array(applicationSources.suffix(1))), + ]), + ] + ), + .init( + targetId: "unitTests", + moduleName: "FixtureAppTests", + sourceRoot: "Tests", + compileCommandsPath: "unitTests.compile-commands.json", + compileCommandsSha256: String(repeating: "b", count: 64), + sourceFiles: unitTestSources, + sourceFilesSha256: AnalyzerTargetPlan.sourceInventoryDigest(unitTestSources), + rulePlans: [ + .init(rule: "capture_variable", mode: .wholeTarget, batches: []), + .init(rule: "unused_import", mode: .batches, batches: [ + .init(batchIndex: 0, requestedPaths: unitTestSources), + ]), + ] + ), + ] + ) +} + +private func capabilities() -> [String: AnalyzerRuleCapability] { + [ + "capture_variable": .init(isCollecting: true, maximumBatchSize: nil), + "unused_import": .init(isCollecting: false, maximumBatchSize: 32), + ] +} + +private func analyzeOptions() -> LintOrAnalyzeOptions { + LintOrAnalyzeOptions( + mode: .analyze, + paths: [URL(filePath: "/tmp/Sources")], + useSTDIN: false, + configurationFiles: [], + strict: false, + lenient: false, + forceExclude: false, + useExcludingByPrefix: false, + useScriptInputFiles: false, + useScriptInputFileLists: false, + benchmark: false, + reporter: "json", + baseline: nil, + writeBaseline: nil, + workingDirectory: nil, + quiet: true, + output: nil, + progress: false, + cachePath: nil, + ignoreCache: true, + enableAllRules: false, + onlyRule: [], + autocorrect: false, + format: false, + disableSourceKit: false, + compilerLogPath: nil, + compileCommands: nil, + checkForUpdates: false + ) +} + +private func workerResult(job: AnalyzerJob) -> AnalyzerWorkerResult { + AnalyzerWorkerResult( + schemaIdentity: AnalyzerWorkerResult.schemaIdentity, + schemaVersion: AnalyzerWorkerResult.currentVersion, + workerId: job.workerId, + jobId: job.jobId, + targetId: job.targetId, + ruleIdentifier: job.ruleIdentifier, + batchIndex: job.batchIndex, + requestedPaths: job.requestedPaths, + compileCommandsSha256: job.compileCommandsSha256, + violations: [], + files: job.requestedPaths.map { URL(filePath: $0) }, + fileTimings: [], + ruleTimings: [], + startOffsetSeconds: 0, + durationSeconds: 1, + exitCode: 0 + ) +} + +private func violation(rule: String, file: String, line: Int) -> StyleViolation { + StyleViolation( + ruleDescription: RuleDescription(identifier: rule, name: rule, description: rule, kind: .lint), + location: Location(file: URL(filePath: file), line: line, character: 1) + ) +} + +private enum TestWorkerError: Error { + case failed +} + +private actor ConcurrencyProbe { + private var activeJobs = 0 + private(set) var maximumActiveJobs = 0 + + func started() { + activeJobs += 1 + maximumActiveJobs = max(maximumActiveJobs, activeJobs) + } + + func finished() { + activeJobs -= 1 + } +} + +private actor CancellationProbe { + private var startedRequests = Set() + private var startWaiters = [CheckedContinuation]() + private var cancellationWaiters = [Int: [CheckedContinuation]]() + private(set) var cancelledRequests = Set() + + func started(_ request: Int) { + startedRequests.insert(request) + guard startedRequests.count == 2 else { return } + startWaiters.forEach { $0.resume() } + startWaiters.removeAll() + } + + func waitUntilBothStarted() async { + guard startedRequests.count < 2 else { return } + await withCheckedContinuation { continuation in + startWaiters.append(continuation) + } + } + + func cancelled(_ request: Int) { + cancelledRequests.insert(request) + cancellationWaiters.removeValue(forKey: request)?.forEach { $0.resume() } + } + + func waitUntilCancelled(_ request: Int) async { + guard !cancelledRequests.contains(request) else { return } + await withCheckedContinuation { continuation in + cancellationWaiters[request, default: []].append(continuation) + } + } +} + +private struct ProcessFixture { + let directory: URL + + init(suffix: String = "") throws { + directory = FileManager.default.temporaryDirectory.appending( + path: "swiftlint-process-runner-tests-\(UUID().uuidString)\(suffix)", + directoryHint: .isDirectory + ) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: false) + } + + func remove() { + try? FileManager.default.removeItem(at: directory) + } + + func runner() -> AnalyzerWorkerProcessRunner { + AnalyzerWorkerProcessRunner( + executableURL: URL(filePath: "/bin/sh"), + currentDirectoryURL: directory, + environment: ProcessInfo.processInfo.environment + ) + } + + func invocation(jobId: String, script: String) -> AnalyzerWorkerInvocation { + let identifier = jobId.replacingOccurrences(of: "/", with: "-") + let requestURL = directory.appending(path: "\(identifier)-request.json") + let resultURL = directory.appending(path: "\(identifier)-result.json") + let standardOutputURL = directory.appending(path: "\(identifier)-stdout.txt") + let standardErrorURL = directory.appending(path: "\(identifier)-stderr.txt") + FileManager.default.createFile(atPath: standardOutputURL.path, contents: nil) + FileManager.default.createFile(atPath: standardErrorURL.path, contents: nil) + var environment = ProcessInfo.processInfo.environment + environment["PID_FILE"] = directory.appending(path: jobId.contains("forwarded") + ? "forwarded-descendant.pid" + : "descendant.pid").path + return AnalyzerWorkerInvocation( + jobId: jobId, + requestURL: requestURL, + resultURL: resultURL, + standardOutputURL: standardOutputURL, + standardErrorURL: standardErrorURL, + arguments: ["-c", script], + environment: environment + ) + } +} + +private func waitForProcessIdentifier(at url: URL) async throws -> pid_t { + for _ in 0..<200 { + if let contents = try? String(contentsOf: url, encoding: .utf8), + let processIdentifier = pid_t(contents) { + return processIdentifier + } + try await Task.sleep(for: .milliseconds(10)) + } + throw TestProcessError.pidNotWritten +} + +private func platformCompilerContextArguments() throws -> [String] { + #if os(macOS) + let process = Process() + process.executableURL = URL(filePath: "/usr/bin/xcrun") + process.arguments = ["--sdk", "macosx", "--show-sdk-path"] + let output = Pipe() + process.standardOutput = output + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0, + let sdkRoot = String( + data: output.fileHandleForReading.readDataToEndOfFile(), + encoding: .utf8 + )?.trimmingCharacters(in: .whitespacesAndNewlines), + !sdkRoot.isEmpty else { + throw AnalyzerProcessRunnerError.invalidExecutable("macOS SDK") + } + #if arch(arm64) + let target = "arm64-apple-macos14.0" + #elseif arch(x86_64) + let target = "x86_64-apple-macos14.0" + #endif + return ["-target", target, "-sdk", sdkRoot] + #else + return [] + #endif +} + +private func testSwiftLintExecutable() throws -> URL { + let packageRoot = URL(filePath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + let candidates = [ + Bundle.main.bundleURL.deletingLastPathComponent().appending(path: "swiftlint"), + packageRoot.appending(path: ".build/debug/swiftlint"), + ] + guard let executable = candidates.first(where: { + FileManager.default.isExecutableFile(atPath: $0.path) + }) else { + throw AnalyzerProcessRunnerError.invalidExecutable("swiftlint test product") + } + return executable +} + +private func processExits(_ processIdentifier: pid_t) async -> Bool { + for _ in 0..<200 { + if kill(processIdentifier, 0) == -1, errno == ESRCH { + return true + } + #if canImport(Glibc) + if let stat = try? String(contentsOfFile: "/proc/\(processIdentifier)/stat", encoding: .utf8), + let commandEnd = stat.range(of: ")", options: .backwards)?.upperBound { + let fields = stat[commandEnd...].split(whereSeparator: \.isWhitespace) + if fields.first == "Z" || fields.first == "X" { + return true + } + } + #endif + try? await Task.sleep(for: .milliseconds(10)) + } + return false +} + +private enum TestProcessError: Error { + case pidNotWritten +}