From 3863edca55bd40d0adf59b4dc296af2f2e2082f3 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 30 Oct 2024 19:14:14 +0000 Subject: [PATCH 01/33] Clean up binary artifacts code --- .../Workspace/Workspace+BinaryArtifacts.swift | 91 +++++++------------ 1 file changed, 35 insertions(+), 56 deletions(-) diff --git a/Sources/Workspace/Workspace+BinaryArtifacts.swift b/Sources/Workspace/Workspace+BinaryArtifacts.swift index de8033c5a57..d2b9cafc76e 100644 --- a/Sources/Workspace/Workspace+BinaryArtifacts.swift +++ b/Sources/Workspace/Workspace+BinaryArtifacts.swift @@ -156,7 +156,7 @@ extension Workspace { let jsonDecoder = JSONDecoder.makeWithDefaults() for indexFile in indexFiles { group.enter() - var request = LegacyHTTPClient.Request(method: .get, url: indexFile.url) + var request = HTTPClient.Request(method: .get, url: indexFile.url) request.options.validResponseCodes = [200] request.options.authorizationProvider = self.authorizationProvider?.httpAuthorizationHeader(for:) self.httpClient.execute(request) { result in @@ -569,7 +569,6 @@ extension Workspace { } public func cancel(deadline: DispatchTime) throws { - try self.httpClient.cancel(deadline: deadline) if let cancellableArchiver = self.archiver as? Cancellable { try cancellableArchiver.cancel(deadline: deadline) } @@ -579,34 +578,27 @@ extension Workspace { artifact: RemoteArtifact, destination: AbsolutePath, observabilityScope: ObservabilityScope, - progress: @escaping @Sendable (Int64, Optional) -> Void, - completion: @escaping (Result) -> Void - ) { + progress: @escaping @Sendable (Int64, Optional) -> Void + ) async throws -> Bool { // not using cache, download directly guard let cachePath = self.cachePath else { self.delegate?.willDownloadBinaryArtifact(from: artifact.url.absoluteString, fromCache: false) - return self.download( + try await self.download( artifact: artifact, destination: destination, observabilityScope: observabilityScope, - progress: progress, - completion: { result in - // not fetched from cache - completion(result.map{ _ in false }) - } + progress: progress ) + + // not fetched from cache + return false } // initialize cache if necessary - do { - if !self.fileSystem.exists(cachePath) { - try self.fileSystem.createDirectory(cachePath, recursive: true) - } - } catch { - return completion(.failure(error)) + if !self.fileSystem.exists(cachePath) { + try self.fileSystem.createDirectory(cachePath, recursive: true) } - // try to fetch from cache, or download and cache // / FIXME: use better escaping of URL let cacheKey = artifact.url.absoluteString.spm_mangledToC99ExtendedIdentifier() @@ -615,50 +607,43 @@ extension Workspace { if self.fileSystem.exists(cachedArtifactPath) { observabilityScope.emit(debug: "copying cached binary artifact for \(artifact.url) from \(cachedArtifactPath)") self.delegate?.willDownloadBinaryArtifact(from: artifact.url.absoluteString, fromCache: true) - return completion( - Result.init(catching: { - // copy from cache to destination - try self.fileSystem.copy(from: cachedArtifactPath, to: destination) - return true // fetched from cache - }) - ) + + // copy from cache to destination + try self.fileSystem.copy(from: cachedArtifactPath, to: destination) + return true // fetched from cache } // download to the cache - observabilityScope.emit(debug: "downloading binary artifact for \(artifact.url) to cached at \(cachedArtifactPath)") - self.download( - artifact: artifact, - destination: cachedArtifactPath, - observabilityScope: observabilityScope, - progress: progress, - completion: { result in - self.delegate?.willDownloadBinaryArtifact(from: artifact.url.absoluteString, fromCache: false) - if case .failure = result { - try? self.fileSystem.removeFileTree(cachedArtifactPath) - } - completion(result.flatMap { - Result.init(catching: { - // copy from cache to destination - try self.fileSystem.copy(from: cachedArtifactPath, to: destination) - return false // not fetched from cache - }) - }) - } - ) + observabilityScope.emit(debug: "downloading binary artifact for \(artifact.url) to cache at \(cachedArtifactPath)") + + self.delegate?.willDownloadBinaryArtifact(from: artifact.url.absoluteString, fromCache: false) + + do { + try await self.download( + artifact: artifact, + destination: cachedArtifactPath, + observabilityScope: observabilityScope, + progress: progress + ) + try self.fileSystem.copy(from: cachedArtifactPath, to: destination) + return false // not fetched from cache + } catch { + try? self.fileSystem.removeFileTree(cachedArtifactPath) + throw error + } } private func download( artifact: RemoteArtifact, destination: AbsolutePath, observabilityScope: ObservabilityScope, - progress: @escaping @Sendable (Int64, Optional) -> Void, - completion: @escaping (Result) -> Void - ) { + progress: @escaping @Sendable (Int64, Optional) -> Void + ) async throws { observabilityScope.emit(debug: "downloading \(artifact.url) to \(destination)") var headers = HTTPClientHeaders() headers.add(name: "Accept", value: "application/octet-stream") - var request = LegacyHTTPClient.Request.download( + var request = HTTPClient.Request.download( url: artifact.url, headers: headers, fileSystem: self.fileSystem, @@ -668,13 +653,7 @@ extension Workspace { request.options.retryStrategy = .exponentialBackoff(maxAttempts: 3, baseDelay: .milliseconds(50)) request.options.validResponseCodes = [200] - self.httpClient.execute( - request, - progress: progress, - completion: { result in - completion(result.map{ _ in Void() }) - } - ) + _ = try await self.httpClient.execute(request, progress: progress) } } } From 1c5af4a3f4ee3fd86333308e7ff45e4f55b8e1db Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 30 Oct 2024 22:20:45 +0000 Subject: [PATCH 02/33] `HTTPClient` in `BinaryArtifacts` WIP --- .../Basics/Concurrency/ThreadSafeBox.swift | 6 + .../Workspace/Workspace+BinaryArtifacts.swift | 436 +++++++++--------- .../Workspace/Workspace+Dependencies.swift | 8 +- .../_InternalTestSupport/MockHTTPClient.swift | 26 +- .../_InternalTestSupport/MockWorkspace.swift | 2 +- 5 files changed, 232 insertions(+), 246 deletions(-) diff --git a/Sources/Basics/Concurrency/ThreadSafeBox.swift b/Sources/Basics/Concurrency/ThreadSafeBox.swift index f719068b440..c05347a46ad 100644 --- a/Sources/Basics/Concurrency/ThreadSafeBox.swift +++ b/Sources/Basics/Concurrency/ThreadSafeBox.swift @@ -30,6 +30,12 @@ public final class ThreadSafeBox { self.underlying = value } } + + public func mutate(body: (inout Value?) throws -> ()) rethrows { + try self.lock.withLock { + try body(&self.underlying) + } + } @discardableResult public func memoize(body: () throws -> Value) rethrows -> Value { diff --git a/Sources/Workspace/Workspace+BinaryArtifacts.swift b/Sources/Workspace/Workspace+BinaryArtifacts.swift index d2b9cafc76e..0731a1a65ff 100644 --- a/Sources/Workspace/Workspace+BinaryArtifacts.swift +++ b/Sources/Workspace/Workspace+BinaryArtifacts.swift @@ -24,12 +24,12 @@ import enum TSCUtility.Diagnostics extension Workspace { // marked public for testing public struct CustomBinaryArtifactsManager { - let httpClient: LegacyHTTPClient? + let httpClient: HTTPClient? let archiver: Archiver? let useCache: Bool? public init( - httpClient: LegacyHTTPClient? = .none, + httpClient: HTTPClient? = .none, archiver: Archiver? = .none, useCache: Bool? = .none ) { @@ -46,7 +46,7 @@ extension Workspace { private let fileSystem: FileSystem private let authorizationProvider: AuthorizationProvider? private let hostToolchain: UserToolchain - private let httpClient: LegacyHTTPClient + private let httpClient: HTTPClient private let archiver: Archiver private let checksumAlgorithm: HashAlgorithm private let cachePath: AbsolutePath? @@ -58,7 +58,7 @@ extension Workspace { hostToolchain: UserToolchain, checksumAlgorithm: HashAlgorithm, cachePath: AbsolutePath?, - customHTTPClient: LegacyHTTPClient?, + customHTTPClient: HTTPClient?, customArchiver: Archiver?, delegate: Delegate? ) { @@ -66,7 +66,7 @@ extension Workspace { self.authorizationProvider = authorizationProvider self.hostToolchain = hostToolchain self.checksumAlgorithm = checksumAlgorithm - self.httpClient = customHTTPClient ?? LegacyHTTPClient() + self.httpClient = customHTTPClient ?? HTTPClient() self.archiver = customArchiver ?? ZipArchiver(fileSystem: fileSystem) self.cachePath = cachePath self.delegate = delegate @@ -139,34 +139,27 @@ extension Workspace { _ artifacts: [RemoteArtifact], artifactsDirectory: AbsolutePath, observabilityScope: ObservabilityScope - ) throws -> [ManagedArtifact] { - let group = DispatchGroup() - let result = ThreadSafeArrayStore() - + ) async throws -> [ManagedArtifact] { // zip files to download // stored in a thread-safe way as we may fetch more from "artifactbundleindex" files - let zipArtifacts = ThreadSafeArrayStore(artifacts.filter { + var zipArtifacts = artifacts.filter { $0.url.pathExtension.lowercased() == "zip" - }) + } // fetch and parse "artifactbundleindex" files, if any let indexFiles = artifacts.filter { $0.url.pathExtension.lowercased() == "artifactbundleindex" } if !indexFiles.isEmpty { let errors = ThreadSafeArrayStore() - let jsonDecoder = JSONDecoder.makeWithDefaults() - for indexFile in indexFiles { - group.enter() - var request = HTTPClient.Request(method: .get, url: indexFile.url) - request.options.validResponseCodes = [200] - request.options.authorizationProvider = self.authorizationProvider?.httpAuthorizationHeader(for:) - self.httpClient.execute(request) { result in - defer { group.leave() } - do { - switch result { - case .failure(let error): - throw error - case .success(let response): + zipArtifacts.append(contentsOf: try await withThrowingTaskGroup(of: RemoteArtifact?.self, returning: [RemoteArtifact].self) { group in + let jsonDecoder = JSONDecoder.makeWithDefaults() + for indexFile in indexFiles { + group.addTask { + var request = HTTPClient.Request(method: .get, url: indexFile.url) + request.options.validResponseCodes = [200] + request.options.authorizationProvider = self.authorizationProvider?.httpAuthorizationHeader(for:) + do { + let response = try await self.httpClient.execute(request) guard let body = response.body else { throw StringError("Body is empty") } @@ -180,224 +173,201 @@ extension Workspace { } let metadata = try jsonDecoder.decode(ArchiveIndexFile.self, from: body) // FIXME: this filter needs to become more sophisticated - guard let supportedArchive = metadata.archives - .first(where: { - $0.fileName.lowercased().hasSuffix(".zip") && $0.supportedTriples - .contains(self.hostToolchain.targetTriple) - }) - else { + guard let supportedArchive = metadata.archives.first(where: { + $0.fileName.lowercased().hasSuffix(".zip") && $0.supportedTriples + .contains(self.hostToolchain.targetTriple) + }) else { throw StringError( "No supported archive was found for '\(self.hostToolchain.targetTriple.tripleString)'" ) } // add relevant archive - zipArtifacts.append( - RemoteArtifact( - packageRef: indexFile.packageRef, - targetName: indexFile.targetName, - url: indexFile.url.deletingLastPathComponent() - .appendingPathComponent(supportedArchive.fileName), - checksum: supportedArchive.checksum - ) + return RemoteArtifact( + packageRef: indexFile.packageRef, + targetName: indexFile.targetName, + url: indexFile.url.deletingLastPathComponent() + .appendingPathComponent(supportedArchive.fileName), + checksum: supportedArchive.checksum + ) + } catch { + errors.append(error) + observabilityScope.emit( + error: "failed retrieving '\(indexFile.url)'", + underlyingError: error ) } - } catch { - errors.append(error) - observabilityScope.emit( - error: "failed retrieving '\(indexFile.url)'", - underlyingError: error - ) + + return nil } } - } - // wait for all "artifactbundleindex" files to be processed - group.wait() + // no reason to continue if we already ran into issues + if !errors.isEmpty { + throw Diagnostics.fatalError + } - // no reason to continue if we already ran into issues - if !errors.isEmpty { - throw Diagnostics.fatalError - } + return try await group.reduce(into: []) { + if let artifact = $1 { + $0.append(artifact) + } + } + }) } - // finally download zip files, if any - for artifact in zipArtifacts.get() { - let destinationDirectory = artifactsDirectory - .appending(components: [artifact.packageRef.identity.description, artifact.targetName]) - guard observabilityScope - .trap({ try fileSystem.createDirectory(destinationDirectory, recursive: true) }) - else { - continue - } + let result = await withTaskGroup(of: ManagedArtifact?.self, returning: [ManagedArtifact].self) { group in + // finally download zip files, if any + for artifact in zipArtifacts { + group.addTask { () -> ManagedArtifact? in + let destinationDirectory = artifactsDirectory + .appending(components: [artifact.packageRef.identity.description, artifact.targetName]) + guard observabilityScope.trap({ try fileSystem.createDirectory(destinationDirectory, recursive: true) }) + else { + return nil + } - let archivePath = destinationDirectory.appending(component: artifact.url.lastPathComponent) - if self.fileSystem.exists(archivePath) { - guard observabilityScope.trap({ try self.fileSystem.removeFileTree(archivePath) }) else { - continue - } - } + let archivePath = destinationDirectory.appending(component: artifact.url.lastPathComponent) + if self.fileSystem.exists(archivePath) { + guard observabilityScope.trap({ try self.fileSystem.removeFileTree(archivePath) }) else { + return nil + } + } - group.enter() - let fetchStart: DispatchTime = .now() - self.fetch( - artifact: artifact, - destination: archivePath, - observabilityScope: observabilityScope, - progress: { bytesDownloaded, totalBytesToDownload in - self.delegate?.downloadingBinaryArtifact( - from: artifact.url.absoluteString, - bytesDownloaded: bytesDownloaded, - totalBytesToDownload: totalBytesToDownload - ) - }, - completion: { fetchResult in - defer { group.leave() } + let fetchStart: DispatchTime = .now() + do { + let cached = try await self.fetch( + artifact: artifact, + destination: archivePath, + observabilityScope: observabilityScope, + progress: { bytesDownloaded, totalBytesToDownload in + self.delegate?.downloadingBinaryArtifact( + from: artifact.url.absoluteString, + bytesDownloaded: bytesDownloaded, + totalBytesToDownload: totalBytesToDownload + ) + } + ) - switch fetchResult { - case .success(let cached): // TODO: Use the same extraction logic for both remote and local archived artifacts. - group.enter() observabilityScope.emit(debug: "validating \(archivePath)") - self.archiver.validate(path: archivePath, completion: { validationResult in - defer { group.leave() } - - switch validationResult { - case .success(let valid): - guard valid else { - observabilityScope - .emit(.artifactInvalidArchive( - artifactURL: artifact.url, - targetName: artifact.targetName - )) - return - } + do { + let valid = try await self.archiver.validate(path: archivePath) - guard let archiveChecksum = observabilityScope - .trap({ try self.checksum(forBinaryArtifactAt: archivePath) }) - else { - return - } - guard archiveChecksum == artifact.checksum else { - observabilityScope.emit(.artifactInvalidChecksum( - targetName: artifact.targetName, - expectedChecksum: artifact.checksum, - actualChecksum: archiveChecksum + guard valid else { + observabilityScope + .emit(.artifactInvalidArchive( + artifactURL: artifact.url, + targetName: artifact.targetName )) - observabilityScope.trap { try self.fileSystem.removeFileTree(archivePath) } - return - } + return nil + } - guard let tempExtractionDirectory = observabilityScope.trap({ () -> AbsolutePath in - let path = artifactsDirectory.appending( - components: "extract", - artifact.packageRef.identity.description, - artifact.targetName, - UUID().uuidString - ) - try self.fileSystem.forceCreateDirectory(at: path) - return path - }) else { - return - } + guard let archiveChecksum = observabilityScope + .trap({ try self.checksum(forBinaryArtifactAt: archivePath) }) + else { + return nil + } + guard archiveChecksum == artifact.checksum else { + observabilityScope.emit(.artifactInvalidChecksum( + targetName: artifact.targetName, + expectedChecksum: artifact.checksum, + actualChecksum: archiveChecksum + )) + observabilityScope.trap { try self.fileSystem.removeFileTree(archivePath) } + return nil + } - group.enter() - observabilityScope - .emit(debug: "extracting \(archivePath) to \(tempExtractionDirectory)") - self.archiver.extract( - from: archivePath, - to: tempExtractionDirectory, - completion: { extractResult in - defer { group.leave() } - - switch extractResult { - case .success: - observabilityScope.trap { - try self.fileSystem.withLock( - on: destinationDirectory, - type: .exclusive - ) { - // strip first level component if needed - if try self.fileSystem.shouldStripFirstLevel( - archiveDirectory: tempExtractionDirectory, - acceptableExtensions: BinaryModule.Kind.allCases - .map(\.fileExtension) - ) { - observabilityScope - .emit( - debug: "stripping first level component from \(tempExtractionDirectory)" - ) - try self.fileSystem - .stripFirstLevel(of: tempExtractionDirectory) - } else { - observabilityScope - .emit( - debug: "no first level component stripping needed for \(tempExtractionDirectory)" - ) - } - let content = try self.fileSystem - .getDirectoryContents(tempExtractionDirectory) - // copy from temp location to actual location - for file in content { - let source = tempExtractionDirectory - .appending(component: file) - let destination = destinationDirectory - .appending(component: file) - if self.fileSystem.exists(destination) { - try self.fileSystem.removeFileTree(destination) - } - try self.fileSystem.copy(from: source, to: destination) - } - } - // remove temp location - try self.fileSystem.removeFileTree(tempExtractionDirectory) - } + guard let tempExtractionDirectory = observabilityScope.trap({ () -> AbsolutePath in + let path = artifactsDirectory.appending( + components: "extract", + artifact.packageRef.identity.description, + artifact.targetName, + UUID().uuidString + ) + try self.fileSystem.forceCreateDirectory(at: path) + return path + }) else { + return nil + } - // derive concrete artifact path and type - guard let (artifactPath, artifactKind) = try? Self.deriveBinaryArtifact( - fileSystem: self.fileSystem, - path: destinationDirectory, - observabilityScope: observabilityScope - ) else { - return observabilityScope - .emit(.remoteArtifactNotFound( - artifactURL: artifact.url, - targetName: artifact.targetName - )) - } + observabilityScope.emit(debug: "extracting \(archivePath) to \(tempExtractionDirectory)") + do { + try await self.archiver.extract( + from: archivePath, + to: tempExtractionDirectory + ) - result.append( - .remote( - packageRef: artifact.packageRef, - targetName: artifact.targetName, - url: artifact.url.absoluteString, - checksum: artifact.checksum, - path: artifactPath, - kind: artifactKind - ) - ) - self.delegate?.didDownloadBinaryArtifact( - from: artifact.url.absoluteString, - result: .success((path: artifactPath, fromCache: cached)), - duration: fetchStart.distance(to: .now()) + defer { + observabilityScope.trap { try self.fileSystem.removeFileTree(archivePath) } + } + observabilityScope.trap { + try self.fileSystem.withLock( + on: destinationDirectory, + type: .exclusive + ) { + // strip first level component if needed + if try self.fileSystem.shouldStripFirstLevel( + archiveDirectory: tempExtractionDirectory, + acceptableExtensions: BinaryModule.Kind.allCases + .map(\.fileExtension) + ) { + observabilityScope.emit( + debug: "stripping first level component from \(tempExtractionDirectory)" ) - case .failure(let error): - observabilityScope.emit(.remoteArtifactFailedExtraction( - artifactURL: artifact.url, - targetName: artifact.targetName, - reason: error.interpolationDescription - )) - self.delegate?.didDownloadBinaryArtifact( - from: artifact.url.absoluteString, - result: .failure(error), - duration: fetchStart.distance(to: .now()) + try self.fileSystem + .stripFirstLevel(of: tempExtractionDirectory) + } else { + observabilityScope.emit( + debug: "no first level component stripping needed for \(tempExtractionDirectory)" ) } - - observabilityScope.trap { try self.fileSystem.removeFileTree(archivePath) } + let content = try self.fileSystem.getDirectoryContents(tempExtractionDirectory) + // copy from temp location to actual location + for file in content { + let source = tempExtractionDirectory + .appending(component: file) + let destination = destinationDirectory + .appending(component: file) + if self.fileSystem.exists(destination) { + try self.fileSystem.removeFileTree(destination) + } + try self.fileSystem.copy(from: source, to: destination) + } } + // remove temp location + try self.fileSystem.removeFileTree(tempExtractionDirectory) + } + + // derive concrete artifact path and type + guard let (artifactPath, artifactKind) = try? Self.deriveBinaryArtifact( + fileSystem: self.fileSystem, + path: destinationDirectory, + observabilityScope: observabilityScope + ) else { + observabilityScope + .emit(.remoteArtifactNotFound( + artifactURL: artifact.url, + targetName: artifact.targetName + )) + return nil + } + + self.delegate?.didDownloadBinaryArtifact( + from: artifact.url.absoluteString, + result: .success((path: artifactPath, fromCache: cached)), + duration: fetchStart.distance(to: .now()) ) - case .failure(let error): - observabilityScope.emit(.artifactFailedValidation( + + return ManagedArtifact.remote( + packageRef: artifact.packageRef, + targetName: artifact.targetName, + url: artifact.url.absoluteString, + checksum: artifact.checksum, + path: artifactPath, + kind: artifactKind + ) + + } catch { + observabilityScope.emit(.remoteArtifactFailedExtraction( artifactURL: artifact.url, targetName: artifact.targetName, reason: error.interpolationDescription @@ -408,8 +378,19 @@ extension Workspace { duration: fetchStart.distance(to: .now()) ) } - }) - case .failure(let error): + } catch { + observabilityScope.emit(.artifactFailedValidation( + artifactURL: artifact.url, + targetName: artifact.targetName, + reason: error.interpolationDescription + )) + self.delegate?.didDownloadBinaryArtifact( + from: artifact.url.absoluteString, + result: .failure(error), + duration: fetchStart.distance(to: .now()) + ) + } + } catch { observabilityScope.trap { try self.fileSystem.removeFileTree(archivePath) } observabilityScope.emit(.artifactFailedDownload( artifactURL: artifact.url, @@ -422,24 +403,30 @@ extension Workspace { duration: fetchStart.distance(to: .now()) ) } + + return nil } - ) - } + } - group.wait() + return await group.reduce(into: []) { + if let artifact = $1 { + $0.append(artifact) + } + } + } if zipArtifacts.count > 0 { delegate?.didDownloadAllBinaryArtifacts() } - return result.get() + return result } func extract( _ artifacts: [ManagedArtifact], artifactsDirectory: AbsolutePath, observabilityScope: ObservabilityScope - ) throws -> [ManagedArtifact] { + ) async throws -> [ManagedArtifact] { let result = ThreadSafeArrayStore() let group = DispatchGroup() @@ -456,10 +443,7 @@ extension Workspace { ) try self.fileSystem.forceCreateDirectory(at: tempExtractionDirectory) - group.enter() self.archiver.extract(from: artifact.path, to: tempExtractionDirectory, completion: { extractResult in - defer { group.leave() } - switch extractResult { case .success: observabilityScope.trap { () in @@ -814,7 +798,7 @@ extension Workspace { manifests: DependencyManifests, addedOrUpdatedPackages: [PackageReference], observabilityScope: ObservabilityScope - ) throws { + ) async throws { let manifestArtifacts = try self.binaryArtifactsManager.parseArtifacts( from: manifests, observabilityScope: observabilityScope @@ -925,7 +909,7 @@ extension Workspace { } // Download the artifacts - let downloadedArtifacts = try self.binaryArtifactsManager.fetch( + let downloadedArtifacts = try await self.binaryArtifactsManager.fetch( artifactsToDownload, artifactsDirectory: self.location.artifactsDirectory, observabilityScope: observabilityScope @@ -933,7 +917,7 @@ extension Workspace { artifactsToAdd.append(contentsOf: downloadedArtifacts) // Extract the local archived artifacts - let extractedLocalArtifacts = try self.binaryArtifactsManager.extract( + let extractedLocalArtifacts = try await self.binaryArtifactsManager.extract( artifactsToExtract, artifactsDirectory: self.location.artifactsDirectory, observabilityScope: observabilityScope diff --git a/Sources/Workspace/Workspace+Dependencies.swift b/Sources/Workspace/Workspace+Dependencies.swift index 51453b3b350..92725dad40e 100644 --- a/Sources/Workspace/Workspace+Dependencies.swift +++ b/Sources/Workspace/Workspace+Dependencies.swift @@ -179,7 +179,7 @@ extension Workspace { // Update the binary target artifacts. let addedOrUpdatedPackages = packageStateChanges.compactMap { $0.1.isAddedOrUpdated ? $0.0 : nil } - try self.updateBinaryArtifacts( + try await self.updateBinaryArtifacts( manifests: updatedDependencyManifests, addedOrUpdatedPackages: addedOrUpdatedPackages, observabilityScope: observabilityScope @@ -449,7 +449,7 @@ extension Workspace { observabilityScope: observabilityScope ) - try self.updateBinaryArtifacts( + try await self.updateBinaryArtifacts( manifests: currentManifests, addedOrUpdatedPackages: [], observabilityScope: observabilityScope @@ -548,7 +548,7 @@ extension Workspace { observabilityScope: observabilityScope ) - try self.updateBinaryArtifacts( + try await self.updateBinaryArtifacts( manifests: currentManifests, addedOrUpdatedPackages: [], observabilityScope: observabilityScope @@ -614,7 +614,7 @@ extension Workspace { ) let addedOrUpdatedPackages = packageStateChanges.compactMap { $0.1.isAddedOrUpdated ? $0.0 : nil } - try self.updateBinaryArtifacts( + try await self.updateBinaryArtifacts( manifests: updatedDependencyManifests, addedOrUpdatedPackages: addedOrUpdatedPackages, observabilityScope: observabilityScope diff --git a/Sources/_InternalTestSupport/MockHTTPClient.swift b/Sources/_InternalTestSupport/MockHTTPClient.swift index c680bcc2612..1eb8cfa5792 100644 --- a/Sources/_InternalTestSupport/MockHTTPClient.swift +++ b/Sources/_InternalTestSupport/MockHTTPClient.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2021 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -12,24 +12,20 @@ import Basics -extension LegacyHTTPClient { - public static func mock(fileSystem: FileSystem) -> LegacyHTTPClient { - let handler: LegacyHTTPClient.Handler = { request, _, completion in +extension HTTPClient { + public static func mock(fileSystem: FileSystem) -> HTTPClient { + HTTPClient { request, _ in switch request.kind { case.generic: - completion(.success(.okay(body: request.url.absoluteString))) + return .okay(body: request.url.absoluteString) + case .download(let fileSystem, let destination): - do { - try fileSystem.writeFileContents( - destination, - string: request.url.absoluteString - ) - completion(.success(.okay(body: request.url.absoluteString))) - } catch { - completion(.failure(error)) - } + try fileSystem.writeFileContents( + destination, + string: request.url.absoluteString + ) + return .okay(body: request.url.absoluteString) } } - return LegacyHTTPClient(handler: handler) } } diff --git a/Sources/_InternalTestSupport/MockWorkspace.swift b/Sources/_InternalTestSupport/MockWorkspace.swift index 406fceba0bc..6909eb2b234 100644 --- a/Sources/_InternalTestSupport/MockWorkspace.swift +++ b/Sources/_InternalTestSupport/MockWorkspace.swift @@ -128,7 +128,7 @@ public final class MockWorkspace { self.sourceControlToRegistryDependencyTransformation = sourceControlToRegistryDependencyTransformation self.defaultRegistry = defaultRegistry self.customBinaryArtifactsManager = customBinaryArtifactsManager ?? .init( - httpClient: LegacyHTTPClient.mock(fileSystem: fileSystem), + httpClient: HTTPClient.mock(fileSystem: fileSystem), archiver: MockArchiver() ) self.customHostToolchain = try UserToolchain.mockHostToolchain(fileSystem) From a913419321d47288242b4681d978d1cee8186274 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 30 Oct 2024 22:20:59 +0000 Subject: [PATCH 03/33] Update `WorkspaceTests.swift` --- Tests/WorkspaceTests/WorkspaceTests.swift | 991 ++++++++++------------ 1 file changed, 446 insertions(+), 545 deletions(-) diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index a0dfe2d7a3d..342a7fe3134 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -6023,31 +6023,27 @@ final class WorkspaceTests: XCTestCase { let a5FrameworkName = "A5.xcframework" // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a4.zip": - contents = [0xA4] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a4.zip": + contents = [0xA4] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + return .okay() + } // create a dummy xcframework directory (with a marker subdirectory) from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -6788,36 +6784,32 @@ final class WorkspaceTests: XCTestCase { let downloads = ThreadSafeKeyValueStore() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } - - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.zip": - contents = [0xA1] - case "a2.zip": - contents = [0xA2] - case "b.zip": - contents = [0xB0] - default: - throw StringError("unexpected url \(request.url)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.zip": + contents = [0xA1] + case "a2.zip": + contents = [0xA2] + case "b.zip": + contents = [0xB0] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads[request.url] = destination - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + downloads[request.url] = destination + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -6981,40 +6973,36 @@ final class WorkspaceTests: XCTestCase { let downloads = ThreadSafeKeyValueStore() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } - - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.zip": - contents = [0xA1] - case "a2.zip": - contents = [0xA2] - case "a3.zip": - contents = [0xA3] - case "a7.zip": - contents = [0xA7] - case "b.zip": - contents = [0xB0] - default: - throw StringError("unexpected url \(request.url)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.zip": + contents = [0xA1] + case "a2.zip": + contents = [0xA2] + case "a3.zip": + contents = [0xA3] + case "a7.zip": + contents = [0xA7] + case "b.zip": + contents = [0xB0] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads[request.url] = destination - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + downloads[request.url] = destination + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -7293,32 +7281,28 @@ final class WorkspaceTests: XCTestCase { let downloads = ThreadSafeArrayStore<(URL, AbsolutePath)>() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.zip": - contents = [0xA1] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.zip": + contents = [0xA1] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads.append((request.url, destination)) - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + downloads.append((request.url, destination)) + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -7419,24 +7403,20 @@ final class WorkspaceTests: XCTestCase { try fs.createDirectory(sandbox, recursive: true) let artifactUrl = "https://a.com/a.zip" - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - // mimics URLSession behavior which write the file even if sends an error message - try fileSystem.writeFileContents( - destination, - bytes: "not found", - atomically: true - ) + // mimics URLSession behavior which write the file even if sends an error message + try fileSystem.writeFileContents( + destination, + bytes: "not found", + atomically: true + ) - completion(.success(.notFound())) - } catch { - completion(.failure(error)) - } - }) + return .notFound() + } let workspace = try await MockWorkspace( sandbox: sandbox, @@ -7491,28 +7471,24 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } - - switch request.url { - case "https://a.com/a1.zip": - completion(.success(.serverError())) - case "https://a.com/a2.zip": - try fileSystem.writeFileContents(destination, bytes: ByteString([0xA2])) - completion(.success(.okay())) - case "https://a.com/a3.zip": - try fileSystem.writeFileContents(destination, bytes: "different contents = different checksum") - completion(.success(.okay())) - default: - throw StringError("unexpected url") - } - } catch { - completion(.failure(error)) + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + switch request.url { + case "https://a.com/a1.zip": + return .serverError() + case "https://a.com/a2.zip": + try fileSystem.writeFileContents(destination, bytes: ByteString([0xA2])) + return .okay() + case "https://a.com/a3.zip": + try fileSystem.writeFileContents(destination, bytes: "different contents = different checksum") + return .okay() + default: + throw StringError("unexpected url") } - }) + } let archiver = MockArchiver(handler: { _, _, destinationPath, completion in XCTAssertEqual(destinationPath.parentDirectory, AbsolutePath("/tmp/ws/.build/artifacts/extract/root/A2")) @@ -7582,29 +7558,25 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } - - switch request.url { - case "https://a.com/a1.zip": - try fileSystem.writeFileContents(destination, bytes: ByteString([0xA1])) - completion(.success(.okay())) - case "https://a.com/a2.zip": - try fileSystem.writeFileContents(destination, bytes: ByteString([0xA2])) - completion(.success(.okay())) - case "https://a.com/a3.zip": - try fileSystem.writeFileContents(destination, bytes: ByteString([0xA3])) - completion(.success(.okay())) - default: - throw StringError("unexpected url") - } - } catch { - completion(.failure(error)) + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + switch request.url { + case "https://a.com/a1.zip": + try fileSystem.writeFileContents(destination, bytes: ByteString([0xA1])) + return .okay() + case "https://a.com/a2.zip": + try fileSystem.writeFileContents(destination, bytes: ByteString([0xA2])) + return .okay() + case "https://a.com/a3.zip": + try fileSystem.writeFileContents(destination, bytes: ByteString([0xA3])) + return .okay() + default: + throw StringError("unexpected url") } - }) + } let archiver = MockArchiver( extractionHandler: { archiver, archivePath, destinationPath, completion in @@ -7694,23 +7666,19 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - switch request.url { - case "https://a.com/a1.zip": - try fileSystem.writeFileContents(destination, bytes: ByteString([0xA1])) - completion(.success(.okay())) - default: - throw StringError("unexpected url") - } - } catch { - completion(.failure(error)) + switch request.url { + case "https://a.com/a1.zip": + try fileSystem.writeFileContents(destination, bytes: ByteString([0xA1])) + return .okay() + default: + throw StringError("unexpected url") } - }) + } let archiver = MockArchiver( extractionHandler: { _, archivePath, destinationPath, completion in @@ -7769,23 +7737,19 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - switch request.url { - case "https://a.com/foo.zip": - try fileSystem.writeFileContents(destination, bytes: ByteString([0xA1])) - completion(.success(.okay())) - default: - throw StringError("unexpected url") - } - } catch { - completion(.failure(error)) + switch request.url { + case "https://a.com/foo.zip": + try fileSystem.writeFileContents(destination, bytes: ByteString([0xA1])) + return .okay() + default: + throw StringError("unexpected url") } - }) + } let archiver = MockArchiver( extractionHandler: { _, archivePath, destinationPath, completion in @@ -7919,9 +7883,9 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let httpClient = LegacyHTTPClient(handler: { _, _, _ in - XCTFail("should not be called") - }) + let httpClient = HTTPClient { _, _ in + throw StringError("should not be called") + } let workspace = try await MockWorkspace( sandbox: sandbox, @@ -7971,33 +7935,29 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a.zip": - contents = [0xA1] - case "b.zip": - contents = [0xB1] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a.zip": + contents = [0xA1] + case "b.zip": + contents = [0xB1] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + return .okay() + } let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in do { @@ -8065,35 +8025,31 @@ final class WorkspaceTests: XCTestCase { func testArtifactDownloadAddsAcceptHeader() async throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - var acceptHeaders: [String] = [] + let acceptHeaders = ThreadSafeBox([String]()) // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } - acceptHeaders.append(request.headers.get("accept").first!) + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + acceptHeaders.mutate { $0?.append(request.headers.get("accept").first!) } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.zip": - contents = [0xA1] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.zip": + contents = [0xA1] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -8136,7 +8092,7 @@ final class WorkspaceTests: XCTestCase { try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(acceptHeaders, [ + XCTAssertEqual(acceptHeaders.get(), [ "application/octet-stream", ]) } @@ -8145,35 +8101,31 @@ final class WorkspaceTests: XCTestCase { func testDownloadedArtifactNoCache() async throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - var downloads = 0 + let downloads = ThreadSafeBox(0) // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.zip": - contents = [0xA1] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.zip": + contents = [0xA1] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads += 1 - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + downloads.increment() + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -8218,55 +8170,51 @@ final class WorkspaceTests: XCTestCase { // should not come from cache try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 1) + XCTAssertEqual(downloads.get(), 1) } // state is there, should not come from local cache try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 1) + XCTAssertEqual(downloads.get(), 1) } // resetting state, should not come from global cache try workspace.resetState() try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 2) + XCTAssertEqual(downloads.get(), 2) } } func testDownloadedArtifactCache() async throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - var downloads = 0 + let downloads = ThreadSafeBox(0) // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.zip": - contents = [0xA1] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.zip": + contents = [0xA1] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads += 1 - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + downloads.increment() + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -8311,20 +8259,20 @@ final class WorkspaceTests: XCTestCase { // should not come from cache try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 1) + XCTAssertEqual(downloads.get(), 1) } // state is there, should not come from local cache try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 1) + XCTAssertEqual(downloads.get(), 1) } // resetting state, should come from global cache try workspace.resetState() try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 1) + XCTAssertEqual(downloads.get(), 1) } // delete global cache, should download again @@ -8332,14 +8280,14 @@ final class WorkspaceTests: XCTestCase { try fs.removeFileTree(fs.swiftPMCacheDirectory) try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 2) + XCTAssertEqual(downloads.get(), 2) } // resetting state, should come from global cache again try workspace.resetState() try await workspace.checkPackageGraph(roots: ["Root"]) { _, diagnostics in XCTAssertNoDiagnostics(diagnostics) - XCTAssertEqual(downloads, 2) + XCTAssertEqual(downloads.get(), 2) } } @@ -8349,35 +8297,31 @@ final class WorkspaceTests: XCTestCase { let downloads = ThreadSafeKeyValueStore() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a.zip": - contents = [0xA] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a.zip": + contents = [0xA] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - if downloads[request.url] != nil { - throw StringError("\(request.url) already requested") - } - downloads[request.url] = destination - completion(.success(.okay())) - } catch { - completion(.failure(error)) + if downloads[request.url] != nil { + throw StringError("\(request.url) already requested") } - }) + downloads[request.url] = destination + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -8530,41 +8474,37 @@ final class WorkspaceTests: XCTestCase { ) // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - // this is to test the test's integrity, as it relied on internal knowledge of the destination path construction - guard expectedDownloadDestination == destination else { - throw StringError("expected destination of \(expectedDownloadDestination)") - } + // this is to test the test's integrity, as it relied on internal knowledge of the destination path construction + guard expectedDownloadDestination == destination else { + throw StringError("expected destination of \(expectedDownloadDestination)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "binary.zip": - contents = [0x01] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "binary.zip": + contents = [0x01] + default: + throw StringError("unexpected url \(request.url)") + } - // in-memory fs does not check for this! - if fileSystem.exists(destination) { - throw StringError("\(destination) already exists") - } + // in-memory fs does not check for this! + if fileSystem.exists(destination) { + throw StringError("\(destination) already exists") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -8637,42 +8577,34 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() let maxConcurrentRequests = 2 - var concurrentRequests = 0 + let concurrentRequests = ThreadSafeBox(0) let concurrentRequestsLock = NSLock() - var configuration = LegacyHTTPClient.Configuration() + var configuration = HTTPClient.Configuration() configuration.maxConcurrentRequests = maxConcurrentRequests - let httpClient = LegacyHTTPClient(configuration: configuration, handler: { request, _, completion in + let httpClient = HTTPClient(configuration: configuration) { request, _ in defer { - concurrentRequestsLock.withLock { - concurrentRequests -= 1 - } + concurrentRequests.decrement() } - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - concurrentRequestsLock.withLock { - concurrentRequests += 1 - if concurrentRequests > maxConcurrentRequests { - XCTFail("too many concurrent requests \(concurrentRequests), expected \(maxConcurrentRequests)") - } - } + concurrentRequests.increment() + if concurrentRequests.get()! > maxConcurrentRequests { + XCTFail("too many concurrent requests \(concurrentRequests), expected \(maxConcurrentRequests)") + } - // returns a dummy zipfile for the requested artifact - try fileSystem.writeFileContents( - destination, - bytes: [0x01], - atomically: true - ) + // returns a dummy zipfile for the requested artifact + try fileSystem.writeFileContents( + destination, + bytes: [0x01], + atomically: true + ) - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { _, archivePath, destinationPath, completion in @@ -8759,36 +8691,32 @@ final class WorkspaceTests: XCTestCase { let downloads = ThreadSafeKeyValueStore() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } - - let contents: [UInt8] - switch request.url.lastPathComponent { - case "flat.zip": - contents = [0x01] - case "nested.zip": - contents = [0x02] - case "nested2.zip": - contents = [0x03] - default: - throw StringError("unexpected url \(request.url)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + let contents: [UInt8] + switch request.url.lastPathComponent { + case "flat.zip": + contents = [0x01] + case "nested.zip": + contents = [0x02] + case "nested2.zip": + contents = [0x03] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads[request.url] = destination - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + downloads[request.url] = destination + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -8923,34 +8851,30 @@ final class WorkspaceTests: XCTestCase { let downloads = ThreadSafeKeyValueStore() // returns a dummy zipfile for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") - } + let httpClient = HTTPClient { request, _ in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.xcframework.zip": - contents = [0xA1] - case "a2.zip.zip": - contents = [0xA2] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.xcframework.zip": + contents = [0xA1] + case "a2.zip.zip": + contents = [0xA2] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads[request.url] = destination - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } - }) + downloads[request.url] = destination + return .okay() + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -9127,50 +9051,43 @@ final class WorkspaceTests: XCTestCase { let ariFilesChecksums = ariFiles.map { checksumAlgorithm.hash($0).hexadecimalRepresentation } // returns a dummy file for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in + let httpClient = HTTPClient { request, _ in switch request.kind { case .generic: - do { - let contents: String - switch request.url.lastPathComponent { - case "a1.artifactbundleindex": - contents = ariFiles[0] - case "a2.artifactbundleindex": - contents = ariFiles[1] - default: - throw StringError("unexpected url \(request.url)") - } - completion(.success(.okay(body: contents))) - } catch { - completion(.failure(error)) + let contents: String + switch request.url.lastPathComponent { + case "a1.artifactbundleindex": + contents = ariFiles[0] + case "a2.artifactbundleindex": + contents = ariFiles[1] + default: + throw StringError("unexpected url \(request.url)") } + return .okay(body: contents) + case .download(let fileSystem, let destination): - do { - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a1.zip": - contents = [0xA1] - case "a2.zip": - contents = [0xA2] - case "b.zip": - contents = [0xB0] - default: - throw StringError("unexpected url \(request.url)") - } + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a1.zip": + contents = [0xA1] + case "a2.zip": + contents = [0xA2] + case "b.zip": + contents = [0xB0] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - downloads[request.url] = destination - completion(.success(.okay())) - } catch { - completion(.failure(error)) - } + downloads[request.url] = destination + return .okay() } - }) + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -9343,9 +9260,9 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() // returns a dummy files for the requested artifact - let httpClient = LegacyHTTPClient(handler: { _, _, completion in - completion(.success(.serverError())) - }) + let httpClient = HTTPClient { _, _ in + return .serverError() + } let workspace = try await MockWorkspace( sandbox: sandbox, @@ -9402,20 +9319,16 @@ final class WorkspaceTests: XCTestCase { let ariChecksums = checksumAlgorithm.hash(ari).hexadecimalRepresentation // returns a dummy files for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - let contents: String - switch request.url.lastPathComponent { - case "a.artifactbundleindex": - contents = ari - default: - throw StringError("unexpected url \(request.url)") - } - completion(.success(.okay(body: contents))) - } catch { - completion(.failure(error)) + let httpClient = HTTPClient { request, _ in + let contents: String + switch request.url.lastPathComponent { + case "a.artifactbundleindex": + contents = ari + default: + throw StringError("unexpected url \(request.url)") } - }) + return .okay(body: contents) + } let workspace = try await MockWorkspace( sandbox: sandbox, @@ -9522,41 +9435,37 @@ final class WorkspaceTests: XCTestCase { let ariChecksums = checksumAlgorithm.hash(ari).hexadecimalRepresentation // returns a dummy files for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - switch request.kind { - case .generic: - let contents: String - switch request.url.lastPathComponent { - case "a.artifactbundleindex": - contents = ari - default: - throw StringError("unexpected url \(request.url)") - } + let httpClient = HTTPClient { request, _ in + switch request.kind { + case .generic: + let contents: String + switch request.url.lastPathComponent { + case "a.artifactbundleindex": + contents = ari + default: + throw StringError("unexpected url \(request.url)") + } - completion(.success(.okay(body: contents))) + return .okay(body: contents) - case .download(let fileSystem, let destination): - let contents: [UInt8] - switch request.url.lastPathComponent { - case "a.zip": - contents = [0x42] - default: - throw StringError("unexpected url \(request.url)") - } + case .download(let fileSystem, let destination): + let contents: [UInt8] + switch request.url.lastPathComponent { + case "a.zip": + contents = [0x42] + default: + throw StringError("unexpected url \(request.url)") + } - try fileSystem.writeFileContents( - destination, - bytes: ByteString(contents), - atomically: true - ) + try fileSystem.writeFileContents( + destination, + bytes: ByteString(contents), + atomically: true + ) - completion(.success(.okay())) - } - } catch { - completion(.failure(error)) + return .okay() } - }) + } // create a dummy xcframework directory from the request archive let archiver = MockArchiver(handler: { archiver, archivePath, destinationPath, completion in @@ -9633,26 +9542,22 @@ final class WorkspaceTests: XCTestCase { let ariChecksums = checksumAlgorithm.hash(ari).hexadecimalRepresentation // returns a dummy files for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { - switch request.kind { - case .generic: - let contents: String - switch request.url.lastPathComponent { - case "a.artifactbundleindex": - contents = ari - default: - throw StringError("unexpected url \(request.url)") - } - completion(.success(.okay(body: contents))) - - case .download: - completion(.success(.notFound())) + let httpClient = HTTPClient { request, _ in + switch request.kind { + case .generic: + let contents: String + switch request.url.lastPathComponent { + case "a.artifactbundleindex": + contents = ari + default: + throw StringError("unexpected url \(request.url)") } - } catch { - completion(.failure(error)) + return .okay(body: contents) + + case .download: + return .notFound() } - }) + } let workspace = try await MockWorkspace( sandbox: sandbox, @@ -9713,8 +9618,7 @@ final class WorkspaceTests: XCTestCase { let ariChecksum = checksumAlgorithm.hash(ari).hexadecimalRepresentation // returns a dummy files for the requested artifact - let httpClient = LegacyHTTPClient(handler: { request, _, completion in - do { + let httpClient = HTTPClient { request, _ in let contents: String switch request.url.lastPathComponent { case "a.artifactbundleindex": @@ -9722,11 +9626,8 @@ final class WorkspaceTests: XCTestCase { default: throw StringError("unexpected url \(request.url)") } - completion(.success(.okay(body: contents))) - } catch { - completion(.failure(error)) - } - }) + return .okay(body: contents) + } let workspace = try await MockWorkspace( sandbox: sandbox, From f60977c99e0967c61335a4fb5bde006231e0b5d7 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 31 Oct 2024 12:19:12 +0000 Subject: [PATCH 04/33] Minor fixups --- .../Workspace/Workspace+BinaryArtifacts.swift | 137 +++++++++--------- 1 file changed, 68 insertions(+), 69 deletions(-) diff --git a/Sources/Workspace/Workspace+BinaryArtifacts.swift b/Sources/Workspace/Workspace+BinaryArtifacts.swift index 0731a1a65ff..ca48810f45e 100644 --- a/Sources/Workspace/Workspace+BinaryArtifacts.swift +++ b/Sources/Workspace/Workspace+BinaryArtifacts.swift @@ -427,98 +427,97 @@ extension Workspace { artifactsDirectory: AbsolutePath, observabilityScope: ObservabilityScope ) async throws -> [ManagedArtifact] { - let result = ThreadSafeArrayStore() - let group = DispatchGroup() - - for artifact in artifacts { - let destinationDirectory = artifactsDirectory - .appending(components: [artifact.packageRef.identity.description, artifact.targetName]) - try fileSystem.createDirectory(destinationDirectory, recursive: true) - - let tempExtractionDirectory = artifactsDirectory.appending( - components: "extract", - artifact.packageRef.identity.description, - artifact.targetName, - UUID().uuidString - ) - try self.fileSystem.forceCreateDirectory(at: tempExtractionDirectory) - - self.archiver.extract(from: artifact.path, to: tempExtractionDirectory, completion: { extractResult in - switch extractResult { - case .success: - observabilityScope.trap { () in - try self.fileSystem.withLock(on: destinationDirectory, type: .exclusive) { - // strip first level component if needed - if try self.fileSystem.shouldStripFirstLevel( - archiveDirectory: tempExtractionDirectory, - acceptableExtensions: BinaryModule.Kind.allCases.map(\.fileExtension) - ) { - observabilityScope - .emit(debug: "stripping first level component from \(tempExtractionDirectory)") - try self.fileSystem.stripFirstLevel(of: tempExtractionDirectory) - } else { - observabilityScope - .emit( + return try await withThrowingTaskGroup(of: ManagedArtifact?.self) { group in + for artifact in artifacts { + group.addTask { () -> ManagedArtifact? in + let destinationDirectory = artifactsDirectory + .appending(components: [artifact.packageRef.identity.description, artifact.targetName]) + try fileSystem.createDirectory(destinationDirectory, recursive: true) + + let tempExtractionDirectory = artifactsDirectory.appending( + components: "extract", + artifact.packageRef.identity.description, + artifact.targetName, + UUID().uuidString + ) + try self.fileSystem.forceCreateDirectory(at: tempExtractionDirectory) + + do { + try await self.archiver.extract(from: artifact.path, to: tempExtractionDirectory) + + return observabilityScope.trap { + try self.fileSystem.withLock(on: destinationDirectory, type: .exclusive) { + // strip first level component if needed + if try self.fileSystem.shouldStripFirstLevel( + archiveDirectory: tempExtractionDirectory, + acceptableExtensions: BinaryModule.Kind.allCases.map(\.fileExtension) + ) { + observabilityScope + .emit(debug: "stripping first level component from \(tempExtractionDirectory)") + try self.fileSystem.stripFirstLevel(of: tempExtractionDirectory) + } else { + observabilityScope.emit( debug: "no first level component stripping needed for \(tempExtractionDirectory)" ) - } - let content = try self.fileSystem.getDirectoryContents(tempExtractionDirectory) - // copy from temp location to actual location - for file in content { - let source = tempExtractionDirectory.appending(component: file) - let destination = destinationDirectory.appending(component: file) - if self.fileSystem.exists(destination) { - try self.fileSystem.removeFileTree(destination) } - try self.fileSystem.copy(from: source, to: destination) + let content = try self.fileSystem.getDirectoryContents(tempExtractionDirectory) + // copy from temp location to actual location + for file in content { + let source = tempExtractionDirectory.appending(component: file) + let destination = destinationDirectory.appending(component: file) + if self.fileSystem.exists(destination) { + try self.fileSystem.removeFileTree(destination) + } + try self.fileSystem.copy(from: source, to: destination) + } } - } - // remove temp location - try self.fileSystem.removeFileTree(tempExtractionDirectory) + // remove temp location + try self.fileSystem.removeFileTree(tempExtractionDirectory) - // derive concrete artifact path and type - guard let (artifactPath, artifactKind) = try Self.deriveBinaryArtifact( - fileSystem: self.fileSystem, - path: destinationDirectory, - observabilityScope: observabilityScope - ) else { - return observabilityScope - .emit(.localArchivedArtifactNotFound( + // derive concrete artifact path and type + guard let (artifactPath, artifactKind) = try Self.deriveBinaryArtifact( + fileSystem: self.fileSystem, + path: destinationDirectory, + observabilityScope: observabilityScope + ) else { + observabilityScope.emit(.localArchivedArtifactNotFound( archivePath: artifact.path, targetName: artifact.targetName )) - } - // compute the checksum - let artifactChecksum = try self.checksum(forBinaryArtifactAt: artifact.path) + return nil + } - result.append( - .local( + // compute the checksum + let artifactChecksum = try self.checksum(forBinaryArtifactAt: artifact.path) + + return .some(ManagedArtifact.local( packageRef: artifact.packageRef, targetName: artifact.targetName, path: artifactPath, kind: artifactKind, checksum: artifactChecksum - ) - ) - } - case .failure(let error): - let reason = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + )) + } + } catch { + let reason = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription - observabilityScope - .emit(.localArtifactFailedExtraction( + observabilityScope.emit(.localArtifactFailedExtraction( artifactPath: artifact.path, targetName: artifact.targetName, reason: reason )) - } - }) - } - group.wait() + return nil + } + } + } - return result.get() + await group.reduce(into: []) { + if let artifact = $1 { $0.append(artifact) } + } + } } package static func checksum( From 4914d9aa72ee6faf3ec7667f5ecde1709b1d4f13 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 31 Oct 2024 18:36:45 +0000 Subject: [PATCH 05/33] Address remaining warnings in `Workspace+BinaryArtifacts.swift` --- Sources/Workspace/Diagnostics.swift | 46 ++++++++++------- .../Workspace/Workspace+BinaryArtifacts.swift | 50 ++++++++++--------- .../Workspace/Workspace+Dependencies.swift | 4 +- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/Sources/Workspace/Diagnostics.swift b/Sources/Workspace/Diagnostics.swift index 70d2ab7d49c..7c6500cd2d2 100644 --- a/Sources/Workspace/Diagnostics.swift +++ b/Sources/Workspace/Diagnostics.swift @@ -123,59 +123,67 @@ extension Basics.Diagnostic { static func customDependencyMissing(packageName: String) -> Self { .warning("dependency '\(packageName)' is missing; retrieving again") } +} + +struct BinaryArtifactsManagerError: Error, CustomStringConvertible { + let description: String + + private init(description: String) { + self.description = description + } static func artifactInvalidArchive(artifactURL: URL, targetName: String) -> Self { - .error( - "invalid archive returned from '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)'" + .init( + description: "invalid archive returned from '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)'" ) } static func artifactChecksumChanged(targetName: String) -> Self { - .error( - "artifact of binary target '\(targetName)' has changed checksum; this is a potential security risk so the new artifact won't be downloaded" + .init( + description: "artifact of binary target '\(targetName)' has changed checksum; this is a potential security risk so the new artifact won't be downloaded" ) } static func artifactInvalidChecksum(targetName: String, expectedChecksum: String, actualChecksum: String?) -> Self { - .error( - "checksum of downloaded artifact of binary target '\(targetName)' (\(actualChecksum ?? "none")) does not match checksum specified by the manifest (\(expectedChecksum))" + .init( + description: "checksum of downloaded artifact of binary target '\(targetName)' (\(actualChecksum ?? "none")) does not match checksum specified by the manifest (\(expectedChecksum))" ) } static func artifactFailedDownload(artifactURL: URL, targetName: String, reason: String) -> Self { - .error( - "failed downloading '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)" + .init( + description: "failed downloading '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)" ) } static func artifactFailedValidation(artifactURL: URL, targetName: String, reason: String) -> Self { - .error( - "failed validating archive from '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)" + .init( + description: "failed validating archive from '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)" ) } static func remoteArtifactFailedExtraction(artifactURL: URL, targetName: String, reason: String) -> Self { - .error( - "failed extracting '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)" + .init( + description: "failed extracting '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)" ) } static func localArtifactFailedExtraction(artifactPath: AbsolutePath, targetName: String, reason: String) -> Self { - .error("failed extracting '\(artifactPath)' which is required by binary target '\(targetName)': \(reason)") + .init(description: "failed extracting '\(artifactPath)' which is required by binary target '\(targetName)': \(reason)") } static func remoteArtifactNotFound(artifactURL: URL, targetName: String) -> Self { - .error( - "downloaded archive of binary target '\(targetName)' from '\(artifactURL.absoluteString)' does not contain a binary artifact." + .init( + description: "downloaded archive of binary target '\(targetName)' from '\(artifactURL.absoluteString)' does not contain a binary artifact." ) } static func localArchivedArtifactNotFound(archivePath: AbsolutePath, targetName: String) -> Self { - .error("local archive of binary target '\(targetName)' at '\(archivePath)' does not contain a binary artifact.") + .init(description: "local archive of binary target '\(targetName)' at '\(archivePath)' does not contain a binary artifact.") } static func localArtifactNotFound(artifactPath: AbsolutePath, targetName: String) -> Self { - .error("local binary target '\(targetName)' at '\(artifactPath)' does not contain a binary artifact.") + .init(description: "local binary target '\(targetName)' at '\(artifactPath)' does not contain a binary artifact.") } static func exhaustedAttempts(missing: [PackageReference]) -> Self { @@ -189,8 +197,8 @@ extension Basics.Diagnostic { return "'\($0.identity)' at \(path)" } } - return .error( - "exhausted attempts to resolve the dependencies graph, with the following dependencies unresolved:\n* \(missing.joined(separator: "\n* "))" + return .init( + description: "exhausted attempts to resolve the dependencies graph, with the following dependencies unresolved:\n* \(missing.joined(separator: "\n* "))" ) } } diff --git a/Sources/Workspace/Workspace+BinaryArtifacts.swift b/Sources/Workspace/Workspace+BinaryArtifacts.swift index ca48810f45e..2db24f3a5b1 100644 --- a/Sources/Workspace/Workspace+BinaryArtifacts.swift +++ b/Sources/Workspace/Workspace+BinaryArtifacts.swift @@ -104,8 +104,12 @@ extension Workspace { path: absolutePath, observabilityScope: observabilityScope ) else { - observabilityScope - .emit(.localArtifactNotFound(artifactPath: absolutePath, targetName: target.name)) + observabilityScope.emit( + BinaryArtifactsManagerError.localArtifactNotFound( + artifactPath: absolutePath, + targetName: target.name + ) + ) continue } localArtifacts.append( @@ -253,11 +257,10 @@ extension Workspace { let valid = try await self.archiver.validate(path: archivePath) guard valid else { - observabilityScope - .emit(.artifactInvalidArchive( - artifactURL: artifact.url, - targetName: artifact.targetName - )) + observabilityScope.emit(BinaryArtifactsManagerError.artifactInvalidArchive( + artifactURL: artifact.url, + targetName: artifact.targetName + )) return nil } @@ -267,7 +270,7 @@ extension Workspace { return nil } guard archiveChecksum == artifact.checksum else { - observabilityScope.emit(.artifactInvalidChecksum( + observabilityScope.emit(BinaryArtifactsManagerError.artifactInvalidChecksum( targetName: artifact.targetName, expectedChecksum: artifact.checksum, actualChecksum: archiveChecksum @@ -343,11 +346,10 @@ extension Workspace { path: destinationDirectory, observabilityScope: observabilityScope ) else { - observabilityScope - .emit(.remoteArtifactNotFound( + observabilityScope.emit(BinaryArtifactsManagerError.remoteArtifactNotFound( artifactURL: artifact.url, targetName: artifact.targetName - )) + )) return nil } @@ -367,7 +369,7 @@ extension Workspace { ) } catch { - observabilityScope.emit(.remoteArtifactFailedExtraction( + observabilityScope.emit(BinaryArtifactsManagerError.remoteArtifactFailedExtraction( artifactURL: artifact.url, targetName: artifact.targetName, reason: error.interpolationDescription @@ -379,7 +381,7 @@ extension Workspace { ) } } catch { - observabilityScope.emit(.artifactFailedValidation( + observabilityScope.emit(BinaryArtifactsManagerError.artifactFailedValidation( artifactURL: artifact.url, targetName: artifact.targetName, reason: error.interpolationDescription @@ -392,7 +394,7 @@ extension Workspace { } } catch { observabilityScope.trap { try self.fileSystem.removeFileTree(archivePath) } - observabilityScope.emit(.artifactFailedDownload( + observabilityScope.emit(BinaryArtifactsManagerError.artifactFailedDownload( artifactURL: artifact.url, targetName: artifact.targetName, reason: error.interpolationDescription @@ -481,29 +483,27 @@ extension Workspace { path: destinationDirectory, observabilityScope: observabilityScope ) else { - observabilityScope.emit(.localArchivedArtifactNotFound( + throw BinaryArtifactsManagerError.localArchivedArtifactNotFound( archivePath: artifact.path, targetName: artifact.targetName - )) - - return nil + ) } // compute the checksum let artifactChecksum = try self.checksum(forBinaryArtifactAt: artifact.path) - return .some(ManagedArtifact.local( + return ManagedArtifact.local( packageRef: artifact.packageRef, targetName: artifact.targetName, path: artifactPath, kind: artifactKind, checksum: artifactChecksum - )) + ) } } catch { let reason = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription - observabilityScope.emit(.localArtifactFailedExtraction( + observabilityScope.emit(BinaryArtifactsManagerError.localArtifactFailedExtraction( artifactPath: artifact.path, targetName: artifact.targetName, reason: reason @@ -514,7 +514,7 @@ extension Workspace { } } - await group.reduce(into: []) { + return try await group.reduce(into: []) { if let artifact = $1 { $0.append(artifact) } } } @@ -840,7 +840,7 @@ extension Workspace { path: artifact.path, observabilityScope: observabilityScope ) else { - observabilityScope.emit(.localArtifactNotFound( + observabilityScope.emit(BinaryArtifactsManagerError.localArtifactNotFound( artifactPath: artifact.path, targetName: artifact.targetName )) @@ -871,7 +871,9 @@ extension Workspace { let urlChanged = artifact.url != URL(string: existingURL) // If the checksum is different but the package wasn't updated, this is a security risk. if !urlChanged && !addedOrUpdatedPackages.contains(artifact.packageRef) { - observabilityScope.emit(.artifactChecksumChanged(targetName: artifact.targetName)) + observabilityScope.emit( + BinaryArtifactsManagerError.artifactChecksumChanged(targetName: artifact.targetName) + ) continue } } diff --git a/Sources/Workspace/Workspace+Dependencies.swift b/Sources/Workspace/Workspace+Dependencies.swift index 92725dad40e..22c57eb7f06 100644 --- a/Sources/Workspace/Workspace+Dependencies.swift +++ b/Sources/Workspace/Workspace+Dependencies.swift @@ -164,7 +164,7 @@ extension Workspace { // If we have missing packages, something is fundamentally wrong with the resolution of the graph let stillMissingPackages = try updatedDependencyManifests.missingPackages guard stillMissingPackages.isEmpty else { - observabilityScope.emit(.exhaustedAttempts(missing: stillMissingPackages)) + observabilityScope.emit(BinaryArtifactsManagerError.exhaustedAttempts(missing: stillMissingPackages)) return nil } @@ -600,7 +600,7 @@ extension Workspace { // If we still have missing packages, something is fundamentally wrong with the resolution of the graph let stillMissingPackages = try updatedDependencyManifests.missingPackages guard stillMissingPackages.isEmpty else { - observabilityScope.emit(.exhaustedAttempts(missing: stillMissingPackages)) + observabilityScope.emit(BinaryArtifactsManagerError.exhaustedAttempts(missing: stillMissingPackages)) return updatedDependencyManifests } From 291705a4c89af5836d2ba6d38c2b8aea042d289d Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 31 Oct 2024 18:40:07 +0000 Subject: [PATCH 06/33] Break up long line for nicer formatting --- Sources/Workspace/Workspace+BinaryArtifacts.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Workspace/Workspace+BinaryArtifacts.swift b/Sources/Workspace/Workspace+BinaryArtifacts.swift index 2db24f3a5b1..2efcf00bb08 100644 --- a/Sources/Workspace/Workspace+BinaryArtifacts.swift +++ b/Sources/Workspace/Workspace+BinaryArtifacts.swift @@ -155,7 +155,10 @@ extension Workspace { if !indexFiles.isEmpty { let errors = ThreadSafeArrayStore() - zipArtifacts.append(contentsOf: try await withThrowingTaskGroup(of: RemoteArtifact?.self, returning: [RemoteArtifact].self) { group in + zipArtifacts.append(contentsOf: try await withThrowingTaskGroup( + of: RemoteArtifact?.self, + returning: [RemoteArtifact].self + ) { group in let jsonDecoder = JSONDecoder.makeWithDefaults() for indexFile in indexFiles { group.addTask { From 82505237f7d13e2ca489382754b4f90074defd19 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 29 Oct 2024 17:01:46 +0000 Subject: [PATCH 07/33] async `RegistryClient` WIP --- Sources/PackageRegistry/RegistryClient.swift | 357 +++++++------------ 1 file changed, 128 insertions(+), 229 deletions(-) diff --git a/Sources/PackageRegistry/RegistryClient.swift b/Sources/PackageRegistry/RegistryClient.swift index 838af6af76f..0340b911e05 100644 --- a/Sources/PackageRegistry/RegistryClient.swift +++ b/Sources/PackageRegistry/RegistryClient.swift @@ -30,7 +30,7 @@ public protocol RegistryClientDelegate { /// Package registry client. /// API specification: https://github.com/swiftlang/swift-package-manager/blob/main/Documentation/PackageRegistry/Registry.md -public final class RegistryClient: Cancellable { +public final class RegistryClient { public typealias Delegate = RegistryClientDelegate private static let apiVersion: APIVersion = .v1 @@ -39,7 +39,7 @@ public final class RegistryClient: Cancellable { private var configuration: RegistryConfiguration private let archiverProvider: (FileSystem) -> Archiver - private let httpClient: LegacyHTTPClient + private let httpClient: HTTPClient private let authorizationProvider: LegacyHTTPClientConfiguration.AuthorizationProvider? private let fingerprintStorage: PackageFingerprintStorage? private let fingerprintCheckingMode: FingerprintCheckingMode @@ -68,7 +68,7 @@ public final class RegistryClient: Cancellable { signingEntityStorage: PackageSigningEntityStorage?, signingEntityCheckingMode: SigningEntityCheckingMode, authorizationProvider: AuthorizationProvider? = .none, - customHTTPClient: LegacyHTTPClient? = .none, + customHTTPClient: HTTPClient? = .none, customArchiverProvider: ((FileSystem) -> Archiver)? = .none, delegate: Delegate?, checksumAlgorithm: HashAlgorithm @@ -97,7 +97,7 @@ public final class RegistryClient: Cancellable { self.authorizationProvider = .none } - self.httpClient = customHTTPClient ?? LegacyHTTPClient() + self.httpClient = customHTTPClient ?? HTTPClient() self.archiverProvider = customArchiverProvider ?? { fileSystem in ZipArchiver(fileSystem: fileSystem) } self.fingerprintStorage = fingerprintStorage self.fingerprintCheckingMode = fingerprintCheckingMode @@ -124,11 +124,6 @@ public final class RegistryClient: Cancellable { } } - /// Cancel any outstanding requests - public func cancel(deadline: DispatchTime) throws { - try self.httpClient.cancel(deadline: deadline) - } - public func changeSigningEntityFromVersion( package: PackageIdentity, version: Version, @@ -1372,13 +1367,10 @@ public final class RegistryClient: Cancellable { scmURL: SourceControlURL, timeout: DispatchTimeInterval?, observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result, Error>) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + callbackQueue: DispatchQueue + ) async throws -> Set { guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } components.appendPathComponents("identifiers") @@ -1387,47 +1379,45 @@ public final class RegistryClient: Cancellable { ] guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } - let request = LegacyHTTPClient.Request( + let request = HTTPClient.Request( method: .get, url: url, headers: [ "Accept": self.acceptHeader(mediaType: .json), ], - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + options: self.defaultRequestOptions(timeout: timeout) ) let start = DispatchTime.now() observabilityScope.emit(info: "looking up identity for \(scmURL) from \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - completion( - result.tryMap { response in - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - let packageIdentities = try response.parseJSON( - Serialization.PackageIdentifiers.self, - decoder: self.jsonDecoder - ) - observabilityScope.emit(debug: "matched identities for \(scmURL): \(packageIdentities)") - return Set(packageIdentities.identifiers.map { - PackageIdentity.plain($0) - }) - case 404: - // 404 is valid, no identities mapped - return [] - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - } - }.mapError { - RegistryError.failedIdentityLookup(registry: registry, scmURL: scmURL, error: $0) - } - ) + + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope + .emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + switch response.statusCode { + case 200: + let packageIdentities = try response.parseJSON( + Serialization.PackageIdentifiers.self, + decoder: self.jsonDecoder + ) + observabilityScope.emit(debug: "matched identities for \(scmURL): \(packageIdentities)") + return Set(packageIdentities.identifiers.map { + PackageIdentity.plain($0) + }) + case 404: + // 404 is valid, no identities mapped + return [] + default: + throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) + } + } catch { + throw RegistryError.failedIdentityLookup(registry: registry, scmURL: scmURL, error: error) } } @@ -1437,54 +1427,30 @@ public final class RegistryClient: Cancellable { observabilityScope: ObservabilityScope, callbackQueue: DispatchQueue ) async throws { - try await withCheckedThrowingContinuation { continuation in - self.login( - loginURL: loginURL, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } - ) - } - } - - @available(*, noasync, message: "Use the async alternative") - public func login( - loginURL: URL, - timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - - let request = LegacyHTTPClient.Request( + let request = HTTPClient.Request( method: .post, url: loginURL, - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + options: self.defaultRequestOptions(timeout: timeout) ) let start = DispatchTime.now() observabilityScope.emit(info: "logging-in into \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - switch result { - case .success(let response): - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - return completion(.success(())) - default: - let error = self.unexpectedStatusError(response, expectedStatus: [200]) - return completion(.failure(RegistryError.loginFailed(url: loginURL, error: error))) - } - case .failure(let error): - return completion(.failure(RegistryError.loginFailed(url: loginURL, error: error))) + + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope + .emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + switch response.statusCode { + case 200: + return + default: + let error = self.unexpectedStatusError(response, expectedStatus: [200]) + throw RegistryError.loginFailed(url: loginURL, error: error) } + } catch { + throw RegistryError.loginFailed(url: loginURL, error: error) } } @@ -1501,70 +1467,31 @@ public final class RegistryClient: Cancellable { fileSystem: FileSystem, observabilityScope: ObservabilityScope, callbackQueue: DispatchQueue - ) async throws -> PublishResult { - try await withCheckedThrowingContinuation { continuation in - self.publish( - registryURL: registryURL, - packageIdentity: packageIdentity, - packageVersion: packageVersion, - packageArchive: packageArchive, - packageMetadata: packageMetadata, - signature: signature, - metadataSignature: metadataSignature, - signatureFormat: signatureFormat, - timeout: timeout, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } - ) - } - } - - @available(*, noasync, message: "Use the async alternative") - public func publish( - registryURL: URL, - packageIdentity: PackageIdentity, - packageVersion: Version, - packageArchive: AbsolutePath, - packageMetadata: AbsolutePath?, - signature: [UInt8]?, - metadataSignature: [UInt8]?, - signatureFormat: SignatureFormat?, - timeout: DispatchTimeInterval? = .none, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + ) async throws -> PublishResult { guard let registryIdentity = packageIdentity.registry else { - return completion(.failure(RegistryError.invalidPackageIdentity(packageIdentity))) + throw RegistryError.invalidPackageIdentity(packageIdentity) } guard var components = URLComponents(url: registryURL, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registryURL))) + throw RegistryError.invalidURL(registryURL) } components.appendPathComponents(registryIdentity.scope.description) components.appendPathComponents(registryIdentity.name.description) components.appendPathComponents(packageVersion.description) guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registryURL))) + throw RegistryError.invalidURL(registryURL) } // TODO: don't load the entire file in memory guard let packageArchiveContent: Data = try? fileSystem.readFileContents(packageArchive) else { - return completion(.failure(RegistryError.failedLoadingPackageArchive(packageArchive))) + throw RegistryError.failedLoadingPackageArchive(packageArchive) } var metadataContent: String? = .none if let packageMetadata { do { metadataContent = try fileSystem.readFileContents(packageMetadata) } catch { - return completion(.failure(RegistryError.failedLoadingPackageMetadata(packageMetadata))) + throw RegistryError.failedLoadingPackageMetadata(packageMetadata) } } @@ -1584,7 +1511,7 @@ public final class RegistryClient: Cancellable { if let signature { guard signatureFormat != nil else { - return completion(.failure(RegistryError.missingSignatureFormat)) + throw RegistryError.missingSignatureFormat } body.append(contentsOf: """ @@ -1612,20 +1539,17 @@ public final class RegistryClient: Cancellable { if signature != nil { guard metadataSignature != nil else { - return completion(.failure( - RegistryError.invalidSignature(reason: "both archive and metadata must be signed") - )) + throw RegistryError.invalidSignature(reason: "both archive and metadata must be signed") } } if let metadataSignature { guard signature != nil else { - return completion(.failure( - RegistryError.invalidSignature(reason: "both archive and metadata must be signed") - )) + throw RegistryError.invalidSignature(reason: "both archive and metadata must be signed") } + guard signatureFormat != nil else { - return completion(.failure(RegistryError.missingSignatureFormat)) + throw RegistryError.missingSignatureFormat } body.append(contentsOf: """ @@ -1643,7 +1567,7 @@ public final class RegistryClient: Cancellable { // footer body.append(contentsOf: "\r\n--\(boundary)--\r\n".utf8) - var request = LegacyHTTPClient.Request( + var request = HTTPClient.Request( method: .put, url: url, headers: [ @@ -1653,7 +1577,7 @@ public final class RegistryClient: Cancellable { "Prefer": "respond-async", ], body: body, - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + options: self.defaultRequestOptions(timeout: timeout) ) if signature != nil, let signatureFormat { @@ -1662,117 +1586,89 @@ public final class RegistryClient: Cancellable { let start = DispatchTime.now() observabilityScope.emit(info: "publishing \(packageIdentity) \(packageVersion) to \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - completion( - result.tryMap { response in - observabilityScope - .emit( - debug: "server response for \(url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 201: - try response.validateAPIVersion() - let location = response.headers.get("Location").first.flatMap { URL(string: $0) } - return PublishResult.published(location) - case 202: - try response.validateAPIVersion() - - guard let location = (response.headers.get("Location").first.flatMap { URL(string: $0) }) else { - throw RegistryError.missingPublishingLocation - } - let retryAfter = response.headers.get("Retry-After").first.flatMap { Int($0) } - return PublishResult.processing(statusURL: location, retryAfter: retryAfter) - default: - throw self.unexpectedStatusError(response, expectedStatus: [201, 202]) - } - }.mapError { - RegistryError.failedPublishing($0) - } + + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope.emit( + debug: "server response for \(url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" ) - } - } - func checkAvailability( - registry: Registry, - timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws -> AvailabilityStatus { - try await withCheckedThrowingContinuation { continuation in - self.checkAvailability( - registry: registry, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) + switch response.statusCode { + case 201: + try response.validateAPIVersion() + let location = response.headers.get("Location").first.flatMap { URL(string: $0) } + return PublishResult.published(location) + case 202: + try response.validateAPIVersion() + + guard let location = (response.headers.get("Location").first.flatMap { URL(string: $0) }) else { + throw RegistryError.missingPublishingLocation } - ) + let retryAfter = response.headers.get("Retry-After").first.flatMap { Int($0) } + return PublishResult.processing(statusURL: location, retryAfter: retryAfter) + default: + throw self.unexpectedStatusError(response, expectedStatus: [201, 202]) + } + } catch { + throw RegistryError.failedPublishing(error) } } // marked internal for testing - @available(*, noasync, message: "Use the async alternative") func checkAvailability( registry: Registry, timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> AvailabilityStatus { guard registry.supportsAvailability else { - return completion(.failure(StringError("registry \(registry.url) does not support availability checks."))) + throw StringError("registry \(registry.url) does not support availability checks.") } guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } components.appendPathComponents("availability") guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } - let request = LegacyHTTPClient.Request( + let request = HTTPClient.Request( method: .get, url: url, - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + options: self.defaultRequestOptions(timeout: timeout) ) let start = DispatchTime.now() observabilityScope.emit(info: "checking availability of \(registry.url) using \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - switch result { - case .success(let response): - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - return completion(.success(.available)) - case let value where AvailabilityStatus.unavailableStatusCodes.contains(value): - return completion(.success(.unavailable)) - default: - if let error = try? response.parseError(decoder: self.jsonDecoder) { - return completion(.success(.error(error.detail))) - } - return completion(.success(.error("unknown server error (\(response.statusCode))"))) + + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope + .emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + switch response.statusCode { + case 200: + return .available + case let value where AvailabilityStatus.unavailableStatusCodes.contains(value): + return .unavailable + default: + if let error = try? response.parseError(decoder: self.jsonDecoder) { + return .error(error.detail) } - case .failure(let error): - return completion(.failure(RegistryError.availabilityCheckFailed(registry: registry, error: error))) + return .error("unknown server error (\(response.statusCode))") } + } catch { + throw RegistryError.availabilityCheckFailed(registry: registry, error: error) } } private func withAvailabilityCheck( registry: Registry, observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, next: @escaping (Error?) -> Void - ) { + ) async throws { let availabilityHandler: (Result) -> Void = { (result: Result) in switch result { @@ -1794,14 +1690,19 @@ public final class RegistryClient: Cancellable { return availabilityHandler(cached.status) } - self.checkAvailability( - registry: registry, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - self.availabilityCache[registry.url] = (status: result, expires: .now() + Self.availabilityCacheTTL) - availabilityHandler(result) + let result: Result + + do { + result = try await .success(self.checkAvailability( + registry: registry, + observabilityScope: observabilityScope + )) + } catch { + result = .failure(error) } + + self.availabilityCache[registry.url] = (status: result, expires: .now() + Self.availabilityCacheTTL) + availabilityHandler(result) } private func unexpectedStatusError( @@ -1842,12 +1743,10 @@ public final class RegistryClient: Cancellable { } private func defaultRequestOptions( - timeout: DispatchTimeInterval? = .none, - callbackQueue: DispatchQueue - ) -> LegacyHTTPClient.Request.Options { - var options = LegacyHTTPClient.Request.Options() + timeout: DispatchTimeInterval? = nil + ) -> HTTPClient.Request.Options { + var options = HTTPClient.Request.Options() options.timeout = timeout - options.callbackQueue = callbackQueue options.authorizationProvider = self.authorizationProvider return options } From 0db79711955210f8fe76b458b0d42d6ae68c8d10 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 30 Oct 2024 19:07:55 +0000 Subject: [PATCH 08/33] `LegacyHTTPClient` removal WIP --- .../FileSystem/FileSystem+Extensions.swift | 5 + .../HTTPClient/URLSessionHTTPClient.swift | 5 + .../Commands/CommandWorkspaceDelegate.swift | 44 +- Sources/PackageLoading/ManifestLoader.swift | 548 +++--- Sources/PackageMetadata/PackageMetadata.swift | 38 +- Sources/PackageRegistry/ChecksumTOFU.swift | 139 +- Sources/PackageRegistry/RegistryClient.swift | 1523 +++++++---------- .../RegistryDownloadsManager.swift | 191 +-- .../PackageRegistry/SignatureValidation.swift | 740 ++++---- .../PackageRegistry/SigningEntityTOFU.swift | 180 +- .../PackageRegistryCommand+Auth.swift | 3 +- .../PackageRegistryCommand+Publish.swift | 3 +- .../FileSystemPackageContainer.swift | 34 +- .../RegistryPackageContainer.swift | 198 +-- .../SourceControlPackageContainer.swift | 35 +- Sources/Workspace/Workspace+Delegation.swift | 43 +- Sources/Workspace/Workspace+Editing.swift | 20 +- Sources/Workspace/Workspace+Manifests.swift | 126 +- Sources/Workspace/Workspace+Registry.swift | 141 +- Sources/Workspace/Workspace.swift | 280 ++- .../MockManifestLoader.swift | 17 +- .../_InternalTestSupport/MockRegistry.swift | 48 +- .../RegistryDownloadsManagerTests.swift | 15 +- .../RegistryPackageContainerTests.swift | 252 ++- Tests/WorkspaceTests/WorkspaceTests.swift | 147 +- 25 files changed, 1920 insertions(+), 2855 deletions(-) diff --git a/Sources/Basics/FileSystem/FileSystem+Extensions.swift b/Sources/Basics/FileSystem/FileSystem+Extensions.swift index 76d49229dc4..0fc4c47911e 100644 --- a/Sources/Basics/FileSystem/FileSystem+Extensions.swift +++ b/Sources/Basics/FileSystem/FileSystem+Extensions.swift @@ -200,6 +200,11 @@ extension FileSystem { try self.withLock(on: path.underlying, type: type, blocking: blocking, body) } + /// Execute the given block while holding the lock. + public func withLock(on path: AbsolutePath, type: FileLock.LockType, blocking: Bool = true, _ body: () async throws -> T) async throws -> T { + try await self.withLock(on: path.underlying, type: type, blocking: blocking, body) + } + /// Returns any known item replacement directories for a given path. These may be used by platform-specific /// libraries to handle atomic file system operations, such as deletion. func itemReplacementDirectories(for path: AbsolutePath) throws -> [AbsolutePath] { diff --git a/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift b/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift index 7c27608749d..450c09370b3 100644 --- a/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift +++ b/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift @@ -18,6 +18,11 @@ import struct TSCUtility.Versioning import FoundationNetworking #endif +protocol HTTPClientImplementation: Sendable { + @Sendable + func execute(request: HTTPClient.Request, progressHandler: HTTPClient.ProgressHandler?) async throws -> HTTPClient.Response +} + final class URLSessionHTTPClient: Sendable { private let dataSession: URLSession private let downloadSession: URLSession diff --git a/Sources/Commands/CommandWorkspaceDelegate.swift b/Sources/Commands/CommandWorkspaceDelegate.swift index accec9fe9a3..b720a015b12 100644 --- a/Sources/Commands/CommandWorkspaceDelegate.swift +++ b/Sources/Commands/CommandWorkspaceDelegate.swift @@ -184,30 +184,34 @@ final class CommandWorkspaceDelegate: WorkspaceDelegate { // registry signature handlers - func onUnsignedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version, completion: (Bool) -> Void) { - self.inputHandler("\(package) \(version) from \(registryURL) is unsigned. okay to proceed? (yes/no) ") { response in - switch response?.lowercased() { - case "yes": - completion(true) // continue - case "no": - completion(false) // stop resolution - default: - self.outputHandler("invalid response: '\(response ?? "")'", false) - completion(false) + func onUnsignedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version) async -> Bool { + await withCheckedContinuation { continuation in + self.inputHandler("\(package) \(version) from \(registryURL) is unsigned. okay to proceed? (yes/no) ") { response in + switch response?.lowercased() { + case "yes": + continuation.resume(returning: true) // continue + case "no": + continuation.resume(returning: false) // stop resolution + default: + self.outputHandler("invalid response: '\(response ?? "")'", false) + continuation.resume(returning: false) + } } } } - func onUntrustedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version, completion: (Bool) -> Void) { - self.inputHandler("\(package) \(version) from \(registryURL) is signed with an untrusted certificate. okay to proceed? (yes/no) ") { response in - switch response?.lowercased() { - case "yes": - completion(true) // continue - case "no": - completion(false) // stop resolution - default: - self.outputHandler("invalid response: '\(response ?? "")'", false) - completion(false) + func onUntrustedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version) async -> Bool { + await withCheckedContinuation { continuation in + self.inputHandler("\(package) \(version) from \(registryURL) is signed with an untrusted certificate. okay to proceed? (yes/no) ") { response in + switch response?.lowercased() { + case "yes": + continuation.resume(returning: true) // continue + case "no": + continuation.resume(returning: false) // stop resolution + default: + self.outputHandler("invalid response: '\(response ?? "")'", false) + continuation.resume(returning: false) + } } } } diff --git a/Sources/PackageLoading/ManifestLoader.swift b/Sources/PackageLoading/ManifestLoader.swift index 3fd515132c5..327270f88c5 100644 --- a/Sources/PackageLoading/ManifestLoader.swift +++ b/Sources/PackageLoading/ManifestLoader.swift @@ -126,9 +126,8 @@ public protocol ManifestLoaderProtocol { fileSystem: FileSystem, observabilityScope: ObservabilityScope, delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) + callbackQueue: DispatchQueue + ) async throws -> Manifest /// Reset any internal cache held by the manifest loader. func resetCache(observabilityScope: ObservabilityScope) @@ -189,50 +188,6 @@ public protocol ManifestLoaderDelegate { // this will first find the most appropriate manifest file in the package directory // bases on the toolchain's tools-version and proceed to load that manifest extension ManifestLoaderProtocol { - public func load( - packagePath: AbsolutePath, - packageIdentity: PackageIdentity, - packageKind: PackageReference.Kind, - packageLocation: String, - packageVersion: (version: Version?, revision: String?)?, - currentToolsVersion: ToolsVersion, - identityResolver: IdentityResolver, - dependencyMapper: DependencyMapper, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - do { - // find the manifest path and parse it's tools-version - let manifestPath = try ManifestLoader.findManifest(packagePath: packagePath, fileSystem: fileSystem, currentToolsVersion: currentToolsVersion) - let manifestToolsVersion = try ToolsVersionParser.parse(manifestPath: manifestPath, fileSystem: fileSystem) - // validate the manifest tools-version against the toolchain tools-version - try manifestToolsVersion.validateToolsVersion(currentToolsVersion, packageIdentity: packageIdentity, packageVersion: packageVersion?.version?.description ?? packageVersion?.revision) - - self.load( - manifestPath: manifestPath, - manifestToolsVersion: manifestToolsVersion, - packageIdentity: packageIdentity, - packageKind: packageKind, - packageLocation: packageLocation, - packageVersion: packageVersion, - identityResolver: identityResolver, - dependencyMapper: dependencyMapper, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue, - completion: completion - ) - } catch { - callbackQueue.async { - completion(.failure(error)) - } - } - } - public func load( packagePath: AbsolutePath, packageIdentity: PackageIdentity, @@ -247,25 +202,26 @@ extension ManifestLoaderProtocol { delegateQueue: DispatchQueue, callbackQueue: DispatchQueue ) async throws -> Manifest { - try await withCheckedThrowingContinuation { continuation in - self.load( - packagePath: packagePath, - packageIdentity: packageIdentity, - packageKind: packageKind, - packageLocation: packageLocation, - packageVersion: packageVersion, - currentToolsVersion: currentToolsVersion, - identityResolver: identityResolver, - dependencyMapper: dependencyMapper, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } - ) - } + // find the manifest path and parse it's tools-version + let manifestPath = try ManifestLoader.findManifest(packagePath: packagePath, fileSystem: fileSystem, currentToolsVersion: currentToolsVersion) + let manifestToolsVersion = try ToolsVersionParser.parse(manifestPath: manifestPath, fileSystem: fileSystem) + // validate the manifest tools-version against the toolchain tools-version + try manifestToolsVersion.validateToolsVersion(currentToolsVersion, packageIdentity: packageIdentity, packageVersion: packageVersion?.version?.description ?? packageVersion?.revision) + + return try await self.load( + manifestPath: manifestPath, + manifestToolsVersion: manifestToolsVersion, + packageIdentity: packageIdentity, + packageKind: packageKind, + packageLocation: packageLocation, + packageVersion: packageVersion, + identityResolver: identityResolver, + dependencyMapper: dependencyMapper, + fileSystem: fileSystem, + observabilityScope: observabilityScope, + delegateQueue: delegateQueue, + callbackQueue: callbackQueue + ) } } @@ -300,6 +256,8 @@ public final class ManifestLoader: ManifestLoaderProtocol { /// OperationQueue to park pending lookups private let evaluationQueue: OperationQueue + private let tokenBucket = TokenBucket(tokens: Concurrency.maxOperations) + public init( toolchain: UserToolchain, serializedDiagnostics: Bool = false, @@ -327,7 +285,7 @@ public final class ManifestLoader: ManifestLoaderProtocol { self.evaluationQueue.maxConcurrentOperationCount = Concurrency.maxOperations self.concurrencySemaphore = DispatchSemaphore(value: Concurrency.maxOperations) } - + public func load( manifestPath: AbsolutePath, manifestToolsVersion: ToolsVersion, @@ -342,43 +300,6 @@ public final class ManifestLoader: ManifestLoaderProtocol { delegateQueue: DispatchQueue, callbackQueue: DispatchQueue ) async throws -> Manifest { - try await withCheckedThrowingContinuation { continuation in - self.load( - manifestPath: manifestPath, - manifestToolsVersion: manifestToolsVersion, - packageIdentity: packageIdentity, - packageKind: packageKind, - packageLocation: packageLocation, - packageVersion: packageVersion, - identityResolver: identityResolver, - dependencyMapper: dependencyMapper, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } - ) - } - } - - @available(*, noasync, message: "Use the async alternative") - public func load( - manifestPath: AbsolutePath, - manifestToolsVersion: ToolsVersion, - packageIdentity: PackageIdentity, - packageKind: PackageReference.Kind, - packageLocation: String, - packageVersion: (version: Version?, revision: String?)?, - identityResolver: IdentityResolver, - dependencyMapper: DependencyMapper, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { // Inform the delegate. let start = DispatchTime.now() delegateQueue.async { @@ -391,12 +312,10 @@ public final class ManifestLoader: ManifestLoaderProtocol { // Validate that the file exists. guard fileSystem.isFile(manifestPath) else { - return callbackQueue.async { - completion(.failure(PackageModel.Package.Error.noManifest(at: manifestPath, version: packageVersion?.version))) - } + throw PackageModel.Package.Error.noManifest(at: manifestPath, version: packageVersion?.version) } - self.loadAndCacheManifest( + let parsedManifest = try await self.loadAndCacheManifest( at: manifestPath, toolsVersion: manifestToolsVersion, packageIdentity: packageIdentity, @@ -410,69 +329,60 @@ public final class ManifestLoader: ManifestLoaderProtocol { delegate: delegate, delegateQueue: delegateQueue, callbackQueue: callbackQueue - ) { parseResult in - do { - dispatchPrecondition(condition: .onQueue(callbackQueue)) - - let parsedManifest = try parseResult.get() - // Convert legacy system packages to the current target‐based model. - var products = parsedManifest.products - var targets = parsedManifest.targets - if products.isEmpty, targets.isEmpty, - fileSystem.isFile(manifestPath.parentDirectory.appending(component: moduleMapFilename)) { - try products.append(ProductDescription( - name: parsedManifest.name, - type: .library(.automatic), - targets: [parsedManifest.name]) - ) - targets.append(try TargetDescription( - name: parsedManifest.name, - path: "", - type: .system, - packageAccess: false, - pkgConfig: parsedManifest.pkgConfig, - providers: parsedManifest.providers - )) - } + ) - let manifest = Manifest( - displayName: parsedManifest.name, - path: manifestPath, - packageKind: packageKind, - packageLocation: packageLocation, - defaultLocalization: parsedManifest.defaultLocalization, - platforms: parsedManifest.platforms, - version: packageVersion?.version, - revision: packageVersion?.revision, - toolsVersion: manifestToolsVersion, - pkgConfig: parsedManifest.pkgConfig, - providers: parsedManifest.providers, - cLanguageStandard: parsedManifest.cLanguageStandard, - cxxLanguageStandard: parsedManifest.cxxLanguageStandard, - swiftLanguageVersions: parsedManifest.swiftLanguageVersions, - dependencies: parsedManifest.dependencies, - products: products, - targets: targets, - traits: parsedManifest.traits - ) + // Convert legacy system packages to the current target‐based model. + var products = parsedManifest.products + var targets = parsedManifest.targets + if products.isEmpty, targets.isEmpty, + fileSystem.isFile(manifestPath.parentDirectory.appending(component: moduleMapFilename)) { + try products.append(ProductDescription( + name: parsedManifest.name, + type: .library(.automatic), + targets: [parsedManifest.name]) + ) + targets.append(try TargetDescription( + name: parsedManifest.name, + path: "", + type: .system, + packageAccess: false, + pkgConfig: parsedManifest.pkgConfig, + providers: parsedManifest.providers + )) + } - // Inform the delegate. - delegateQueue.async { - self.delegate?.didLoad( - packageIdentity: packageIdentity, - packageLocation: packageLocation, - manifestPath: manifestPath, - duration: start.distance(to: .now()) - ) - } + let manifest = Manifest( + displayName: parsedManifest.name, + path: manifestPath, + packageKind: packageKind, + packageLocation: packageLocation, + defaultLocalization: parsedManifest.defaultLocalization, + platforms: parsedManifest.platforms, + version: packageVersion?.version, + revision: packageVersion?.revision, + toolsVersion: manifestToolsVersion, + pkgConfig: parsedManifest.pkgConfig, + providers: parsedManifest.providers, + cLanguageStandard: parsedManifest.cLanguageStandard, + cxxLanguageStandard: parsedManifest.cxxLanguageStandard, + swiftLanguageVersions: parsedManifest.swiftLanguageVersions, + dependencies: parsedManifest.dependencies, + products: products, + targets: targets, + traits: parsedManifest.traits + ) - completion(.success(manifest)) - } catch { - callbackQueue.async { - completion(.failure(error)) - } - } + // Inform the delegate. + delegateQueue.async { + self.delegate?.didLoad( + packageIdentity: packageIdentity, + packageLocation: packageLocation, + manifestPath: manifestPath, + duration: start.distance(to: .now()) + ) } + + return manifest } /// Load the JSON string for the given manifest. @@ -553,47 +463,23 @@ public final class ManifestLoader: ManifestLoaderProtocol { observabilityScope: ObservabilityScope, delegate: Delegate?, delegateQueue: DispatchQueue?, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - // put callback on right queue - var completion = completion - do { - let previousCompletion = completion - completion = { result in callbackQueue.async { previousCompletion(result) } } - } - - let key : CacheKey - do { - key = try CacheKey( - packageIdentity: packageIdentity, - packageLocation: packageLocation, - manifestPath: path, - toolsVersion: toolsVersion, - env: Environment.current.cachable, - swiftpmVersion: SwiftVersion.current.displayString, - extraManifestFlags: self.extraManifestFlags, - fileSystem: fileSystem - ) - } catch { - return completion(.failure(error)) - } + callbackQueue: DispatchQueue + ) async throws -> ManifestJSONParser.Result { + let key = try CacheKey( + packageIdentity: packageIdentity, + packageLocation: packageLocation, + manifestPath: path, + toolsVersion: toolsVersion, + env: Environment.current.cachable, + swiftpmVersion: SwiftVersion.current.displayString, + extraManifestFlags: self.extraManifestFlags, + fileSystem: fileSystem + ) // try from in-memory cache if self.useInMemoryCache, let parsedManifest = self.memoryCache[key] { observabilityScope.emit(debug: "loading manifest for '\(packageIdentity)' v. \(packageVersion?.description ?? "unknown") from memory cache") - return completion(.success(parsedManifest)) - } - - // make sure callback record results in-memory - do { - let previousCompletion = completion - completion = { result in - if self.useInMemoryCache, case .success(let parsedManifest) = result { - self.memoryCache[key] = parsedManifest - } - previousCompletion(result) - } + return parsedManifest } // initialize db cache @@ -610,19 +496,14 @@ public final class ManifestLoader: ManifestLoaderProtocol { ) } - // make sure callback closes db cache - do { - let previousCompletion = completion - completion = { result in - do { - try dbCache?.close() - } catch { - observabilityScope.emit( - warning: "failed closing manifest db cache", - underlyingError: error - ) - } - previousCompletion(result) + defer { + do { + try dbCache?.close() + } catch { + observabilityScope.emit( + warning: "failed closing manifest db cache", + underlyingError: error + ) } } @@ -645,7 +526,11 @@ public final class ManifestLoader: ManifestLoaderProtocol { delegate: delegate, delegateQueue: delegateQueue ) - return completion(.success(parsedManifest)) + + if self.useInMemoryCache { + self.memoryCache[key] = parsedManifest + } + return parsedManifest } } catch { observabilityScope.emit( @@ -656,104 +541,75 @@ public final class ManifestLoader: ManifestLoaderProtocol { // shells out and compiles the manifest, finally output a JSON observabilityScope.emit(debug: "evaluating manifest for '\(packageIdentity)' v. \(packageVersion?.description ?? "unknown")") - do { - try self.evaluateManifest( - packageIdentity: key.packageIdentity, - packageLocation: packageLocation, - manifestPath: key.manifestPath, - manifestContents: key.manifestContents, - toolsVersion: key.toolsVersion, - observabilityScope: observabilityScope, - delegate: delegate, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue - ) { result in - dispatchPrecondition(condition: .onQueue(callbackQueue)) - do { - let evaluationResult = try result.get() - // only cache successfully parsed manifests - let parsedManifest = try self.parseManifest( - evaluationResult, - packageIdentity: packageIdentity, - packageKind: packageKind, - packagePath: path.parentDirectory, - packageLocation: packageLocation, - toolsVersion: toolsVersion, - identityResolver: identityResolver, - dependencyMapper: dependencyMapper, - fileSystem: fileSystem, - emitCompilerOutput: true, - observabilityScope: observabilityScope, - delegate: delegate, - delegateQueue: delegateQueue - ) + let evaluationResult = try await self.evaluateManifest( + packageIdentity: key.packageIdentity, + packageLocation: packageLocation, + manifestPath: key.manifestPath, + manifestContents: key.manifestContents, + toolsVersion: key.toolsVersion, + observabilityScope: observabilityScope, + delegate: delegate, + delegateQueue: delegateQueue, + callbackQueue: callbackQueue + ) - do { - self.memoryCache[key] = parsedManifest - try dbCache?.put(key: key.sha256Checksum, value: evaluationResult, observabilityScope: observabilityScope) - } catch { - observabilityScope.emit( - warning: "failed storing manifest for '\(key.packageIdentity)' in cache", - underlyingError: error - ) - } + // only cache successfully parsed manifests + let parsedManifest = try self.parseManifest( + evaluationResult, + packageIdentity: packageIdentity, + packageKind: packageKind, + packagePath: path.parentDirectory, + packageLocation: packageLocation, + toolsVersion: toolsVersion, + identityResolver: identityResolver, + dependencyMapper: dependencyMapper, + fileSystem: fileSystem, + emitCompilerOutput: true, + observabilityScope: observabilityScope, + delegate: delegate, + delegateQueue: delegateQueue + ) - return completion(.success(parsedManifest)) - } catch { - return completion(.failure(error)) - } - } + do { + self.memoryCache[key] = parsedManifest + try dbCache?.put(key: key.sha256Checksum, value: evaluationResult, observabilityScope: observabilityScope) } catch { - return completion(.failure(error)) + observabilityScope.emit( + warning: "failed storing manifest for '\(key.packageIdentity)' in cache", + underlyingError: error + ) } + + return parsedManifest } private func validateImports( manifestPath: AbsolutePath, - toolsVersion: ToolsVersion, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void) { - // If there are no import restrictions, we do not need to validate. - guard let importRestrictions = self.importRestrictions, toolsVersion >= importRestrictions.startingToolsVersion else { - return callbackQueue.async { - completion(.success(())) - } - } - - // Allowed are the expected defaults, plus anything allowed by the configured restrictions. - let allowedImports = ["PackageDescription", "Swift", - "SwiftOnoneSupport", "_SwiftConcurrencyShims"] + importRestrictions.allowedImports - - // wrap the completion to free concurrency control semaphore - let completion: (Result) -> Void = { result in - self.concurrencySemaphore.signal() - callbackQueue.async { - completion(result) - } - } + toolsVersion: ToolsVersion + ) async throws { + // If there are no import restrictions, we do not need to validate. + guard let importRestrictions = self.importRestrictions, toolsVersion >= importRestrictions.startingToolsVersion else { + return + } - // we must not block the calling thread (for concurrency control) so nesting this in a queue - self.evaluationQueue.addOperation { - // park the evaluation thread based on the max concurrency allowed - self.concurrencySemaphore.wait() + // Allowed are the expected defaults, plus anything allowed by the configured restrictions. + let allowedImports = ["PackageDescription", "Swift", + "SwiftOnoneSupport", "_SwiftConcurrencyShims"] + importRestrictions.allowedImports - let importScanner = SwiftcImportScanner(swiftCompilerEnvironment: self.toolchain.swiftCompilerEnvironment, - swiftCompilerFlags: self.extraManifestFlags, - swiftCompilerPath: self.toolchain.swiftCompilerPathForManifests) + // we must not block the calling thread (for concurrency control) so scheduling this with a token bucket + try await self.tokenBucket.withToken { + let importScanner = SwiftcImportScanner(swiftCompilerEnvironment: self.toolchain.swiftCompilerEnvironment, + swiftCompilerFlags: self.extraManifestFlags, + swiftCompilerPath: self.toolchain.swiftCompilerPathForManifests) - Task { - let result = try await importScanner.scanImports(manifestPath) - let imports = result.filter { !allowedImports.contains($0) } - guard imports.isEmpty else { - callbackQueue.async { - completion(.failure(ManifestParseError.importsRestrictedModules(imports))) - } - return - } - } + let result = try await importScanner.scanImports(manifestPath) + let imports = result.filter { !allowedImports.contains($0) } + guard imports.isEmpty else { + throw ManifestParseError.importsRestrictedModules(imports) } } + } /// Compiler the manifest at the given path and retrieve the JSON. fileprivate func evaluateManifest( @@ -765,9 +621,8 @@ public final class ManifestLoader: ManifestLoaderProtocol { observabilityScope: ObservabilityScope, delegate: Delegate?, delegateQueue: DispatchQueue?, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) throws { + callbackQueue: DispatchQueue + ) async throws -> EvaluationResult { let manifestPreamble: ByteString if toolsVersion >= .v5_8 { manifestPreamble = ByteString() @@ -775,58 +630,43 @@ public final class ManifestLoader: ManifestLoaderProtocol { manifestPreamble = ByteString("\nimport Foundation") } - do { - try withTemporaryDirectory { tempDir, cleanupTempDir in - let manifestTempFilePath = tempDir.appending("manifest.swift") - // Since this isn't overwriting the original file, append Foundation library - // import to avoid having diagnostics being displayed on the incorrect line. - try localFileSystem.writeFileContents(manifestTempFilePath, bytes: ByteString(manifestContents + manifestPreamble.contents)) - - let vfsOverlayTempFilePath = tempDir.appending("vfs.yaml") - try VFSOverlay(roots: [ - VFSOverlay.File( - name: manifestPath._normalized.replacingOccurrences(of: #"\"#, with: #"\\"#), - externalContents: manifestTempFilePath._nativePathString(escaped: true) - ) - ]).write(to: vfsOverlayTempFilePath, fileSystem: localFileSystem) + return try await withTemporaryDirectory(removeTreeOnDeinit: true) { tempDir in + let manifestTempFilePath = tempDir.appending("manifest.swift") + // Since this isn't overwriting the original file, append Foundation library + // import to avoid having diagnostics being displayed on the incorrect line. + try localFileSystem.writeFileContents(manifestTempFilePath, bytes: ByteString(manifestContents + manifestPreamble.contents)) + + let vfsOverlayTempFilePath = tempDir.appending("vfs.yaml") + try VFSOverlay(roots: [ + VFSOverlay.File( + name: manifestPath._normalized.replacingOccurrences(of: #"\"#, with: #"\\"#), + externalContents: manifestTempFilePath._nativePathString(escaped: true) + ) + ]).write(to: vfsOverlayTempFilePath, fileSystem: localFileSystem) - validateImports( - manifestPath: manifestTempFilePath, - toolsVersion: toolsVersion, - callbackQueue: callbackQueue - ) { result in - dispatchPrecondition(condition: .onQueue(callbackQueue)) - - do { - try result.get() - - try self.evaluateManifest( - at: manifestPath, - vfsOverlayPath: vfsOverlayTempFilePath, - packageIdentity: packageIdentity, - packageLocation: packageLocation, - toolsVersion: toolsVersion, - observabilityScope: observabilityScope, - delegate: delegate, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue - ) { result in - dispatchPrecondition(condition: .onQueue(callbackQueue)) - cleanupTempDir(tempDir) - completion(result) - } - } catch { - cleanupTempDir(tempDir) - callbackQueue.async { - completion(.failure(error)) - } - } + try await validateImports( + manifestPath: manifestTempFilePath, + toolsVersion: toolsVersion + ) + + return try await withCheckedThrowingContinuation { continuation in + do { + try self.evaluateManifest( + at: manifestPath, + vfsOverlayPath: vfsOverlayTempFilePath, + packageIdentity: packageIdentity, + packageLocation: packageLocation, + toolsVersion: toolsVersion, + observabilityScope: observabilityScope, + delegate: delegate, + delegateQueue: delegateQueue, + callbackQueue: callbackQueue, + completion: { continuation.resume(with: $0) } + ) + } catch { + continuation.resume(throwing: error) } } - } catch { - callbackQueue.async { - completion(.failure(error)) - } } } diff --git a/Sources/PackageMetadata/PackageMetadata.swift b/Sources/PackageMetadata/PackageMetadata.swift index 7d5dac640ee..9af67ec9a46 100644 --- a/Sources/PackageMetadata/PackageMetadata.swift +++ b/Sources/PackageMetadata/PackageMetadata.swift @@ -174,8 +174,8 @@ public struct PackageSearchClient { package: package, version: version, fileSystem: self.fileSystem, - observabilityScope: observabilityScope, - callbackQueue: DispatchQueue.sharedConcurrent) + observabilityScope: observabilityScope + ) return Metadata( licenseURL: metadata.licenseURL, @@ -285,7 +285,7 @@ public struct PackageSearchClient { } let metadata: RegistryClient.PackageMetadata do { - metadata = try await self.registryClient.getPackageMetadata(package: identity, observabilityScope: observabilityScope, callbackQueue: DispatchQueue.sharedConcurrent) + metadata = try await self.registryClient.getPackageMetadata(package: identity, observabilityScope: observabilityScope) } catch { return try await fetchStandalonePackageByURL(error) } @@ -324,16 +324,12 @@ public struct PackageSearchClient { public func lookupIdentities( scmURL: SourceControlURL, timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result, Error>) -> Void - ) { - registryClient.lookupIdentities( + observabilityScope: ObservabilityScope + ) async throws -> Set { + try await self.registryClient.lookupIdentities( scmURL: scmURL, timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion + observabilityScope: observabilityScope ) } @@ -343,21 +339,15 @@ public struct PackageSearchClient { observabilityScope: ObservabilityScope, callbackQueue: DispatchQueue, completion: @escaping (Result, Error>) -> Void - ) { - registryClient.getPackageMetadata( + ) async throws -> Set { + let metadata = try await self.registryClient.getPackageMetadata( package: package, timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - do { - let metadata = try result.get() - let alternateLocations = metadata.alternateLocations ?? [] - return completion(.success(Set(alternateLocations))) - } catch { - return completion(.failure(error)) - } - } + observabilityScope: observabilityScope + ) + + let alternateLocations = metadata.alternateLocations ?? [] + return Set(alternateLocations) } } diff --git a/Sources/PackageRegistry/ChecksumTOFU.swift b/Sources/PackageRegistry/ChecksumTOFU.swift index 8715928dd5f..4d7c9cbc1c9 100644 --- a/Sources/PackageRegistry/ChecksumTOFU.swift +++ b/Sources/PackageRegistry/ChecksumTOFU.swift @@ -36,66 +36,31 @@ struct PackageVersionChecksumTOFU { self.versionMetadataProvider = versionMetadataProvider } - // MARK: - source archive func validateSourceArchive( registry: Registry, package: PackageIdentity.RegistryIdentity, version: Version, checksum: String, timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws { - try await withCheckedThrowingContinuation { continuation in - self.validateSourceArchive( - registry: registry, - package: package, - version: version, - checksum: checksum, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } - ) - } - } - - @available(*, noasync, message: "Use the async alternative") - func validateSourceArchive( - registry: Registry, - package: PackageIdentity.RegistryIdentity, - version: Version, - checksum: String, - timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - self.getExpectedChecksum( + let expectedChecksum = try await self.getExpectedChecksum( registry: registry, package: package, version: version, timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - completion( - result.tryMap { expectedChecksum in - if checksum != expectedChecksum { - switch self.fingerprintCheckingMode { - case .strict: - throw RegistryError.invalidChecksum(expected: expectedChecksum, actual: checksum) - case .warn: - observabilityScope - .emit( - warning: "the checksum \(checksum) for source archive of \(package) \(version) does not match previously recorded value \(expectedChecksum)" - ) - } - } - } - ) + observabilityScope: observabilityScope + ) + + if checksum != expectedChecksum { + switch self.fingerprintCheckingMode { + case .strict: + throw RegistryError.invalidChecksum(expected: expectedChecksum, actual: checksum) + case .warn: + observabilityScope.emit( + warning: "the checksum \(checksum) for source archive of \(package) \(version) does not match previously recorded value \(expectedChecksum)" + ) + } } } @@ -104,57 +69,50 @@ struct PackageVersionChecksumTOFU { package: PackageIdentity.RegistryIdentity, version: Version, timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) async throws -> String { // We either use a previously recorded checksum, or fetch it from the registry. if let savedChecksum = try? self.readFromStorage(package: package, version: version, contentType: .sourceCode, observabilityScope: observabilityScope) { - return completion(.success(savedChecksum)) + return savedChecksum } // Try fetching checksum from registry if: // - No storage available // - Checksum not found in storage // - Reading from storage resulted in error - Task { - do { - let versionMetadata = try await self.versionMetadataProvider(package, version) - guard let sourceArchiveResource = versionMetadata.sourceArchive else { - throw RegistryError.missingSourceArchive - } - guard let checksum = sourceArchiveResource.checksum else { - throw RegistryError.sourceArchiveMissingChecksum( - registry: registry, - package: package.underlying, - version: version - ) - } - do { - try self.writeToStorage( - registry: registry, - package: package, - version: version, - checksum: checksum, - contentType: .sourceCode, - observabilityScope: observabilityScope - ) - completion(.success(checksum)) - } catch { - completion(.failure(error)) - } - } catch { - completion(.failure(RegistryError.failedRetrievingReleaseChecksum( + do { + let versionMetadata = try await self.versionMetadataProvider(package, version) + guard let sourceArchiveResource = versionMetadata.sourceArchive else { + throw RegistryError.missingSourceArchive + } + guard let checksum = sourceArchiveResource.checksum else { + throw RegistryError.sourceArchiveMissingChecksum( registry: registry, package: package.underlying, - version: version, - error: error - ))) + version: version + ) } + + try self.writeToStorage( + registry: registry, + package: package, + version: version, + checksum: checksum, + contentType: .sourceCode, + observabilityScope: observabilityScope + ) + + return checksum + } catch { + throw RegistryError.failedRetrievingReleaseChecksum( + registry: registry, + package: package.underlying, + version: version, + error: error + ) } } - @available(*, noasync, message: "Use the async alternative") func validateManifest( registry: Registry, package: PackageIdentity.RegistryIdentity, @@ -247,15 +205,14 @@ struct PackageVersionChecksumTOFU { fingerprint: fingerprint, observabilityScope: observabilityScope ) - } catch PackageFingerprintStorageError.conflict(_, let existing){ + } catch PackageFingerprintStorageError.conflict(_, let existing) { switch self.fingerprintCheckingMode { case .strict: throw RegistryError.checksumChanged(latest: checksum, previous: existing.value) case .warn: - observabilityScope - .emit( - warning: "the checksum \(checksum) for \(contentType) of \(package) \(version) from \(registry.url.absoluteString) does not match previously recorded value \(existing.value) from \(String(describing: existing.origin.url?.absoluteString))" - ) + observabilityScope.emit( + warning: "the checksum \(checksum) for \(contentType) of \(package) \(version) from \(registry.url.absoluteString) does not match previously recorded value \(existing.value) from \(String(describing: existing.origin.url?.absoluteString))" + ) } } } diff --git a/Sources/PackageRegistry/RegistryClient.swift b/Sources/PackageRegistry/RegistryClient.swift index 0340b911e05..00efc091914 100644 --- a/Sources/PackageRegistry/RegistryClient.swift +++ b/Sources/PackageRegistry/RegistryClient.swift @@ -24,8 +24,8 @@ import protocol TSCBasic.HashAlgorithm import struct TSCUtility.Version public protocol RegistryClientDelegate { - func onUnsigned(registry: Registry, package: PackageIdentity, version: Version, completion: (Bool) -> Void) - func onUntrusted(registry: Registry, package: PackageIdentity, version: Version, completion: (Bool) -> Void) + func onUnsigned(registry: Registry, package: PackageIdentity, version: Version) async -> Bool + func onUntrusted(registry: Registry, package: PackageIdentity, version: Version) async -> Bool } /// Package registry client. @@ -143,66 +143,39 @@ public final class RegistryClient { public func getPackageMetadata( package: PackageIdentity, timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws -> PackageMetadata { - try await withCheckedThrowingContinuation { continuation in - self.getPackageMetadata( - package: package, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } - ) - } - } - - @available(*, noasync, message: "Use the async alternative") - public func getPackageMetadata( - package: PackageIdentity, - timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - guard let registryIdentity = package.registry else { - return completion(.failure(RegistryError.invalidPackageIdentity(package))) + throw RegistryError.invalidPackageIdentity(package) } guard let registry = self.configuration.registry(for: registryIdentity.scope) else { - return completion(.failure(RegistryError.registryNotConfigured(scope: registryIdentity.scope))) + throw RegistryError.registryNotConfigured(scope: registryIdentity.scope) } observabilityScope.emit(debug: "registry for \(package): \(registry)") let underlying = { - self._getPackageMetadata( + try await self._getPackageMetadata( registry: registry, package: registryIdentity, timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion + observabilityScope: observabilityScope ) } if registry.supportsAvailability { - self.withAvailabilityCheck( + return try await self.withAvailabilityCheck( registry: registry, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue + observabilityScope: observabilityScope ) { error in if let error { - return completion(.failure(error)) + throw error } - underlying() + return try await underlying() } } else { - underlying() + return try await underlying() } } @@ -211,137 +184,100 @@ public final class RegistryClient { registry: Registry, package: PackageIdentity.RegistryIdentity, timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> PackageMetadata { guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } components.appendPathComponents("\(package.scope)", "\(package.name)") guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } - let request = LegacyHTTPClient.Request( + let request = HTTPClient.Request( method: .get, url: url, headers: [ "Accept": self.acceptHeader(mediaType: .json), ], - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + options: self.defaultRequestOptions(timeout: timeout) ) let start = DispatchTime.now() observabilityScope.emit(info: "retrieving \(package) metadata from \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - completion( - result.tryMap { response in - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - let packageMetadata = try response.parseJSON( - Serialization.PackageMetadata.self, - decoder: self.jsonDecoder - ) - let versions = packageMetadata.releases.filter { $0.value.problem == nil } - .compactMap { Version($0.key) } - .sorted(by: >) + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope + .emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + switch response.statusCode { + case 200: + let packageMetadata = try response.parseJSON( + Serialization.PackageMetadata.self, + decoder: self.jsonDecoder + ) - let alternateLocations = try response.headers.parseAlternativeLocationLinks() + let versions = packageMetadata.releases.filter { $0.value.problem == nil } + .compactMap { Version($0.key) } + .sorted(by: >) - return PackageMetadata( - registry: registry, - versions: versions, - alternateLocations: alternateLocations?.map(\.url) - ) - case 404: - throw RegistryError.packageNotFound - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - } - }.mapError { - RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: $0) - } - ) - } - } + let alternateLocations = try response.headers.parseAlternativeLocationLinks() - public func getPackageVersionMetadata( - package: PackageIdentity, - version: Version, - timeout: DispatchTimeInterval? = .none, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws -> PackageVersionMetadata { - try await withCheckedThrowingContinuation { continuation in - self.getPackageVersionMetadata( - package: package, - version: version, - timeout: timeout, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } - ) + return PackageMetadata( + registry: registry, + versions: versions, + alternateLocations: alternateLocations?.map(\.url) + ) + case 404: + throw RegistryError.packageNotFound + default: + throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) + } + } catch { + throw RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: error) } } - @available(*, noasync, message: "Use the async alternative") public func getPackageVersionMetadata( package: PackageIdentity, version: Version, timeout: DispatchTimeInterval? = .none, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> PackageVersionMetadata { guard let registryIdentity = package.registry else { - return completion(.failure(RegistryError.invalidPackageIdentity(package))) + throw RegistryError.invalidPackageIdentity(package) } guard let registry = self.configuration.registry(for: registryIdentity.scope) else { - return completion(.failure(RegistryError.registryNotConfigured(scope: registryIdentity.scope))) + throw RegistryError.registryNotConfigured(scope: registryIdentity.scope) } let underlying = { - self._getPackageVersionMetadata( + try await self._getPackageVersionMetadata( registry: registry, package: registryIdentity, version: version, timeout: timeout, fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion + observabilityScope: observabilityScope ) } if registry.supportsAvailability { - self.withAvailabilityCheck( + return try await self.withAvailabilityCheck( registry: registry, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue + observabilityScope: observabilityScope ) { error in if let error { - return completion(.failure(error)) + throw error } - underlying() + return try await underlying() } } else { - underlying() + return try await underlying() } } @@ -352,91 +288,73 @@ public final class RegistryClient { version: Version, timeout: DispatchTimeInterval?, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - self._getRawPackageVersionMetadata( + observabilityScope: ObservabilityScope + ) async throws -> PackageVersionMetadata { + let versionMetadata = try await self._getRawPackageVersionMetadata( registry: registry, package: package, version: version, timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - switch result { - case .failure(let failure): - completion(.failure(failure)) - case .success(let versionMetadata): - Task { - // WIP: async map the signing entity - - var resourceSigning: [(resource: RegistryClient.Serialization.VersionMetadata.Resource, signingEntity: SigningEntity?)] = [] - for resource in versionMetadata.resources { - guard let signing = resource.signing, - let signatureData = Data(base64Encoded: signing.signatureBase64Encoded), - let signatureFormat = SignatureFormat(rawValue: signing.signatureFormat) else { - resourceSigning.append((resource, nil)) - continue - } - let configuration = self.configuration.signing(for: package, registry: registry) - - let result = try? await withCheckedThrowingContinuation { continuation in - SignatureValidation.extractSigningEntity( - signature: [UInt8](signatureData), - signatureFormat: signatureFormat, - configuration: configuration, - fileSystem: fileSystem, - completion: { - continuation.resume(with: $0) - } - ) - } - resourceSigning.append((resource, result)) - } + observabilityScope: observabilityScope + ) - let packageVersionMetadata = PackageVersionMetadata( - registry: registry, - licenseURL: versionMetadata.metadata?.licenseURL.flatMap { URL(string: $0) }, - readmeURL: versionMetadata.metadata?.readmeURL.flatMap { URL(string: $0) }, - repositoryURLs: versionMetadata.metadata?.repositoryURLs?.compactMap { SourceControlURL($0) }, - resources: resourceSigning.map { - .init( - name: $0.resource.name, - type: $0.resource.type, - checksum: $0.resource.checksum, - signing: $0.resource.signing.flatMap { - PackageVersionMetadata.Signing( - signatureBase64Encoded: $0.signatureBase64Encoded, - signatureFormat: $0.signatureFormat - ) - }, - signingEntity: $0.signingEntity - ) - }, - author: versionMetadata.metadata?.author.map { - .init( - name: $0.name, - email: $0.email, - description: $0.description, - organization: $0.organization.map { - .init( - name: $0.name, - email: $0.email, - description: $0.description, - url: $0.url.flatMap { URL(string: $0) } - ) - }, - url: $0.url.flatMap { URL(string: $0) } - ) - }, - description: versionMetadata.metadata?.description, - publishedAt: versionMetadata.metadata?.originalPublicationTime ?? versionMetadata.publishedAt - ) - completion(.success(packageVersionMetadata)) - } + var resourceSigning: [(resource: RegistryClient.Serialization.VersionMetadata.Resource, signingEntity: SigningEntity?)] = [] + for resource in versionMetadata.resources { + guard let signing = resource.signing, + let signatureData = Data(base64Encoded: signing.signatureBase64Encoded), + let signatureFormat = SignatureFormat(rawValue: signing.signatureFormat) else { + resourceSigning.append((resource, nil)) + continue } + let configuration = self.configuration.signing(for: package, registry: registry) + + let result = try? await SignatureValidation.extractSigningEntity( + signature: [UInt8](signatureData), + signatureFormat: signatureFormat, + configuration: configuration, + fileSystem: fileSystem + ) + resourceSigning.append((resource, result)) } + + return PackageVersionMetadata( + registry: registry, + licenseURL: versionMetadata.metadata?.licenseURL.flatMap { URL(string: $0) }, + readmeURL: versionMetadata.metadata?.readmeURL.flatMap { URL(string: $0) }, + repositoryURLs: versionMetadata.metadata?.repositoryURLs?.compactMap { SourceControlURL($0) }, + resources: resourceSigning.map { + .init( + name: $0.resource.name, + type: $0.resource.type, + checksum: $0.resource.checksum, + signing: $0.resource.signing.flatMap { + PackageVersionMetadata.Signing( + signatureBase64Encoded: $0.signatureBase64Encoded, + signatureFormat: $0.signatureFormat + ) + }, + signingEntity: $0.signingEntity + ) + }, + author: versionMetadata.metadata?.author.map { + .init( + name: $0.name, + email: $0.email, + description: $0.description, + organization: $0.organization.map { + .init( + name: $0.name, + email: $0.email, + description: $0.description, + url: $0.url.flatMap { URL(string: $0) } + ) + }, + url: $0.url.flatMap { URL(string: $0) } + ) + }, + description: versionMetadata.metadata?.description, + publishedAt: versionMetadata.metadata?.originalPublicationTime ?? versionMetadata.publishedAt + ) } private func _getRawPackageVersionMetadata( @@ -444,134 +362,99 @@ public final class RegistryClient { package: PackageIdentity.RegistryIdentity, version: Version, timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> Serialization.VersionMetadata { let cacheKey = MetadataCacheKey(registry: registry, package: package) if let cached = self.metadataCache[cacheKey], cached.expires < .now() { - return completion(.success(cached.metadata)) + return cached.metadata } guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } components.appendPathComponents("\(package.scope)", "\(package.name)", "\(version)") guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registry.url))) + throw RegistryError.invalidURL(registry.url) } - let request = LegacyHTTPClient.Request( + let request = HTTPClient.Request( method: .get, url: url, headers: [ "Accept": self.acceptHeader(mediaType: .json), ], - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + options: self.defaultRequestOptions(timeout: timeout) ) let start = DispatchTime.now() observabilityScope.emit(info: "retrieving \(package) \(version) metadata from \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - completion( - result.tryMap { response in - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - let metadata = try response.parseJSON( - Serialization.VersionMetadata.self, - decoder: self.jsonDecoder - ) - self.metadataCache[cacheKey] = (metadata: metadata, expires: .now() + Self.metadataCacheTTL) - return metadata - case 404: - throw RegistryError.packageVersionNotFound - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - } - }.mapError { - RegistryError.failedRetrievingReleaseInfo( - registry: registry, - package: package.underlying, - version: version, - error: $0 - ) - } - ) - } - } - public func getAvailableManifests( - package: PackageIdentity, - version: Version, - timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws -> [String: (toolsVersion: ToolsVersion, content: String?)]{ - try await withCheckedThrowingContinuation { continuation in - self.getAvailableManifests( - package: package, + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope + .emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + switch response.statusCode { + case 200: + let metadata = try response.parseJSON( + Serialization.VersionMetadata.self, + decoder: self.jsonDecoder + ) + self.metadataCache[cacheKey] = (metadata: metadata, expires: .now() + Self.metadataCacheTTL) + return metadata + case 404: + throw RegistryError.packageVersionNotFound + default: + throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) + } + } catch { + throw RegistryError.failedRetrievingReleaseInfo( + registry: registry, + package: package.underlying, version: version, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } + error: error ) } } - @available(*, noasync, message: "Use the async alternative") public func getAvailableManifests( package: PackageIdentity, version: Version, timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result<[String: (toolsVersion: ToolsVersion, content: String?)], Error>) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> [String: (toolsVersion: ToolsVersion, content: String?)] { guard let registryIdentity = package.registry else { - return completion(.failure(RegistryError.invalidPackageIdentity(package))) + throw RegistryError.invalidPackageIdentity(package) } guard let registry = self.configuration.registry(for: registryIdentity.scope) else { - return completion(.failure(RegistryError.registryNotConfigured(scope: registryIdentity.scope))) + throw RegistryError.registryNotConfigured(scope: registryIdentity.scope) } let underlying = { - self._getAvailableManifests( + try await self._getAvailableManifests( registry: registry, package: registryIdentity, version: version, timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion + observabilityScope: observabilityScope ) } if registry.supportsAvailability { - self.withAvailabilityCheck( + return try await self.withAvailabilityCheck( registry: registry, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue + observabilityScope: observabilityScope ) { error in if let error { - return completion(.failure(error)) + throw error } - underlying() + return try await underlying() } } else { - underlying() + return try await underlying() } } @@ -581,247 +464,174 @@ public final class RegistryClient { package: PackageIdentity.RegistryIdentity, version: Version, timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result<[String: (toolsVersion: ToolsVersion, content: String?)], Error>) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> [String: (toolsVersion: ToolsVersion, content: String?)] { // first get the release metadata to see if archive is signed (therefore manifest is also signed) - self._getPackageVersionMetadata( + let versionMetadata = try await self._getPackageVersionMetadata( registry: registry, package: package, version: version, timeout: timeout, fileSystem: localFileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - switch result { - case .success(let versionMetadata): - guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registry.url))) - } - components.appendPathComponents( - "\(package.scope)", - "\(package.name)", - "\(version)", - Manifest.filename - ) + observabilityScope: observabilityScope + ) + guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { + throw RegistryError.invalidURL(registry.url) + } + components.appendPathComponents( + "\(package.scope)", + "\(package.name)", + "\(version)", + Manifest.filename + ) - guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registry.url))) - } + guard let url = components.url else { + throw RegistryError.invalidURL(registry.url) + } + + let request = HTTPClient.Request( + method: .get, + url: url, + headers: [ + "Accept": self.acceptHeader(mediaType: .swift), + ], + options: self.defaultRequestOptions(timeout: timeout) + ) + + // signature validation helper + let signatureValidation = SignatureValidation( + skipSignatureValidation: self.skipSignatureValidation, + signingEntityStorage: self.signingEntityStorage, + signingEntityCheckingMode: self.signingEntityCheckingMode, + versionMetadataProvider: { _, _ in versionMetadata }, + delegate: RegistryClientSignatureValidationDelegate(underlying: self.delegate) + ) - let request = LegacyHTTPClient.Request( - method: .get, - url: url, - headers: [ - "Accept": self.acceptHeader(mediaType: .swift), - ], - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + // checksum TOFU validation helper + let checksumTOFU = PackageVersionChecksumTOFU( + fingerprintStorage: self.fingerprintStorage, + fingerprintCheckingMode: self.fingerprintCheckingMode, + versionMetadataProvider: { _, _ in versionMetadata } + ) + + let start = DispatchTime.now() + observabilityScope + .emit(info: "retrieving available manifests for \(package) \(version) from \(request.url)") + + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + + switch response.statusCode { + case 200: + try response.validateAPIVersion() + try response.validateContentType(.swift) + + guard let data = response.body else { + throw RegistryError.invalidResponse + } + let manifestContent = String(decoding: data, as: UTF8.self) + + _ = try await signatureValidation.validate( + registry: registry, + package: package, + version: version, + toolsVersion: .none, + manifestContent: manifestContent, + configuration: self.configuration.signing(for: package, registry: registry), + timeout: timeout, + fileSystem: localFileSystem, + observabilityScope: observabilityScope ) - // signature validation helper - let signatureValidation = SignatureValidation( - skipSignatureValidation: self.skipSignatureValidation, - signingEntityStorage: self.signingEntityStorage, - signingEntityCheckingMode: self.signingEntityCheckingMode, - versionMetadataProvider: { _, _ in versionMetadata }, - delegate: RegistryClientSignatureValidationDelegate(underlying: self.delegate) + // TODO: expose Data based API on checksumAlgorithm + let actualChecksum = self.checksumAlgorithm.hash(.init(data)) + .hexadecimalRepresentation + + try checksumTOFU.validateManifest( + registry: registry, + package: package, + version: version, + toolsVersion: .none, + checksum: actualChecksum, + timeout: timeout, + observabilityScope: observabilityScope ) - // checksum TOFU validation helper - let checksumTOFU = PackageVersionChecksumTOFU( - fingerprintStorage: self.fingerprintStorage, - fingerprintCheckingMode: self.fingerprintCheckingMode, - versionMetadataProvider: { _, _ in versionMetadata } + var result = [String: (toolsVersion: ToolsVersion, content: String?)]() + let toolsVersion = try ToolsVersionParser + .parse(utf8String: manifestContent) + result[Manifest.filename] = ( + toolsVersion: toolsVersion, + content: manifestContent ) - let start = DispatchTime.now() - observabilityScope - .emit(info: "retrieving available manifests for \(package) \(version) from \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - switch result { - case .success(let response): - do { - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - try response.validateAPIVersion() - try response.validateContentType(.swift) - - guard let data = response.body else { - throw RegistryError.invalidResponse - } - let manifestContent = String(decoding: data, as: UTF8.self) - - signatureValidation.validate( - registry: registry, - package: package, - version: version, - toolsVersion: .none, - manifestContent: manifestContent, - configuration: self.configuration.signing(for: package, registry: registry), - timeout: timeout, - fileSystem: localFileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { signatureResult in - switch signatureResult { - case .success: - // TODO: expose Data based API on checksumAlgorithm - let actualChecksum = self.checksumAlgorithm.hash(.init(data)) - .hexadecimalRepresentation - - do { - try checksumTOFU.validateManifest( - registry: registry, - package: package, - version: version, - toolsVersion: .none, - checksum: actualChecksum, - timeout: timeout, - observabilityScope: observabilityScope - ) - do { - var result = - [String: (toolsVersion: ToolsVersion, content: String?)]() - let toolsVersion = try ToolsVersionParser - .parse(utf8String: manifestContent) - result[Manifest.filename] = ( - toolsVersion: toolsVersion, - content: manifestContent - ) - - let alternativeManifests = try response.headers.parseManifestLinks() - for alternativeManifest in alternativeManifests { - result[alternativeManifest.filename] = ( - toolsVersion: alternativeManifest.toolsVersion, - content: .none - ) - } - completion(.success(result)) - } catch { - completion(.failure( - RegistryError.failedRetrievingManifest( - registry: registry, - package: package.underlying, - version: version, - error: error - ) - )) - } - } catch { - completion(.failure(error)) - } - case .failure(let error): - completion(.failure(error)) - } - } - case 404: - throw RegistryError.packageVersionNotFound - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - } - } catch { - completion(.failure( - RegistryError.failedRetrievingManifest( - registry: registry, - package: package.underlying, - version: version, - error: error - ) - )) - } - case .failure(let error): - completion(.failure( - RegistryError.failedRetrievingManifest( - registry: registry, - package: package.underlying, - version: version, - error: error - ) - )) - } + let alternativeManifests = try response.headers.parseManifestLinks() + for alternativeManifest in alternativeManifests { + result[alternativeManifest.filename] = ( + toolsVersion: alternativeManifest.toolsVersion, + content: .none + ) } - case .failure(let error): - completion(.failure(error)) + return result + + case 404: + throw RegistryError.packageVersionNotFound + + default: + throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) } - } - } - public func getManifestContent( - package: PackageIdentity, - version: Version, - customToolsVersion: ToolsVersion?, - timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws -> String { - try await withCheckedThrowingContinuation { continuation in - self.getManifestContent( - package: package, + } catch { + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, version: version, - customToolsVersion: customToolsVersion, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } + error: error ) } } - @available(*, noasync, message: "Use the async alternative") public func getManifestContent( package: PackageIdentity, version: Version, customToolsVersion: ToolsVersion?, timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> String { guard let registryIdentity = package.registry else { - return completion(.failure(RegistryError.invalidPackageIdentity(package))) + throw RegistryError.invalidPackageIdentity(package) } guard let registry = self.configuration.registry(for: registryIdentity.scope) else { - return completion(.failure(RegistryError.registryNotConfigured(scope: registryIdentity.scope))) + throw RegistryError.registryNotConfigured(scope: registryIdentity.scope) } let underlying = { - self._getManifestContent( + try await self._getManifestContent( registry: registry, package: registryIdentity, version: version, customToolsVersion: customToolsVersion, timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion + observabilityScope: observabilityScope ) } if registry.supportsAvailability { - self.withAvailabilityCheck( + return try await self.withAvailabilityCheck( registry: registry, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue + observabilityScope: observabilityScope ) { error in if let error { - return completion(.failure(error)) + throw error } - underlying() + return try await underlying() } } else { - underlying() + return try await underlying() } } @@ -832,207 +642,143 @@ public final class RegistryClient { version: Version, customToolsVersion: ToolsVersion?, timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> String { // first get the release metadata to see if archive is signed (therefore manifest is also signed) - self._getPackageVersionMetadata( + let versionMetadata = try await self._getPackageVersionMetadata( registry: registry, package: package, version: version, timeout: timeout, fileSystem: localFileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - switch result { - case .success(let versionMetadata): - guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registry.url))) - } - components.appendPathComponents( - "\(package.scope)", - "\(package.name)", - "\(version)", - Manifest.filename - ) + observabilityScope: observabilityScope + ) + guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { + throw RegistryError.invalidURL(registry.url) + } + components.appendPathComponents( + "\(package.scope)", + "\(package.name)", + "\(version)", + Manifest.filename + ) - if let toolsVersion = customToolsVersion { - components.queryItems = [ - URLQueryItem(name: "swift-version", value: toolsVersion.description), - ] - } + if let toolsVersion = customToolsVersion { + components.queryItems = [ + URLQueryItem(name: "swift-version", value: toolsVersion.description), + ] + } - guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registry.url))) - } + guard let url = components.url else { + throw RegistryError.invalidURL(registry.url) + } + + let request = HTTPClient.Request( + method: .get, + url: url, + headers: [ + "Accept": self.acceptHeader(mediaType: .swift), + ], + options: self.defaultRequestOptions(timeout: timeout) + ) - let request = LegacyHTTPClient.Request( - method: .get, - url: url, - headers: [ - "Accept": self.acceptHeader(mediaType: .swift), - ], - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue) + // signature validation helper + let signatureValidation = SignatureValidation( + skipSignatureValidation: self.skipSignatureValidation, + signingEntityStorage: self.signingEntityStorage, + signingEntityCheckingMode: self.signingEntityCheckingMode, + versionMetadataProvider: { _, _ in versionMetadata }, + delegate: RegistryClientSignatureValidationDelegate(underlying: self.delegate) + ) + + // checksum TOFU validation helper + let checksumTOFU = PackageVersionChecksumTOFU( + fingerprintStorage: self.fingerprintStorage, + fingerprintCheckingMode: self.fingerprintCheckingMode, + versionMetadataProvider: { _, _ in versionMetadata } + ) + + let start = DispatchTime.now() + observabilityScope.emit(info: "retrieving \(package) \(version) manifest from \(request.url)") + + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope + .emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" ) + switch response.statusCode { + case 200: + try response.validateAPIVersion(isOptional: true) + try response.validateContentType(.swift) - // signature validation helper - let signatureValidation = SignatureValidation( - skipSignatureValidation: self.skipSignatureValidation, - signingEntityStorage: self.signingEntityStorage, - signingEntityCheckingMode: self.signingEntityCheckingMode, - versionMetadataProvider: { _, _ in versionMetadata }, - delegate: RegistryClientSignatureValidationDelegate(underlying: self.delegate) + guard let data = response.body else { + throw RegistryError.invalidResponse + } + let manifestContent = String(decoding: data, as: UTF8.self) + + _ = try await signatureValidation.validate( + registry: registry, + package: package, + version: version, + toolsVersion: customToolsVersion, + manifestContent: manifestContent, + configuration: self.configuration.signing(for: package, registry: registry), + timeout: timeout, + fileSystem: localFileSystem, + observabilityScope: observabilityScope ) - // checksum TOFU validation helper - let checksumTOFU = PackageVersionChecksumTOFU( - fingerprintStorage: self.fingerprintStorage, - fingerprintCheckingMode: self.fingerprintCheckingMode, - versionMetadataProvider: { _, _ in versionMetadata } + // TODO: expose Data based API on checksumAlgorithm + let actualChecksum = self.checksumAlgorithm.hash(.init(data)) + .hexadecimalRepresentation + + try checksumTOFU.validateManifest( + registry: registry, + package: package, + version: version, + toolsVersion: customToolsVersion, + checksum: actualChecksum, + timeout: timeout, + observabilityScope: observabilityScope ) - let start = DispatchTime.now() - observabilityScope.emit(info: "retrieving \(package) \(version) manifest from \(request.url)") - self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - switch result { - case .success(let response): - do { - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - try response.validateAPIVersion(isOptional: true) - try response.validateContentType(.swift) - - guard let data = response.body else { - throw RegistryError.invalidResponse - } - let manifestContent = String(decoding: data, as: UTF8.self) - - signatureValidation.validate( - registry: registry, - package: package, - version: version, - toolsVersion: customToolsVersion, - manifestContent: manifestContent, - configuration: self.configuration.signing(for: package, registry: registry), - timeout: timeout, - fileSystem: localFileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { signatureResult in - switch signatureResult { - case .success: - // TODO: expose Data based API on checksumAlgorithm - let actualChecksum = self.checksumAlgorithm.hash(.init(data)) - .hexadecimalRepresentation - - do { - try checksumTOFU.validateManifest( - registry: registry, - package: package, - version: version, - toolsVersion: customToolsVersion, - checksum: actualChecksum, - timeout: timeout, - observabilityScope: observabilityScope - ) - completion(.success(manifestContent)) - } catch { - completion(.failure(error)) - } - case .failure(let error): - completion(.failure(error)) - } - } - case 404: - throw RegistryError.packageVersionNotFound - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - } - } catch { - completion(.failure( - RegistryError.failedRetrievingManifest( - registry: registry, - package: package.underlying, - version: version, - error: error - ) - )) - } - case .failure(let error): - completion(.failure( - RegistryError.failedRetrievingManifest( - registry: registry, - package: package.underlying, - version: version, - error: error - ) - )) - } - } - case .failure(let error): - completion(.failure(error)) + return manifestContent + + case 404: + throw RegistryError.packageVersionNotFound + default: + throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) } - } - } - public func downloadSourceArchive( - package: PackageIdentity, - version: Version, - destinationPath: AbsolutePath, - progressHandler: (@Sendable (_ bytesReceived: Int64, _ totalBytes: Int64?) -> Void)?, - timeout: DispatchTimeInterval? = .none, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws { - try await withCheckedThrowingContinuation { continuation in - self.downloadSourceArchive( - package: package, + } catch { + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, version: version, - destinationPath: destinationPath, - progressHandler: progressHandler, - timeout: timeout, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) - } + error: error ) } } - @available(*, noasync, message: "Use the async alternative") public func downloadSourceArchive( package: PackageIdentity, version: Version, destinationPath: AbsolutePath, progressHandler: (@Sendable (_ bytesReceived: Int64, _ totalBytes: Int64?) -> Void)?, - timeout: DispatchTimeInterval? = .none, + timeout: DispatchTimeInterval? = nil, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws { guard let registryIdentity = package.registry else { - return completion(.failure(RegistryError.invalidPackageIdentity(package))) + throw RegistryError.invalidPackageIdentity(package) } guard let registry = self.configuration.registry(for: registryIdentity.scope) else { - return completion(.failure(RegistryError.registryNotConfigured(scope: registryIdentity.scope))) + throw RegistryError.registryNotConfigured(scope: registryIdentity.scope) } let underlying = { - self._downloadSourceArchive( + try await self._downloadSourceArchive( registry: registry, package: registryIdentity, version: version, @@ -1040,25 +786,22 @@ public final class RegistryClient { progressHandler: progressHandler, timeout: timeout, fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion + observabilityScope: observabilityScope ) } if registry.supportsAvailability { - self.withAvailabilityCheck( + try await self.withAvailabilityCheck( registry: registry, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue + observabilityScope: observabilityScope ) { error in if let error { - return completion(.failure(error)) + throw error } - underlying() + try await underlying() } } else { - underlying() + try await underlying() } } @@ -1071,293 +814,213 @@ public final class RegistryClient { progressHandler: (@Sendable (_ bytesReceived: Int64, _ totalBytes: Int64?) -> Void)?, timeout: DispatchTimeInterval?, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws { // first get the release metadata // TODO: this should be included in the archive to save the extra HTTP call - self._getPackageVersionMetadata( + let versionMetadata = try await self._getPackageVersionMetadata( registry: registry, package: package, version: version, timeout: timeout, fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - switch result { - case .success(let versionMetadata): - // download archive - guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { - return completion(.failure(RegistryError.invalidURL(registry.url))) - } - components.appendPathComponents("\(package.scope)", "\(package.name)", "\(version).zip") + observabilityScope: observabilityScope + ) - guard let url = components.url else { - return completion(.failure(RegistryError.invalidURL(registry.url))) - } + // download archive + guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { + throw RegistryError.invalidURL(registry.url) + } + components.appendPathComponents("\(package.scope)", "\(package.name)", "\(version).zip") - // prepare target download locations - let downloadPath = destinationPath.appending(extension: "zip") - do { - // prepare directories - if !fileSystem.exists(downloadPath.parentDirectory) { - try fileSystem.createDirectory(downloadPath.parentDirectory, recursive: true) - } - // clear out download path if exists - try fileSystem.removeFileTree(downloadPath) - // validate that the destination does not already exist - guard !fileSystem.exists(destinationPath) else { - throw RegistryError.pathAlreadyExists(destinationPath) - } - } catch { - return completion(.failure(error)) - } + guard let url = components.url else { + throw RegistryError.invalidURL(registry.url) + } + + // prepare target download locations + let downloadPath = destinationPath.appending(extension: "zip") + // prepare directories + if !fileSystem.exists(downloadPath.parentDirectory) { + try fileSystem.createDirectory(downloadPath.parentDirectory, recursive: true) + } + // clear out download path if exists + try fileSystem.removeFileTree(downloadPath) + // validate that the destination does not already exist + guard !fileSystem.exists(destinationPath) else { + throw RegistryError.pathAlreadyExists(destinationPath) + } + + // signature validation helper + let signatureValidation = SignatureValidation( + skipSignatureValidation: self.skipSignatureValidation, + signingEntityStorage: self.signingEntityStorage, + signingEntityCheckingMode: self.signingEntityCheckingMode, + versionMetadataProvider: { _, _ in versionMetadata }, + delegate: RegistryClientSignatureValidationDelegate(underlying: self.delegate) + ) + + // checksum TOFU validation helper + let checksumTOFU = PackageVersionChecksumTOFU( + fingerprintStorage: self.fingerprintStorage, + fingerprintCheckingMode: self.fingerprintCheckingMode, + versionMetadataProvider: { _, _ in versionMetadata } + ) + + let request = HTTPClient.Request.download( + url: url, + headers: [ + "Accept": self.acceptHeader(mediaType: .zip), + ], + options: self.defaultRequestOptions(timeout: timeout), + fileSystem: fileSystem, + destination: downloadPath + ) + + let downloadStart = DispatchTime.now() + observabilityScope.emit(info: "downloading \(package) \(version) source archive from \(request.url)") + do { + let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: progressHandler) + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(downloadStart.distance(to: .now()).descriptionInSeconds)" + ) - // signature validation helper - let signatureValidation = SignatureValidation( - skipSignatureValidation: self.skipSignatureValidation, - signingEntityStorage: self.signingEntityStorage, - signingEntityCheckingMode: self.signingEntityCheckingMode, - versionMetadataProvider: { _, _ in versionMetadata }, - delegate: RegistryClientSignatureValidationDelegate(underlying: self.delegate) + switch response.statusCode { + case 200: + try response.validateAPIVersion(isOptional: true) + try response.validateContentType(.zip) + + let archiveContent: Data = try fileSystem.readFileContents(downloadPath) + // TODO: expose Data based API on checksumAlgorithm + let actualChecksum = self.checksumAlgorithm.hash(.init(archiveContent)) + .hexadecimalRepresentation + + observabilityScope.emit( + debug: "performing TOFU checks on \(package) \(version) source archive (checksum: '\(actualChecksum)'" + ) + let signingEntity = try await signatureValidation.validate( + registry: registry, + package: package, + version: version, + content: archiveContent, + configuration: self.configuration.signing(for: package, registry: registry), + timeout: timeout, + fileSystem: fileSystem, + observabilityScope: observabilityScope ) - // checksum TOFU validation helper - let checksumTOFU = PackageVersionChecksumTOFU( - fingerprintStorage: self.fingerprintStorage, - fingerprintCheckingMode: self.fingerprintCheckingMode, - versionMetadataProvider: { _, _ in versionMetadata } + try await checksumTOFU.validateSourceArchive( + registry: registry, + package: package, + version: version, + checksum: actualChecksum, + timeout: timeout, + observabilityScope: observabilityScope ) - let request = LegacyHTTPClient.Request.download( - url: url, - headers: [ - "Accept": self.acceptHeader(mediaType: .zip), - ], - options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue), - fileSystem: fileSystem, - destination: downloadPath + // validate that the destination does not already exist + // (again, as this is async) + guard !fileSystem.exists(destinationPath) else { + throw RegistryError.pathAlreadyExists(destinationPath) + } + try fileSystem.createDirectory( + destinationPath, + recursive: true ) + // extract the content + let extractStart = DispatchTime.now() + observabilityScope + .emit( + debug: "extracting \(package) \(version) source archive to '\(destinationPath)'" + ) + let archiver = self.archiverProvider(fileSystem) - let downloadStart = DispatchTime.now() - observabilityScope.emit(info: "downloading \(package) \(version) source archive from \(request.url)") - self.httpClient - .execute(request, observabilityScope: observabilityScope, progress: progressHandler) { result in - switch result { - case .success(let response): - do { - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(downloadStart.distance(to: .now()).descriptionInSeconds)" - ) - switch response.statusCode { - case 200: - try response.validateAPIVersion(isOptional: true) - try response.validateContentType(.zip) - - do { - let archiveContent: Data = try fileSystem.readFileContents(downloadPath) - // TODO: expose Data based API on checksumAlgorithm - let actualChecksum = self.checksumAlgorithm.hash(.init(archiveContent)) - .hexadecimalRepresentation - - observabilityScope - .emit( - debug: "performing TOFU checks on \(package) \(version) source archive (checksum: '\(actualChecksum)'" - ) - signatureValidation.validate( - registry: registry, - package: package, - version: version, - content: archiveContent, - configuration: self.configuration.signing(for: package, registry: registry), - timeout: timeout, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { signatureResult in - switch signatureResult { - case .success(let signingEntity): - checksumTOFU.validateSourceArchive( - registry: registry, - package: package, - version: version, - checksum: actualChecksum, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { checksumResult in - switch checksumResult { - case .success: - do { - // validate that the destination does not already exist - // (again, as this - // is - // async) - guard !fileSystem.exists(destinationPath) else { - throw RegistryError.pathAlreadyExists(destinationPath) - } - try fileSystem.createDirectory( - destinationPath, - recursive: true - ) - // extract the content - let extractStart = DispatchTime.now() - observabilityScope - .emit( - debug: "extracting \(package) \(version) source archive to '\(destinationPath)'" - ) - let archiver = self.archiverProvider(fileSystem) - // TODO: Bail if archive contains relative paths or overlapping files - archiver - .extract( - from: downloadPath, - to: destinationPath - ) { result in - defer { - try? fileSystem.removeFileTree(downloadPath) - } - observabilityScope - .emit( - debug: "extracted \(package) \(version) source archive to '\(destinationPath)' in \(extractStart.distance(to: .now()).descriptionInSeconds)" - ) - completion(result.tryMap { - // strip first level component - try fileSystem - .stripFirstLevel(of: destinationPath) - // write down copy of version metadata - let registryMetadataPath = destinationPath - .appending( - component: RegistryReleaseMetadataStorage - .fileName - ) - observabilityScope - .emit( - debug: "saving \(package) \(version) metadata to '\(registryMetadataPath)'" - ) - try RegistryReleaseMetadataStorage.save( - metadata: versionMetadata, - signingEntity: signingEntity, - to: registryMetadataPath, - fileSystem: fileSystem - ) - }.mapError { error in - StringError( - "failed extracting '\(downloadPath)' to '\(destinationPath)': \(error.interpolationDescription)" - ) - }) - } - } catch { - completion(.failure( - RegistryError - .failedDownloadingSourceArchive( - registry: registry, - package: package.underlying, - version: version, - error: error - ) - )) - } - case .failure(let error): - completion(.failure(error)) - } - } - case .failure(let error): - completion(.failure(error)) - } - } - } catch { - throw RegistryError.failedToComputeChecksum(error) - } - case 404: - throw RegistryError.packageVersionNotFound - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - } - } catch { - completion(.failure(RegistryError.failedDownloadingSourceArchive( - registry: registry, - package: package.underlying, - version: version, - error: error - ))) - } - case .failure(let error): - completion(.failure(RegistryError.failedDownloadingSourceArchive( - registry: registry, - package: package.underlying, - version: version, - error: error - ))) - } + do { + // TODO: Bail if archive contains relative paths or overlapping files + try await archiver.extract( + from: downloadPath, + to: destinationPath + ) + defer { + try? fileSystem.removeFileTree(downloadPath) } - case .failure(let error): - completion(.failure(error)) - } - } - } + observabilityScope.emit( + debug: "extracted \(package) \(version) source archive to '\(destinationPath)' in \(extractStart.distance(to: .now()).descriptionInSeconds)" + ) - public func lookupIdentities( - scmURL: SourceControlURL, - timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws -> Set { - try await withCheckedThrowingContinuation { continuation in - self.lookupIdentities( - scmURL: scmURL, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { - continuation.resume(with: $0) + // strip first level component + try fileSystem + .stripFirstLevel(of: destinationPath) + // write down copy of version metadata + let registryMetadataPath = destinationPath + .appending( + component: RegistryReleaseMetadataStorage + .fileName + ) + observabilityScope + .emit( + debug: "saving \(package) \(version) metadata to '\(registryMetadataPath)'" + ) + try RegistryReleaseMetadataStorage.save( + metadata: versionMetadata, + signingEntity: signingEntity, + to: registryMetadataPath, + fileSystem: fileSystem + ) + } catch { + throw StringError( + "failed extracting '\(downloadPath)' to '\(destinationPath)': \(error.interpolationDescription)" + ) } + + case 404: + throw RegistryError.packageVersionNotFound + + default: + throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) + + } + } catch { + throw RegistryError.failedDownloadingSourceArchive( + registry: registry, + package: package.underlying, + version: version, + error: error ) } } - @available(*, noasync, message: "Use the async alternative") public func lookupIdentities( scmURL: SourceControlURL, timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result, Error>) -> Void - ) { - let completion = self.makeAsync(completion, on: callbackQueue) - + observabilityScope: ObservabilityScope + ) async throws -> Set { guard let registry = self.configuration.defaultRegistry else { - return completion(.failure(RegistryError.registryNotConfigured(scope: nil))) - } - - let underlying = { - self._lookupIdentities( - registry: registry, - scmURL: scmURL, - timeout: timeout, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion - ) + throw RegistryError.registryNotConfigured(scope: nil) } if registry.supportsAvailability { - self.withAvailabilityCheck( + return try await self.withAvailabilityCheck( registry: registry, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue + observabilityScope: observabilityScope ) { error in if let error { - return completion(.failure(error)) + throw error } - underlying() + + return try await self._lookupIdentities( + registry: registry, + scmURL: scmURL, + timeout: timeout, + observabilityScope: observabilityScope + ) } } else { - underlying() + return try await self._lookupIdentities( + registry: registry, + scmURL: scmURL, + timeout: timeout, + observabilityScope: observabilityScope + ) } } @@ -1366,8 +1029,7 @@ public final class RegistryClient { registry: Registry, scmURL: SourceControlURL, timeout: DispatchTimeInterval?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws -> Set { guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else { throw RegistryError.invalidURL(registry.url) @@ -1424,8 +1086,7 @@ public final class RegistryClient { public func login( loginURL: URL, timeout: DispatchTimeInterval? = .none, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws { let request = HTTPClient.Request( method: .post, @@ -1465,8 +1126,7 @@ public final class RegistryClient { signatureFormat: SignatureFormat?, timeout: DispatchTimeInterval? = .none, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws -> PublishResult { guard let registryIdentity = packageIdentity.registry else { throw RegistryError.invalidPackageIdentity(packageIdentity) @@ -1664,30 +1324,29 @@ public final class RegistryClient { } } - private func withAvailabilityCheck( + private func withAvailabilityCheck( registry: Registry, observabilityScope: ObservabilityScope, - next: @escaping (Error?) -> Void - ) async throws { - let availabilityHandler: (Result) - -> Void = { (result: Result) in + next: @escaping (Error?) async throws -> T + ) async throws -> T { + let availabilityHandler = { (result: Result) in switch result { case .success(let status): switch status { case .available: - return next(.none) + return try await next(.none) case .unavailable: - return next(RegistryError.registryNotAvailable(registry)) + return try await next(RegistryError.registryNotAvailable(registry)) case .error(let description): - return next(StringError(description)) + return try await next(StringError(description)) } case .failure(let error): - return next(error) + return try await next(error) } } if let cached = self.availabilityCache[registry.url], cached.expires < .now() { - return availabilityHandler(cached.status) + return try await availabilityHandler(cached.status) } let result: Result @@ -1702,7 +1361,7 @@ public final class RegistryClient { } self.availabilityCache[registry.url] = (status: result, expires: .now() + Self.availabilityCacheTTL) - availabilityHandler(result) + return try await availabilityHandler(result) } private func unexpectedStatusError( @@ -2487,54 +2146,52 @@ private struct RegistryClientSignatureValidationDelegate: SignatureValidation.De func onUnsigned( registry: Registry, package: PackageModel.PackageIdentity, - version: TSCUtility.Version, - completion: (Bool) -> Void - ) { + version: TSCUtility.Version + ) async -> Bool { let responseCacheKey = ResponseCacheKey(registry: registry, package: package, version: version) if let cachedResponse = self.onUnsignedResponseCache[responseCacheKey] { - return completion(cachedResponse) + return cachedResponse } if let underlying { - underlying.onUnsigned( + let response = await underlying.onUnsigned( registry: registry, package: package, version: version - ) { response in - self.onUnsignedResponseCache[responseCacheKey] = response - completion(response) - } + ) + + self.onUnsignedResponseCache[responseCacheKey] = response + return response } else { // true == continue resolution // false == stop dependency resolution - completion(false) + return false } } func onUntrusted( registry: Registry, package: PackageModel.PackageIdentity, - version: TSCUtility.Version, - completion: (Bool) -> Void - ) { + version: TSCUtility.Version + ) async -> Bool { let responseCacheKey = ResponseCacheKey(registry: registry, package: package, version: version) if let cachedResponse = self.onUntrustedResponseCache[responseCacheKey] { - return completion(cachedResponse) + return cachedResponse } if let underlying { - underlying.onUntrusted( + let response = await underlying.onUntrusted( registry: registry, package: package, version: version - ) { response in - self.onUntrustedResponseCache[responseCacheKey] = response - completion(response) - } + ) + + self.onUntrustedResponseCache[responseCacheKey] = response + return response } else { // true == continue resolution // false == stop dependency resolution - completion(false) + return false } } diff --git a/Sources/PackageRegistry/RegistryDownloadsManager.swift b/Sources/PackageRegistry/RegistryDownloadsManager.swift index cee0967f05f..191d924b937 100644 --- a/Sources/PackageRegistry/RegistryDownloadsManager.swift +++ b/Sources/PackageRegistry/RegistryDownloadsManager.swift @@ -19,7 +19,7 @@ import PackageModel import struct TSCUtility.Version -public class RegistryDownloadsManager: Cancellable { +public actor RegistryDownloadsManager { public typealias Delegate = RegistryDownloadsManagerDelegate private let fileSystem: FileSystem @@ -28,8 +28,7 @@ public class RegistryDownloadsManager: Cancellable { private let registryClient: RegistryClient private let delegate: Delegate? - private var pendingLookups = [PackageIdentity: DispatchGroup]() - private var pendingLookupsLock = NSLock() + private var pendingLookups = [PackageIdentity: [CheckedContinuation]]() public init( fileSystem: FileSystem, @@ -44,117 +43,78 @@ public class RegistryDownloadsManager: Cancellable { self.registryClient = registryClient self.delegate = delegate } - - public func lookup( - package: PackageIdentity, - version: Version, - observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue - ) async throws -> AbsolutePath { - try await withCheckedThrowingContinuation { continuation in - self.lookup( - package: package, - version: version, - observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue, - completion: { continuation.resume(with: $0) } - ) - } - } - @available(*, noasync, message: "Use the async alternative") public func lookup( package: PackageIdentity, version: Version, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - // wrap the callback in the requested queue - let completion = { result in callbackQueue.async { completion(result) } } - + delegateQueue: DispatchQueue + ) async throws -> AbsolutePath { let packageRelativePath: RelativePath let packagePath: AbsolutePath - do { - packageRelativePath = try package.downloadPath(version: version) - packagePath = self.path.appending(packageRelativePath) + packageRelativePath = try package.downloadPath(version: version) + packagePath = self.path.appending(packageRelativePath) - // TODO: we can do some finger-print checking to improve the validation - // already exists and valid, we can exit early - if try self.fileSystem.validPackageDirectory(packagePath) { - return completion(.success(packagePath)) - } - } catch { - return completion(.failure(error)) + // TODO: we can do some finger-print checking to improve the validation + // already exists and valid, we can exit early + if try self.fileSystem.validPackageDirectory(packagePath) { + return packagePath } // next we check if there is a pending lookup - self.pendingLookupsLock.lock() - if let pendingLookup = self.pendingLookups[package] { - self.pendingLookupsLock.unlock() + if self.pendingLookups.keys.contains(package) { // chain onto the pending lookup - pendingLookup.notify(queue: callbackQueue) { - // at this point the previous lookup should be complete and we can re-lookup - self.lookup( - package: package, - version: version, - observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue, - completion: completion - ) + return await withCheckedContinuation { + self.pendingLookups[package]?.append($0) } } else { - // record the pending lookup - assert(self.pendingLookups[package] == nil) - let group = DispatchGroup() - group.enter() - self.pendingLookups[package] = group - self.pendingLookupsLock.unlock() - // inform delegate that we are starting to fetch // calculate if cached (for delegate call) outside queue as it may change while queue is processing let isCached = self.cachePath.map { self.fileSystem.exists($0.appending(packageRelativePath)) } ?? false + let delegate = self.delegate delegateQueue.async { let details = FetchDetails(fromCache: isCached, updatedCache: false) - self.delegate?.willFetch(package: package, version: version, fetchDetails: details) + delegate?.willFetch(package: package, version: version, fetchDetails: details) } // make sure destination is free. try? self.fileSystem.removeFileTree(packagePath) let start = DispatchTime.now() - self.downloadAndPopulateCache( - package: package, - version: version, - packagePath: packagePath, - observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue - ) { result in - // inform delegate that we finished to fetch - let duration = start.distance(to: .now()) - delegateQueue.async { - self.delegate?.didFetch(package: package, version: version, result: result, duration: duration) + let result: Result + do { + result = try await .success(self.downloadAndPopulateCache( + package: package, + version: version, + packagePath: packagePath, + observabilityScope: observabilityScope, + delegateQueue: delegateQueue + )) + } catch { + result = .failure(error) + } + + // inform delegate that we finished to fetch + let duration = start.distance(to: .now()) + delegateQueue.async { + delegate?.didFetch(package: package, version: version, result: result, duration: duration) + } + + // remove the pending lookup + defer { + if let pendingLookups = self.pendingLookups[package] { + for lookup in pendingLookups { + lookup.resume(returning: packagePath) + } } - // remove the pending lookup - self.pendingLookupsLock.lock() - self.pendingLookups[package]?.leave() + self.pendingLookups[package] = nil - self.pendingLookupsLock.unlock() - // and done - completion(result.map { _ in packagePath }) } - } - } - /// Cancel any outstanding requests - public func cancel(deadline: DispatchTime) throws { - try self.registryClient.cancel(deadline: deadline) + // and done + return packagePath + } } private func downloadAndPopulateCache( @@ -162,17 +122,15 @@ public class RegistryDownloadsManager: Cancellable { version: Version, packagePath: AbsolutePath, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping @Sendable (Result) -> Void - ) { + delegateQueue: DispatchQueue + ) async throws -> FetchDetails { if let cachePath { do { let relativePath = try package.downloadPath(version: version) let cachedPackagePath = cachePath.appending(relativePath) try self.initializeCacheIfNeeded(cachePath: cachePath) - try self.fileSystem.withLock(on: cachedPackagePath, type: .exclusive) { + return try await self.fileSystem.withLock(on: cachedPackagePath, type: .exclusive) { // download the package into the cache unless already exists if try self.fileSystem.validPackageDirectory(cachedPackagePath) { // extra validation to defend from racy edge cases @@ -182,31 +140,28 @@ public class RegistryDownloadsManager: Cancellable { // copy the package from the cache into the package path. try self.fileSystem.createDirectory(packagePath.parentDirectory, recursive: true) try self.fileSystem.copy(from: cachedPackagePath, to: packagePath) - completion(.success(.init(fromCache: true, updatedCache: false))) + return FetchDetails(fromCache: true, updatedCache: false) } else { // it is possible that we already created the directory before from failed attempts, so clear leftover data if present. try? self.fileSystem.removeFileTree(cachedPackagePath) // download the package from the registry - self.registryClient.downloadSourceArchive( + try await self.registryClient.downloadSourceArchive( package: package, version: version, destinationPath: cachedPackagePath, progressHandler: updateDownloadProgress, fileSystem: self.fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - completion(result.tryMap { - // extra validation to defend from racy edge cases - if self.fileSystem.exists(packagePath) { - throw StringError("\(packagePath) already exists unexpectedly") - } - // copy the package from the cache into the package path. - try self.fileSystem.createDirectory(packagePath.parentDirectory, recursive: true) - try self.fileSystem.copy(from: cachedPackagePath, to: packagePath) - return FetchDetails(fromCache: true, updatedCache: true) - }) + observabilityScope: observabilityScope + ) + + // extra validation to defend from racy edge cases + if self.fileSystem.exists(packagePath) { + throw StringError("\(packagePath) already exists unexpectedly") } + // copy the package from the cache into the package path. + try self.fileSystem.createDirectory(packagePath.parentDirectory, recursive: true) + try self.fileSystem.copy(from: cachedPackagePath, to: packagePath) + return FetchDetails(fromCache: true, updatedCache: true) } } } catch { @@ -217,33 +172,31 @@ public class RegistryDownloadsManager: Cancellable { ) // it is possible that we already created the directory from failed attempts, so clear leftover data if present. try? self.fileSystem.removeFileTree(packagePath) - self.registryClient.downloadSourceArchive( + _ = try await self.registryClient.downloadSourceArchive( package: package, version: version, destinationPath: packagePath, progressHandler: updateDownloadProgress, fileSystem: self.fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - completion(result.map { FetchDetails(fromCache: false, updatedCache: false) }) - } + observabilityScope: observabilityScope + ) + + return FetchDetails(fromCache: false, updatedCache: false) } } else { // it is possible that we already created the directory from failed attempts, so clear leftover data if present. try? self.fileSystem.removeFileTree(packagePath) // download without populating the cache when no `cachePath` is set. - self.registryClient.downloadSourceArchive( + _ = try await self.registryClient.downloadSourceArchive( package: package, version: version, destinationPath: packagePath, progressHandler: updateDownloadProgress, fileSystem: self.fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - completion(result.map { FetchDetails(fromCache: false, updatedCache: false) }) - } + observabilityScope: observabilityScope + ) + + return FetchDetails(fromCache: false, updatedCache: false) } // utility to update progress @@ -260,13 +213,13 @@ public class RegistryDownloadsManager: Cancellable { } } - public func remove(package: PackageIdentity) throws { + public nonisolated func remove(package: PackageIdentity) throws { let relativePath = try package.downloadPath() let packagesPath = self.path.appending(relativePath) try self.fileSystem.removeFileTree(packagesPath) } - public func reset(observabilityScope: ObservabilityScope) { + public nonisolated func reset(observabilityScope: ObservabilityScope) { do { try self.fileSystem.removeFileTree(self.path) } catch { @@ -277,7 +230,7 @@ public class RegistryDownloadsManager: Cancellable { } } - public func purgeCache(observabilityScope: ObservabilityScope) { + public nonisolated func purgeCache(observabilityScope: ObservabilityScope) { guard let cachePath else { return } diff --git a/Sources/PackageRegistry/SignatureValidation.swift b/Sources/PackageRegistry/SignatureValidation.swift index 9ffd915b578..106c1674d6d 100644 --- a/Sources/PackageRegistry/SignatureValidation.swift +++ b/Sources/PackageRegistry/SignatureValidation.swift @@ -22,8 +22,8 @@ import PackageSigning import struct TSCUtility.Version protocol SignatureValidationDelegate { - func onUnsigned(registry: Registry, package: PackageIdentity, version: Version, completion: (Bool) -> Void) - func onUntrusted(registry: Registry, package: PackageIdentity, version: Version, completion: (Bool) -> Void) + func onUnsigned(registry: Registry, package: PackageIdentity, version: Version) async -> Bool + func onUntrusted(registry: Registry, package: PackageIdentity, version: Version) async -> Bool } struct SignatureValidation { @@ -61,43 +61,13 @@ struct SignatureValidation { configuration: RegistryConfiguration.Security.Signing, timeout: DispatchTimeInterval?, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws -> SigningEntity? { - try await withCheckedThrowingContinuation { continuation in - self.validate( - registry: registry, - package: package, - version: version, - content: content, - configuration: configuration, - timeout: timeout, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { continuation.resume(with: $0) } - ) - } - } - - @available(*, noasync, message: "Use the async alternative") - func validate( - registry: Registry, - package: PackageIdentity.RegistryIdentity, - version: Version, - content: Data, - configuration: RegistryConfiguration.Security.Signing, - timeout: DispatchTimeInterval?, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping @Sendable (Result) -> Void - ) { guard !self.skipSignatureValidation else { - return completion(.success(.none)) + return nil } - self.getAndValidateSourceArchiveSignature( + let signingEntity = try await self.getAndValidateSourceArchiveSignature( registry: registry, package: package, version: version, @@ -105,27 +75,20 @@ struct SignatureValidation { configuration: configuration, timeout: timeout, fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - switch result { - case .success(let signingEntity): - // Always do signing entity TOFU check at the end, - // whether the package is signed or not. - self.signingEntityTOFU.validate( - registry: registry, - package: package, - version: version, - signingEntity: signingEntity, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { _ in - completion(.success(signingEntity)) - } - case .failure(let error): - completion(.failure(error)) - } - } + observabilityScope: observabilityScope + ) + + // Always do signing entity TOFU check at the end, + // whether the package is signed or not. + try self.signingEntityTOFU.validate( + registry: registry, + package: package, + version: version, + signingEntity: signingEntity, + observabilityScope: observabilityScope + ) + + return signingEntity } private func getAndValidateSourceArchiveSignature( @@ -136,99 +99,94 @@ struct SignatureValidation { configuration: RegistryConfiguration.Security.Signing, timeout: DispatchTimeInterval?, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping @Sendable (Result) -> Void - ) { - Task { - do { - let versionMetadata = try await self.versionMetadataProvider(package, version) + observabilityScope: ObservabilityScope + ) async throws -> SigningEntity? { + do { + let versionMetadata = try await self.versionMetadataProvider(package, version) - guard let sourceArchiveResource = versionMetadata.sourceArchive else { - throw RegistryError.missingSourceArchive - } - guard let signatureBase64Encoded = sourceArchiveResource.signing?.signatureBase64Encoded else { - throw RegistryError.sourceArchiveNotSigned( - registry: registry, - package: package.underlying, - version: version - ) - } + guard let sourceArchiveResource = versionMetadata.sourceArchive else { + throw RegistryError.missingSourceArchive + } + guard let signatureBase64Encoded = sourceArchiveResource.signing?.signatureBase64Encoded else { + throw RegistryError.sourceArchiveNotSigned( + registry: registry, + package: package.underlying, + version: version + ) + } - guard let signatureData = Data(base64Encoded: signatureBase64Encoded) else { - throw RegistryError.failedLoadingSignature - } - guard let signatureFormatString = sourceArchiveResource.signing?.signatureFormat else { - throw RegistryError.missingSignatureFormat - } - guard let signatureFormat = SignatureFormat(rawValue: signatureFormatString) else { - throw RegistryError.unknownSignatureFormat(signatureFormatString) + guard let signatureData = Data(base64Encoded: signatureBase64Encoded) else { + throw RegistryError.failedLoadingSignature + } + guard let signatureFormatString = sourceArchiveResource.signing?.signatureFormat else { + throw RegistryError.missingSignatureFormat + } + guard let signatureFormat = SignatureFormat(rawValue: signatureFormatString) else { + throw RegistryError.unknownSignatureFormat(signatureFormatString) + } + + return try await self.validateSourceArchiveSignature( + registry: registry, + package: package, + version: version, + signature: Array(signatureData), + signatureFormat: signatureFormat, + content: Array(content), + configuration: configuration, + fileSystem: fileSystem, + observabilityScope: observabilityScope + ) + } catch RegistryError.sourceArchiveNotSigned { + observabilityScope.emit( + info: "\(package) \(version) from \(registry) is unsigned", + metadata: .registryPackageMetadata(identity: package) + ) + guard let onUnsigned = configuration.onUnsigned else { + throw RegistryError.missingConfiguration(details: "security.signing.onUnsigned") + } + + let sourceArchiveNotSignedError = RegistryError.sourceArchiveNotSigned( + registry: registry, + package: package.underlying, + version: version + ) + + switch onUnsigned { + case .prompt: + let `continue` = await self.delegate.onUnsigned(registry: registry, package: package.underlying, version: version) + if `continue` { + return nil + } else { + throw sourceArchiveNotSignedError } - self.validateSourceArchiveSignature( - registry: registry, - package: package, - version: version, - signature: Array(signatureData), - signatureFormat: signatureFormat, - content: Array(content), - configuration: configuration, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - completion: completion - ) - } catch RegistryError.sourceArchiveNotSigned { + case .error: + throw sourceArchiveNotSignedError + case .warn: observabilityScope.emit( - info: "\(package) \(version) from \(registry) is unsigned", + warning: "\(sourceArchiveNotSignedError)", metadata: .registryPackageMetadata(identity: package) ) - guard let onUnsigned = configuration.onUnsigned else { - return completion(.failure(RegistryError.missingConfiguration(details: "security.signing.onUnsigned"))) - } + return nil - let sourceArchiveNotSignedError = RegistryError.sourceArchiveNotSigned( - registry: registry, - package: package.underlying, - version: version - ) - - switch onUnsigned { - case .prompt: - self.delegate - .onUnsigned(registry: registry, package: package.underlying, version: version) { `continue` in - if `continue` { - completion(.success(.none)) - } else { - completion(.failure(sourceArchiveNotSignedError)) - } - } - case .error: - completion(.failure(sourceArchiveNotSignedError)) - case .warn: - observabilityScope.emit( - warning: "\(sourceArchiveNotSignedError)", - metadata: .registryPackageMetadata(identity: package) - ) - completion(.success(.none)) - case .silentAllow: - // Continue without logging - completion(.success(.none)) - } - } catch RegistryError.failedRetrievingReleaseInfo(_, _, _, let error) { - completion(.failure(RegistryError.failedRetrievingSourceArchiveSignature( - registry: registry, - package: package.underlying, - version: version, - error: error - ))) - } catch { - completion(.failure(RegistryError.failedRetrievingSourceArchiveSignature( - registry: registry, - package: package.underlying, - version: version, - error: error - ))) + case .silentAllow: + // Continue without logging + return nil } + } catch RegistryError.failedRetrievingReleaseInfo(_, _, _, let error) { + throw RegistryError.failedRetrievingSourceArchiveSignature( + registry: registry, + package: package.underlying, + version: version, + error: error + ) + } catch { + throw RegistryError.failedRetrievingSourceArchiveSignature( + registry: registry, + package: package.underlying, + version: version, + error: error + ) } } @@ -241,109 +199,78 @@ struct SignatureValidation { content: [UInt8], configuration: RegistryConfiguration.Security.Signing, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - completion: @escaping @Sendable (Result) -> Void - ) { - Task { - do { - let signatureStatus = try await SignatureProvider.status( - signature: signature, - content: content, - format: signatureFormat, - verifierConfiguration: try VerifierConfiguration.from(configuration, fileSystem: fileSystem), - observabilityScope: observabilityScope + observabilityScope: ObservabilityScope + ) async throws -> SigningEntity? { + do { + let signatureStatus = try await SignatureProvider.status( + signature: signature, + content: content, + format: signatureFormat, + verifierConfiguration: try VerifierConfiguration.from(configuration, fileSystem: fileSystem), + observabilityScope: observabilityScope + ) + + switch signatureStatus { + case .valid(let signingEntity): + observabilityScope.emit( + info: "\(package) \(version) from \(registry) is signed with a valid entity '\(signingEntity)'" ) + return signingEntity - switch signatureStatus { - case .valid(let signingEntity): - observabilityScope - .emit( - info: "\(package) \(version) from \(registry) is signed with a valid entity '\(signingEntity)'" - ) - completion(.success(signingEntity)) - case .invalid(let reason): - completion(.failure(RegistryError.invalidSignature(reason: reason))) - case .certificateInvalid(let reason): - completion(.failure(RegistryError.invalidSigningCertificate(reason: reason))) - case .certificateNotTrusted(let signingEntity): - observabilityScope - .emit( - info: "\(package) \(version) from \(registry) signing entity '\(signingEntity)' is untrusted", - metadata: .registryPackageMetadata(identity: package) - ) - - guard let onUntrusted = configuration.onUntrustedCertificate else { - return completion(.failure( - RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") - )) - } + case .invalid(let reason): + throw RegistryError.invalidSignature(reason: reason) + + case .certificateInvalid(let reason): + throw RegistryError.invalidSigningCertificate(reason: reason) + + case .certificateNotTrusted(let signingEntity): + observabilityScope.emit( + info: "\(package) \(version) from \(registry) signing entity '\(signingEntity)' is untrusted", + metadata: .registryPackageMetadata(identity: package) + ) - let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) - - switch onUntrusted { - case .prompt: - self.delegate - .onUntrusted( - registry: registry, - package: package.underlying, - version: version - ) { `continue` in - if `continue` { - completion(.success(.none)) - } else { - completion(.failure(signerNotTrustedError)) - } - } - case .error: - completion(.failure(signerNotTrustedError)) - case .warn: - observabilityScope.emit( - warning: "\(signerNotTrustedError)", - metadata: .registryPackageMetadata(identity: package) - ) - completion(.success(.none)) - case .silentAllow: - // Continue without logging - completion(.success(.none)) + guard let onUntrusted = configuration.onUntrustedCertificate else { + throw RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") + } + + let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) + + switch onUntrusted { + case .prompt: + let `continue` = await self.delegate.onUntrusted( + registry: registry, + package: package.underlying, + version: version + ) + + if `continue` { + return nil + } else { + throw signerNotTrustedError } + + case .error: + throw signerNotTrustedError + + case .warn: + observabilityScope.emit( + warning: "\(signerNotTrustedError)", + metadata: .registryPackageMetadata(identity: package) + ) + return nil + + case .silentAllow: + // Continue without logging + return nil } - } catch { - completion(.failure(RegistryError.failedToValidateSignature(error))) } + } catch { + throw RegistryError.failedToValidateSignature(error) } } // MARK: - manifests - func validate( - registry: Registry, - package: PackageIdentity.RegistryIdentity, - version: Version, - toolsVersion: ToolsVersion?, - manifestContent: String, - configuration: RegistryConfiguration.Security.Signing, - timeout: DispatchTimeInterval?, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws -> SigningEntity? { - try await withCheckedThrowingContinuation { continuation in - self.validate( - registry: registry, - package: package, - version: version, - toolsVersion: toolsVersion, - manifestContent: manifestContent, - configuration: configuration, - timeout: timeout, - fileSystem:fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { continuation.resume(with: $0) } - ) - } - } - @available(*, noasync, message: "Use the async alternative") func validate( registry: Registry, package: PackageIdentity.RegistryIdentity, @@ -353,15 +280,13 @@ struct SignatureValidation { configuration: RegistryConfiguration.Security.Signing, timeout: DispatchTimeInterval?, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping @Sendable (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) async throws -> SigningEntity? { guard !self.skipSignatureValidation else { - return completion(.success(.none)) + return nil } - self.getAndValidateManifestSignature( + let signingEntity = try await self.getAndValidateManifestSignature( registry: registry, package: package, version: version, @@ -370,27 +295,20 @@ struct SignatureValidation { configuration: configuration, timeout: timeout, fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - switch result { - case .success(let signingEntity): - // Always do signing entity TOFU check at the end, - // whether the manifest is signed or not. - self.signingEntityTOFU.validate( - registry: registry, - package: package, - version: version, - signingEntity: signingEntity, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { _ in - completion(.success(signingEntity)) - } - case .failure(let error): - completion(.failure(error)) - } - } + observabilityScope: observabilityScope + ) + + // Always do signing entity TOFU check at the end, + // whether the manifest is signed or not. + try self.signingEntityTOFU.validate( + registry: registry, + package: package, + version: version, + signingEntity: signingEntity, + observabilityScope: observabilityScope + ) + + return signingEntity } private func getAndValidateManifestSignature( @@ -402,101 +320,93 @@ struct SignatureValidation { configuration: RegistryConfiguration.Security.Signing, timeout: DispatchTimeInterval?, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping @Sendable (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) async throws -> SigningEntity? { let manifestName = toolsVersion.map { "Package@swift-\($0).swift" } ?? Manifest.filename - Task { - do { - let versionMetadata = try await self.versionMetadataProvider(package, version) - - guard let sourceArchiveResource = versionMetadata.sourceArchive else { - observabilityScope - .emit( - debug: "cannot determine if \(manifestName) should be signed because source archive for \(package) \(version) is not found in \(registry)", - metadata: .registryPackageMetadata(identity: package) - ) - return completion(.success(.none)) - } - guard sourceArchiveResource.signing?.signatureBase64Encoded != nil else { - throw RegistryError.sourceArchiveNotSigned( - registry: registry, - package: package.underlying, - version: version - ) - } - - // source archive is signed, so the manifest must also be signed - guard let manifestSignature = try ManifestSignatureParser.parse(utf8String: manifestContent) else { - return completion(.failure(RegistryError.manifestNotSigned( - registry: registry, - package: package.underlying, - version: version, - toolsVersion: toolsVersion - ))) - } - - guard let signatureFormat = SignatureFormat(rawValue: manifestSignature.signatureFormat) else { - return completion(.failure(RegistryError.unknownSignatureFormat(manifestSignature.signatureFormat))) - } + do { + let versionMetadata = try await self.versionMetadataProvider(package, version) - self.validateManifestSignature( + guard let sourceArchiveResource = versionMetadata.sourceArchive else { + observabilityScope + .emit( + debug: "cannot determine if \(manifestName) should be signed because source archive for \(package) \(version) is not found in \(registry)", + metadata: .registryPackageMetadata(identity: package) + ) + return nil + } + guard sourceArchiveResource.signing?.signatureBase64Encoded != nil else { + throw RegistryError.sourceArchiveNotSigned( registry: registry, - package: package, - version: version, - manifestName: manifestName, - signature: manifestSignature.signature, - signatureFormat: signatureFormat, - content: manifestSignature.contents, - configuration: configuration, - fileSystem: fileSystem, - observabilityScope: observabilityScope, - completion: completion - ) - } catch RegistryError.sourceArchiveNotSigned { - observabilityScope.emit( - debug: "\(manifestName) is not signed because source archive for \(package) \(version) from \(registry) is not signed", - metadata: .registryPackageMetadata(identity: package) + package: package.underlying, + version: version ) - guard let onUnsigned = configuration.onUnsigned else { - return completion(.failure(RegistryError.missingConfiguration(details: "security.signing.onUnsigned"))) - } + } - let sourceArchiveNotSignedError = RegistryError.sourceArchiveNotSigned( + // source archive is signed, so the manifest must also be signed + guard let manifestSignature = try ManifestSignatureParser.parse(utf8String: manifestContent) else { + throw RegistryError.manifestNotSigned( registry: registry, package: package.underlying, - version: version + version: version, + toolsVersion: toolsVersion ) + } - // Prompt if configured, otherwise just continue (this differs - // from source archive to minimize duplicate loggings). - switch onUnsigned { - case .prompt: - self.delegate - .onUnsigned(registry: registry, package: package.underlying, version: version) { `continue` in - if `continue` { - completion(.success(.none)) - } else { - completion(.failure(sourceArchiveNotSignedError)) - } - } - default: - completion(.success(.none)) + guard let signatureFormat = SignatureFormat(rawValue: manifestSignature.signatureFormat) else { + throw RegistryError.unknownSignatureFormat(manifestSignature.signatureFormat) + } + + return try await self.validateManifestSignature( + registry: registry, + package: package, + version: version, + manifestName: manifestName, + signature: manifestSignature.signature, + signatureFormat: signatureFormat, + content: manifestSignature.contents, + configuration: configuration, + fileSystem: fileSystem, + observabilityScope: observabilityScope + ) + } catch RegistryError.sourceArchiveNotSigned { + observabilityScope.emit( + debug: "\(manifestName) is not signed because source archive for \(package) \(version) from \(registry) is not signed", + metadata: .registryPackageMetadata(identity: package) + ) + guard let onUnsigned = configuration.onUnsigned else { + throw RegistryError.missingConfiguration(details: "security.signing.onUnsigned") + } + + let sourceArchiveNotSignedError = RegistryError.sourceArchiveNotSigned( + registry: registry, + package: package.underlying, + version: version + ) + + // Prompt if configured, otherwise just continue (this differs + // from source archive to minimize duplicate loggings). + switch onUnsigned { + case .prompt: + let `continue` = await self.delegate.onUnsigned(registry: registry, package: package.underlying, version: version) + if `continue` { + return nil + } else { + throw sourceArchiveNotSignedError } - } catch ManifestSignatureParser.Error.malformedManifestSignature { - completion(.failure(RegistryError.invalidSignature(reason: "manifest signature is malformed"))) - } catch { - observabilityScope - .emit( - debug: "cannot determine if \(manifestName) should be signed because retrieval of source archive signature for \(package) \(version) from \(registry) failed", - metadata: .registryPackageMetadata(identity: package), - underlyingError: error - ) - completion(.success(.none)) + + default: + return nil } + } catch ManifestSignatureParser.Error.malformedManifestSignature { + throw RegistryError.invalidSignature(reason: "manifest signature is malformed") + } catch { + observabilityScope.emit( + debug: "cannot determine if \(manifestName) should be signed because retrieval of source archive signature for \(package) \(version) from \(registry) failed", + metadata: .registryPackageMetadata(identity: package), + underlyingError: error + ) + return nil } - } private func validateManifestSignature( @@ -509,68 +419,64 @@ struct SignatureValidation { content: [UInt8], configuration: RegistryConfiguration.Security.Signing, fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - completion: @escaping @Sendable (Result) -> Void - ) { - Task { - do { - let signatureStatus = try await SignatureProvider.status( - signature: signature, - content: content, - format: signatureFormat, - verifierConfiguration: try VerifierConfiguration.from(configuration, fileSystem: fileSystem), - observabilityScope: observabilityScope + observabilityScope: ObservabilityScope + ) async throws -> SigningEntity? { + do { + let signatureStatus = try await SignatureProvider.status( + signature: signature, + content: content, + format: signatureFormat, + verifierConfiguration: try VerifierConfiguration.from(configuration, fileSystem: fileSystem), + observabilityScope: observabilityScope + ) + + switch signatureStatus { + case .valid(let signingEntity): + observabilityScope.emit( + info: "\(package) \(version) \(manifestName) from \(registry) is signed with a valid entity '\(signingEntity)'" ) + return signingEntity - switch signatureStatus { - case .valid(let signingEntity): - observabilityScope - .emit( - info: "\(package) \(version) \(manifestName) from \(registry) is signed with a valid entity '\(signingEntity)'" - ) - completion(.success(signingEntity)) - case .invalid(let reason): - completion(.failure(RegistryError.invalidSignature(reason: reason))) - case .certificateInvalid(let reason): - completion(.failure(RegistryError.invalidSigningCertificate(reason: reason))) - case .certificateNotTrusted(let signingEntity): - observabilityScope - .emit( - debug: "the signer '\(signingEntity)' of \(package) \(version) \(manifestName) from \(registry) is not trusted", - metadata: .registryPackageMetadata(identity: package) - ) - - guard let onUntrusted = configuration.onUntrustedCertificate else { - return completion(.failure( - RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") - )) - } + case .invalid(let reason): + throw RegistryError.invalidSignature(reason: reason) + + case .certificateInvalid(let reason): + throw RegistryError.invalidSigningCertificate(reason: reason) + + case .certificateNotTrusted(let signingEntity): + observabilityScope.emit( + debug: "the signer '\(signingEntity)' of \(package) \(version) \(manifestName) from \(registry) is not trusted", + metadata: .registryPackageMetadata(identity: package) + ) + + guard let onUntrusted = configuration.onUntrustedCertificate else { + throw RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") + } + + let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) - let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) - - // Prompt if configured, otherwise just continue (this differs - // from source archive to minimize duplicate loggings). - switch onUntrusted { - case .prompt: - self.delegate - .onUntrusted( - registry: registry, - package: package.underlying, - version: version - ) { `continue` in - if `continue` { - completion(.success(.none)) - } else { - completion(.failure(signerNotTrustedError)) - } - } - default: - completion(.success(.none)) + // Prompt if configured, otherwise just continue (this differs + // from source archive to minimize duplicate loggings). + switch onUntrusted { + case .prompt: + let `continue` = await self.delegate.onUntrusted( + registry: registry, + package: package.underlying, + version: version + ) + + if `continue` { + return nil + } else { + throw signerNotTrustedError } + + default: + return nil } - } catch { - completion(.failure(RegistryError.failedToValidateSignature(error))) } + } catch { + throw RegistryError.failedToValidateSignature(error) } } @@ -581,37 +487,13 @@ struct SignatureValidation { signatureFormat: SignatureFormat, configuration: RegistryConfiguration.Security.Signing, fileSystem: FileSystem - ) async throws -> SigningEntity? { - try await withCheckedThrowingContinuation { continuation in - SignatureValidation.extractSigningEntity( - signature: signature, - signatureFormat: signatureFormat, - configuration: configuration, - fileSystem: fileSystem, - completion: { continuation.resume(with: $0) } - ) - } - } - static func extractSigningEntity( - signature: [UInt8], - signatureFormat: SignatureFormat, - configuration: RegistryConfiguration.Security.Signing, - fileSystem: FileSystem, - completion: @escaping @Sendable (Result) -> Void - ) { - Task { - do { - let verifierConfiguration = try VerifierConfiguration.from(configuration, fileSystem: fileSystem) - let signingEntity = try await SignatureProvider.extractSigningEntity( - signature: signature, - format: signatureFormat, - verifierConfiguration: verifierConfiguration - ) - completion(.success(signingEntity)) - } catch { - completion(.failure(error)) - } - } + ) async throws -> SigningEntity? { + let verifierConfiguration = try VerifierConfiguration.from(configuration, fileSystem: fileSystem) + return try await SignatureProvider.extractSigningEntity( + signature: signature, + format: signatureFormat, + verifierConfiguration: verifierConfiguration + ) } } diff --git a/Sources/PackageRegistry/SigningEntityTOFU.swift b/Sources/PackageRegistry/SigningEntityTOFU.swift index 80ac0b37ee9..914eed82968 100644 --- a/Sources/PackageRegistry/SigningEntityTOFU.swift +++ b/Sources/PackageRegistry/SigningEntityTOFU.swift @@ -36,78 +36,43 @@ struct PackageSigningEntityTOFU { package: PackageIdentity.RegistryIdentity, version: Version, signingEntity: SigningEntity?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue - ) async throws { - try await withCheckedThrowingContinuation { continuation in - self.validate( - registry: registry, - package: package, - version: version, - signingEntity: signingEntity, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: { continuation.resume(with: $0) } - ) + observabilityScope: ObservabilityScope + ) throws { + guard let signingEntityStorage else { + return } - } - @available(*, noasync, message: "Use the async alternative") - func validate( - registry: Registry, - package: PackageIdentity.RegistryIdentity, - version: Version, - signingEntity: SigningEntity?, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - guard let signingEntityStorage else { - return completion(.success(())) + let packageSigners: PackageSigners + do { + packageSigners = try signingEntityStorage.get(package: package.underlying, observabilityScope: observabilityScope) + } catch { + observabilityScope.emit( + error: "Failed to get signing entity for \(package) from storage", + underlyingError: error + ) + throw error } - Task { - let packageSigners: PackageSigners - do { - packageSigners = try signingEntityStorage.get(package: package.underlying, observabilityScope: observabilityScope) - } catch { - observabilityScope.emit( - error: "Failed to get signing entity for \(package) from storage", - underlyingError: error - ) - return completion(.failure(error)) - } - self.validateSigningEntity( - registry: registry, - package: package, - version: version, - signingEntity: signingEntity, - packageSigners: packageSigners, - observabilityScope: observabilityScope - ) { validateResult in - switch validateResult { - case .success(let shouldWrite): - // We only use certain type(s) of signing entity for TOFU - guard shouldWrite, let signingEntity = signingEntity, case .recognized = signingEntity else { - return completion(.success(())) - } - do { - try self.writeToStorage( - registry: registry, - package: package, - version: version, - signingEntity: signingEntity, - observabilityScope: observabilityScope - ) - return completion(.success(())) - } catch { - return completion(.failure(error)) - } + let shouldWrite = try self.validateSigningEntity( + registry: registry, + package: package, + version: version, + signingEntity: signingEntity, + packageSigners: packageSigners, + observabilityScope: observabilityScope + ) - case .failure(let error): - completion(.failure(error)) - } - } + // We only use certain type(s) of signing entity for TOFU + guard shouldWrite, let signingEntity = signingEntity, case .recognized = signingEntity else { + return } + + try self.writeToStorage( + registry: registry, + package: package, + version: version, + signingEntity: signingEntity, + observabilityScope: observabilityScope + ) } private func validateSigningEntity( @@ -116,14 +81,13 @@ struct PackageSigningEntityTOFU { version: Version, signingEntity: SigningEntity?, packageSigners: PackageSigners, - observabilityScope: ObservabilityScope, - completion: @escaping (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) throws -> Bool { // Package is never signed. // If signingEntity is nil, it means package remains unsigned, which is OK. (none -> none) // Otherwise, package has gained a signer, which is also OK. (none -> some) if packageSigners.isEmpty { - return completion(.success(true)) + return true } // If we get to this point, it means we have seen a signed version of the package. @@ -139,27 +103,23 @@ struct PackageSigningEntityTOFU { // - If signingEntity is nil, it could mean the package author has stopped signing the package. // - If signingEntity is non-nil, it could mean the package has changed ownership and the new owner // is re-signing all of the package versions. - do { - try self.handleSigningEntityForPackageVersionChanged( - registry: registry, - package: package, - version: version, - latest: signingEntity, - existing: signingEntitiesForVersion.first!, // !-safe since signingEntitiesForVersion is non-empty - observabilityScope: observabilityScope - ) - return completion(.success(false)) - } catch { - return completion(.failure(error)) - } + try self.handleSigningEntityForPackageVersionChanged( + registry: registry, + package: package, + version: version, + latest: signingEntity, + existing: signingEntitiesForVersion.first!, // !-safe since signingEntitiesForVersion is non-empty + observabilityScope: observabilityScope + ) + return true } // Signer remains the same for the version - return completion(.success(false)) + return false } // Check signer(s) of other version(s) switch signingEntity { - // Is the package changing from one signer to another? + // Is the package changing from one signer to another? case .some(let signingEntity): // Does the package have an expected signer? if let expectedSigner = packageSigners.expectedSigner, @@ -167,7 +127,7 @@ struct PackageSigningEntityTOFU { { // Signer is as expected if signingEntity == expectedSigner.signingEntity { - return completion(.success(true)) + return true } // If the signer is different from expected but has been seen before, // we allow versions before its highest known version to be signed @@ -181,10 +141,11 @@ struct PackageSigningEntityTOFU { let highestKnownVersion = knownSigner.versions.sorted(by: >).first, version < highestKnownVersion { - return completion(.success(true)) + return true } + // Different signer than expected - self.handleSigningEntityForPackageChanged( + try self.handleSigningEntityForPackageChanged( registry: registry, package: package, version: version, @@ -192,13 +153,13 @@ struct PackageSigningEntityTOFU { existing: expectedSigner.signingEntity, existingVersion: expectedSigner.fromVersion, observabilityScope: observabilityScope - ) { result in - completion(result.tryMap { false }) - } + ) + + return false } else { // There might be other signers, but if we have seen this signer before, allow it. if packageSigners.signers[signingEntity] != nil { - return completion(.success(true)) + return true } let otherSigningEntities = packageSigners.signers.keys.filter { $0 != signingEntity } @@ -206,7 +167,7 @@ struct PackageSigningEntityTOFU { // We have not seen this signer before, and there is at least one other signer already. // TODO: This could indicate a legitimate change in package ownership if let existingVersion = packageSigners.signers[otherSigningEntity]?.versions.sorted(by: >).first { - return self.handleSigningEntityForPackageChanged( + try self.handleSigningEntityForPackageChanged( registry: registry, package: package, version: version, @@ -214,16 +175,16 @@ struct PackageSigningEntityTOFU { existing: otherSigningEntity, existingVersion: existingVersion, observabilityScope: observabilityScope - ) { result in - completion(result.tryMap { false }) - } + ) + + return false } } // Package doesn't have any other signer besides the given one, which is good. - completion(.success(true)) + return true } - // Or is the package going from having a signer to .none? + // Or is the package going from having a signer to .none? case .none: let versionSigningEntities = packageSigners.versionSigningEntities // If the given version is semantically newer than any signed version, @@ -248,7 +209,7 @@ struct PackageSigningEntityTOFU { .sorted(by: >) for olderSignedVersion in olderSignedVersions { if let olderVersionSigner = versionSigningEntities[olderSignedVersion]?.first { - return self.handleSigningEntityForPackageChanged( + try self.handleSigningEntityForPackageChanged( registry: registry, package: package, version: version, @@ -256,13 +217,14 @@ struct PackageSigningEntityTOFU { existing: olderVersionSigner, existingVersion: olderSignedVersion, observabilityScope: observabilityScope - ) { result in - completion(result.tryMap { false }) - } + ) + + return false } } + // Assume the given version is an older version before package started getting signed - completion(.success(false)) + return false } } @@ -329,25 +291,23 @@ struct PackageSigningEntityTOFU { latest: SigningEntity?, existing: SigningEntity, existingVersion: Version, - observabilityScope: ObservabilityScope, - completion: @escaping (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) throws { switch self.signingEntityCheckingMode { case .strict: - completion(.failure(RegistryError.signingEntityForPackageChanged( + throw RegistryError.signingEntityForPackageChanged( registry: registry, package: package.underlying, version: version, latest: latest, previous: existing, previousVersion: existingVersion - ))) + ) case .warn: observabilityScope .emit( warning: "the signing entity '\(String(describing: latest))' from \(registry) for \(package) version \(version) is different from the previously recorded value '\(existing)' for version \(existingVersion)" ) - completion(.success(())) } } } diff --git a/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift b/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift index 41df217cee2..8608ddb7fb1 100644 --- a/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift +++ b/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift @@ -259,8 +259,7 @@ extension PackageRegistryCommand { try await registryClient.login( loginURL: loginURL, timeout: .seconds(5), - observabilityScope: swiftCommandState.observabilityScope, - callbackQueue: .sharedConcurrent + observabilityScope: swiftCommandState.observabilityScope ) print("Login successful.") diff --git a/Sources/PackageRegistryCommand/PackageRegistryCommand+Publish.swift b/Sources/PackageRegistryCommand/PackageRegistryCommand+Publish.swift index 0a06f2cdf3f..412b9809c14 100644 --- a/Sources/PackageRegistryCommand/PackageRegistryCommand+Publish.swift +++ b/Sources/PackageRegistryCommand/PackageRegistryCommand+Publish.swift @@ -221,8 +221,7 @@ extension PackageRegistryCommand { metadataSignature: metadataSignature, signatureFormat: self.signatureFormat, fileSystem: localFileSystem, - observabilityScope: swiftCommandState.observabilityScope, - callbackQueue: .sharedConcurrent + observabilityScope: swiftCommandState.observabilityScope ) switch result { diff --git a/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift b/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift index b804384c860..15d61bd95d9 100644 --- a/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift +++ b/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift @@ -78,26 +78,20 @@ public struct FileSystemPackageContainer: PackageContainer { } // Load the manifest. - // FIXME: this should not block - return try await withCheckedThrowingContinuation { continuation in - manifestLoader.load( - packagePath: packagePath, - packageIdentity: self.package.identity, - packageKind: self.package.kind, - packageLocation: self.package.locationString, - packageVersion: nil, - currentToolsVersion: self.currentToolsVersion, - identityResolver: self.identityResolver, - dependencyMapper: self.dependencyMapper, - fileSystem: self.fileSystem, - observabilityScope: self.observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent, - completion: { - continuation.resume(with: $0) - } - ) - } + return try await manifestLoader.load( + packagePath: packagePath, + packageIdentity: self.package.identity, + packageKind: self.package.kind, + packageLocation: self.package.locationString, + packageVersion: nil, + currentToolsVersion: self.currentToolsVersion, + identityResolver: self.identityResolver, + dependencyMapper: self.dependencyMapper, + fileSystem: self.fileSystem, + observabilityScope: self.observabilityScope, + delegateQueue: .sharedConcurrent, + callbackQueue: .sharedConcurrent + ) } } diff --git a/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift b/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift index 705f4eea23c..65ad4b15358 100644 --- a/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift +++ b/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift @@ -72,11 +72,7 @@ public class RegistryPackageContainer: PackageContainer { public func toolsVersion(for version: Version) async throws -> ToolsVersion { try await self.toolsVersionsCache.memoize(version) { - let result = try await withCheckedThrowingContinuation { continuation in - self.getAvailableManifestsFilesystem(version: version, completion: { - continuation.resume(with: $0) - }) - } + let result = try await self.getAvailableManifestsFilesystem(version: version) // find the manifest path and parse it's tools-version let manifestPath = try ManifestLoader.findManifest(packagePath: .root, fileSystem: result.fileSystem, currentToolsVersion: self.currentToolsVersion) return try ToolsVersionParser.parse(manifestPath: manifestPath, fileSystem: result.fileSystem) @@ -85,16 +81,10 @@ public class RegistryPackageContainer: PackageContainer { public func versionsDescending() async throws -> [Version] { try await self.knownVersionsCache.memoize { - let metadata = try await withCheckedThrowingContinuation { continuation in - self.registryClient.getPackageMetadata( - package: self.package.identity, - observabilityScope: self.observabilityScope, - callbackQueue: .sharedConcurrent, - completion: { - continuation.resume(with: $0) - } - ) - } + let metadata = try await self.registryClient.getPackageMetadata( + package: self.package.identity, + observabilityScope: self.observabilityScope + ) return metadata.versions.sorted(by: >) } } @@ -130,126 +120,100 @@ public class RegistryPackageContainer: PackageContainer { return self.package } - // marked internal for testing - internal func loadManifest(version: Version) async throws -> Manifest { + // left internal for testing + func loadManifest(version: Version) async throws -> Manifest { return try await self.manifestsCache.memoize(version) { - try await withCheckedThrowingContinuation { continuation in - self.loadManifest(version: version, completion: { - continuation.resume(with: $0) - }) - } + try await self.loadManifest(version: version) } } - private func loadManifest(version: Version, completion: @escaping (Result) -> Void) { - self.getAvailableManifestsFilesystem(version: version) { result in - switch result { - case .failure(let error): - return completion(.failure(error)) - case .success(let result): - do { - let manifests = result.manifests - let fileSystem = result.fileSystem + private func loadUnmemoizedManifest(version: Version) async throws -> Manifest { + let result = try await self.getAvailableManifestsFilesystem(version: version) + let manifests = result.manifests + let fileSystem = result.fileSystem - // first, decide the tools-version we should use - guard let defaultManifestToolsVersion = manifests.first(where: { $0.key == Manifest.filename })?.value.toolsVersion else { - throw StringError("Could not find the '\(Manifest.filename)' file for '\(self.package.identity)' '\(version)'") - } - // find the preferred manifest path and parse it's tools-version - let preferredToolsVersionManifestPath = try ManifestLoader.findManifest(packagePath: .root, fileSystem: fileSystem, currentToolsVersion: self.currentToolsVersion) - let preferredToolsVersion = try ToolsVersionParser.parse(manifestPath: preferredToolsVersionManifestPath, fileSystem: fileSystem) - // load the manifest content - let loadManifest = { - self.manifestLoader.load( - packagePath: .root, - packageIdentity: self.package.identity, - packageKind: self.package.kind, - packageLocation: self.package.locationString, - packageVersion: (version: version, revision: nil), - currentToolsVersion: self.currentToolsVersion, - identityResolver: self.identityResolver, - dependencyMapper: self.dependencyMapper, - fileSystem: result.fileSystem, - observabilityScope: self.observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent, - completion: completion - ) - } + // first, decide the tools-version we should use + guard let defaultManifestToolsVersion = manifests.first(where: { $0.key == Manifest.filename })?.value.toolsVersion else { + throw StringError("Could not find the '\(Manifest.filename)' file for '\(self.package.identity)' '\(version)'") + } + // find the preferred manifest path and parse it's tools-version + let preferredToolsVersionManifestPath = try ManifestLoader.findManifest(packagePath: .root, fileSystem: fileSystem, currentToolsVersion: self.currentToolsVersion) + let preferredToolsVersion = try ToolsVersionParser.parse(manifestPath: preferredToolsVersionManifestPath, fileSystem: fileSystem) + // load the manifest content + let loadManifest = { + try await self.manifestLoader.load( + packagePath: .root, + packageIdentity: self.package.identity, + packageKind: self.package.kind, + packageLocation: self.package.locationString, + packageVersion: (version: version, revision: nil), + currentToolsVersion: self.currentToolsVersion, + identityResolver: self.identityResolver, + dependencyMapper: self.dependencyMapper, + fileSystem: result.fileSystem, + observabilityScope: self.observabilityScope, + delegateQueue: .sharedConcurrent, + callbackQueue: .sharedConcurrent + ) + } - if preferredToolsVersion == defaultManifestToolsVersion { - // default tools version - we already have the content on disk from getAvailableManifestsFileSystem() - loadManifest() - } else { - // custom tools-version, we need to fetch the content from the server - self.registryClient.getManifestContent( - package: self.package.identity, - version: version, - customToolsVersion: preferredToolsVersion, - observabilityScope: self.observabilityScope, - callbackQueue: .sharedConcurrent - ) { result in - switch result { - case .failure(let error): - return completion(.failure(error)) - case .success(let manifestContent): - do { - // find the fake manifest so we can replace it with the real manifest content - guard let placeholderManifestFileName = try fileSystem.getDirectoryContents(.root).first(where: { file in - if file == Manifest.basename + "@swift-\(preferredToolsVersion).swift" { - return true - } else if preferredToolsVersion.patch == 0, file == Manifest.basename + "@swift-\(preferredToolsVersion.major).\(preferredToolsVersion.minor).swift" { - return true - } else { - return false - } - }) else { - throw StringError("failed locating placeholder manifest for \(preferredToolsVersion)") - } - // replace the fake manifest with the real manifest content - let manifestPath = AbsolutePath.root.appending(component: placeholderManifestFileName) - try fileSystem.removeFileTree(manifestPath) - try fileSystem.writeFileContents(manifestPath, string: manifestContent) - // finally, load the manifest - loadManifest() - } catch { - return completion(.failure(error)) - } - } - } - } - } catch { - return completion(.failure(error)) + if preferredToolsVersion == defaultManifestToolsVersion { + // default tools version - we already have the content on disk from getAvailableManifestsFileSystem() + return try await loadManifest() + } else { + // custom tools-version, we need to fetch the content from the server + let manifestContent = try await self.registryClient.getManifestContent( + package: self.package.identity, + version: version, + customToolsVersion: preferredToolsVersion, + observabilityScope: self.observabilityScope + ) + + // find the fake manifest so we can replace it with the real manifest content + guard let placeholderManifestFileName = try fileSystem.getDirectoryContents(.root).first(where: { file in + if file == Manifest.basename + "@swift-\(preferredToolsVersion).swift" { + return true + } else if preferredToolsVersion.patch == 0, file == Manifest.basename + "@swift-\(preferredToolsVersion.major).\(preferredToolsVersion.minor).swift" { + return true + } else { + return false } + }) else { + throw StringError("failed locating placeholder manifest for \(preferredToolsVersion)") } + // replace the fake manifest with the real manifest content + let manifestPath = AbsolutePath.root.appending(component: placeholderManifestFileName) + try fileSystem.removeFileTree(manifestPath) + try fileSystem.writeFileContents(manifestPath, string: manifestContent) + // finally, load the manifest + return try await loadManifest() } } - private func getAvailableManifestsFilesystem(version: Version, completion: @escaping (Result<(manifests: [String: (toolsVersion: ToolsVersion, content: String?)], fileSystem: FileSystem), Error>) -> Void) { + private func getAvailableManifestsFilesystem( + version: Version + ) async throws -> (manifests: [String: (toolsVersion: ToolsVersion, content: String?)], fileSystem: FileSystem) { // try cached first if let availableManifests = self.availableManifestsCache[version] { - return completion(.success(availableManifests)) + return availableManifests } // get from server - self.registryClient.getAvailableManifests( + let manifests = try await self.registryClient.getAvailableManifests( package: self.package.identity, version: version, - observabilityScope: self.observabilityScope, - callbackQueue: .sharedConcurrent - ) { result in - completion(result.tryMap { manifests in - // ToolsVersionLoader is designed to scan files to decide which is the best tools-version - // as such, this writes a fake manifest based on the information returned by the registry - // with only the header line which is all that is needed by ToolsVersionLoader - let fileSystem = InMemoryFileSystem() - for manifest in manifests { - let content = manifest.value.content ?? "// swift-tools-version:\(manifest.value.toolsVersion)" - try fileSystem.writeFileContents(AbsolutePath.root.appending(component: manifest.key), string: content) - } - self.availableManifestsCache[version] = (manifests: manifests, fileSystem: fileSystem) - return (manifests: manifests, fileSystem: fileSystem) - }) + observabilityScope: self.observabilityScope + ) + + // ToolsVersionLoader is designed to scan files to decide which is the best tools-version + // as such, this writes a fake manifest based on the information returned by the registry + // with only the header line which is all that is needed by ToolsVersionLoader + let fileSystem = InMemoryFileSystem() + for manifest in manifests { + let content = manifest.value.content ?? "// swift-tools-version:\(manifest.value.toolsVersion)" + try fileSystem.writeFileContents(AbsolutePath.root.appending(component: manifest.key), string: content) } + self.availableManifestsCache[version] = (manifests: manifests, fileSystem: fileSystem) + return (manifests: manifests, fileSystem: fileSystem) } } diff --git a/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift b/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift index 080a3a4399c..da1f43b7245 100644 --- a/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift +++ b/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift @@ -394,27 +394,20 @@ internal final class SourceControlPackageContainer: PackageContainer, CustomStri } private func loadManifest(fileSystem: FileSystem, version: Version?, revision: String) async throws -> Manifest { - // Load the manifest. - // FIXME: this should not block - return try await withCheckedThrowingContinuation { continuation in - self.manifestLoader.load( - packagePath: .root, - packageIdentity: self.package.identity, - packageKind: self.package.kind, - packageLocation: self.package.locationString, - packageVersion: (version: version, revision: revision), - currentToolsVersion: self.currentToolsVersion, - identityResolver: self.identityResolver, - dependencyMapper: self.dependencyMapper, - fileSystem: fileSystem, - observabilityScope: self.observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent, - completion: { - continuation.resume(with: $0) - } - ) - } + try await self.manifestLoader.load( + packagePath: .root, + packageIdentity: self.package.identity, + packageKind: self.package.kind, + packageLocation: self.package.locationString, + packageVersion: (version: version, revision: revision), + currentToolsVersion: self.currentToolsVersion, + identityResolver: self.identityResolver, + dependencyMapper: self.dependencyMapper, + fileSystem: fileSystem, + observabilityScope: self.observabilityScope, + delegateQueue: .sharedConcurrent, + callbackQueue: .sharedConcurrent + ) } public var isRemoteContainer: Bool? { diff --git a/Sources/Workspace/Workspace+Delegation.swift b/Sources/Workspace/Workspace+Delegation.swift index dac189887fd..39b9047f094 100644 --- a/Sources/Workspace/Workspace+Delegation.swift +++ b/Sources/Workspace/Workspace+Delegation.swift @@ -142,15 +142,14 @@ public protocol WorkspaceDelegate: AnyObject { func onUnsignedRegistryPackage( registryURL: URL, package: PackageModel.PackageIdentity, - version: TSCUtility.Version, - completion: (Bool) -> Void - ) + version: TSCUtility.Version + ) async -> Bool + func onUntrustedRegistryPackage( registryURL: URL, package: PackageModel.PackageIdentity, - version: TSCUtility.Version, - completion: (Bool) -> Void - ) + version: TSCUtility.Version + ) async -> Bool /// The workspace has started updating dependencies func willUpdateDependencies() @@ -173,23 +172,21 @@ extension WorkspaceDelegate { public func onUnsignedRegistryPackage( registryURL: URL, package: PackageModel.PackageIdentity, - version: TSCUtility.Version, - completion: (Bool) -> Void - ) { + version: TSCUtility.Version + ) async -> Bool { // true == continue resolution // false == stop dependency resolution - completion(true) + return true } public func onUntrustedRegistryPackage( registryURL: URL, package: PackageModel.PackageIdentity, - version: TSCUtility.Version, - completion: (Bool) -> Void - ) { + version: TSCUtility.Version + ) async -> Bool { // true == continue resolution // false == stop dependency resolution - completion(true) + return true } } @@ -361,33 +358,31 @@ struct WorkspaceRegistryClientDelegate: RegistryClient.Delegate { self.workspaceDelegate = workspaceDelegate } - func onUnsigned(registry: Registry, package: PackageIdentity, version: Version, completion: (Bool) -> Void) { + func onUnsigned(registry: Registry, package: PackageIdentity, version: Version) async -> Bool { if let delegate = self.workspaceDelegate { - delegate.onUnsignedRegistryPackage( + return await delegate.onUnsignedRegistryPackage( registryURL: registry.url, package: package, - version: version, - completion: completion + version: version ) } else { // true == continue resolution // false == stop dependency resolution - completion(true) + return true } } - func onUntrusted(registry: Registry, package: PackageIdentity, version: Version, completion: (Bool) -> Void) { + func onUntrusted(registry: Registry, package: PackageIdentity, version: Version) async -> Bool { if let delegate = self.workspaceDelegate { - delegate.onUntrustedRegistryPackage( + return await delegate.onUntrustedRegistryPackage( registryURL: registry.url, package: package, - version: version, - completion: completion + version: version ) } else { // true == continue resolution // false == stop dependency resolution - completion(true) + return true } } } diff --git a/Sources/Workspace/Workspace+Editing.swift b/Sources/Workspace/Workspace+Editing.swift index 966db0108eb..957b03149fe 100644 --- a/Sources/Workspace/Workspace+Editing.swift +++ b/Sources/Workspace/Workspace+Editing.swift @@ -63,19 +63,13 @@ extension Workspace { // If there is something present at the destination, we confirm it has // a valid manifest with name canonical location as the package we are trying to edit. if fileSystem.exists(destination) { - // FIXME: this should not block - let manifest = try await withCheckedThrowingContinuation { continuation in - self.loadManifest( - packageIdentity: dependency.packageRef.identity, - packageKind: .fileSystem(destination), - packagePath: destination, - packageLocation: dependency.packageRef.locationString, - observabilityScope: observabilityScope, - completion: { - continuation.resume(with: $0) - } - ) - } + let manifest = try await self.loadManifest( + packageIdentity: dependency.packageRef.identity, + packageKind: .fileSystem(destination), + packagePath: destination, + packageLocation: dependency.packageRef.locationString, + observabilityScope: observabilityScope + ) guard dependency.packageRef.canonicalLocation == manifest.canonicalPackageLocation else { return observabilityScope diff --git a/Sources/Workspace/Workspace+Manifests.swift b/Sources/Workspace/Workspace+Manifests.swift index 234c7071b63..cb4088b52ab 100644 --- a/Sources/Workspace/Workspace+Manifests.swift +++ b/Sources/Workspace/Workspace+Manifests.swift @@ -666,8 +666,7 @@ extension Workspace { } // Load and return the manifest. - return await withCheckedContinuation { continuation in - self.loadManifest( + return try? await self.loadManifest( packageIdentity: managedDependency.packageRef.identity, packageKind: packageKind, packagePath: packagePath, @@ -675,10 +674,7 @@ extension Workspace { packageVersion: packageVersion, fileSystem: fileSystem, observabilityScope: observabilityScope - ) { result in - continuation.resume(returning: try? result.get()) - } - } + ) } /// Load the manifest at a given path. @@ -691,9 +687,8 @@ extension Workspace { packageLocation: String, packageVersion: Version? = nil, fileSystem: FileSystem? = nil, - observabilityScope: ObservabilityScope, - completion: @escaping (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) async throws -> Manifest { let fileSystem = fileSystem ?? self.fileSystem // Load the manifest, bracketed by the calls to the delegate callbacks. @@ -711,61 +706,70 @@ extension Workspace { var manifestLoadingDiagnostics = [Diagnostic]() + defer { + manifestLoadingScope.emit(manifestLoadingDiagnostics) + } + let start = DispatchTime.now() - self.manifestLoader.load( - packagePath: packagePath, - packageIdentity: packageIdentity, - packageKind: packageKind, - packageLocation: packageLocation, - packageVersion: packageVersion.map { (version: $0, revision: nil) }, - currentToolsVersion: self.currentToolsVersion, - identityResolver: self.identityResolver, - dependencyMapper: self.dependencyMapper, - fileSystem: fileSystem, - observabilityScope: manifestLoadingScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent - ) { result in + do { + let manifest = try await self.manifestLoader.load( + packagePath: packagePath, + packageIdentity: packageIdentity, + packageKind: packageKind, + packageLocation: packageLocation, + packageVersion: packageVersion.map { (version: $0, revision: nil) }, + currentToolsVersion: self.currentToolsVersion, + identityResolver: self.identityResolver, + dependencyMapper: self.dependencyMapper, + fileSystem: fileSystem, + observabilityScope: manifestLoadingScope, + delegateQueue: .sharedConcurrent, + callbackQueue: .sharedConcurrent + ) let duration = start.distance(to: .now()) - var result = result - switch result { - case .failure(let error): - manifestLoadingDiagnostics.append(.error(error)) - self.delegate?.didLoadManifest( - packageIdentity: packageIdentity, - packagePath: packagePath, - url: packageLocation, - version: packageVersion, - packageKind: packageKind, - manifest: nil, - diagnostics: manifestLoadingDiagnostics, - duration: duration - ) - case .success(let manifest): - let validator = ManifestValidator( - manifest: manifest, - sourceControlValidator: self.repositoryManager, - fileSystem: self.fileSystem - ) - let validationIssues = validator.validate() - if !validationIssues.isEmpty { - // Diagnostics.fatalError indicates that a more specific diagnostic has already been added. - result = .failure(Diagnostics.fatalError) - manifestLoadingDiagnostics.append(contentsOf: validationIssues) - } - self.delegate?.didLoadManifest( - packageIdentity: packageIdentity, - packagePath: packagePath, - url: packageLocation, - version: packageVersion, - packageKind: packageKind, - manifest: manifest, - diagnostics: manifestLoadingDiagnostics, - duration: duration - ) + + let validator = ManifestValidator( + manifest: manifest, + sourceControlValidator: self.repositoryManager, + fileSystem: self.fileSystem + ) + let validationIssues = validator.validate() + if !validationIssues.isEmpty { + manifestLoadingDiagnostics.append(contentsOf: validationIssues) } - manifestLoadingScope.emit(manifestLoadingDiagnostics) - completion(result) + + self.delegate?.didLoadManifest( + packageIdentity: packageIdentity, + packagePath: packagePath, + url: packageLocation, + version: packageVersion, + packageKind: packageKind, + manifest: manifest, + diagnostics: manifestLoadingDiagnostics, + duration: duration + ) + + if !validationIssues.isEmpty { + // Diagnostics.fatalError indicates that a more specific diagnostic has already been added. + throw Diagnostics.fatalError + } + + return manifest + } catch { + let duration = start.distance(to: .now()) + manifestLoadingDiagnostics.append(.error(error)) + self.delegate?.didLoadManifest( + packageIdentity: packageIdentity, + packagePath: packagePath, + url: packageLocation, + version: packageVersion, + packageKind: packageKind, + manifest: nil, + diagnostics: manifestLoadingDiagnostics, + duration: duration + ) + + throw error } } diff --git a/Sources/Workspace/Workspace+Registry.swift b/Sources/Workspace/Workspace+Registry.swift index 8a022a9540a..809d37e8f70 100644 --- a/Sources/Workspace/Workspace+Registry.swift +++ b/Sources/Workspace/Workspace+Registry.swift @@ -85,10 +85,9 @@ extension Workspace { fileSystem: any FileSystem, observabilityScope: ObservabilityScope, delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - self.underlying.load( + callbackQueue: DispatchQueue + ) async throws -> Manifest { + let manifest = try await self.underlying.load( manifestPath: manifestPath, manifestToolsVersion: manifestToolsVersion, packageIdentity: packageIdentity, @@ -101,20 +100,13 @@ extension Workspace { observabilityScope: observabilityScope, delegateQueue: delegateQueue, callbackQueue: callbackQueue - ) { result in - switch result { - case .failure(let error): - completion(.failure(error)) - case .success(let manifest): - self.transformSourceControlDependenciesToRegistry( - manifest: manifest, - transformationMode: transformationMode, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: completion - ) - } - } + ) + + return try await self.transformSourceControlDependenciesToRegistry( + manifest: manifest, + transformationMode: transformationMode, + observabilityScope: observabilityScope + ) } func resetCache(observabilityScope: ObservabilityScope) { @@ -128,52 +120,42 @@ extension Workspace { private func transformSourceControlDependenciesToRegistry( manifest: Manifest, transformationMode: TransformationMode, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - let sync = DispatchGroup() - let transformations = ThreadSafeKeyValueStore() - for dependency in manifest.dependencies { - if case .sourceControl(let settings) = dependency, case .remote(let url) = settings.location { - sync.enter() - self.mapRegistryIdentity( - url: url, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - defer { sync.leave() } - switch result { - case .failure(let error): - // do not raise error, only report it as warning - observabilityScope.emit( - warning: "failed querying registry identity for '\(url)'", - underlyingError: error - ) - case .success(.some(let identity)): - transformations[dependency] = identity - case .success(.none): - // no identity found - break + observabilityScope: ObservabilityScope + ) async throws -> Manifest { + let transformations = await withTaskGroup(of: (PackageDependency, PackageIdentity?).self) { group in + for dependency in manifest.dependencies { + group.addTask { + if case .sourceControl(let settings) = dependency, case .remote(let url) = settings.location { + do { + let identity = try await self.mapRegistryIdentity( + url: url, + observabilityScope: observabilityScope + ) + + return (dependency, identity) + } catch { + // do not raise error, only report it as warning + observabilityScope.emit( + warning: "failed querying registry identity for '\(url)'", + underlyingError: error + ) + } } + + return (dependency, nil) } } + + return await group.reduce(into: [:]) { $0[$1.0] = $1.1 } } // update the manifest with the transformed dependencies - sync.notify(queue: callbackQueue) { - do { - let updatedManifest = try self.transformManifest( - manifest: manifest, - transformations: transformations.get(), - transformationMode: transformationMode, - observabilityScope: observabilityScope - ) - completion(.success(updatedManifest)) - } catch { - return completion(.failure(error)) - } - } + return try self.transformManifest( + manifest: manifest, + transformations: transformations, + transformationMode: transformationMode, + observabilityScope: observabilityScope + ) } private func transformManifest( @@ -322,35 +304,31 @@ extension Workspace { private func mapRegistryIdentity( url: SourceControlURL, - observabilityScope: ObservabilityScope, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) async throws -> PackageIdentity? { if let cached = self.identityLookupCache[url], cached.expirationTime > .now() { switch cached.result { case .success(let identity): - return completion(.success(identity)) + return identity case .failure: // server error, do not try again - return completion(.success(.none)) + return nil } } - self.registryClient.lookupIdentities( - scmURL: url, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue - ) { result in - switch result { - case .failure(let error): - self.identityLookupCache[url] = (result: .failure(error), expirationTime: .now() + self.cacheTTL) - completion(.failure(error)) - case .success(let identities): - // FIXME: returns first result... need to consider how to address multiple ones - let identity = identities.sorted().first - self.identityLookupCache[url] = (result: .success(identity), expirationTime: .now() + self.cacheTTL) - completion(.success(identity)) - } + do { + let identities = try await self.registryClient.lookupIdentities( + scmURL: url, + observabilityScope: observabilityScope + ) + + // FIXME: returns first result... need to consider how to address multiple ones + let identity = identities.sorted().first + self.identityLookupCache[url] = (result: .success(identity), expirationTime: .now() + self.cacheTTL) + return identity + } catch { + self.identityLookupCache[url] = (result: .failure(error), expirationTime: .now() + self.cacheTTL) + throw error } } @@ -398,8 +376,7 @@ extension Workspace { package: package.identity, version: version, observabilityScope: observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) // Record the new state. @@ -445,6 +422,6 @@ extension Workspace { try self.fileSystem.removeFileTree(downloadPath) // remove the local copy - try registryDownloadsManager.remove(package: dependency.packageRef.identity) + try self.registryDownloadsManager.remove(package: dependency.packageRef.identity) } } diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index e2bc004b593..06a160938e7 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -527,8 +527,6 @@ public class Workspace { registryClient: registryClient, delegate: delegate.map(WorkspaceRegistryDownloadsManagerDelegate.init(workspaceDelegate:)) ) - // register the registry dependencies downloader with the cancellation handler - cancellator?.register(name: "registry downloads", handler: registryDownloadsManager) if let transformationMode = RegistryAwareManifestLoader .TransformationMode(configuration.sourceControlToRegistryDependencyTransformation) @@ -999,53 +997,32 @@ extension Workspace { packages: [AbsolutePath], observabilityScope: ObservabilityScope ) async throws -> [AbsolutePath: Manifest] { - try await withCheckedThrowingContinuation { continuation in - self.loadRootManifests(packages: packages, observabilityScope: observabilityScope) { result in - continuation.resume(with: result) - } - } - } - - /// Loads and returns manifests at the given paths. - @available(*, noasync, message: "Use the async alternative") - public func loadRootManifests( - packages: [AbsolutePath], - observabilityScope: ObservabilityScope, - completion: @escaping (Result<[AbsolutePath: Manifest], Error>) -> Void - ) { - let lock = NSLock() - let sync = DispatchGroup() - var rootManifests = [AbsolutePath: Manifest]() - Set(packages).forEach { package in - sync.enter() - // TODO: this does not use the identity resolver which is probably fine since its the root packages - self.loadManifest( - packageIdentity: PackageIdentity(path: package), - packageKind: .root(package), - packagePath: package, - packageLocation: package.pathString, - observabilityScope: observabilityScope - ) { result in - defer { sync.leave() } - if case .success(let manifest) = result { - lock.withLock { - rootManifests[package] = manifest - } + let rootManifests = try await withThrowingTaskGroup(of: (AbsolutePath, Manifest?).self) { group in + for package in Set(packages) { + group.addTask { + // TODO: this does not use the identity resolver which is probably fine since its the root packages + (package, try await self.loadManifest( + packageIdentity: PackageIdentity(path: package), + packageKind: .root(package), + packagePath: package, + packageLocation: package.pathString, + observabilityScope: observabilityScope + )) } } - } - sync.notify(queue: .sharedConcurrent) { - // Check for duplicate root packages. - let duplicateRoots = rootManifests.values.spm_findDuplicateElements(by: \.displayName) - if !duplicateRoots.isEmpty { - let name = duplicateRoots[0][0].displayName - observabilityScope.emit(error: "found multiple top-level packages named '\(name)'") - return completion(.success([:])) - } + return try await group.reduce(into: [:]) { $0[$1.0] = $1.1 } + } - completion(.success(rootManifests)) + // Check for duplicate root packages. + let duplicateRoots = rootManifests.values.spm_findDuplicateElements(by: \.displayName) + if !duplicateRoots.isEmpty { + let name = duplicateRoots[0][0].displayName + observabilityScope.emit(error: "found multiple top-level packages named '\(name)'") + return [:] } + + return rootManifests } /// Loads and returns manifest at the given path. @@ -1053,105 +1030,77 @@ extension Workspace { at path: AbsolutePath, observabilityScope: ObservabilityScope ) async throws -> Manifest { - try await withCheckedThrowingContinuation { continuation in - self.loadRootManifest(at: path, observabilityScope: observabilityScope) { result in - continuation.resume(with: result) - } + let manifests = try await self.loadRootManifests(packages: [path], observabilityScope: observabilityScope) + + // normally, we call loadRootManifests which attempts to load any manifest it can and report errors via + // diagnostics + // in this case, we want to load a specific manifest, so if the diagnostics contains an error we want to + // throw + guard !observabilityScope.errorsReported else { + throw Diagnostics.fatalError } - } - - /// Loads and returns manifest at the given path. - public func loadRootManifest( - at path: AbsolutePath, - observabilityScope: ObservabilityScope, - completion: @escaping (Result) -> Void - ) { - self.loadRootManifests(packages: [path], observabilityScope: observabilityScope) { result in - completion(result.tryMap { - // normally, we call loadRootManifests which attempts to load any manifest it can and report errors via - // diagnostics - // in this case, we want to load a specific manifest, so if the diagnostics contains an error we want to - // throw - guard !observabilityScope.errorsReported else { - throw Diagnostics.fatalError - } - guard let manifest = $0[path] else { - throw InternalError("Unknown manifest for '\(path)'") - } - return manifest - }) + guard let manifest = manifests[path] else { + throw InternalError("Unknown manifest for '\(path)'") } - } - /// Loads root package - public func loadRootPackage(at path: AbsolutePath, observabilityScope: ObservabilityScope) async throws -> Package { - try await withCheckedThrowingContinuation { continuation in - self.loadRootPackage(at: path, observabilityScope: observabilityScope) { result in - continuation.resume(with: result) - } - } + return manifest } /// Loads root package public func loadRootPackage( at path: AbsolutePath, - observabilityScope: ObservabilityScope, - completion: @escaping (Result) -> Void - ) { - self.loadRootManifest(at: path, observabilityScope: observabilityScope) { result in - let result = result.tryMap { manifest -> Package in - let identity = try self.identityResolver.resolveIdentity(for: manifest.packageKind) - - // radar/82263304 - // compute binary artifacts for the sake of constructing a project model - // note this does not actually download remote artifacts and as such does not have the artifact's type - // or path - let binaryArtifacts = try manifest.targets.filter { $0.type == .binary } - .reduce(into: [String: BinaryArtifact]()) { partial, target in - if let path = target.path { - let artifactPath = try manifest.path.parentDirectory - .appending(RelativePath(validating: path)) - guard let (_, artifactKind) = try BinaryArtifactsManager.deriveBinaryArtifact( - fileSystem: self.fileSystem, - path: artifactPath, - observabilityScope: observabilityScope - ) else { - throw StringError("\(artifactPath) does not contain binary artifact") - } - partial[target.name] = BinaryArtifact( - kind: artifactKind, - originURL: .none, - path: artifactPath - ) - } else if let url = target.url.flatMap(URL.init(string:)) { - let fakePath = try manifest.path.parentDirectory.appending(components: "remote", "archive") - .appending(RelativePath(validating: url.lastPathComponent)) - partial[target.name] = BinaryArtifact( - kind: .unknown, - originURL: url.absoluteString, - path: fakePath - ) - } else { - throw InternalError("a binary target should have either a path or a URL and a checksum") - } + observabilityScope: ObservabilityScope + ) async throws -> Package { + let manifest = try await self.loadRootManifest(at: path, observabilityScope: observabilityScope) + let identity = try self.identityResolver.resolveIdentity(for: manifest.packageKind) + + // radar/82263304 + // compute binary artifacts for the sake of constructing a project model + // note this does not actually download remote artifacts and as such does not have the artifact's type + // or path + let binaryArtifacts = try manifest.targets.filter { $0.type == .binary } + .reduce(into: [String: BinaryArtifact]()) { partial, target in + if let path = target.path { + let artifactPath = try manifest.path.parentDirectory + .appending(RelativePath(validating: path)) + guard let (_, artifactKind) = try BinaryArtifactsManager.deriveBinaryArtifact( + fileSystem: self.fileSystem, + path: artifactPath, + observabilityScope: observabilityScope + ) else { + throw StringError("\(artifactPath) does not contain binary artifact") } - - let builder = PackageBuilder( - identity: identity, - manifest: manifest, - productFilter: .everything, - path: path, - additionalFileRules: [], - binaryArtifacts: binaryArtifacts, - fileSystem: self.fileSystem, - observabilityScope: observabilityScope, - // For now we enable all traits - enabledTraits: Set(manifest.traits.map { $0.name }) - ) - return try builder.construct() + partial[target.name] = BinaryArtifact( + kind: artifactKind, + originURL: .none, + path: artifactPath + ) + } else if let url = target.url.flatMap(URL.init(string:)) { + let fakePath = try manifest.path.parentDirectory.appending(components: "remote", "archive") + .appending(RelativePath(validating: url.lastPathComponent)) + partial[target.name] = BinaryArtifact( + kind: .unknown, + originURL: url.absoluteString, + path: fakePath + ) + } else { + throw InternalError("a binary target should have either a path or a URL and a checksum") + } } - completion(result) - } + + let builder = PackageBuilder( + identity: identity, + manifest: manifest, + productFilter: .everything, + path: path, + additionalFileRules: [], + binaryArtifacts: binaryArtifacts, + fileSystem: self.fileSystem, + observabilityScope: observabilityScope, + // For now we enable all traits + enabledTraits: Set(manifest.traits.map { $0.name }) + ) + return try builder.construct() } public func loadPluginImports( @@ -1184,58 +1133,41 @@ extension Workspace { return importList } - public func loadPackage( - with identity: PackageIdentity, - packageGraph: ModulesGraph, - observabilityScope: ObservabilityScope - ) async throws -> Package { - try await withCheckedThrowingContinuation { continuation in - self.loadPackage(with: identity, packageGraph: packageGraph, observabilityScope: observabilityScope, completion: { - continuation.resume(with: $0) - }) - } - } - /// Loads a single package in the context of a previously loaded graph. This can be useful for incremental loading /// in a longer-lived program, like an IDE. - @available(*, noasync, message: "Use the async alternative") public func loadPackage( with identity: PackageIdentity, packageGraph: ModulesGraph, - observabilityScope: ObservabilityScope, - completion: @escaping (Result) -> Void - ) { + observabilityScope: ObservabilityScope + ) async throws -> Package { guard let previousPackage = packageGraph.package(for: identity) else { - return completion(.failure(StringError("could not find package with identity \(identity)"))) + throw StringError("could not find package with identity \(identity)") } - self.loadManifest( + let manifest = try await self.loadManifest( packageIdentity: identity, packageKind: previousPackage.underlying.manifest.packageKind, packagePath: previousPackage.path, packageLocation: previousPackage.underlying.manifest.packageLocation, observabilityScope: observabilityScope - ) { result in - let result = result.tryMap { manifest -> Package in - let builder = PackageBuilder( - identity: identity, - manifest: manifest, - productFilter: .everything, - // TODO: this will not be correct when reloading a transitive dependencies if `ENABLE_TARGET_BASED_DEPENDENCY_RESOLUTION` is enabled - path: previousPackage.path, - additionalFileRules: self.configuration.additionalFileRules, - binaryArtifacts: packageGraph.binaryArtifacts[identity] ?? [:], - shouldCreateMultipleTestProducts: self.configuration.shouldCreateMultipleTestProducts, - createREPLProduct: self.configuration.createREPLProduct, - fileSystem: self.fileSystem, - observabilityScope: observabilityScope, - // For now we enable all traits - enabledTraits: Set(manifest.traits.map { $0.name }) - ) - return try builder.construct() - } - completion(result) - } + ) + + let builder = PackageBuilder( + identity: identity, + manifest: manifest, + productFilter: .everything, + // TODO: this will not be correct when reloading a transitive dependencies if `ENABLE_TARGET_BASED_DEPENDENCY_RESOLUTION` is enabled + path: previousPackage.path, + additionalFileRules: self.configuration.additionalFileRules, + binaryArtifacts: packageGraph.binaryArtifacts[identity] ?? [:], + shouldCreateMultipleTestProducts: self.configuration.shouldCreateMultipleTestProducts, + createREPLProduct: self.configuration.createREPLProduct, + fileSystem: self.fileSystem, + observabilityScope: observabilityScope, + // For now we enable all traits + enabledTraits: Set(manifest.traits.map { $0.name }) + ) + return try builder.construct() } } @@ -1491,11 +1423,7 @@ private func warnToStderr(_ message: String) { } // used for manifest validation -#if compiler(<6.0) extension RepositoryManager: ManifestSourceControlValidator {} -#else -extension RepositoryManager: @retroactive ManifestSourceControlValidator {} -#endif extension ContainerUpdateStrategy { var repositoryUpdateStrategy: RepositoryUpdateStrategy { diff --git a/Sources/_InternalTestSupport/MockManifestLoader.swift b/Sources/_InternalTestSupport/MockManifestLoader.swift index 3245d7d8737..43fa9fd82c4 100644 --- a/Sources/_InternalTestSupport/MockManifestLoader.swift +++ b/Sources/_InternalTestSupport/MockManifestLoader.swift @@ -62,16 +62,13 @@ public final class MockManifestLoader: ManifestLoaderProtocol { fileSystem: FileSystem, observabilityScope: ObservabilityScope, delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { - callbackQueue.async { - let key = Key(url: packageLocation, version: packageVersion?.version) - if let result = self.manifests[key] { - return completion(.success(result)) - } else { - return completion(.failure(MockManifestLoaderError.unknownRequest("\(key)"))) - } + callbackQueue: DispatchQueue + ) throws -> Manifest { + let key = Key(url: packageLocation, version: packageVersion?.version) + if let result = self.manifests[key] { + return result + } else { + throw MockManifestLoaderError.unknownRequest("\(key)") } } diff --git a/Sources/_InternalTestSupport/MockRegistry.swift b/Sources/_InternalTestSupport/MockRegistry.swift index 5bb1a553184..5c03d1dd00e 100644 --- a/Sources/_InternalTestSupport/MockRegistry.swift +++ b/Sources/_InternalTestSupport/MockRegistry.swift @@ -67,7 +67,7 @@ public class MockRegistry { signingEntityStorage: signingEntityStorage, signingEntityCheckingMode: .strict, authorizationProvider: .none, - customHTTPClient: LegacyHTTPClient(handler: self.httpHandler), + customHTTPClient: HTTPClient(implementation: self.httpHandler), customArchiverProvider: { fileSystem in MockRegistryArchiver(fileSystem: fileSystem) }, delegate: .none, checksumAlgorithm: checksumAlgorithm @@ -114,35 +114,29 @@ public class MockRegistry { } } + @Sendable func httpHandler( - request: LegacyHTTPClient.Request, - progress: LegacyHTTPClient.ProgressHandler?, - completion: @escaping (Result) -> Void - ) { - do { - guard request.url.absoluteString.hasPrefix(self.baseURL.absoluteString) else { - throw StringError("url outside mock registry \(self.baseURL)") - } + request: HTTPClient.Request, + progress: HTTPClient.ProgressHandler? + ) async throws -> HTTPClient.Response { + guard request.url.absoluteString.hasPrefix(self.baseURL.absoluteString) else { + throw StringError("url outside mock registry \(self.baseURL)") + } - switch request.kind { - case .generic: - let response = try self.handleRequest(request: request) - completion(.success(response)) - case .download(let fileSystem, let destination): - let response = try self.handleDownloadRequest( - request: request, - progress: progress, - fileSystem: fileSystem, - destination: destination - ) - completion(.success(response)) - } - } catch { - completion(.failure(error)) + switch request.kind { + case .generic: + return try self.handleRequest(request: request) + case .download(let fileSystem, let destination): + return try self.handleDownloadRequest( + request: request, + progress: progress, + fileSystem: fileSystem, + destination: destination + ) } } - private func handleRequest(request: LegacyHTTPClient.Request) throws -> LegacyHTTPClient.Response { + private func handleRequest(request: HTTPClient.Request) throws -> LegacyHTTPClient.Response { let routeComponents = request.url.absoluteString.dropFirst(self.baseURL.absoluteString.count + 1) .split(separator: "/") switch routeComponents.count { @@ -303,8 +297,8 @@ public class MockRegistry { } private func handleDownloadRequest( - request: LegacyHTTPClient.Request, - progress: LegacyHTTPClient.ProgressHandler?, + request: HTTPClient.Request, + progress: HTTPClient.ProgressHandler?, fileSystem: FileSystem, destination: AbsolutePath ) throws -> HTTPClientResponse { diff --git a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift index c5a8490b35e..dc580359669 100644 --- a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift @@ -126,7 +126,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { // remove the package do { - try manager.remove(package: package) + try await manager.remove(package: package) delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) @@ -210,7 +210,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { // remove the "local" package, should come from cache do { - try manager.remove(package: package) + try await manager.remove(package: package) delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) @@ -232,8 +232,8 @@ final class RegistryDownloadsManagerTests: XCTestCase { // remove the "local" package, and purge cache do { - try manager.remove(package: package) - manager.purgeCache(observabilityScope: observability.topScope) + try await manager.remove(package: package) + await manager.purgeCache(observabilityScope: observability.topScope) delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) @@ -295,7 +295,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { for packageVersion in packageVersions { group.addTask { delegate.prepare(fetchExpected: true) - results[packageVersion] = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope, delegateQueue: .sharedConcurrent, callbackQueue: .sharedConcurrent) + results[packageVersion] = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope, delegateQueue: .sharedConcurrent) } } try await group.waitForAll() @@ -335,7 +335,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { group.addTask { delegate.prepare(fetchExpected: index < concurrency / repeatRatio) let packageVersion = Version(index % (concurrency / repeatRatio), 0 , 0) - results[packageVersion] = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope, delegateQueue: .sharedConcurrent, callbackQueue: .sharedConcurrent) + results[packageVersion] = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope, delegateQueue: .sharedConcurrent) } } try await group.waitForAll() @@ -415,8 +415,7 @@ extension RegistryDownloadsManager { package: package, version: version, observabilityScope: observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) } } diff --git a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift index 94cb2616904..91178942302 100644 --- a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift +++ b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift @@ -38,7 +38,7 @@ final class RegistryPackageContainerTests: XCTestCase { packageVersion: packageVersion, packagePath: packagePath, fileSystem: fs, - releasesRequestHandler: { request, _ , completion in + releasesRequestHandler: { request, _ in let metadata = RegistryClient.Serialization.PackageMetadata( releases: [ "1.0.0": .init(url: .none, problem: .none), @@ -47,18 +47,17 @@ final class RegistryPackageContainerTests: XCTestCase { "1.0.3": .init(url: .none, problem: .none) ] ) - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/json" - ], - body: try! JSONEncoder.makeWithDefaults().encode(metadata) - ) - )) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/json" + ], + body: try! JSONEncoder.makeWithDefaults().encode(metadata) + ) }, - manifestRequestHandler: { request, _ , completion in + manifestRequestHandler: { request, _ in let toolsVersion: ToolsVersion switch request.url.deletingLastPathComponent().lastPathComponent { case "1.0.0": @@ -72,16 +71,15 @@ final class RegistryPackageContainerTests: XCTestCase { default: toolsVersion = .current } - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "text/x-swift" - ], - body: Data("// swift-tools-version:\(toolsVersion)".utf8) - ) - )) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "text/x-swift" + ], + body: Data("// swift-tools-version:\(toolsVersion)".utf8) + ) } ) @@ -135,21 +133,19 @@ final class RegistryPackageContainerTests: XCTestCase { packageVersion: packageVersion, packagePath: packagePath, fileSystem: fs, - manifestRequestHandler: { request, _ , completion in - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "text/x-swift", - "Link": """ - \(self.manifestLink(packageIdentity, .v5_4)), - \(self.manifestLink(packageIdentity, .v5_5)), - """ - ], - body: Data("// swift-tools-version:\(ToolsVersion.v5_3)".utf8) - ) - )) + manifestRequestHandler: { request, _ in + HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "text/x-swift", + "Link": """ + \(self.manifestLink(packageIdentity, .v5_4)), + \(self.manifestLink(packageIdentity, .v5_5)), + """ + ], + body: Data("// swift-tools-version:\(ToolsVersion.v5_3)".utf8) + ) } ) @@ -232,25 +228,24 @@ final class RegistryPackageContainerTests: XCTestCase { packageVersion: packageVersion, packagePath: packagePath, fileSystem: fs, - manifestRequestHandler: { request, _ , completion in + manifestRequestHandler: { request, _ in let requestedVersionString = request.url.query?.spm_dropPrefix("swift-version=") let requestedVersion = (requestedVersionString.flatMap{ ToolsVersion(string: $0) }) ?? .v5_3 guard supportedVersions.contains(requestedVersion) else { - return completion(.failure(StringError("invalid version \(requestedVersion)"))) + throw StringError("invalid version \(requestedVersion)") } - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "text/x-swift", - "Link": (supportedVersions.subtracting([requestedVersion])).map { - self.manifestLink(packageIdentity, $0) - }.joined(separator: ",\n") - ], - body: Data("// swift-tools-version:\(requestedVersion)".utf8) - ) - )) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "text/x-swift", + "Link": (supportedVersions.subtracting([requestedVersion])).map { + self.manifestLink(packageIdentity, $0) + }.joined(separator: ",\n") + ], + body: Data("// swift-tools-version:\(requestedVersion)".utf8) + ) } ) @@ -265,29 +260,28 @@ final class RegistryPackageContainerTests: XCTestCase { ) struct MockManifestLoader: ManifestLoaderProtocol { - func load(manifestPath: AbsolutePath, - manifestToolsVersion: ToolsVersion, - packageIdentity: PackageIdentity, - packageKind: PackageReference.Kind, - packageLocation: String, - packageVersion: (version: Version?, revision: String?)?, - identityResolver: IdentityResolver, - dependencyMapper: DependencyMapper, - fileSystem: FileSystem, - observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void) { - completion(.success( - Manifest.createManifest( - displayName: packageIdentity.description, - path: manifestPath, - packageKind: packageKind, - packageLocation: packageLocation, - platforms: [], - toolsVersion: manifestToolsVersion - ) - )) + func load( + manifestPath: AbsolutePath, + manifestToolsVersion: ToolsVersion, + packageIdentity: PackageIdentity, + packageKind: PackageReference.Kind, + packageLocation: String, + packageVersion: (version: Version?, revision: String?)?, + identityResolver: IdentityResolver, + dependencyMapper: DependencyMapper, + fileSystem: FileSystem, + observabilityScope: ObservabilityScope, + delegateQueue: DispatchQueue, + callbackQueue: DispatchQueue + ) async throws -> Manifest { + Manifest.createManifest( + displayName: packageIdentity.description, + path: manifestPath, + packageKind: packageKind, + packageLocation: packageLocation, + platforms: [], + toolsVersion: manifestToolsVersion + ) } func resetCache(observabilityScope: ObservabilityScope) {} @@ -342,10 +336,10 @@ final class RegistryPackageContainerTests: XCTestCase { packagePath: AbsolutePath, fileSystem: FileSystem, configuration: PackageRegistry.RegistryConfiguration? = .none, - releasesRequestHandler: LegacyHTTPClient.Handler? = .none, - versionMetadataRequestHandler: LegacyHTTPClient.Handler? = .none, - manifestRequestHandler: LegacyHTTPClient.Handler? = .none, - downloadArchiveRequestHandler: LegacyHTTPClient.Handler? = .none, + releasesRequestHandler: HTTPClient.Implementation? = .none, + versionMetadataRequestHandler: HTTPClient.Implementation? = .none, + manifestRequestHandler: HTTPClient.Implementation? = .none, + downloadArchiveRequestHandler: HTTPClient.Implementation? = .none, archiver: Archiver? = .none ) throws -> RegistryClient { let jsonEncoder = JSONEncoder.makeWithDefaults() @@ -361,23 +355,22 @@ final class RegistryPackageContainerTests: XCTestCase { configuration!.defaultRegistry = .init(url: "http://localhost", supportsAvailability: false) } - let releasesRequestHandler = releasesRequestHandler ?? { request, _ , completion in + let releasesRequestHandler = releasesRequestHandler ?? { request, _ in let metadata = RegistryClient.Serialization.PackageMetadata( releases: [packageVersion.description: .init(url: .none, problem: .none)] ) - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/json" - ], - body: try! jsonEncoder.encode(metadata) - ) - )) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/json" + ], + body: try! jsonEncoder.encode(metadata) + ) } - let versionMetadataRequestHandler = versionMetadataRequestHandler ?? { request, _ , completion in + let versionMetadataRequestHandler = versionMetadataRequestHandler ?? { request, _ in let metadata = RegistryClient.Serialization.VersionMetadata( id: packageIdentity.description, version: packageVersion.description, @@ -392,32 +385,29 @@ final class RegistryPackageContainerTests: XCTestCase { metadata: .init(description: ""), publishedAt: nil ) - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/json" - ], - body: try! jsonEncoder.encode(metadata) - ) - )) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/json" + ], + body: try! jsonEncoder.encode(metadata) + ) } - let manifestRequestHandler = manifestRequestHandler ?? { request, _ , completion in - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "text/x-swift" - ], - body: Data("// swift-tools-version:\(ToolsVersion.current)".utf8) - ) - )) + let manifestRequestHandler = manifestRequestHandler ?? { request, _ in + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "text/x-swift" + ], + body: Data("// swift-tools-version:\(ToolsVersion.current)".utf8) + ) } - let downloadArchiveRequestHandler = downloadArchiveRequestHandler ?? { request, _ , completion in + let downloadArchiveRequestHandler = downloadArchiveRequestHandler ?? { request, _ in // meh let path = packagePath .appending(components: ".build", "registry", "downloads", registryIdentity.scope.description, registryIdentity.name.description) @@ -425,16 +415,14 @@ final class RegistryPackageContainerTests: XCTestCase { try! fileSystem.createDirectory(path.parentDirectory, recursive: true) try! fileSystem.writeFileContents(path, string: "") - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/zip" - ], - body: Data("".utf8) - ) - )) + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/zip" + ], + body: Data("".utf8) + ) } let archiver = archiver ?? MockArchiver(handler: { archiver, from, to, completion in @@ -454,32 +442,32 @@ final class RegistryPackageContainerTests: XCTestCase { signingEntityStorage: .none, signingEntityCheckingMode: .strict, authorizationProvider: .none, - customHTTPClient: LegacyHTTPClient(configuration: .init(), handler: { request, progress , completion in + customHTTPClient: HTTPClient(implementation: { request, progress in var pathComponents = request.url.pathComponents if pathComponents.first == "/" { pathComponents = Array(pathComponents.dropFirst()) } guard pathComponents.count >= 2 else { - return completion(.failure(StringError("invalid url \(request.url)"))) + throw StringError("invalid url \(request.url)") } guard pathComponents[0] == registryIdentity.scope.description else { - return completion(.failure(StringError("invalid url \(request.url)"))) + throw StringError("invalid url \(request.url)") } guard pathComponents[1] == registryIdentity.name.description else { - return completion(.failure(StringError("invalid url \(request.url)"))) + throw StringError("invalid url \(request.url)") } switch pathComponents.count { case 2: - releasesRequestHandler(request, progress, completion) + return try await releasesRequestHandler(request, progress) case 3 where pathComponents[2].hasSuffix(".zip"): - downloadArchiveRequestHandler(request, progress, completion) + return try await downloadArchiveRequestHandler(request, progress) case 3: - versionMetadataRequestHandler(request, progress, completion) + return try await versionMetadataRequestHandler(request, progress) case 4 where pathComponents[3].hasSuffix(".swift"): - manifestRequestHandler(request, progress, completion) + return try await manifestRequestHandler(request, progress) default: - completion(.failure(StringError("unexpected url \(request.url)"))) + throw StringError("unexpected url \(request.url)") } }), customArchiverProvider: { _ in archiver }, diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 342a7fe3134..b5114ec995b 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -12169,26 +12169,19 @@ final class WorkspaceTests: XCTestCase { fileSystem: FileSystem, observabilityScope: ObservabilityScope, delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) { + callbackQueue: DispatchQueue + ) throws -> Manifest { if let error { - callbackQueue.async { - completion(.failure(error)) - } + throw error } else { - callbackQueue.async { - completion(.success( - Manifest.createManifest( - displayName: packageIdentity.description, - path: manifestPath, - packageKind: packageKind, - packageLocation: packageLocation, - platforms: [], - toolsVersion: manifestToolsVersion - ) - )) - } + return Manifest.createManifest( + displayName: packageIdentity.description, + path: manifestPath, + packageKind: packageKind, + packageLocation: packageLocation, + platforms: [], + toolsVersion: manifestToolsVersion + ) } } @@ -14319,8 +14312,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - releasesRequestHandler: { _, _, completion in - completion(.failure(StringError("boom"))) + releasesRequestHandler: { _, _ in + throw StringError("boom") }, fileSystem: fs ) @@ -14341,8 +14334,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - releasesRequestHandler: { _, _, completion in - completion(.success(.serverError())) + releasesRequestHandler: { _, _ in + .serverError() }, fileSystem: fs ) @@ -14404,8 +14397,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - versionMetadataRequestHandler: { _, _, completion in - completion(.failure(StringError("boom"))) + versionMetadataRequestHandler: { _, _ in + throw StringError("boom") }, fileSystem: fs ) @@ -14600,8 +14593,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - downloadArchiveRequestHandler: { _, _, completion in - completion(.success(.serverError())) + downloadArchiveRequestHandler: { _, _ in + .serverError() }, fileSystem: fs ) @@ -15121,10 +15114,10 @@ final class WorkspaceTests: XCTestCase { signingEntityStorage: PackageSigningEntityStorage? = .none, signingEntityCheckingMode: SigningEntityCheckingMode = .strict, authorizationProvider: AuthorizationProvider? = .none, - releasesRequestHandler: LegacyHTTPClient.Handler? = .none, - versionMetadataRequestHandler: LegacyHTTPClient.Handler? = .none, - manifestRequestHandler: LegacyHTTPClient.Handler? = .none, - downloadArchiveRequestHandler: LegacyHTTPClient.Handler? = .none, + releasesRequestHandler: HTTPClient.Implementation? = .none, + versionMetadataRequestHandler: HTTPClient.Implementation? = .none, + manifestRequestHandler: HTTPClient.Implementation? = .none, + downloadArchiveRequestHandler: HTTPClient.Implementation? = .none, archiver: Archiver? = .none, fileSystem: FileSystem ) throws -> RegistryClient { @@ -15141,23 +15134,22 @@ final class WorkspaceTests: XCTestCase { return configuration }() - let releasesRequestHandler = releasesRequestHandler ?? { _, _, completion in + let releasesRequestHandler = releasesRequestHandler ?? { _, _ in let metadata = RegistryClient.Serialization.PackageMetadata( releases: [packageVersion.description: .init(url: .none, problem: .none)] ) - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/json", - ], - body: try! jsonEncoder.encode(metadata) - ) - )) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/json", + ], + body: try! jsonEncoder.encode(metadata) + ) } - let versionMetadataRequestHandler = versionMetadataRequestHandler ?? { _, _, completion in + let versionMetadataRequestHandler = versionMetadataRequestHandler ?? { _, _ in let metadata = RegistryClient.Serialization.VersionMetadata( id: packageIdentity.description, version: packageVersion.description, @@ -15176,32 +15168,29 @@ final class WorkspaceTests: XCTestCase { ), publishedAt: nil ) - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/json", - ], - body: try! jsonEncoder.encode(metadata) - ) - )) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/json", + ], + body: try! jsonEncoder.encode(metadata) + ) } - let manifestRequestHandler = manifestRequestHandler ?? { _, _, completion in - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "text/x-swift", - ], - body: Data("// swift-tools-version:\(ToolsVersion.current)".utf8) - ) - )) + let manifestRequestHandler = manifestRequestHandler ?? { _, _ in + HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "text/x-swift", + ], + body: Data("// swift-tools-version:\(ToolsVersion.current)".utf8) + ) } - let downloadArchiveRequestHandler = downloadArchiveRequestHandler ?? { request, _, completion in + let downloadArchiveRequestHandler = downloadArchiveRequestHandler ?? { request, _ in switch request.kind { case .download(let fileSystem, let destination): // creates a dummy zipfile which is required by the archiver step @@ -15211,16 +15200,14 @@ final class WorkspaceTests: XCTestCase { preconditionFailure("invalid request") } - completion(.success( - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/zip", - ], - body: Data("".utf8) - ) - )) + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/zip", + ], + body: Data("".utf8) + ) } let archiver = archiver ?? MockArchiver(handler: { _, _, to, completion in @@ -15259,22 +15246,22 @@ final class WorkspaceTests: XCTestCase { signingEntityStorage: signingEntityStorage, signingEntityCheckingMode: signingEntityCheckingMode, authorizationProvider: authorizationProvider, - customHTTPClient: LegacyHTTPClient(configuration: .init(), handler: { request, progress, completion in + customHTTPClient: HTTPClient(implementation: { request, progress in switch request.url.path { // request to get package releases case "/\(identity.scope)/\(identity.name)": - releasesRequestHandler(request, progress, completion) + try await releasesRequestHandler(request, progress) // request to get package version metadata case "/\(identity.scope)/\(identity.name)/\(packageVersion)": - versionMetadataRequestHandler(request, progress, completion) + try await versionMetadataRequestHandler(request, progress) // request to get package manifest case "/\(identity.scope)/\(identity.name)/\(packageVersion)/Package.swift": - manifestRequestHandler(request, progress, completion) + try await manifestRequestHandler(request, progress) // request to get download the version source archive case "/\(identity.scope)/\(identity.name)/\(packageVersion).zip": - downloadArchiveRequestHandler(request, progress, completion) + try await downloadArchiveRequestHandler(request, progress) default: - completion(.failure(StringError("unexpected url \(request.url)"))) + throw StringError("unexpected url \(request.url)") } }), customArchiverProvider: { _ in archiver }, From dd35171c4300a28eac130255f52a4094262bf37d Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 31 Oct 2024 18:54:08 +0000 Subject: [PATCH 09/33] Fix one use of `HTTPClient` in `RegistryClientTests` --- .../RegistryClientTests.swift | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Tests/PackageRegistryTests/RegistryClientTests.swift b/Tests/PackageRegistryTests/RegistryClientTests.swift index 8d350b89c4d..1d8e1070dd4 100644 --- a/Tests/PackageRegistryTests/RegistryClientTests.swift +++ b/Tests/PackageRegistryTests/RegistryClientTests.swift @@ -1899,7 +1899,7 @@ final class RegistryClientTests: XCTestCase { SourceControlURL("git@github.com:\(identity.scope)/\(identity.name).git"), ] - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -1924,7 +1924,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1932,14 +1932,14 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) case (.download(let fileSystem, let path), .get, downloadURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") let data = Data(emptyZipFile.contents) try! fileSystem.writeFileContents(path, data: data) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1955,15 +1955,12 @@ final class RegistryClientTests: XCTestCase { ), ]), body: nil - ))) + ) default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) From 238fd150e257d320a2837be97a5d1cd57ba287a4 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 1 Nov 2024 12:51:58 +0000 Subject: [PATCH 10/33] Fix build errors --- .../PackageRegistry/SigningEntityTOFU.swift | 14 +- Tests/CommandsTests/PackageCommandTests.swift | 10 +- .../PackageSigningEntityTOFUTests.swift | 58 +- .../PackageVersionChecksumTOFUTests.swift | 110 +-- .../RegistryClientTests.swift | 706 +++++++----------- .../SignatureValidationTests.swift | 217 ++---- Tests/WorkspaceTests/WorkspaceTests.swift | 16 +- 7 files changed, 422 insertions(+), 709 deletions(-) diff --git a/Sources/PackageRegistry/SigningEntityTOFU.swift b/Sources/PackageRegistry/SigningEntityTOFU.swift index 914eed82968..f3c57d73f60 100644 --- a/Sources/PackageRegistry/SigningEntityTOFU.swift +++ b/Sources/PackageRegistry/SigningEntityTOFU.swift @@ -277,10 +277,9 @@ struct PackageSigningEntityTOFU { previous: existing ) case .warn: - observabilityScope - .emit( - warning: "the signing entity '\(String(describing: latest))' from \(registry) for \(package) version \(version) is different from the previously recorded value '\(existing)'" - ) + observabilityScope.emit( + warning: "the signing entity '\(String(describing: latest))' from \(registry) for \(package) version \(version) is different from the previously recorded value '\(existing)'" + ) } } @@ -304,10 +303,9 @@ struct PackageSigningEntityTOFU { previousVersion: existingVersion ) case .warn: - observabilityScope - .emit( - warning: "the signing entity '\(String(describing: latest))' from \(registry) for \(package) version \(version) is different from the previously recorded value '\(existing)' for version \(existingVersion)" - ) + observabilityScope.emit( + warning: "the signing entity '\(String(describing: latest))' from \(registry) for \(package) version \(version) is different from the previously recorded value '\(existing)' for version \(existingVersion)" + ) } } } diff --git a/Tests/CommandsTests/PackageCommandTests.swift b/Tests/CommandsTests/PackageCommandTests.swift index 699ba5e8aae..69341f740f2 100644 --- a/Tests/CommandsTests/PackageCommandTests.swift +++ b/Tests/CommandsTests/PackageCommandTests.swift @@ -2506,27 +2506,27 @@ final class PackageCommandTests: CommandsTestCase { // Overall configuration: debug, plugin build request: debug -> without testability try await fixture(name: "Miscellaneous/Plugins/CommandPluginTestStub") { fixturePath in - try await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "debug", "check-testability", "InternalModule", "debug", "true"], packagePath: fixturePath)) + await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "debug", "check-testability", "InternalModule", "debug", "true"], packagePath: fixturePath)) } // Overall configuration: debug, plugin build request: release -> without testability try await fixture(name: "Miscellaneous/Plugins/CommandPluginTestStub") { fixturePath in - try await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "debug", "check-testability", "InternalModule", "release", "false"], packagePath: fixturePath)) + await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "debug", "check-testability", "InternalModule", "release", "false"], packagePath: fixturePath)) } // Overall configuration: release, plugin build request: debug -> with testability try await fixture(name: "Miscellaneous/Plugins/CommandPluginTestStub") { fixturePath in - try await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "release", "check-testability", "InternalModule", "debug", "true"], packagePath: fixturePath)) + await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "release", "check-testability", "InternalModule", "debug", "true"], packagePath: fixturePath)) } // Overall configuration: release, plugin build request: release -> with testability try await fixture(name: "Miscellaneous/Plugins/CommandPluginTestStub") { fixturePath in - try await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "release", "check-testability", "InternalModule", "release", "false"], packagePath: fixturePath)) + await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "release", "check-testability", "InternalModule", "release", "false"], packagePath: fixturePath)) } // Overall configuration: release, plugin build request: release including tests -> with testability try await fixture(name: "Miscellaneous/Plugins/CommandPluginTestStub") { fixturePath in - try await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "release", "check-testability", "all-with-tests", "release", "true"], packagePath: fixturePath)) + await XCTAssertAsyncNoThrow(try await SwiftPM.Package.execute(["-c", "release", "check-testability", "all-with-tests", "release", "true"], packagePath: fixturePath)) } } diff --git a/Tests/PackageRegistryTests/PackageSigningEntityTOFUTests.swift b/Tests/PackageRegistryTests/PackageSigningEntityTOFUTests.swift index da07b344073..5200a30dd66 100644 --- a/Tests/PackageRegistryTests/PackageSigningEntityTOFUTests.swift +++ b/Tests/PackageRegistryTests/PackageSigningEntityTOFUTests.swift @@ -43,7 +43,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Package doesn't have any recorded signer. // It should be ok to assign one. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -75,7 +75,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Package doesn't have any recorded signer. // It should be ok to continue not to have one. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -110,7 +110,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Package doesn't have any recorded signer. // It should be ok to continue not to have one. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -155,7 +155,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has "J. Appleseed" as signer for package version. // Signer remaining the same should be ok. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -200,7 +200,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has "J. Smith" as signer for package version. // The given signer "J. Appleseed" is different so it should fail. await XCTAssertAsyncThrowsError( - try await tofu.validate( + try tofu.validate( registry: registry, package: package, version: version, @@ -262,7 +262,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has "J. Smith" as signer for package version. // The given signer "J. Appleseed" is different, but because // of .warn mode, no error is thrown. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -314,8 +314,8 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has "J. Smith" as signer for package version. // The given signer is nil which is different so it should fail. - await XCTAssertAsyncThrowsError( - try await tofu.validate( + XCTAssertThrowsError( + try tofu.validate( registry: registry, package: package, version: version, @@ -369,7 +369,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has "J. Appleseed" as signer for package v2.0.0. // Signer remaining the same should be ok. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -422,8 +422,8 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has "J. Smith" as signer for package v2.0.0. // The given signer "J. Appleseed" is different so it should fail. - await XCTAssertAsyncThrowsError( - try await tofu.validate( + XCTAssertThrowsError( + try tofu.validate( registry: registry, package: package, version: version, @@ -494,7 +494,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has "J. Smith" as signer for package v2.0.0. // The given signer "J. Appleseed" is different, but because // of .warn mode, no error is thrown. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -548,7 +548,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has versions 1.5.0 and 2.0.0 signed. The given version 1.1.1 is // "older" than both, and we allow nil signer in this case, assuming // this is before package started being signed. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -596,8 +596,8 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Storage has versions 1.5.0 and 2.0.0 signed. The given version 1.6.1 is // "newer" than 1.5.0, which we don't allow, because we assume from 1.5.0 // onwards all versions are signed. - await XCTAssertAsyncThrowsError( - try await tofu.validate( + XCTAssertThrowsError( + try tofu.validate( registry: registry, package: package, version: version, @@ -663,7 +663,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // "newer" than 1.5.0, which we don't allow, because we assume from 1.5.0 // onwards all versions are signed. However, because of .warn mode, // no error is thrown. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -719,7 +719,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // We allow this with the assumption that package signing might not have // begun until a later 2.x version, so until we encounter a signed 2.x version, // we assume none of them is signed. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -767,7 +767,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // Package has expected signer starting from v1.5.0. // The given v2.0.0 is newer than v1.5.0, and signer // matches the expected signer. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -830,7 +830,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // the given signer was recorded previously for v2.2.0. // The given v2.0.0 is before v2.2.0, and we allow the same // signer for older versions. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -894,8 +894,8 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // the given signer was recorded previously for v2.2.0, but // the given v2.3.0 is after v2.2.0, which we don't allow // because we assume the signer has "stopped" signing at v2.2.0. - await XCTAssertAsyncThrowsError( - try await tofu.validate( + XCTAssertThrowsError( + try tofu.validate( registry: registry, package: package, version: version, @@ -947,8 +947,8 @@ final class PackageSigningEntityTOFUTests: XCTestCase { ) // This triggers a storage write conflict - await XCTAssertAsyncThrowsError( - try await tofu.validate( + XCTAssertThrowsError( + try tofu.validate( registry: registry, package: package, version: version, @@ -984,7 +984,7 @@ final class PackageSigningEntityTOFUTests: XCTestCase { // This triggers a storage write conflict, but // because of .warn mode, no error is thrown. - _ = try await tofu.validate( + _ = try tofu.validate( registry: registry, package: package, version: version, @@ -1004,16 +1004,14 @@ extension PackageSigningEntityTOFU { registry: Registry, package: PackageIdentity.RegistryIdentity, version: Version, - signingEntity: SigningEntity?, - observabilityScope: ObservabilityScope? = nil - ) async throws { - try await self.validate( + signingEntity: SigningEntity? + ) throws { + try self.validate( registry: registry, package: package, version: version, signingEntity: signingEntity, - observabilityScope: observabilityScope ?? ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } } diff --git a/Tests/PackageRegistryTests/PackageVersionChecksumTOFUTests.swift b/Tests/PackageRegistryTests/PackageVersionChecksumTOFUTests.swift index 7d4257d4773..9d779844dca 100644 --- a/Tests/PackageRegistryTests/PackageVersionChecksumTOFUTests.swift +++ b/Tests/PackageRegistryTests/PackageVersionChecksumTOFUTests.swift @@ -31,7 +31,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Get package version metadata endpoint will be called to fetch expected checksum - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -53,7 +53,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -61,16 +61,13 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -121,7 +118,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let metadataURL = URL("\(registryURL)/\(package.scope)/\(package.name)/\(version)") let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -143,7 +140,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -151,16 +148,13 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -206,7 +200,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let metadataURL = URL("\(registryURL)/\(package.scope)/\(package.name)/\(version)") let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -228,7 +222,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -236,16 +230,13 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -300,9 +291,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { errorDescription: "not found" ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -353,9 +342,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { errorDescription: UUID().uuidString ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient{ try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -400,9 +387,7 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let serverErrorHandler = UnavailableServerErrorHandler(registryURL: registryURL) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: true) var configuration = RegistryConfiguration() @@ -446,14 +431,10 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Checksum already exists in storage so API will not be called - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("Unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("Unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -504,14 +485,10 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Checksum already exists in storage so API will not be called - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("Unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("Unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -570,14 +547,10 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Checksum already exists in storage so API will not be called - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("Unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("Unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -637,14 +610,10 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let version = Version("1.1.1") // Registry API doesn't include manifest checksum so we don't call it - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("Unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("Unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -716,14 +685,10 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Registry API doesn't include manifest checksum so we don't call it - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("Unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("Unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -776,14 +741,10 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Registry API doesn't include manifest checksum so we don't call it - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("Unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("Unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -844,14 +805,10 @@ final class PackageVersionChecksumTOFUTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Registry API doesn't include manifest checksum so we don't call it - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("Unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("Unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -921,8 +878,7 @@ extension PackageVersionChecksumTOFU { version: version, checksum: checksum, timeout: nil, - observabilityScope: observabilityScope ?? ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: observabilityScope ?? ObservabilitySystem.NOOP ) } diff --git a/Tests/PackageRegistryTests/RegistryClientTests.swift b/Tests/PackageRegistryTests/RegistryClientTests.swift index 1d8e1070dd4..b89ba36d5cd 100644 --- a/Tests/PackageRegistryTests/RegistryClientTests.swift +++ b/Tests/PackageRegistryTests/RegistryClientTests.swift @@ -30,7 +30,7 @@ final class RegistryClientTests: XCTestCase { let identity = PackageIdentity.plain("mona.LinkedList") let releasesURL = URL("\(registryURL)/\(identity.registry!.scope)/\(identity.registry!.name)") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, releasesURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -63,7 +63,7 @@ final class RegistryClientTests: XCTestCase { ; rel="alternate" """ - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -72,16 +72,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Link", value: links), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -108,9 +105,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -139,9 +134,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -169,9 +162,7 @@ final class RegistryClientTests: XCTestCase { let serverErrorHandler = UnavailableServerErrorHandler(registryURL: registryURL) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: true) var configuration = RegistryConfiguration() @@ -192,7 +183,7 @@ final class RegistryClientTests: XCTestCase { let version = Version("1.1.1") let releaseURL = URL("\(registryURL)/\(identity.registry!.scope)/\(identity.registry!.name)/\(version)") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, releaseURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -223,7 +214,7 @@ final class RegistryClientTests: XCTestCase { } """#.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -231,16 +222,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -276,9 +264,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -314,9 +300,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -349,9 +333,7 @@ final class RegistryClientTests: XCTestCase { let serverErrorHandler = UnavailableServerErrorHandler(registryURL: registryURL) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: true) var configuration = RegistryConfiguration() @@ -397,7 +379,7 @@ final class RegistryClientTests: XCTestCase { ) """ - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, metadataURL): let data = """ @@ -426,7 +408,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -434,7 +416,8 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") @@ -446,7 +429,7 @@ final class RegistryClientTests: XCTestCase { ; rel="alternate"; filename="Package@swift-5.3.swift"; swift-tools-version="5.3" """ - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(defaultManifestData.count)"), @@ -455,16 +438,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Link", value: links), ]), body: defaultManifestData - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -517,7 +497,7 @@ final class RegistryClientTests: XCTestCase { ) """ - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, metadataURL): let data = """ @@ -546,7 +526,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -554,7 +534,8 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") @@ -566,7 +547,7 @@ final class RegistryClientTests: XCTestCase { ; rel="alternate"; filename="Package@swift-5.3.swift"; swift-tools-version="5.3" """ - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(defaultManifestData.count)"), @@ -575,16 +556,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Link", value: links), ]), body: defaultManifestData - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -656,7 +634,7 @@ final class RegistryClientTests: XCTestCase { ) """ - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, metadataURL): let data = """ @@ -685,7 +663,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -693,7 +671,8 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") @@ -705,7 +684,7 @@ final class RegistryClientTests: XCTestCase { ; rel="alternate"; filename="Package@swift-5.3.swift"; swift-tools-version="5.3" """ - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(defaultManifestData.count)"), @@ -714,16 +693,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Link", value: links), ]), body: defaultManifestData - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -791,7 +767,7 @@ final class RegistryClientTests: XCTestCase { ) """ - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, metadataURL): let data = """ @@ -820,7 +796,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -828,7 +804,8 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") @@ -840,7 +817,7 @@ final class RegistryClientTests: XCTestCase { ; rel="alternate"; filename="Package@swift-5.3.swift"; swift-tools-version="5.3" """ - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(defaultManifestData.count)"), @@ -849,16 +826,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Link", value: links), ]), body: defaultManifestData - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -925,7 +899,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: "not found" ) - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -937,7 +911,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -945,16 +919,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - serverErrorHandler.handle(request: request, progress: nil, completion: completion) + return try serverErrorHandler.handle(request: request, progress: nil) } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -988,7 +959,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -1000,7 +971,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1008,16 +979,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - serverErrorHandler.handle(request: request, progress: nil, completion: completion) + return try serverErrorHandler.handle(request: request, progress: nil) } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -1044,9 +1012,7 @@ final class RegistryClientTests: XCTestCase { let serverErrorHandler = UnavailableServerErrorHandler(registryURL: registryURL) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: true) var configuration = RegistryConfiguration() @@ -1072,7 +1038,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in var components = URLComponents(url: request.url, resolvingAgainstBaseURL: false)! let toolsVersion = components.queryItems?.first { $0.name == "swift-version" } .flatMap { ToolsVersion(string: $0.value!) } ?? ToolsVersion.current @@ -1107,7 +1073,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1115,7 +1081,8 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") @@ -1127,7 +1094,7 @@ final class RegistryClientTests: XCTestCase { let package = Package() """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1135,16 +1102,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -1197,7 +1161,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in var components = URLComponents(url: request.url, resolvingAgainstBaseURL: false)! let toolsVersion = components.queryItems?.first { $0.name == "swift-version" } .flatMap { ToolsVersion(string: $0.value!) } ?? ToolsVersion.current @@ -1232,7 +1196,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1240,7 +1204,8 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") @@ -1252,7 +1217,7 @@ final class RegistryClientTests: XCTestCase { let package = Package() """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1260,16 +1225,13 @@ final class RegistryClientTests: XCTestCase { // Omit `Content-Version` header ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -1312,7 +1274,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in var components = URLComponents(url: request.url, resolvingAgainstBaseURL: false)! let toolsVersion = components.queryItems?.first { $0.name == "swift-version" } .flatMap { ToolsVersion(string: $0.value!) } ?? ToolsVersion.current @@ -1347,7 +1309,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1355,13 +1317,14 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") let data = Data(manifestContent(toolsVersion: toolsVersion).utf8) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1369,16 +1332,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -1446,7 +1406,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in var components = URLComponents(url: request.url, resolvingAgainstBaseURL: false)! let toolsVersion = components.queryItems?.first { $0.name == "swift-version" } .flatMap { ToolsVersion(string: $0.value!) } ?? ToolsVersion.current @@ -1481,7 +1441,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1489,13 +1449,14 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") let data = Data(manifestContent(toolsVersion: toolsVersion).utf8) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1503,16 +1464,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -1580,7 +1538,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in var components = URLComponents(url: request.url, resolvingAgainstBaseURL: false)! let toolsVersion = components.queryItems?.first { $0.name == "swift-version" } .flatMap { ToolsVersion(string: $0.value!) } ?? ToolsVersion.current @@ -1615,7 +1573,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1623,13 +1581,14 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.get, manifestURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+swift") let data = Data(manifestContent(toolsVersion: toolsVersion).utf8) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1637,16 +1596,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -1734,7 +1690,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: "not found" ) - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -1746,7 +1702,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1754,16 +1710,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - serverErrorHandler.handle(request: request, progress: nil, completion: completion) + return try serverErrorHandler.handle(request: request, progress: nil) } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -1800,7 +1753,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -1812,7 +1765,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -1820,16 +1773,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - serverErrorHandler.handle(request: request, progress: nil, completion: completion) + return try serverErrorHandler.handle(request: request, progress: nil) } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -1859,9 +1809,7 @@ final class RegistryClientTests: XCTestCase { let serverErrorHandler = UnavailableServerErrorHandler(registryURL: registryURL) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: true) var configuration = RegistryConfiguration() @@ -2023,7 +1971,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -2052,7 +2000,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2060,14 +2008,14 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) case (.download(let fileSystem, let path), .get, downloadURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") let data = Data(emptyZipFile.contents) try! fileSystem.writeFileContents(path, data: data) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2080,16 +2028,13 @@ final class RegistryClientTests: XCTestCase { ), ]), body: nil - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -2154,7 +2099,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -2183,7 +2128,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2191,14 +2136,15 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.download(let fileSystem, let path), .get, downloadURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") let data = Data(emptyZipFile.contents) try! fileSystem.writeFileContents(path, data: data) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2211,16 +2157,13 @@ final class RegistryClientTests: XCTestCase { ), ]), body: nil - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -2291,7 +2234,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -2320,7 +2263,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2328,14 +2271,15 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + case (.download(let fileSystem, let path), .get, downloadURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") let data = Data(emptyZipFile.contents) try! fileSystem.writeFileContents(path, data: data) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2348,16 +2292,13 @@ final class RegistryClientTests: XCTestCase { ), ]), body: nil - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -2431,7 +2372,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.download(let fileSystem, let path), .get, downloadURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") @@ -2439,7 +2380,7 @@ final class RegistryClientTests: XCTestCase { let data = Data(emptyZipFile.contents) try! fileSystem.writeFileContents(path, data: data) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2452,7 +2393,8 @@ final class RegistryClientTests: XCTestCase { ), ]), body: nil - ))) + ) + // `downloadSourceArchive` calls this API to fetch checksum case (.generic, .get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -2474,7 +2416,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2482,16 +2424,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -2556,7 +2495,7 @@ final class RegistryClientTests: XCTestCase { let checksumAlgorithm: HashAlgorithm = MockHashAlgorithm() let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.download(let fileSystem, let path), .get, downloadURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") @@ -2564,7 +2503,7 @@ final class RegistryClientTests: XCTestCase { let data = Data(emptyZipFile.contents) try! fileSystem.writeFileContents(path, data: data) - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2577,7 +2516,8 @@ final class RegistryClientTests: XCTestCase { ), ]), body: nil - ))) + ) + // `downloadSourceArchive` calls this API to fetch checksum case (.generic, .get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -2599,7 +2539,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2607,16 +2547,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.security = .testDefault @@ -2674,7 +2611,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: "not found" ) - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -2686,7 +2623,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2694,16 +2631,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - serverErrorHandler.handle(request: request, progress: nil, completion: completion) + return try serverErrorHandler.handle(request: request, progress: nil) } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -2755,7 +2689,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.kind, request.method, request.url) { case (.generic, .get, metadataURL): let data = """ @@ -2767,7 +2701,7 @@ final class RegistryClientTests: XCTestCase { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2775,16 +2709,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - serverErrorHandler.handle(request: request, progress: nil, completion: completion) + return try serverErrorHandler.handle(request: request, progress: nil) } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -2830,9 +2761,7 @@ final class RegistryClientTests: XCTestCase { let serverErrorHandler = UnavailableServerErrorHandler(registryURL: registryURL) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: true) var configuration = RegistryConfiguration() @@ -2872,7 +2801,7 @@ final class RegistryClientTests: XCTestCase { let packageURL = SourceControlURL("https://example.com/mona/LinkedList") let identifiersURL = URL("\(registryURL)/identifiers?url=\(packageURL.absoluteString)") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, identifiersURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -2885,7 +2814,7 @@ final class RegistryClientTests: XCTestCase { } """#.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2893,16 +2822,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -2916,20 +2842,17 @@ final class RegistryClientTests: XCTestCase { let packageURL = SourceControlURL("https://example.com/mona/LinkedList") let identifiersURL = URL("\(registryURL)/identifiers?url=\(packageURL.absoluteString)") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, identifiersURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") - completion(.success(.notFound())) + return .notFound() + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -2950,9 +2873,7 @@ final class RegistryClientTests: XCTestCase { errorDescription: UUID().uuidString ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -2979,7 +2900,7 @@ final class RegistryClientTests: XCTestCase { let token = "top-sekret" - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, identifiersURL): XCTAssertEqual(request.headers.get("Authorization").first, "Bearer \(token)") @@ -2993,7 +2914,7 @@ final class RegistryClientTests: XCTestCase { } """#.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -3001,16 +2922,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.registryAuthentication[registryURL.host!] = .init(type: .token) @@ -3034,7 +2952,7 @@ final class RegistryClientTests: XCTestCase { let user = "jappleseed" let password = "top-sekret" - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, identifiersURL): XCTAssertEqual( @@ -3051,7 +2969,7 @@ final class RegistryClientTests: XCTestCase { } """#.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -3059,16 +2977,13 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.registryAuthentication[registryURL.host!] = .init(type: .basic) @@ -3090,26 +3005,23 @@ final class RegistryClientTests: XCTestCase { let token = "top-sekret" - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.post, loginURL): XCTAssertEqual(request.headers.get("Authorization").first, "Bearer \(token)") - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Version", value: "1"), ]) - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.registryAuthentication[registryURL.host!] = .init(type: .token) @@ -3128,26 +3040,23 @@ final class RegistryClientTests: XCTestCase { let registryURL = URL("https://packages.example.com") let loginURL = URL("\(registryURL)/login") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.post, loginURL): XCTAssertNil(request.headers.get("Authorization").first) - completion(.success(.init( + return .init( statusCode: 401, headers: .init([ .init(name: "Content-Version", value: "1"), ]) - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3169,26 +3078,23 @@ final class RegistryClientTests: XCTestCase { let token = "top-sekret" - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.post, loginURL): XCTAssertNotNil(request.headers.get("Authorization").first) - completion(.success(.init( + return .init( statusCode: 501, headers: .init([ .init(name: "Content-Version", value: "1"), ]) - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) configuration.registryAuthentication[registryURL.host!] = .init(type: .token) @@ -3219,7 +3125,7 @@ final class RegistryClientTests: XCTestCase { let archiveContent = UUID().uuidString let metadataContent = UUID().uuidString - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.put, publishURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -3230,16 +3136,17 @@ final class RegistryClientTests: XCTestCase { XCTAssertMatch(body, .contains(archiveContent)) XCTAssertMatch(body, .contains(metadataContent)) - completion(.success(.init( + return .init( statusCode: 201, headers: .init([ .init(name: "Location", value: expectedLocation.absoluteString), .init(name: "Content-Version", value: "1"), ]), body: .none - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } @@ -3250,10 +3157,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending("\(identity)-\(version)-metadata.json") try localFileSystem.writeFileContents(metadataPath, string: metadataContent) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3286,7 +3189,7 @@ final class RegistryClientTests: XCTestCase { let archiveContent = UUID().uuidString let metadataContent = UUID().uuidString - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.put, publishURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -3297,7 +3200,7 @@ final class RegistryClientTests: XCTestCase { XCTAssertMatch(body, .contains(archiveContent)) XCTAssertMatch(body, .contains(metadataContent)) - completion(.success(.init( + return .init( statusCode: 202, headers: .init([ .init(name: "Location", value: expectedLocation.absoluteString), @@ -3305,9 +3208,10 @@ final class RegistryClientTests: XCTestCase { .init(name: "Content-Version", value: "1"), ]), body: .none - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } @@ -3318,10 +3222,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending("\(identity)-\(version)-metadata.json") try localFileSystem.writeFileContents(metadataPath, string: metadataContent) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3356,7 +3256,7 @@ final class RegistryClientTests: XCTestCase { let metadataSignature = UUID().uuidString let signatureFormat = SignatureFormat.cms_1_0_0 - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.put, publishURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -3369,16 +3269,17 @@ final class RegistryClientTests: XCTestCase { XCTAssertMatch(body, .contains(signature)) XCTAssertMatch(body, .contains(metadataSignature)) - completion(.success(.init( + return .init( statusCode: 201, headers: .init([ .init(name: "Location", value: expectedLocation.absoluteString), .init(name: "Content-Version", value: "1"), ]), body: .none - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } @@ -3389,10 +3290,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending(component: "\(identity)-\(version)-metadata.json") try localFileSystem.writeFileContents(metadataPath, string: metadataContent) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3423,8 +3320,8 @@ final class RegistryClientTests: XCTestCase { let signature = UUID().uuidString let metadataSignature = UUID().uuidString - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("should not be called"))) + let httpClient = HTTPClient { _, _ in + throw StringError("should not be called") } try withTemporaryDirectory { temporaryDirectory in @@ -3434,10 +3331,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending(component: "\(identity)-\(version)-metadata.json") try localFileSystem.writeFileContents(metadataPath, string: metadataContent) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3470,8 +3363,8 @@ final class RegistryClientTests: XCTestCase { let signature = UUID().uuidString let signatureFormat = SignatureFormat.cms_1_0_0 - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("should not be called"))) + let httpClient = HTTPClient { _, _ in + throw StringError("should not be called") } try withTemporaryDirectory { temporaryDirectory in @@ -3481,10 +3374,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending(component: "\(identity)-\(version)-metadata.json") try localFileSystem.writeFileContents(metadataPath, string: metadataContent) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3517,8 +3406,8 @@ final class RegistryClientTests: XCTestCase { let metadataSignature = UUID().uuidString let signatureFormat = SignatureFormat.cms_1_0_0 - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("should not be called"))) + let httpClient = HTTPClient { _, _ in + throw StringError("should not be called") } try withTemporaryDirectory { temporaryDirectory in @@ -3528,10 +3417,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending(component: "\(identity)-\(version)-metadata.json") try localFileSystem.writeFileContents(metadataPath, string: metadataContent) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3574,9 +3459,7 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending("\(identity)-\(version)-metadata.json") try localFileSystem.writeFileContents(metadataPath, bytes: []) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3613,8 +3496,8 @@ final class RegistryClientTests: XCTestCase { let identity = PackageIdentity.plain("mona.LinkedList") let version = Version("1.1.1") - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("should not be called"))) + let httpClient = HTTPClient { _, _ in + throw StringError("should not be called") } try withTemporaryDirectory { temporaryDirectory in @@ -3623,10 +3506,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending("\(identity)-\(version)-metadata.json") - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3654,8 +3533,8 @@ final class RegistryClientTests: XCTestCase { let identity = PackageIdentity.plain("mona.LinkedList") let version = Version("1.1.1") - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("should not be called"))) + let httpClient = HTTPClient { _, _ in + throw StringError("should not be called") } try withTemporaryDirectory { temporaryDirectory in @@ -3664,10 +3543,6 @@ final class RegistryClientTests: XCTestCase { let metadataPath = temporaryDirectory.appending("\(identity)-\(version)-metadata.json") - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) @@ -3694,19 +3569,15 @@ final class RegistryClientTests: XCTestCase { let registryURL = URL("https://packages.example.com") let availabilityURL = URL("\(registryURL)/availability") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, availabilityURL): - completion(.success(.okay())) + return .okay() default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: true) let registryClient = makeRegistryClient( @@ -3723,19 +3594,15 @@ final class RegistryClientTests: XCTestCase { let availabilityURL = URL("\(registryURL)/availability") for unavailableStatus in RegistryClient.AvailabilityStatus.unavailableStatusCodes { - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, availabilityURL): - completion(.success(.init(statusCode: unavailableStatus))) + return .init(statusCode: unavailableStatus) default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: true) let registryClient = makeRegistryClient( @@ -3752,19 +3619,15 @@ final class RegistryClientTests: XCTestCase { let registryURL = URL("https://packages.example.com") let availabilityURL = URL("\(registryURL)/availability") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, availabilityURL): - completion(.success(.serverError(reason: "boom"))) + return .serverError(reason: "boom") default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: true) let registryClient = makeRegistryClient( @@ -3780,19 +3643,15 @@ final class RegistryClientTests: XCTestCase { let registryURL = URL("https://packages.example.com") let availabilityURL = URL("\(registryURL)/availability") - let handler: LegacyHTTPClient.Handler = { request, _, completion in + let httpClient = HTTPClient { request, _ in switch (request.method, request.url) { case (.get, availabilityURL): - completion(.success(.serverError(reason: "boom"))) + return .serverError(reason: "boom") default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) let registryClient = makeRegistryClient( @@ -3815,8 +3674,7 @@ extension RegistryClient { fileprivate func getPackageMetadata(package: PackageIdentity) async throws -> RegistryClient.PackageMetadata { try await self.getPackageMetadata( package: package, - observabilityScope: ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } @@ -3828,8 +3686,7 @@ extension RegistryClient { package: package, version: version, fileSystem: InMemoryFileSystem(), - observabilityScope: ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } @@ -3837,45 +3694,35 @@ extension RegistryClient { package: PackageIdentity.RegistryIdentity, version: Version ) async throws -> PackageVersionMetadata { - return try await withCheckedThrowingContinuation { continuation in - self.getPackageVersionMetadata( - package: package.underlying, - version: version, - fileSystem: InMemoryFileSystem(), - observabilityScope: ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent, - completion: { - continuation.resume(with: $0) - } - ) - } + try await self.getPackageVersionMetadata( + package: package.underlying, + version: version, + fileSystem: InMemoryFileSystem(), + observabilityScope: ObservabilitySystem.NOOP + ) } fileprivate func getAvailableManifests( package: PackageIdentity, - version: Version, - observabilityScope: ObservabilityScope = ObservabilitySystem.NOOP + version: Version ) async throws -> [String: (toolsVersion: ToolsVersion, content: String?)] { try await self.getAvailableManifests( package: package, version: version, - observabilityScope: observabilityScope, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } fileprivate func getManifestContent( package: PackageIdentity, version: Version, - customToolsVersion: ToolsVersion?, - observabilityScope: ObservabilityScope = ObservabilitySystem.NOOP + customToolsVersion: ToolsVersion? ) async throws -> String { try await self.getManifestContent( package: package, version: version, customToolsVersion: customToolsVersion, - observabilityScope: observabilityScope, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } @@ -3892,24 +3739,21 @@ extension RegistryClient { destinationPath: destinationPath, progressHandler: .none, fileSystem: fileSystem, - observabilityScope: observabilityScope, - callbackQueue: .sharedConcurrent + observabilityScope: observabilityScope ) } fileprivate func lookupIdentities(scmURL: SourceControlURL) async throws -> Set { try await self.lookupIdentities( scmURL: scmURL, - observabilityScope: ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } fileprivate func login(loginURL: URL) async throws { try await self.login( loginURL: loginURL, - observabilityScope: ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } @@ -3934,23 +3778,21 @@ extension RegistryClient { metadataSignature: metadataSignature, signatureFormat: signatureFormat, fileSystem: fileSystem, - observabilityScope: ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } func checkAvailability(registry: Registry) async throws -> AvailabilityStatus { try await self.checkAvailability( registry: registry, - observabilityScope: ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: ObservabilitySystem.NOOP ) } } func makeRegistryClient( configuration: RegistryConfiguration, - httpClient: LegacyHTTPClient, + httpClient: HTTPClient, authorizationProvider: AuthorizationProvider? = .none, fingerprintStorage: PackageFingerprintStorage = MockPackageFingerprintStorage(), fingerprintCheckingMode: FingerprintCheckingMode = .strict, @@ -4001,10 +3843,9 @@ struct ServerErrorHandler { } func handle( - request: LegacyHTTPClient.Request, - progress: LegacyHTTPClient.ProgressHandler?, - completion: @escaping ((Result) -> Void) - ) { + request: HTTPClient.Request, + progress: HTTPClient.ProgressHandler? + ) throws -> HTTPClient.Response { let data = """ { "detail": "\(self.errorDescription)" @@ -4014,21 +3855,17 @@ struct ServerErrorHandler { if request.method == self.method && request.url == self.url { - completion( - .success(.init( - statusCode: self.errorCode, - headers: .init([ - .init(name: "Content-Length", value: "\(data.count)"), - .init(name: "Content-Type", value: "application/problem+json"), - .init(name: "Content-Version", value: "1"), - ]), - body: data - )) + return .init( + statusCode: self.errorCode, + headers: .init([ + .init(name: "Content-Length", value: "\(data.count)"), + .init(name: "Content-Type", value: "application/problem+json"), + .init(name: "Content-Version", value: "1"), + ]), + body: data ) } else { - completion( - .failure(StringError("unexpected request")) - ) + throw StringError("unexpected request") } } } @@ -4040,20 +3877,15 @@ struct UnavailableServerErrorHandler { } func handle( - request: LegacyHTTPClient.Request, - progress: LegacyHTTPClient.ProgressHandler?, - completion: @escaping ((Result) -> Void) - ) { + request: HTTPClient.Request, + progress: HTTPClient.ProgressHandler? + ) throws -> HTTPClient.Response { if request.method == .get && request.url == URL("\(self.registryURL)/availability") { - completion( - .success(.init( - statusCode: RegistryClient.AvailabilityStatus.unavailableStatusCodes.first! - )) + return .init( + statusCode: RegistryClient.AvailabilityStatus.unavailableStatusCodes.first! ) } else { - completion( - .failure(StringError("unexpected request")) - ) + throw StringError("unexpected request") } } } diff --git a/Tests/PackageRegistryTests/SignatureValidationTests.swift b/Tests/PackageRegistryTests/SignatureValidationTests.swift index dd1f691aee1..0b1963624e7 100644 --- a/Tests/PackageRegistryTests/SignatureValidationTests.swift +++ b/Tests/PackageRegistryTests/SignatureValidationTests.swift @@ -42,13 +42,10 @@ final class SignatureValidationTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -106,13 +103,10 @@ final class SignatureValidationTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -172,13 +166,10 @@ final class SignatureValidationTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -265,9 +256,7 @@ final class SignatureValidationTests: XCTestCase { errorDescription: "not found" ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -324,13 +313,10 @@ final class SignatureValidationTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -413,13 +399,10 @@ final class SignatureValidationTests: XCTestCase { let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -485,9 +468,7 @@ final class SignatureValidationTests: XCTestCase { errorDescription: "not found" ) - let httpClient = LegacyHTTPClient(handler: serverErrorHandler.handle) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + let httpClient = HTTPClient { try serverErrorHandler.handle(request: $0, progress: $1) } let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -557,15 +538,13 @@ final class SignatureValidationTests: XCTestCase { ) // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -647,15 +626,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -731,15 +707,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -812,15 +785,12 @@ final class SignatureValidationTests: XCTestCase { ) // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -890,15 +860,12 @@ final class SignatureValidationTests: XCTestCase { let signatureFormat = SignatureFormat.cms_1_0_0 // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -951,14 +918,10 @@ final class SignatureValidationTests: XCTestCase { let version = Version("1.1.1") // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = { _, _, completion in - completion(.failure(StringError("unexpected request"))) + let httpClient = HTTPClient { _, _ in + throw StringError("unexpected request") } - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none - let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() configuration.defaultRegistry = registry @@ -1020,15 +983,12 @@ final class SignatureValidationTests: XCTestCase { ) // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1114,15 +1074,12 @@ final class SignatureValidationTests: XCTestCase { ) // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1199,15 +1156,12 @@ final class SignatureValidationTests: XCTestCase { ) // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1308,15 +1262,12 @@ final class SignatureValidationTests: XCTestCase { ) // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1405,15 +1356,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1499,15 +1447,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1581,15 +1526,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1663,15 +1605,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1768,15 +1707,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1889,15 +1825,12 @@ final class SignatureValidationTests: XCTestCase { """ // Get metadata endpoint will be called to see if package version is signed - let handler: LegacyHTTPClient.Handler = LegacyHTTPClient.packageReleaseMetadataAPIHandler( + let httpClient = HTTPClient(implementation: HTTPClient.packageReleaseMetadataAPIHandler( metadataURL: metadataURL, checksum: checksum, signatureBytes: signatureBytes, signatureFormat: signatureFormat - ) - let httpClient = LegacyHTTPClient(handler: handler) - httpClient.configuration.circuitBreakerStrategy = .none - httpClient.configuration.retryStrategy = .none + )) let registry = Registry(url: registryURL, supportsAvailability: false) var configuration = RegistryConfiguration() @@ -1944,7 +1877,7 @@ final class SignatureValidationTests: XCTestCase { registry: registry, package: package, version: version, - toolsVersion: .none, + toolsVersion: nil, manifestContent: manifestContent, configuration: configuration.signing(for: package, registry: registry), observabilityScope: observability.topScope @@ -2027,8 +1960,7 @@ extension SignatureValidation { configuration: configuration, timeout: nil, fileSystem: localFileSystem, - observabilityScope: observabilityScope ?? ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: observabilityScope ?? ObservabilitySystem.NOOP ) } @@ -2050,8 +1982,7 @@ extension SignatureValidation { configuration: configuration, timeout: nil, fileSystem: localFileSystem, - observabilityScope: observabilityScope ?? ObservabilitySystem.NOOP, - callbackQueue: .sharedConcurrent + observabilityScope: observabilityScope ?? ObservabilitySystem.NOOP ) } } @@ -2060,19 +1991,17 @@ private struct RejectingSignatureValidationDelegate: SignatureValidation.Delegat func onUnsigned( registry: Registry, package: PackageIdentity, - version: Version, - completion: (Bool) -> Void - ) { - completion(false) + version: Version + ) -> Bool { + return false } func onUntrusted( registry: Registry, package: PackageIdentity, - version: Version, - completion: (Bool) -> Void - ) { - completion(false) + version: Version + ) -> Bool { + return false } } @@ -2080,19 +2009,17 @@ private struct AcceptingSignatureValidationDelegate: SignatureValidation.Delegat func onUnsigned( registry: Registry, package: PackageIdentity, - version: Version, - completion: (Bool) -> Void - ) { - completion(true) + version: Version + ) -> Bool { + return true } func onUntrusted( registry: Registry, package: PackageIdentity, - version: Version, - completion: (Bool) -> Void - ) { - completion(true) + version: Version + ) -> Bool { + return true } } @@ -2105,12 +2032,12 @@ extension PackageSigningEntityStorage { } } -extension LegacyHTTPClient { +extension HTTPClient { static func packageReleaseMetadataAPIHandler( metadataURL: URL, checksum: String - ) -> LegacyHTTPClient.Handler { - { request, _, completion in + ) -> HTTPClient.Implementation { + { request, _ in switch (request.method, request.url) { case (.get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -2132,7 +2059,7 @@ extension LegacyHTTPClient { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2140,9 +2067,10 @@ extension LegacyHTTPClient { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } } @@ -2152,8 +2080,8 @@ extension LegacyHTTPClient { checksum: String, signatureBytes: [UInt8], signatureFormat: SignatureFormat - ) -> LegacyHTTPClient.Handler { - { request, _, completion in + ) -> HTTPClient.Implementation { + { request, _ in switch (request.method, request.url) { case (.get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -2179,7 +2107,7 @@ extension LegacyHTTPClient { } """.data(using: .utf8)! - completion(.success(.init( + return .init( statusCode: 200, headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), @@ -2187,9 +2115,10 @@ extension LegacyHTTPClient { .init(name: "Content-Version", value: "1"), ]), body: data - ))) + ) + default: - completion(.failure(StringError("method and url should match"))) + throw StringError("method and url should match") } } } diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index b5114ec995b..a12710f5ee8 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -14421,8 +14421,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - versionMetadataRequestHandler: { _, _, completion in - completion(.success(.serverError())) + versionMetadataRequestHandler: { _, _ in + return .serverError() }, fileSystem: fs ) @@ -14484,8 +14484,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - manifestRequestHandler: { _, _, completion in - completion(.failure(StringError("boom"))) + manifestRequestHandler: { _, _ in + throw StringError("boom") }, fileSystem: fs ) @@ -14506,8 +14506,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - manifestRequestHandler: { _, _, completion in - completion(.success(.serverError())) + manifestRequestHandler: { _, _ in + return .serverError() }, fileSystem: fs ) @@ -14569,8 +14569,8 @@ final class WorkspaceTests: XCTestCase { let registryClient = try makeRegistryClient( packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", - downloadArchiveRequestHandler: { _, _, completion in - completion(.failure(StringError("boom"))) + downloadArchiveRequestHandler: { _, _ in + throw StringError("boom") }, fileSystem: fs ) From 6731eb540e4c72f7081ec89060b91fa51c5dfb70 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 1 Nov 2024 15:05:14 +0000 Subject: [PATCH 11/33] Fix recursion deadlock in `RegistryPackageContainer`, cleanup warnings --- .../PackageContainer/RegistryPackageContainer.swift | 2 +- Sources/Workspace/Workspace+Manifests.swift | 1 - .../RegistryDownloadsManagerTests.swift | 6 +++--- Tests/WorkspaceTests/WorkspaceTests.swift | 1 - 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift b/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift index 65ad4b15358..4d4a6f99115 100644 --- a/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift +++ b/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift @@ -123,7 +123,7 @@ public class RegistryPackageContainer: PackageContainer { // left internal for testing func loadManifest(version: Version) async throws -> Manifest { return try await self.manifestsCache.memoize(version) { - try await self.loadManifest(version: version) + try await self.loadUnmemoizedManifest(version: version) } } diff --git a/Sources/Workspace/Workspace+Manifests.swift b/Sources/Workspace/Workspace+Manifests.swift index cb4088b52ab..217eca0d020 100644 --- a/Sources/Workspace/Workspace+Manifests.swift +++ b/Sources/Workspace/Workspace+Manifests.swift @@ -587,7 +587,6 @@ extension Workspace { /// Loads the given manifests, if it is present in the managed dependencies. /// - private func loadManagedManifests( for packages: [PackageReference], observabilityScope: ObservabilityScope diff --git a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift index dc580359669..b0fb1d9ecd5 100644 --- a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift @@ -126,7 +126,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { // remove the package do { - try await manager.remove(package: package) + try manager.remove(package: package) delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) @@ -232,8 +232,8 @@ final class RegistryDownloadsManagerTests: XCTestCase { // remove the "local" package, and purge cache do { - try await manager.remove(package: package) - await manager.purgeCache(observabilityScope: observability.topScope) + try manager.remove(package: package) + manager.purgeCache(observabilityScope: observability.topScope) delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index a12710f5ee8..544c16be596 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -8578,7 +8578,6 @@ final class WorkspaceTests: XCTestCase { let maxConcurrentRequests = 2 let concurrentRequests = ThreadSafeBox(0) - let concurrentRequestsLock = NSLock() var configuration = HTTPClient.Configuration() configuration.maxConcurrentRequests = maxConcurrentRequests From 085bad1183786b2d00dfa973dea9d2d59f293efd Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 1 Nov 2024 15:53:26 +0000 Subject: [PATCH 12/33] Add async `withLock` to `InMemoryFileSystem` --- .../FileSystem/InMemoryFileSystem.swift | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/Sources/Basics/FileSystem/InMemoryFileSystem.swift b/Sources/Basics/FileSystem/InMemoryFileSystem.swift index 56e6450aa55..8c56d72f5e9 100644 --- a/Sources/Basics/FileSystem/InMemoryFileSystem.swift +++ b/Sources/Basics/FileSystem/InMemoryFileSystem.swift @@ -8,8 +8,8 @@ See http://swift.org/CONTRIBUTORS.txt for Swift project authors */ -import class Foundation.NSLock import class Dispatch.DispatchQueue +import class Foundation.NSLock import struct TSCBasic.AbsolutePath import struct TSCBasic.ByteString import class TSCBasic.FileLock @@ -23,7 +23,7 @@ public final class InMemoryFileSystem: FileSystem { private class Node { /// The actual node data. let contents: NodeContents - + /// Whether the node has executable bit enabled. var isExecutable: Bool @@ -86,7 +86,7 @@ public final class InMemoryFileSystem: FileSystem { /// tests. private let lock = NSLock() /// A map that keeps weak references to all locked files. - private var lockFiles = Dictionary>() + private var lockFiles = [TSCBasic.AbsolutePath: WeakReference]() /// Used to access lockFiles in a thread safe manner. private let lockFilesLock = NSLock() @@ -488,10 +488,39 @@ public final class InMemoryFileSystem: FileSystem { } } - return try fileQueue.sync(flags: type == .exclusive ? .barrier : .init() , execute: body) + return try fileQueue.sync(flags: type == .exclusive ? .barrier : .init(), execute: body) + } + + public func withLock( + on path: TSCBasic.AbsolutePath, + type: FileLock.LockType = .exclusive, + blocking: Bool, + _ body: () async throws -> T + ) async throws -> T { + let resolvedPath: TSCBasic.AbsolutePath = try self.lock.withLock { + if case .symlink(let destination) = try getNode(path)?.contents { + try .init(validating: destination, relativeTo: path.parentDirectory) + } else { + path + } + } + + // FIXME: code calling this function should be migrated to `AsyncFileSystem` instead. + self.lockFilesLock.lock() + + let result = try await body() + + self.lockFilesLock.unlock() + + return result } - - public func withLock(on path: TSCBasic.AbsolutePath, type: FileLock.LockType, blocking: Bool, _ body: () throws -> T) throws -> T { + + public func withLock( + on path: TSCBasic.AbsolutePath, + type: FileLock.LockType, + blocking: Bool, + _ body: () throws -> T + ) throws -> T { try self.withLock(on: path, type: type, body) } } From 7cc81afcd9ccca9d2c7b955c4eafc86ed1718618 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 1 Nov 2024 16:09:59 +0000 Subject: [PATCH 13/33] Make sure `Workspace.loadRootManifests` doesn't throw --- Sources/Workspace/Workspace.swift | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 06a160938e7..c527215c49a 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -996,12 +996,12 @@ extension Workspace { public func loadRootManifests( packages: [AbsolutePath], observabilityScope: ObservabilityScope - ) async throws -> [AbsolutePath: Manifest] { - let rootManifests = try await withThrowingTaskGroup(of: (AbsolutePath, Manifest?).self) { group in + ) async -> [AbsolutePath: Manifest] { + let rootManifests = await withTaskGroup(of: (AbsolutePath, Manifest?).self) { group in for package in Set(packages) { group.addTask { // TODO: this does not use the identity resolver which is probably fine since its the root packages - (package, try await self.loadManifest( + (package, try? await self.loadManifest( packageIdentity: PackageIdentity(path: package), packageKind: .root(package), packagePath: package, @@ -1011,7 +1011,7 @@ extension Workspace { } } - return try await group.reduce(into: [:]) { $0[$1.0] = $1.1 } + return await group.reduce(into: [:]) { $0[$1.0] = $1.1 } } // Check for duplicate root packages. @@ -1030,7 +1030,7 @@ extension Workspace { at path: AbsolutePath, observabilityScope: ObservabilityScope ) async throws -> Manifest { - let manifests = try await self.loadRootManifests(packages: [path], observabilityScope: observabilityScope) + let manifests = await self.loadRootManifests(packages: [path], observabilityScope: observabilityScope) // normally, we call loadRootManifests which attempts to load any manifest it can and report errors via // diagnostics From ad8b6126403678cf4f6746e84962301351ab48a6 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 1 Nov 2024 17:00:33 +0000 Subject: [PATCH 14/33] Don't duplicate `Diagnostics.fatalError` in diagnostics --- Sources/Workspace/Workspace+Manifests.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/Workspace/Workspace+Manifests.swift b/Sources/Workspace/Workspace+Manifests.swift index 217eca0d020..3a32cb3e64b 100644 --- a/Sources/Workspace/Workspace+Manifests.swift +++ b/Sources/Workspace/Workspace+Manifests.swift @@ -756,7 +756,14 @@ extension Workspace { return manifest } catch { let duration = start.distance(to: .now()) - manifestLoadingDiagnostics.append(.error(error)) + + switch error { + case Diagnostics.fatalError: + break + default: + manifestLoadingDiagnostics.append(.error(error)) + } + self.delegate?.didLoadManifest( packageIdentity: packageIdentity, packagePath: packagePath, From 4c00304c996ea095974aeafaef37ffcd39c6ca9b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 1 Nov 2024 18:06:14 +0000 Subject: [PATCH 15/33] Fix `InMemoryFileSystem.withLock` deadlock --- .../Basics/FileSystem/InMemoryFileSystem.swift | 15 +++++++++------ Sources/PackageRegistry/RegistryClient.swift | 17 +++++++---------- .../RegistryDownloadsManager.swift | 8 +++++--- .../_InternalTestSupport/MockWorkspace.swift | 2 +- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Sources/Basics/FileSystem/InMemoryFileSystem.swift b/Sources/Basics/FileSystem/InMemoryFileSystem.swift index 8c56d72f5e9..f503441b399 100644 --- a/Sources/Basics/FileSystem/InMemoryFileSystem.swift +++ b/Sources/Basics/FileSystem/InMemoryFileSystem.swift @@ -90,6 +90,8 @@ public final class InMemoryFileSystem: FileSystem { /// Used to access lockFiles in a thread safe manner. private let lockFilesLock = NSLock() + private let asyncFilesLock = [TSCBasic.AbsolutePath: NSLock]() + /// Exclusive file system lock vended to clients through `withLock()`. /// Used to ensure that DispatchQueues are released when they are no longer in use. private struct WeakReference { @@ -491,26 +493,27 @@ public final class InMemoryFileSystem: FileSystem { return try fileQueue.sync(flags: type == .exclusive ? .barrier : .init(), execute: body) } + @available(*, deprecated, message: "Use of this overload can lead to deadlocks, use `AsyncFileSystem` instead.") public func withLock( on path: TSCBasic.AbsolutePath, type: FileLock.LockType = .exclusive, blocking: Bool, _ body: () async throws -> T ) async throws -> T { - let resolvedPath: TSCBasic.AbsolutePath = try self.lock.withLock { - if case .symlink(let destination) = try getNode(path)?.contents { - try .init(validating: destination, relativeTo: path.parentDirectory) + let resolvedPath: TSCBasic.AbsolutePath = try lock.withLock { + if case let .symlink(destination) = try getNode(path)?.contents { + return try .init(validating: destination, relativeTo: path.parentDirectory) } else { - path + return path } } // FIXME: code calling this function should be migrated to `AsyncFileSystem` instead. - self.lockFilesLock.lock() + self.asyncFilesLock[resolvedPath, default: NSLock()].lock() let result = try await body() - self.lockFilesLock.unlock() + self.asyncFilesLock[resolvedPath, default: NSLock()].unlock() return result } diff --git a/Sources/PackageRegistry/RegistryClient.swift b/Sources/PackageRegistry/RegistryClient.swift index 00efc091914..bd9f760b4d0 100644 --- a/Sources/PackageRegistry/RegistryClient.swift +++ b/Sources/PackageRegistry/RegistryClient.swift @@ -895,7 +895,7 @@ public final class RegistryClient { .hexadecimalRepresentation observabilityScope.emit( - debug: "performing TOFU checks on \(package) \(version) source archive (checksum: '\(actualChecksum)'" + debug: "performing TOFU checks on \(package) \(version) source archive (checksum: '\(actualChecksum)')" ) let signingEntity = try await signatureValidation.validate( registry: registry, @@ -951,15 +951,12 @@ public final class RegistryClient { try fileSystem .stripFirstLevel(of: destinationPath) // write down copy of version metadata - let registryMetadataPath = destinationPath - .appending( - component: RegistryReleaseMetadataStorage - .fileName - ) - observabilityScope - .emit( - debug: "saving \(package) \(version) metadata to '\(registryMetadataPath)'" - ) + let registryMetadataPath = destinationPath.appending( + component: RegistryReleaseMetadataStorage.fileName + ) + observabilityScope.emit( + debug: "saving \(package) \(version) metadata to '\(registryMetadataPath)'" + ) try RegistryReleaseMetadataStorage.save( metadata: versionMetadata, signingEntity: signingEntity, diff --git a/Sources/PackageRegistry/RegistryDownloadsManager.swift b/Sources/PackageRegistry/RegistryDownloadsManager.swift index 191d924b937..d4f7f6c8f32 100644 --- a/Sources/PackageRegistry/RegistryDownloadsManager.swift +++ b/Sources/PackageRegistry/RegistryDownloadsManager.swift @@ -66,9 +66,11 @@ public actor RegistryDownloadsManager { if self.pendingLookups.keys.contains(package) { // chain onto the pending lookup return await withCheckedContinuation { - self.pendingLookups[package]?.append($0) + self.pendingLookups[package]!.append($0) } } else { + self.pendingLookups[package] = [] + // inform delegate that we are starting to fetch // calculate if cached (for delegate call) outside queue as it may change while queue is processing let isCached = self.cachePath.map { self.fileSystem.exists($0.appending(packageRelativePath)) } ?? false @@ -107,9 +109,9 @@ public actor RegistryDownloadsManager { for lookup in pendingLookups { lookup.resume(returning: packagePath) } - } - self.pendingLookups[package] = nil + self.pendingLookups[package] = nil + } } // and done diff --git a/Sources/_InternalTestSupport/MockWorkspace.swift b/Sources/_InternalTestSupport/MockWorkspace.swift index 6909eb2b234..ac9db25ab30 100644 --- a/Sources/_InternalTestSupport/MockWorkspace.swift +++ b/Sources/_InternalTestSupport/MockWorkspace.swift @@ -569,7 +569,7 @@ public final class MockWorkspace { let resolvedPackagesStore = try workspace.resolvedPackagesStore.load() let rootInput = PackageGraphRootInput(packages: try rootPaths(for: roots.map { $0.name }), dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) From 58900735f5bd91ba846227407a54a4f32774006d Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 1 Nov 2024 18:31:58 +0000 Subject: [PATCH 16/33] Fix formatting in `Workspace+Manifests.swift` --- Sources/Workspace/Workspace+Manifests.swift | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Sources/Workspace/Workspace+Manifests.swift b/Sources/Workspace/Workspace+Manifests.swift index 3a32cb3e64b..780a957e90d 100644 --- a/Sources/Workspace/Workspace+Manifests.swift +++ b/Sources/Workspace/Workspace+Manifests.swift @@ -666,13 +666,13 @@ extension Workspace { // Load and return the manifest. return try? await self.loadManifest( - packageIdentity: managedDependency.packageRef.identity, - packageKind: packageKind, - packagePath: packagePath, - packageLocation: managedDependency.packageRef.locationString, - packageVersion: packageVersion, - fileSystem: fileSystem, - observabilityScope: observabilityScope + packageIdentity: managedDependency.packageRef.identity, + packageKind: packageKind, + packagePath: packagePath, + packageLocation: managedDependency.packageRef.locationString, + packageVersion: packageVersion, + fileSystem: fileSystem, + observabilityScope: observabilityScope ) } From d91c0f8eee95e0b32b7971a5ec1314eb7e4facb2 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 4 Nov 2024 10:18:47 +0000 Subject: [PATCH 17/33] Document new `ThreadSafeBox.mutate` overload --- Sources/Basics/Concurrency/ThreadSafeBox.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/Basics/Concurrency/ThreadSafeBox.swift b/Sources/Basics/Concurrency/ThreadSafeBox.swift index c05347a46ad..21d510ed202 100644 --- a/Sources/Basics/Concurrency/ThreadSafeBox.swift +++ b/Sources/Basics/Concurrency/ThreadSafeBox.swift @@ -31,6 +31,8 @@ public final class ThreadSafeBox { } } + /// Modifies value stored in the box in-place, potentially avoiding copies. + /// - Parameter body: function applied to the stored value that modifies it. public func mutate(body: (inout Value?) throws -> ()) rethrows { try self.lock.withLock { try body(&self.underlying) From c12aff1c16d99992bf682aee1c54a936ab44340c Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 4 Nov 2024 12:50:27 +0000 Subject: [PATCH 18/33] Fix remaining `WorkspaceRegistryTests` failures --- .../RegistryDownloadsManager.swift | 11 +- Sources/Workspace/Workspace+Registry.swift | 1 - .../WorkspaceRegistryTests.swift | 2860 ++++++++++++++++ Tests/WorkspaceTests/WorkspaceTests.swift | 2954 +---------------- 4 files changed, 2929 insertions(+), 2897 deletions(-) create mode 100644 Tests/WorkspaceTests/WorkspaceRegistryTests.swift diff --git a/Sources/PackageRegistry/RegistryDownloadsManager.swift b/Sources/PackageRegistry/RegistryDownloadsManager.swift index d4f7f6c8f32..a2fcba77bdf 100644 --- a/Sources/PackageRegistry/RegistryDownloadsManager.swift +++ b/Sources/PackageRegistry/RegistryDownloadsManager.swift @@ -84,6 +84,8 @@ public actor RegistryDownloadsManager { try? self.fileSystem.removeFileTree(packagePath) let start = DispatchTime.now() + + // `Result` type is used by the `didFetch` delegate method called below. let result: Result do { result = try await .success(self.downloadAndPopulateCache( @@ -115,6 +117,7 @@ public actor RegistryDownloadsManager { } // and done + _ = try result.get() return packagePath } } @@ -167,6 +170,11 @@ public actor RegistryDownloadsManager { } } } catch { + if error is RegistryError { + // Avoid handling `RegistryError`s here, propagate those back in the call stack + throw error + } + // download without populating the cache in the case of an error. observabilityScope.emit( warning: "skipping cache due to an error", @@ -203,7 +211,8 @@ public actor RegistryDownloadsManager { // utility to update progress - @Sendable func updateDownloadProgress(downloaded: Int64, total: Int64?) { + @Sendable + func updateDownloadProgress(downloaded: Int64, total: Int64?) { delegateQueue.async { self.delegate?.fetching( package: package, diff --git a/Sources/Workspace/Workspace+Registry.swift b/Sources/Workspace/Workspace+Registry.swift index 809d37e8f70..7eec1c5edb8 100644 --- a/Sources/Workspace/Workspace+Registry.swift +++ b/Sources/Workspace/Workspace+Registry.swift @@ -371,7 +371,6 @@ extension Workspace { at version: Version, observabilityScope: ObservabilityScope ) async throws -> AbsolutePath { - // FIXME: this should not block let downloadPath = try await self.registryDownloadsManager.lookup( package: package.identity, version: version, diff --git a/Tests/WorkspaceTests/WorkspaceRegistryTests.swift b/Tests/WorkspaceTests/WorkspaceRegistryTests.swift new file mode 100644 index 00000000000..545bf95c285 --- /dev/null +++ b/Tests/WorkspaceTests/WorkspaceRegistryTests.swift @@ -0,0 +1,2860 @@ +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import _InternalTestSupport +import Basics +import PackageFingerprint +@testable import PackageGraph +import PackageModel +import PackageRegistry +import PackageSigning +@testable import Workspace +import XCTest + +import struct TSCUtility.Version + +final class WorkspaceRegistryTests: XCTestCase { + func testPackageMirrorURLToRegistry() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let mirrors = try DependencyMirrors() + try mirrors.set(mirror: "org.bar-mirror", for: "https://scm.com/org/bar") + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Foo", + targets: [ + MockTarget(name: "Foo", dependencies: [ + .product(name: "Bar", package: "bar"), + .product(name: "Baz", package: "baz"), + ]), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + dependencies: [ + .sourceControl(url: "https://scm.com/org/bar", requirement: .upToNextMajor(from: "1.0.0")), + .sourceControl(url: "https://scm.com/org/baz", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "BarMirror", + identity: "org.bar-mirror", + targets: [ + MockTarget(name: "Bar"), + ], + products: [ + MockProduct(name: "Bar", modules: ["Bar"]), + ], + versions: ["1.0.0", "1.5.0"] + ), + MockPackage( + name: "Baz", + url: "https://scm.com/org/baz", + targets: [ + MockTarget(name: "Baz"), + ], + products: [ + MockProduct(name: "Baz", modules: ["Baz"]), + ], + versions: ["1.0.0", "1.6.0"] + ), + ], + mirrors: mirrors + ) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { graph, diagnostics in + PackageGraphTester(graph) { result in + result.check(roots: "Foo") + result.check(packages: "org.bar-mirror", "baz", "foo") + result.check(modules: "Bar", "Baz", "Foo") + } + XCTAssertNoDiagnostics(diagnostics) + } + workspace.checkManagedDependencies { result in + result.check(dependency: "org.bar-mirror", at: .registryDownload("1.5.0")) + result.check(dependency: "baz", at: .checkout(.version("1.6.0"))) + result.check(notPresent: "bar") + } + } + + func testPackageMirrorRegistryToURL() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let mirrors = try DependencyMirrors() + try mirrors.set(mirror: "https://scm.com/org/bar-mirror", for: "org.bar") + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Foo", + targets: [ + MockTarget(name: "Foo", dependencies: [ + .product(name: "Bar", package: "org.bar"), + .product(name: "Baz", package: "org.baz"), + ]), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + dependencies: [ + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.baz", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "BarMirror", + url: "https://scm.com/org/bar-mirror", + targets: [ + MockTarget(name: "Bar"), + ], + products: [ + MockProduct(name: "Bar", modules: ["Bar"]), + ], + versions: ["1.0.0", "1.5.0"] + ), + MockPackage( + name: "Baz", + identity: "org.baz", + targets: [ + MockTarget(name: "Baz"), + ], + products: [ + MockProduct(name: "Baz", modules: ["Baz"]), + ], + versions: ["1.0.0", "1.6.0"] + ), + ], + mirrors: mirrors + ) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { graph, diagnostics in + PackageGraphTester(graph) { result in + result.check(roots: "Foo") + result.check(packages: "bar-mirror", "org.baz", "foo") + result.check(modules: "Bar", "Baz", "Foo") + } + XCTAssertNoDiagnostics(diagnostics) + } + workspace.checkManagedDependencies { result in + result.check(dependency: "bar-mirror", at: .checkout(.version("1.5.0"))) + result.check(dependency: "org.baz", at: .registryDownload("1.6.0")) + result.check(notPresent: "org.bar") + } + } + + func testBasicResolutionFromRegistry() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget1", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + MockTarget( + name: "MyTarget2", + dependencies: [ + .product(name: "Bar", package: "org.bar"), + ] + ), + ], + products: [ + MockProduct(name: "MyProduct", modules: ["MyTarget1", "MyTarget2"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "2.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", "1.5.1"] + ), + MockPackage( + name: "Bar", + identity: "org.bar", + targets: [ + MockTarget(name: "Bar"), + ], + products: [ + MockProduct(name: "Bar", modules: ["Bar"]), + ], + versions: ["2.0.0", "2.1.0", "2.2.0"] + ), + ] + ) + + try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "MyPackage") + result.check(packages: "org.bar", "org.foo", "mypackage") + result.check(modules: "Foo", "Bar", "MyTarget1", "MyTarget2") + result.checkTarget("MyTarget1") { result in result.check(dependencies: "Foo") } + result.checkTarget("MyTarget2") { result in result.check(dependencies: "Bar") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.5.1")) + result.check(dependency: "org.bar", at: .registryDownload("2.2.0")) + } + + // Check the load-package callbacks. + XCTAssertMatch( + workspace.delegate.events, + [ + "will load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", + ] + ) + XCTAssertMatch( + workspace.delegate.events, + [ + "did load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", + ] + ) + XCTAssertMatch( + workspace.delegate.events, + ["will load manifest for registry package: org.foo (identity: org.foo)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["did load manifest for registry package: org.foo (identity: org.foo)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["will load manifest for registry package: org.bar (identity: org.bar)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["did load manifest for registry package: org.bar (identity: org.bar)"] + ) + } + + func testBasicTransitiveResolutionFromRegistry() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget1", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + MockTarget( + name: "MyTarget2", + dependencies: [ + .product(name: "Bar", package: "org.bar"), + ] + ), + ], + products: [ + MockProduct(name: "MyProduct", modules: ["MyTarget1", "MyTarget2"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "2.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget( + name: "Foo", + dependencies: [ + .product(name: "Baz", package: "org.baz"), + ] + ), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + dependencies: [ + .registry(identity: "org.baz", requirement: .range("2.0.0" ..< "4.0.0")), + ], + versions: ["1.0.0", "1.1.0"] + ), + MockPackage( + name: "Bar", + identity: "org.bar", + targets: [ + MockTarget( + name: "Bar", + dependencies: [ + .product(name: "Baz", package: "org.baz"), + ] + ), + ], + products: [ + MockProduct(name: "Bar", modules: ["Bar"]), + ], + dependencies: [ + .registry(identity: "org.baz", requirement: .upToNextMajor(from: "3.0.0")), + ], + versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] + ), + MockPackage( + name: "Baz", + identity: "org.baz", + targets: [ + MockTarget(name: "Baz"), + ], + products: [ + MockProduct(name: "Baz", modules: ["Baz"]), + ], + versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0", "3.0.0", "3.1.0"] + ), + ] + ) + + try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "MyPackage") + result.check(packages: "org.bar", "org.baz", "org.foo", "mypackage") + result.check(modules: "Foo", "Bar", "Baz", "MyTarget1", "MyTarget2") + result.checkTarget("MyTarget1") { result in result.check(dependencies: "Foo") } + result.checkTarget("MyTarget2") { result in result.check(dependencies: "Bar") } + result.checkTarget("Foo") { result in result.check(dependencies: "Baz") } + result.checkTarget("Bar") { result in result.check(dependencies: "Baz") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.1.0")) + result.check(dependency: "org.bar", at: .registryDownload("2.1.0")) + result.check(dependency: "org.baz", at: .registryDownload("3.1.0")) + } + + // Check the load-package callbacks. + XCTAssertMatch( + workspace.delegate.events, + [ + "will load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", + ] + ) + XCTAssertMatch( + workspace.delegate.events, + [ + "did load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", + ] + ) + XCTAssertMatch( + workspace.delegate.events, + ["will load manifest for registry package: org.foo (identity: org.foo)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["did load manifest for registry package: org.foo (identity: org.foo)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["will load manifest for registry package: org.bar (identity: org.bar)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["did load manifest for registry package: org.bar (identity: org.bar)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["will load manifest for registry package: org.baz (identity: org.baz)"] + ) + XCTAssertMatch( + workspace.delegate.events, + ["did load manifest for registry package: org.baz (identity: org.baz)"] + ) + } + + // no dups + func testResolutionMixedRegistryAndSourceControl1() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + .product(name: "BarProduct", package: "org.bar"), + ]), + ], + products: [], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0"] + ), + MockPackage( + name: "BarPackage", + identity: "org.bar", + targets: [ + MockTarget(name: "BarTarget"), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + versions: ["1.0.0", "1.1.0"] + ), + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "foo", at: .checkout(.version("1.2.0"))) + result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .checkout(.version("1.2.0"))) + result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) + result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) + } + } + } + + // duplicate package at root level + func testResolutionMixedRegistryAndSourceControl2() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + ]), + ], + products: [], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + + + await XCTAssertAsyncThrowsError(try await workspace.checkPackageGraph(roots: ["root"]) { _, _ in + }) { error in + XCTAssertEqual((error as? PackageGraphError)?.description, "multiple packages (\'foo\' (from \'https://git/org/foo\'), \'org.foo\') declare products with a conflicting name: \'FooProduct’; product names need to be unique across the package graph") + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: "'root' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'", + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + // TODO: this error message should be improved + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: "'root' dependency on 'org.foo' conflicts with dependency on 'org.foo' which has the same identity 'org.foo'", + severity: .error + ) + } + } + } + } + + // mixed graph root --> dep1 scm + // --> dep2 scm --> dep1 registry + func testResolutionMixedRegistryAndSourceControl3() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + .product(name: "BarProduct", package: "bar"), + ]), + ], + products: [], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), + .sourceControl(url: "https://git/org/bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] + ), + MockPackage( + name: "BarPackage", + url: "https://git/org/bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ], + versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] + ), + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + 'bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. + """), + severity: .warning + ) + } + + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .checkout(.version("1.1.0"))) + result.check(dependency: "bar", at: .checkout(.version("1.1.0"))) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.1.0")) + result.check(dependency: "bar", at: .checkout(.version("1.1.0"))) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.1.0")) + result.check(dependency: "bar", at: .checkout(.version("1.1.0"))) + } + } + } + + // mixed graph root --> dep1 scm + // --> dep2 registry --> dep1 registry + func testResolutionMixedRegistryAndSourceControl4() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + .product(name: "BarProduct", package: "org.bar"), + ]), + ], + products: [], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.1.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0"] + ), + MockPackage( + name: "BarPackage", + identity: "org.bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.2.0")), + ], + versions: ["1.0.0", "1.1.0"] + ), + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + 'org.bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. + """), + severity: .warning + ) + } + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .checkout(.version("1.2.0"))) + result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) + result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) + } + } + } + + // mixed graph root --> dep1 scm + // --> dep2 scm --> dep1 registry incompatible version + func testResolutionMixedRegistryAndSourceControl5() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + .product(name: "BarProduct", package: "bar"), + ]), + ], + products: [], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), + .sourceControl(url: "https://git/org/bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "2.0.0"] + ), + MockPackage( + name: "BarPackage", + url: "https://git/org/bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "2.0.0")), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "2.0.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: + """ + Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'bar' 1.0.0..<2.0.0. + 'bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'bar' match the requirement 1.0.1..<2.0.0 and 'bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. + """, + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: + """ + Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'bar' 1.0.0..<2.0.0. + 'bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'bar' match the requirement 1.0.1..<2.0.0 and 'bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. + """, + severity: .error + ) + } + } + } + } + + // mixed graph root --> dep1 registry + // --> dep2 registry --> dep1 scm + func testResolutionMixedRegistryAndSourceControl6() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + .product(name: "BarProduct", package: "org.bar"), + ]), + ], + products: [], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.1.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0"] + ), + MockPackage( + name: "BarPackage", + identity: "org.bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.2.0")), + ], + versions: ["1.0.0", "1.1.0"] + ), + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + 'org.bar' dependency on 'https://git/org/foo' conflicts with dependency on 'org.foo' which has the same identity 'org.foo'. + """), + severity: .warning + ) + if ToolsVersion.current >= .v5_8 { + result.check( + diagnostic: .contains(""" + product 'FooProduct' required by package 'org.bar' target 'BarTarget' not found in package 'foo'. + """), + severity: .error + ) + } + } + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) + result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) + result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) + } + } + } + + // mixed graph root --> dep1 registry + // --> dep2 registry --> dep1 scm incompatible version + func testResolutionMixedRegistryAndSourceControl7() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + .product(name: "BarProduct", package: "org.bar"), + ]), + ], + products: [], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "2.0.0"] + ), + MockPackage( + name: "BarPackage", + identity: "org.bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "2.0.0")), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0", "2.0.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: + """ + Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. + 'org.bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. + """, + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: + """ + Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. + 'org.bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. + """, + severity: .error + ) + } + } + } + } + + // mixed graph root --> dep1 registry --> dep3 scm + // --> dep2 registry --> dep3 registry + func testResolutionMixedRegistryAndSourceControl8() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + .product(name: "BarProduct", package: "org.bar"), + ]), + ], + products: [], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + identity: "org.foo", + targets: [ + MockTarget(name: "FooTarget", dependencies: [ + .product(name: "BazProduct", package: "baz"), + ]), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + dependencies: [ + .sourceControl(url: "https://git/org/baz", requirement: .upToNextMajor(from: "1.1.0")), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "BarPackage", + identity: "org.bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "BazProduct", package: "org.baz"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .registry(identity: "org.baz", requirement: .upToNextMajor(from: "1.0.0")), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "BazPackage", + url: "https://git/org/baz", + targets: [ + MockTarget(name: "BazTarget"), + ], + products: [ + MockProduct(name: "BazProduct", modules: ["BazTarget"]), + ], + versions: ["1.0.0", "1.1.0"] + ), + MockPackage( + name: "BazPackage", + identity: "org.baz", + alternativeURLs: ["https://git/org/baz"], + targets: [ + MockTarget(name: "BazTarget"), + ], + products: [ + MockProduct(name: "BazProduct", modules: ["BazTarget"]), + ], + versions: ["1.0.0", "1.1.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'BazTarget' appear in registry package 'org.baz' and source control package 'baz' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + 'org.foo' dependency on 'https://git/org/baz' conflicts with dependency on 'org.baz' which has the same identity 'org.baz'. + """), + severity: .warning + ) + if ToolsVersion.current >= .v5_8 { + result.check( + diagnostic: .contains(""" + product 'BazProduct' required by package 'org.foo' target 'FooTarget' not found in package 'baz'. + """), + severity: .error + ) + } + } + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.baz", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "BazTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + if ToolsVersion.current < .v5_8 { + result.checkTarget("FooTarget") { result in result.check(dependencies: "BazProduct") } + } + result.checkTarget("BarTarget") { result in result.check(dependencies: "BazProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.0.0")) + result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) + result.check(dependency: "org.baz", at: .registryDownload("1.1.0")) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.baz", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "BazTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + result.checkTarget("FooTarget") { result in result.check(dependencies: "BazProduct") } + result.checkTarget("BarTarget") { result in result.check(dependencies: "BazProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.0.0")) + result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) + result.check(dependency: "org.baz", at: .registryDownload("1.1.0")) + } + } + } + + // mixed graph root --> dep1 registry --> dep3 scm + // --> dep2 registry --> dep3 registry incompatible version + func testResolutionMixedRegistryAndSourceControl9() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + .product(name: "BarProduct", package: "org.bar"), + ]), + ], + products: [], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + identity: "org.foo", + targets: [ + MockTarget(name: "FooTarget", dependencies: [ + .product(name: "BazProduct", package: "baz"), + ]), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + dependencies: [ + .sourceControl(url: "https://git/org/baz", requirement: .upToNextMajor(from: "1.0.0")), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "BarPackage", + identity: "org.bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "BazProduct", package: "org.baz"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .registry(identity: "org.baz", requirement: .upToNextMajor(from: "2.0.0")), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "BazPackage", + url: "https://git/org/baz", + targets: [ + MockTarget(name: "BazTarget"), + ], + products: [ + MockProduct(name: "BazProduct", modules: ["BazTarget"]), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "BazPackage", + identity: "org.baz", + alternativeURLs: ["https://git/org/baz"], + targets: [ + MockTarget(name: "BazTarget"), + ], + products: [ + MockProduct(name: "BazProduct", modules: ["BazTarget"]), + ], + versions: ["1.0.0", "2.0.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'BazTarget' appear in registry package 'org.baz' and source control package 'baz' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: """ + Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. + 'org.bar' is incompatible with 'org.foo' because 'org.foo' 1.0.0 depends on 'org.baz' 1.0.0..<2.0.0 and no versions of 'org.foo' match the requirement 1.0.1..<2.0.0. + 'org.bar' >= 1.0.0 practically depends on 'org.baz' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.baz' 2.0.0..<3.0.0. + """, + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: """ + Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. + 'org.bar' is incompatible with 'org.foo' because 'org.foo' 1.0.0 depends on 'org.baz' 1.0.0..<2.0.0 and no versions of 'org.foo' match the requirement 1.0.1..<2.0.0. + 'org.bar' >= 1.0.0 practically depends on 'org.baz' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.baz' 2.0.0..<3.0.0. + """, + severity: .error + ) + } + } + } + } + + // mixed graph root --> dep1 scm branch + // --> dep2 registry --> dep1 registry + func testResolutionMixedRegistryAndSourceControl10() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "Root", + path: "root", + targets: [ + MockTarget(name: "RootTarget", dependencies: [ + .product(name: "FooProduct", package: "foo"), + .product(name: "BarProduct", package: "org.bar"), + ]), + ], + products: [], + dependencies: [ + .sourceControl(url: "https://git/org/foo", requirement: .branch("experiment")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), + ], + toolsVersion: .v5_6 + ), + ], + packages: [ + MockPackage( + name: "FooPackage", + url: "https://git/org/foo", + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["experiment"] + ), + MockPackage( + name: "BarPackage", + identity: "org.bar", + targets: [ + MockTarget(name: "BarTarget", dependencies: [ + .product(name: "FooProduct", package: "org.foo"), + ]), + ], + products: [ + MockProduct(name: "BarProduct", modules: ["BarTarget"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ], + versions: ["1.0.0"] + ), + MockPackage( + name: "FooPackage", + identity: "org.foo", + alternativeURLs: ["https://git/org/foo"], + targets: [ + MockTarget(name: "FooTarget"), + ], + products: [ + MockProduct(name: "FooProduct", modules: ["FooTarget"]), + ], + versions: ["1.0.0"] + ), + ] + ) + + do { + workspace.sourceControlToRegistryDependencyTransformation = .disabled + try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' + """), + severity: .error + ) + } + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .identity + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + 'org.bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. + """), + severity: .warning + ) + } + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .checkout(.branch("experiment"))) + result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) + } + } + + // reset + try workspace.closeWorkspace() + + do { + workspace.sourceControlToRegistryDependencyTransformation = .swizzle + + try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .contains(""" + 'org.bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. + """), + severity: .warning + ) + } + PackageGraphTester(graph) { result in + result.check(roots: "Root") + result.check(packages: "org.bar", "org.foo", "Root") + result.check(modules: "FooTarget", "BarTarget", "RootTarget") + result + .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } + } + } + + workspace.checkManagedDependencies { result in + result + .check( + dependency: "org.foo", + at: .checkout(.branch("experiment")) + ) // we cannot swizzle branch based deps + result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) + } + } + } + + func testRegistryMissingConfigurationErrors() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + configuration: .init(), + fileSystem: fs + ) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0"] + ), + ], + registryClient: registryClient + ) + + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check(diagnostic: .equal("no registry configured for 'org' scope"), severity: .error) + } + } + } + + func testRegistryReleasesServerErrors() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0"] + ), + ] + ) + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + releasesRequestHandler: { _, _ in + throw StringError("boom") + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal("failed fetching org.foo releases list from http://localhost: boom"), + severity: .error + ) + } + } + } + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + releasesRequestHandler: { _, _ in + .serverError() + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal( + "failed fetching org.foo releases list from http://localhost: server error 500: Internal Server Error" + ), + severity: .error + ) + } + } + } + } + + func testRegistryReleaseChecksumServerErrors() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0"] + ), + ] + ) + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + versionMetadataRequestHandler: { _, _ in + throw StringError("boom") + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal( + "failed fetching org.foo version 1.0.0 release information from http://localhost: boom" + ), + severity: .error + ) + } + } + } + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + versionMetadataRequestHandler: { _, _ in + return .serverError() + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal( + "failed fetching org.foo version 1.0.0 release information from http://localhost: server error 500: Internal Server Error" + ), + severity: .error + ) + } + } + } + } + + func testRegistryManifestServerErrors() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0"] + ), + ] + ) + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + manifestRequestHandler: { _, _ in + throw StringError("boom") + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal("failed retrieving org.foo version 1.0.0 manifest from http://localhost: boom"), + severity: .error + ) + } + } + } + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + manifestRequestHandler: { _, _ in + return .serverError() + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal( + "failed retrieving org.foo version 1.0.0 manifest from http://localhost: server error 500: Internal Server Error" + ), + severity: .error + ) + } + } + } + } + + func testRegistryDownloadServerErrors() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0"] + ), + ] + ) + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + downloadArchiveRequestHandler: { _, _ in + throw StringError("boom") + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal( + "failed downloading org.foo version 1.0.0 source archive from http://localhost: boom" + ), + severity: .error + ) + } + } + } + + do { + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + downloadArchiveRequestHandler: { _, _ in + .serverError() + }, + fileSystem: fs + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .equal( + "failed downloading org.foo version 1.0.0 source archive from http://localhost: server error 500: Internal Server Error" + ), + severity: .error + ) + } + } + } + } + + func testRegistryArchiveErrors() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + archiver: MockArchiver(handler: { _, _, _, completion in + completion(.failure(StringError("boom"))) + }), + fileSystem: fs + ) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0"] + ), + ], + registryClient: registryClient + ) + + try workspace.closeWorkspace() + workspace.registryClient = registryClient + await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in + testDiagnostics(diagnostics) { result in + result.check( + diagnostic: .regex( + "failed extracting '.*[\\\\/]registry[\\\\/]downloads[\\\\/]org[\\\\/]foo[\\\\/]1.0.0.zip' to '.*[\\\\/]registry[\\\\/]downloads[\\\\/]org[\\\\/]foo[\\\\/]1.0.0': boom" + ), + severity: .error + ) + } + } + } + + func testRegistryMetadata() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + let registryURL = URL("https://packages.example.com") + var registryConfiguration = RegistryConfiguration() + registryConfiguration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) + registryConfiguration.security = RegistryConfiguration.Security() + registryConfiguration.security!.default.signing = RegistryConfiguration.Security.Signing() + registryConfiguration.security!.default.signing!.onUnsigned = .silentAllow + + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.5.1", + targets: ["Foo"], + configuration: registryConfiguration, + fileSystem: fs + ) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + products: [ + MockProduct(name: "MyProduct", modules: ["MyTarget"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + registryClient: registryClient + ) + + // for mock manifest loader to work with an actual registry download + // we populate the mock manifest with a pointer to the correct download location + let defaultLocations = try Workspace.Location(forRootPackage: sandbox, fileSystem: fs) + let packagePath = defaultLocations.registryDownloadDirectory.appending(components: ["org", "foo", "1.5.1"]) + workspace.manifestLoader.manifests[.init(url: "org.foo", version: "1.5.1")] = + Manifest.createManifest( + displayName: "Foo", + path: packagePath.appending(component: Manifest.filename), + packageKind: .registry("org.foo"), + packageLocation: "org.foo", + toolsVersion: .current, + products: [ + try .init(name: "Foo", type: .library(.automatic), targets: ["Foo"]), + ], + targets: [ + try .init(name: "Foo"), + ] + ) + + try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + guard let foo = result.find(package: "org.foo") else { + return XCTFail("missing package") + } + XCTAssertNotNil(foo.registryMetadata, "expecting registry metadata") + XCTAssertEqual(foo.registryMetadata?.source, .registry(registryURL)) + XCTAssertMatch(foo.registryMetadata?.metadata.description, .contains("org.foo")) + XCTAssertMatch(foo.registryMetadata?.metadata.readmeURL?.absoluteString, .contains("org.foo")) + XCTAssertMatch(foo.registryMetadata?.metadata.licenseURL?.absoluteString, .contains("org.foo")) + } + } + + workspace.checkManagedDependencies { result in + result.check(dependency: "org.foo", at: .registryDownload("1.5.1")) + } + } + + func testRegistryDefaultRegistryConfiguration() async throws { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + var configuration = RegistryConfiguration() + configuration.security = .testDefault + + let registryClient = try makeRegistryClient( + packageIdentity: .plain("org.foo"), + packageVersion: "1.0.0", + configuration: configuration, + fileSystem: fs + ) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0"] + ), + ], + registryClient: registryClient, + defaultRegistry: .init( + url: "http://some-registry.com", + supportsAvailability: false + ) + ) + + try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + XCTAssertNotNil(result.find(package: "org.foo"), "missing package") + } + } + } + + // MARK: - Expected signing entity verification + + func createBasicRegistryWorkspace(metadata: [String: RegistryReleaseMetadata], mirrors: DependencyMirrors? = nil) async throws -> MockWorkspace { + let sandbox = AbsolutePath("/tmp/ws/") + let fs = InMemoryFileSystem() + + return try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + MockPackage( + name: "MyPackage", + targets: [ + MockTarget( + name: "MyTarget1", + dependencies: [ + .product(name: "Foo", package: "org.foo"), + ] + ), + MockTarget( + name: "MyTarget2", + dependencies: [ + .product(name: "Bar", package: "org.bar"), + ] + ), + ], + products: [ + MockProduct(name: "MyProduct", modules: ["MyTarget1", "MyTarget2"]), + ], + dependencies: [ + .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), + .registry(identity: "org.bar", requirement: .upToNextMajor(from: "2.0.0")), + ] + ), + ], + packages: [ + MockPackage( + name: "Foo", + identity: "org.foo", + metadata: metadata["org.foo"], + targets: [ + MockTarget(name: "Foo"), + ], + products: [ + MockProduct(name: "Foo", modules: ["Foo"]), + ], + versions: ["1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", "1.5.1"] + ), + MockPackage( + name: "Bar", + identity: "org.bar", + metadata: metadata["org.bar"], + targets: [ + MockTarget(name: "Bar"), + ], + products: [ + MockProduct(name: "Bar", modules: ["Bar"]), + ], + versions: ["2.0.0", "2.1.0", "2.2.0"] + ), + MockPackage( + name: "BarMirror", + url: "https://scm.com/org/bar-mirror", + targets: [ + MockTarget(name: "Bar"), + ], + products: [ + MockProduct(name: "Bar", modules: ["Bar"]), + ], + versions: ["2.0.0", "2.1.0", "2.2.0"] + ), + MockPackage( + name: "BarMirrorRegistry", + identity: "ecorp.bar", + metadata: metadata["ecorp.bar"], + targets: [ + MockTarget(name: "Bar"), + ], + products: [ + MockProduct(name: "Bar", modules: ["Bar"]), + ], + versions: ["2.0.0", "2.1.0", "2.2.0"] + ), + ], + mirrors: mirrors + ) + } + + func testSigningEntityVerification_SignedCorrectly() async throws { + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ") + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: ["org.bar": actualMetadata]) + + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("org.bar"): try XCTUnwrap(actualMetadata.signature?.signedBy), + ]) { _, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + } + } + + func testSigningEntityVerification_SignedIncorrectly() async throws { + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ") + ) + let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( + type: "adp", + commonName: "John Doe", + organization: "Evil Corp", + identity: "ABC" + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: ["org.bar": actualMetadata]) + + do { + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("org.bar"): expectedSigningEntity, + ]) { _, _ in } + XCTFail("should not succeed") + } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual){ + XCTAssertEqual(actual, actualMetadata.signature?.signedBy) + XCTAssertEqual(expected, expectedSigningEntity) + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func testSigningEntityVerification_Unsigned() async throws { + let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( + type: "adp", + commonName: "Jane Doe", + organization: "Example Corp", + identity: "XYZ" + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: [:]) + + do { + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("org.bar"): expectedSigningEntity, + ]) { _, _ in } + XCTFail("should not succeed") + } catch Workspace.SigningError.unsigned(_, let expected) { + XCTAssertEqual(expected, expectedSigningEntity) + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func testSigningEntityVerification_NotFound() async throws { + let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( + type: "adp", + commonName: "Jane Doe", + organization: "Example Corp", + identity: "XYZ" + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: [:]) + + do { + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("foo.bar"): expectedSigningEntity, + ]) { _, _ in } + XCTFail("should not succeed") + } catch Workspace.SigningError.expectedIdentityNotFound(let package) { + XCTAssertEqual(package.description, "foo.bar") + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func testSigningEntityVerification_MirroredSignedCorrectly() async throws { + let mirrors = try DependencyMirrors() + try mirrors.set(mirror: "ecorp.bar", for: "org.bar") + + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ") + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: ["ecorp.bar": actualMetadata], mirrors: mirrors) + + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("org.bar"): try XCTUnwrap(actualMetadata.signature?.signedBy), + ]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + XCTAssertNotNil(result.find(package: "ecorp.bar"), "missing package") + XCTAssertNil(result.find(package: "org.bar"), "unexpectedly present package") + } + } + } + + func testSigningEntityVerification_MirrorSignedIncorrectly() async throws { + let mirrors = try DependencyMirrors() + try mirrors.set(mirror: "ecorp.bar", for: "org.bar") + + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ") + ) + let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( + type: "adp", + commonName: "John Doe", + organization: "Evil Corp", + identity: "ABC" + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: ["ecorp.bar": actualMetadata], mirrors: mirrors) + + do { + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("org.bar"): expectedSigningEntity, + ]) { _, _ in } + XCTFail("should not succeed") + } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual){ + XCTAssertEqual(actual, actualMetadata.signature?.signedBy) + XCTAssertEqual(expected, expectedSigningEntity) + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func testSigningEntityVerification_MirroredUnsigned() async throws { + let mirrors = try DependencyMirrors() + try mirrors.set(mirror: "ecorp.bar", for: "org.bar") + + let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( + type: "adp", + commonName: "Jane Doe", + organization: "Example Corp", + identity: "XYZ" + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: [:], mirrors: mirrors) + + do { + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("org.bar"): expectedSigningEntity, + ]) { _, _ in } + XCTFail("should not succeed") + } catch Workspace.SigningError.unsigned(_, let expected) { + XCTAssertEqual(expected, expectedSigningEntity) + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func testSigningEntityVerification_MirroredToSCM() async throws { + let mirrors = try DependencyMirrors() + try mirrors.set(mirror: "https://scm.com/org/bar-mirror", for: "org.bar") + + let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( + type: "adp", + commonName: "Jane Doe", + organization: "Example Corp", + identity: "XYZ" + ) + + let workspace = try await createBasicRegistryWorkspace(metadata: [:], mirrors: mirrors) + + do { + try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ + PackageIdentity.plain("org.bar"): expectedSigningEntity, + ]) { _, _ in } + XCTFail("should not succeed") + } catch Workspace.SigningError.expectedSignedMirroredToSourceControl(_, let expected) { + XCTAssertEqual(expected, expectedSigningEntity) + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func makeRegistryClient( + packageIdentity: PackageIdentity, + packageVersion: Version, + targets: [String] = [], + configuration: PackageRegistry.RegistryConfiguration? = .none, + identityResolver: IdentityResolver? = .none, + fingerprintStorage: PackageFingerprintStorage? = .none, + fingerprintCheckingMode: FingerprintCheckingMode = .strict, + signingEntityStorage: PackageSigningEntityStorage? = .none, + signingEntityCheckingMode: SigningEntityCheckingMode = .strict, + authorizationProvider: AuthorizationProvider? = .none, + releasesRequestHandler: HTTPClient.Implementation? = .none, + versionMetadataRequestHandler: HTTPClient.Implementation? = .none, + manifestRequestHandler: HTTPClient.Implementation? = .none, + downloadArchiveRequestHandler: HTTPClient.Implementation? = .none, + archiver: Archiver? = .none, + fileSystem: FileSystem + ) throws -> RegistryClient { + let jsonEncoder = JSONEncoder.makeWithDefaults() + + guard let identity = packageIdentity.registry else { + throw StringError("Invalid package identifier: '\(packageIdentity)'") + } + + let configuration = configuration ?? { + var configuration = PackageRegistry.RegistryConfiguration() + configuration.defaultRegistry = .init(url: "http://localhost", supportsAvailability: false) + configuration.security = .testDefault + return configuration + }() + + let releasesRequestHandler = releasesRequestHandler ?? { _, _ in + let metadata = RegistryClient.Serialization.PackageMetadata( + releases: [packageVersion.description: .init(url: .none, problem: .none)] + ) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/json", + ], + body: try! jsonEncoder.encode(metadata) + ) + } + + let versionMetadataRequestHandler = versionMetadataRequestHandler ?? { _, _ in + let metadata = RegistryClient.Serialization.VersionMetadata( + id: packageIdentity.description, + version: packageVersion.description, + resources: [ + .init( + name: "source-archive", + type: "application/zip", + checksum: "", + signing: nil + ), + ], + metadata: .init( + description: "package \(identity) description", + licenseURL: "/\(identity)/license", + readmeURL: "/\(identity)/readme" + ), + publishedAt: nil + ) + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/json", + ], + body: try! jsonEncoder.encode(metadata) + ) + } + + let manifestRequestHandler = manifestRequestHandler ?? { _, _ in + HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "text/x-swift", + ], + body: Data("// swift-tools-version:\(ToolsVersion.current)".utf8) + ) + } + + let downloadArchiveRequestHandler = downloadArchiveRequestHandler ?? { request, _ in + switch request.kind { + case .download(let fileSystem, let destination): + // creates a dummy zipfile which is required by the archiver step + try! fileSystem.createDirectory(destination.parentDirectory, recursive: true) + try! fileSystem.writeFileContents(destination, string: "") + default: + preconditionFailure("invalid request") + } + + return HTTPClientResponse( + statusCode: 200, + headers: [ + "Content-Version": "1", + "Content-Type": "application/zip", + ], + body: Data("".utf8) + ) + } + + let archiver = archiver ?? MockArchiver(handler: { _, _, to, completion in + do { + let packagePath = to.appending("top") + try fileSystem.createDirectory(packagePath, recursive: true) + try fileSystem.writeFileContents(packagePath.appending(component: Manifest.filename), bytes: []) + try ToolsVersionSpecificationWriter.rewriteSpecification( + manifestDirectory: packagePath, + toolsVersion: .current, + fileSystem: fileSystem + ) + for target in targets { + try fileSystem.createDirectory( + packagePath.appending(components: "Sources", target), + recursive: true + ) + try fileSystem.writeFileContents( + packagePath.appending(components: ["Sources", target, "file.swift"]), + bytes: [] + ) + } + completion(.success(())) + } catch { + completion(.failure(error)) + } + }) + let fingerprintStorage = fingerprintStorage ?? MockPackageFingerprintStorage() + let signingEntityStorage = signingEntityStorage ?? MockPackageSigningEntityStorage() + + return RegistryClient( + configuration: configuration, + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: fingerprintCheckingMode, + skipSignatureValidation: false, + signingEntityStorage: signingEntityStorage, + signingEntityCheckingMode: signingEntityCheckingMode, + authorizationProvider: authorizationProvider, + customHTTPClient: HTTPClient(implementation: { request, progress in + switch request.url.path { + // request to get package releases + case "/\(identity.scope)/\(identity.name)": + try await releasesRequestHandler(request, progress) + // request to get package version metadata + case "/\(identity.scope)/\(identity.name)/\(packageVersion)": + try await versionMetadataRequestHandler(request, progress) + // request to get package manifest + case "/\(identity.scope)/\(identity.name)/\(packageVersion)/Package.swift": + try await manifestRequestHandler(request, progress) + // request to get download the version source archive + case "/\(identity.scope)/\(identity.name)/\(packageVersion).zip": + try await downloadArchiveRequestHandler(request, progress) + default: + throw StringError("unexpected url \(request.url)") + } + }), + customArchiverProvider: { _ in archiver }, + delegate: .none, + checksumAlgorithm: MockHashAlgorithm() + ) + } +} + +fileprivate extension RegistryReleaseMetadata { + static func createWithSigningEntity(_ entity: RegistryReleaseMetadata.SigningEntity) -> RegistryReleaseMetadata { + return self.init( + source: .registry(URL(string: "https://example.com")!), + metadata: .init(scmRepositoryURLs: nil), + signature: .init(signedBy: entity, format: "xyz", value: []) + ) + } +} diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 544c16be596..2e054a6e503 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -242,7 +242,7 @@ final class WorkspaceTests: XCTestCase { delegate: MockWorkspaceDelegate() ) let rootInput = PackageGraphRootInput(packages: [pkgDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -4649,146 +4649,6 @@ final class WorkspaceTests: XCTestCase { } } - func testPackageMirrorURLToRegistry() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let mirrors = try DependencyMirrors() - try mirrors.set(mirror: "org.bar-mirror", for: "https://scm.com/org/bar") - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Foo", - targets: [ - MockTarget(name: "Foo", dependencies: [ - .product(name: "Bar", package: "bar"), - .product(name: "Baz", package: "baz"), - ]), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - dependencies: [ - .sourceControl(url: "https://scm.com/org/bar", requirement: .upToNextMajor(from: "1.0.0")), - .sourceControl(url: "https://scm.com/org/baz", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "BarMirror", - identity: "org.bar-mirror", - targets: [ - MockTarget(name: "Bar"), - ], - products: [ - MockProduct(name: "Bar", modules: ["Bar"]), - ], - versions: ["1.0.0", "1.5.0"] - ), - MockPackage( - name: "Baz", - url: "https://scm.com/org/baz", - targets: [ - MockTarget(name: "Baz"), - ], - products: [ - MockProduct(name: "Baz", modules: ["Baz"]), - ], - versions: ["1.0.0", "1.6.0"] - ), - ], - mirrors: mirrors - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { graph, diagnostics in - PackageGraphTester(graph) { result in - result.check(roots: "Foo") - result.check(packages: "org.bar-mirror", "baz", "foo") - result.check(modules: "Bar", "Baz", "Foo") - } - XCTAssertNoDiagnostics(diagnostics) - } - workspace.checkManagedDependencies { result in - result.check(dependency: "org.bar-mirror", at: .registryDownload("1.5.0")) - result.check(dependency: "baz", at: .checkout(.version("1.6.0"))) - result.check(notPresent: "bar") - } - } - - func testPackageMirrorRegistryToURL() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let mirrors = try DependencyMirrors() - try mirrors.set(mirror: "https://scm.com/org/bar-mirror", for: "org.bar") - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Foo", - targets: [ - MockTarget(name: "Foo", dependencies: [ - .product(name: "Bar", package: "org.bar"), - .product(name: "Baz", package: "org.baz"), - ]), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - dependencies: [ - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.baz", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "BarMirror", - url: "https://scm.com/org/bar-mirror", - targets: [ - MockTarget(name: "Bar"), - ], - products: [ - MockProduct(name: "Bar", modules: ["Bar"]), - ], - versions: ["1.0.0", "1.5.0"] - ), - MockPackage( - name: "Baz", - identity: "org.baz", - targets: [ - MockTarget(name: "Baz"), - ], - products: [ - MockProduct(name: "Baz", modules: ["Baz"]), - ], - versions: ["1.0.0", "1.6.0"] - ), - ], - mirrors: mirrors - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { graph, diagnostics in - PackageGraphTester(graph) { result in - result.check(roots: "Foo") - result.check(packages: "bar-mirror", "org.baz", "foo") - result.check(modules: "Bar", "Baz", "Foo") - } - XCTAssertNoDiagnostics(diagnostics) - } - workspace.checkManagedDependencies { result in - result.check(dependency: "bar-mirror", at: .checkout(.version("1.5.0"))) - result.check(dependency: "org.baz", at: .registryDownload("1.6.0")) - result.check(notPresent: "org.bar") - } - } - // In this test, we get into a state where an entry in the resolved // file for a transitive dependency whose URL is later changed to // something else, while keeping the same package identity. @@ -12493,2782 +12353,96 @@ final class WorkspaceTests: XCTestCase { ) } - func testBasicResolutionFromRegistry() async throws { + func testCustomPackageContainerProvider() async throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() + let customFS = InMemoryFileSystem() + // write a manifest + try customFS.writeFileContents(.root.appending(component: Manifest.filename), bytes: "") + try ToolsVersionSpecificationWriter.rewriteSpecification( + manifestDirectory: .root, + toolsVersion: .current, + fileSystem: customFS + ) + // write the sources + let sourcesDir = AbsolutePath("/Sources") + let targetDir = sourcesDir.appending("Baz") + try customFS.createDirectory(targetDir, recursive: true) + try customFS.writeFileContents(targetDir.appending("file.swift"), bytes: "") + + let bazURL = SourceControlURL("https://example.com/baz") + let bazPackageReference = PackageReference( + identity: PackageIdentity(url: bazURL), + kind: .remoteSourceControl(bazURL) + ) + let bazContainer = MockPackageContainer( + package: bazPackageReference, + dependencies: ["1.0.0": []], + fileSystem: customFS, + customRetrievalPath: .root + ) + + let fooPath = AbsolutePath("/tmp/ws/Foo") + let fooPackageReference = PackageReference(identity: PackageIdentity(path: fooPath), kind: .root(fooPath)) + let fooContainer = MockPackageContainer(package: fooPackageReference) + let workspace = try await MockWorkspace( sandbox: sandbox, fileSystem: fs, roots: [ MockPackage( - name: "MyPackage", + name: "Foo", targets: [ - MockTarget( - name: "MyTarget1", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - MockTarget( - name: "MyTarget2", - dependencies: [ - .product(name: "Bar", package: "org.bar"), - ] - ), + MockTarget(name: "Foo", dependencies: ["Bar"]), + MockTarget(name: "Bar", dependencies: [.product(name: "Baz", package: "baz")]), + MockTarget(name: "BarTests", dependencies: ["Bar"], type: .test), ], products: [ - MockProduct(name: "MyProduct", modules: ["MyTarget1", "MyTarget2"]), + MockProduct(name: "Foo", modules: ["Foo", "Bar"]), ], dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "2.0.0")), + .sourceControl(url: bazURL, requirement: .upToNextMajor(from: "1.0.0")), ] ), ], packages: [ MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", "1.5.1"] - ), - MockPackage( - name: "Bar", - identity: "org.bar", + name: "Baz", + url: bazURL.absoluteString, targets: [ - MockTarget(name: "Bar"), + MockTarget(name: "Baz"), ], products: [ - MockProduct(name: "Bar", modules: ["Bar"]), + MockProduct(name: "Baz", modules: ["Baz"]), ], - versions: ["2.0.0", "2.1.0", "2.2.0"] + versions: ["1.0.0"] ), - ] + ], + customPackageContainerProvider: MockPackageContainerProvider(containers: [fooContainer, bazContainer]) ) - try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) + let deps: [MockDependency] = [ + .sourceControl(url: bazURL, requirement: .exact("1.0.0")), + ] + try await workspace.checkPackageGraph(roots: ["Foo"], deps: deps) { graph, diagnostics in PackageGraphTester(graph) { result in - result.check(roots: "MyPackage") - result.check(packages: "org.bar", "org.foo", "mypackage") - result.check(modules: "Foo", "Bar", "MyTarget1", "MyTarget2") - result.checkTarget("MyTarget1") { result in result.check(dependencies: "Foo") } - result.checkTarget("MyTarget2") { result in result.check(dependencies: "Bar") } + result.check(roots: "Foo") + result.check(packages: "Baz", "Foo") + result.check(modules: "Bar", "Baz", "Foo") + result.check(testModules: "BarTests") + result.checkTarget("Foo") { result in result.check(dependencies: "Bar") } + result.checkTarget("Bar") { result in result.check(dependencies: "Baz") } + result.checkTarget("BarTests") { result in result.check(dependencies: "Bar") } } + XCTAssertNoDiagnostics(diagnostics) } - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.5.1")) - result.check(dependency: "org.bar", at: .registryDownload("2.2.0")) + result.check(dependency: "baz", at: .custom(Version(1, 0, 0), .root)) } - - // Check the load-package callbacks. - XCTAssertMatch( - workspace.delegate.events, - [ - "will load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", - ] - ) - XCTAssertMatch( - workspace.delegate.events, - [ - "did load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", - ] - ) - XCTAssertMatch( - workspace.delegate.events, - ["will load manifest for registry package: org.foo (identity: org.foo)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["did load manifest for registry package: org.foo (identity: org.foo)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["will load manifest for registry package: org.bar (identity: org.bar)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["did load manifest for registry package: org.bar (identity: org.bar)"] - ) } - func testBasicTransitiveResolutionFromRegistry() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget1", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - MockTarget( - name: "MyTarget2", - dependencies: [ - .product(name: "Bar", package: "org.bar"), - ] - ), - ], - products: [ - MockProduct(name: "MyProduct", modules: ["MyTarget1", "MyTarget2"]), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "2.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget( - name: "Foo", - dependencies: [ - .product(name: "Baz", package: "org.baz"), - ] - ), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - dependencies: [ - .registry(identity: "org.baz", requirement: .range("2.0.0" ..< "4.0.0")), - ], - versions: ["1.0.0", "1.1.0"] - ), - MockPackage( - name: "Bar", - identity: "org.bar", - targets: [ - MockTarget( - name: "Bar", - dependencies: [ - .product(name: "Baz", package: "org.baz"), - ] - ), - ], - products: [ - MockProduct(name: "Bar", modules: ["Bar"]), - ], - dependencies: [ - .registry(identity: "org.baz", requirement: .upToNextMajor(from: "3.0.0")), - ], - versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] - ), - MockPackage( - name: "Baz", - identity: "org.baz", - targets: [ - MockTarget(name: "Baz"), - ], - products: [ - MockProduct(name: "Baz", modules: ["Baz"]), - ], - versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0", "3.0.0", "3.1.0"] - ), - ] - ) - - try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "MyPackage") - result.check(packages: "org.bar", "org.baz", "org.foo", "mypackage") - result.check(modules: "Foo", "Bar", "Baz", "MyTarget1", "MyTarget2") - result.checkTarget("MyTarget1") { result in result.check(dependencies: "Foo") } - result.checkTarget("MyTarget2") { result in result.check(dependencies: "Bar") } - result.checkTarget("Foo") { result in result.check(dependencies: "Baz") } - result.checkTarget("Bar") { result in result.check(dependencies: "Baz") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.1.0")) - result.check(dependency: "org.bar", at: .registryDownload("2.1.0")) - result.check(dependency: "org.baz", at: .registryDownload("3.1.0")) - } - - // Check the load-package callbacks. - XCTAssertMatch( - workspace.delegate.events, - [ - "will load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", - ] - ) - XCTAssertMatch( - workspace.delegate.events, - [ - "did load manifest for root package: \(sandbox.appending(components: "roots", "MyPackage")) (identity: mypackage)", - ] - ) - XCTAssertMatch( - workspace.delegate.events, - ["will load manifest for registry package: org.foo (identity: org.foo)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["did load manifest for registry package: org.foo (identity: org.foo)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["will load manifest for registry package: org.bar (identity: org.bar)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["did load manifest for registry package: org.bar (identity: org.bar)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["will load manifest for registry package: org.baz (identity: org.baz)"] - ) - XCTAssertMatch( - workspace.delegate.events, - ["did load manifest for registry package: org.baz (identity: org.baz)"] - ) - } - - // no dups - func testResolutionMixedRegistryAndSourceControl1() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - .product(name: "BarProduct", package: "org.bar"), - ]), - ], - products: [], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0"] - ), - MockPackage( - name: "BarPackage", - identity: "org.bar", - targets: [ - MockTarget(name: "BarTarget"), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - versions: ["1.0.0", "1.1.0"] - ), - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "foo", at: .checkout(.version("1.2.0"))) - result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .checkout(.version("1.2.0"))) - result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) - result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) - } - } - } - - // duplicate package at root level - func testResolutionMixedRegistryAndSourceControl2() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - ]), - ], - products: [], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - - - await XCTAssertAsyncThrowsError(try await workspace.checkPackageGraph(roots: ["root"]) { _, _ in - }) { error in - XCTAssertEqual((error as? PackageGraphError)?.description, "multiple packages (\'foo\' (from \'https://git/org/foo\'), \'org.foo\') declare products with a conflicting name: \'FooProduct’; product names need to be unique across the package graph") - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: "'root' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'", - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - // TODO: this error message should be improved - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: "'root' dependency on 'org.foo' conflicts with dependency on 'org.foo' which has the same identity 'org.foo'", - severity: .error - ) - } - } - } - } - - // mixed graph root --> dep1 scm - // --> dep2 scm --> dep1 registry - func testResolutionMixedRegistryAndSourceControl3() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - .product(name: "BarProduct", package: "bar"), - ]), - ], - products: [], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), - .sourceControl(url: "https://git/org/bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] - ), - MockPackage( - name: "BarPackage", - url: "https://git/org/bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ], - versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] - ), - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "2.0.0", "2.1.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - 'bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. - """), - severity: .warning - ) - } - - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .checkout(.version("1.1.0"))) - result.check(dependency: "bar", at: .checkout(.version("1.1.0"))) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.1.0")) - result.check(dependency: "bar", at: .checkout(.version("1.1.0"))) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.1.0")) - result.check(dependency: "bar", at: .checkout(.version("1.1.0"))) - } - } - } - - // mixed graph root --> dep1 scm - // --> dep2 registry --> dep1 registry - func testResolutionMixedRegistryAndSourceControl4() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - .product(name: "BarProduct", package: "org.bar"), - ]), - ], - products: [], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.1.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0"] - ), - MockPackage( - name: "BarPackage", - identity: "org.bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.2.0")), - ], - versions: ["1.0.0", "1.1.0"] - ), - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - 'org.bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. - """), - severity: .warning - ) - } - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .checkout(.version("1.2.0"))) - result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) - result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) - } - } - } - - // mixed graph root --> dep1 scm - // --> dep2 scm --> dep1 registry incompatible version - func testResolutionMixedRegistryAndSourceControl5() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - .product(name: "BarProduct", package: "bar"), - ]), - ], - products: [], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.0.0")), - .sourceControl(url: "https://git/org/bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "2.0.0"] - ), - MockPackage( - name: "BarPackage", - url: "https://git/org/bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "2.0.0")), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "2.0.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: - """ - Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'bar' 1.0.0..<2.0.0. - 'bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'bar' match the requirement 1.0.1..<2.0.0 and 'bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. - """, - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: - """ - Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'bar' 1.0.0..<2.0.0. - 'bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'bar' match the requirement 1.0.1..<2.0.0 and 'bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. - """, - severity: .error - ) - } - } - } - } - - // mixed graph root --> dep1 registry - // --> dep2 registry --> dep1 scm - func testResolutionMixedRegistryAndSourceControl6() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - .product(name: "BarProduct", package: "org.bar"), - ]), - ], - products: [], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.1.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0"] - ), - MockPackage( - name: "BarPackage", - identity: "org.bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "1.2.0")), - ], - versions: ["1.0.0", "1.1.0"] - ), - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - 'org.bar' dependency on 'https://git/org/foo' conflicts with dependency on 'org.foo' which has the same identity 'org.foo'. - """), - severity: .warning - ) - if ToolsVersion.current >= .v5_8 { - result.check( - diagnostic: .contains(""" - product 'FooProduct' required by package 'org.bar' target 'BarTarget' not found in package 'foo'. - """), - severity: .error - ) - } - } - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) - result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.2.0")) - result.check(dependency: "org.bar", at: .registryDownload("1.1.0")) - } - } - } - - // mixed graph root --> dep1 registry - // --> dep2 registry --> dep1 scm incompatible version - func testResolutionMixedRegistryAndSourceControl7() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - .product(name: "BarProduct", package: "org.bar"), - ]), - ], - products: [], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "2.0.0"] - ), - MockPackage( - name: "BarPackage", - identity: "org.bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .upToNextMajor(from: "2.0.0")), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0", "2.0.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: - """ - Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. - 'org.bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. - """, - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: - """ - Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. - 'org.bar' >= 1.0.0 practically depends on 'org.foo' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.foo' 2.0.0..<3.0.0. - """, - severity: .error - ) - } - } - } - } - - // mixed graph root --> dep1 registry --> dep3 scm - // --> dep2 registry --> dep3 registry - func testResolutionMixedRegistryAndSourceControl8() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - .product(name: "BarProduct", package: "org.bar"), - ]), - ], - products: [], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - identity: "org.foo", - targets: [ - MockTarget(name: "FooTarget", dependencies: [ - .product(name: "BazProduct", package: "baz"), - ]), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - dependencies: [ - .sourceControl(url: "https://git/org/baz", requirement: .upToNextMajor(from: "1.1.0")), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "BarPackage", - identity: "org.bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "BazProduct", package: "org.baz"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .registry(identity: "org.baz", requirement: .upToNextMajor(from: "1.0.0")), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "BazPackage", - url: "https://git/org/baz", - targets: [ - MockTarget(name: "BazTarget"), - ], - products: [ - MockProduct(name: "BazProduct", modules: ["BazTarget"]), - ], - versions: ["1.0.0", "1.1.0"] - ), - MockPackage( - name: "BazPackage", - identity: "org.baz", - alternativeURLs: ["https://git/org/baz"], - targets: [ - MockTarget(name: "BazTarget"), - ], - products: [ - MockProduct(name: "BazProduct", modules: ["BazTarget"]), - ], - versions: ["1.0.0", "1.1.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'BazTarget' appear in registry package 'org.baz' and source control package 'baz' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - 'org.foo' dependency on 'https://git/org/baz' conflicts with dependency on 'org.baz' which has the same identity 'org.baz'. - """), - severity: .warning - ) - if ToolsVersion.current >= .v5_8 { - result.check( - diagnostic: .contains(""" - product 'BazProduct' required by package 'org.foo' target 'FooTarget' not found in package 'baz'. - """), - severity: .error - ) - } - } - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.baz", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "BazTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - if ToolsVersion.current < .v5_8 { - result.checkTarget("FooTarget") { result in result.check(dependencies: "BazProduct") } - } - result.checkTarget("BarTarget") { result in result.check(dependencies: "BazProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.0.0")) - result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) - result.check(dependency: "org.baz", at: .registryDownload("1.1.0")) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.baz", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "BazTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - result.checkTarget("FooTarget") { result in result.check(dependencies: "BazProduct") } - result.checkTarget("BarTarget") { result in result.check(dependencies: "BazProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.0.0")) - result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) - result.check(dependency: "org.baz", at: .registryDownload("1.1.0")) - } - } - } - - // mixed graph root --> dep1 registry --> dep3 scm - // --> dep2 registry --> dep3 registry incompatible version - func testResolutionMixedRegistryAndSourceControl9() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - .product(name: "BarProduct", package: "org.bar"), - ]), - ], - products: [], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - identity: "org.foo", - targets: [ - MockTarget(name: "FooTarget", dependencies: [ - .product(name: "BazProduct", package: "baz"), - ]), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - dependencies: [ - .sourceControl(url: "https://git/org/baz", requirement: .upToNextMajor(from: "1.0.0")), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "BarPackage", - identity: "org.bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "BazProduct", package: "org.baz"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .registry(identity: "org.baz", requirement: .upToNextMajor(from: "2.0.0")), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "BazPackage", - url: "https://git/org/baz", - targets: [ - MockTarget(name: "BazTarget"), - ], - products: [ - MockProduct(name: "BazProduct", modules: ["BazTarget"]), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "BazPackage", - identity: "org.baz", - alternativeURLs: ["https://git/org/baz"], - targets: [ - MockTarget(name: "BazTarget"), - ], - products: [ - MockProduct(name: "BazProduct", modules: ["BazTarget"]), - ], - versions: ["1.0.0", "2.0.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'BazTarget' appear in registry package 'org.baz' and source control package 'baz' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: """ - Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. - 'org.bar' is incompatible with 'org.foo' because 'org.foo' 1.0.0 depends on 'org.baz' 1.0.0..<2.0.0 and no versions of 'org.foo' match the requirement 1.0.1..<2.0.0. - 'org.bar' >= 1.0.0 practically depends on 'org.baz' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.baz' 2.0.0..<3.0.0. - """, - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: """ - Dependencies could not be resolved because root depends on 'org.foo' 1.0.0..<2.0.0 and root depends on 'org.bar' 1.0.0..<2.0.0. - 'org.bar' is incompatible with 'org.foo' because 'org.foo' 1.0.0 depends on 'org.baz' 1.0.0..<2.0.0 and no versions of 'org.foo' match the requirement 1.0.1..<2.0.0. - 'org.bar' >= 1.0.0 practically depends on 'org.baz' 2.0.0..<3.0.0 because no versions of 'org.bar' match the requirement 1.0.1..<2.0.0 and 'org.bar' 1.0.0 depends on 'org.baz' 2.0.0..<3.0.0. - """, - severity: .error - ) - } - } - } - } - - // mixed graph root --> dep1 scm branch - // --> dep2 registry --> dep1 registry - func testResolutionMixedRegistryAndSourceControl10() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Root", - path: "root", - targets: [ - MockTarget(name: "RootTarget", dependencies: [ - .product(name: "FooProduct", package: "foo"), - .product(name: "BarProduct", package: "org.bar"), - ]), - ], - products: [], - dependencies: [ - .sourceControl(url: "https://git/org/foo", requirement: .branch("experiment")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "1.0.0")), - ], - toolsVersion: .v5_6 - ), - ], - packages: [ - MockPackage( - name: "FooPackage", - url: "https://git/org/foo", - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["experiment"] - ), - MockPackage( - name: "BarPackage", - identity: "org.bar", - targets: [ - MockTarget(name: "BarTarget", dependencies: [ - .product(name: "FooProduct", package: "org.foo"), - ]), - ], - products: [ - MockProduct(name: "BarProduct", modules: ["BarTarget"]), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ], - versions: ["1.0.0"] - ), - MockPackage( - name: "FooPackage", - identity: "org.foo", - alternativeURLs: ["https://git/org/foo"], - targets: [ - MockTarget(name: "FooTarget"), - ], - products: [ - MockProduct(name: "FooProduct", modules: ["FooTarget"]), - ], - versions: ["1.0.0"] - ), - ] - ) - - do { - workspace.sourceControlToRegistryDependencyTransformation = .disabled - try await workspace.checkPackageGraph(roots: ["root"]) { _, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - multiple similar targets 'FooTarget' appear in registry package 'org.foo' and source control package 'foo' - """), - severity: .error - ) - } - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .identity - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - 'org.bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. - """), - severity: .warning - ) - } - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .checkout(.branch("experiment"))) - result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) - } - } - - // reset - try workspace.closeWorkspace() - - do { - workspace.sourceControlToRegistryDependencyTransformation = .swizzle - - try await workspace.checkPackageGraph(roots: ["root"]) { graph, diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .contains(""" - 'org.bar' dependency on 'org.foo' conflicts with dependency on 'https://git/org/foo' which has the same identity 'org.foo'. - """), - severity: .warning - ) - } - PackageGraphTester(graph) { result in - result.check(roots: "Root") - result.check(packages: "org.bar", "org.foo", "Root") - result.check(modules: "FooTarget", "BarTarget", "RootTarget") - result - .checkTarget("RootTarget") { result in result.check(dependencies: "BarProduct", "FooProduct") } - } - } - - workspace.checkManagedDependencies { result in - result - .check( - dependency: "org.foo", - at: .checkout(.branch("experiment")) - ) // we cannot swizzle branch based deps - result.check(dependency: "org.bar", at: .registryDownload("1.0.0")) - } - } - } - - func testCustomPackageContainerProvider() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let customFS = InMemoryFileSystem() - // write a manifest - try customFS.writeFileContents(.root.appending(component: Manifest.filename), bytes: "") - try ToolsVersionSpecificationWriter.rewriteSpecification( - manifestDirectory: .root, - toolsVersion: .current, - fileSystem: customFS - ) - // write the sources - let sourcesDir = AbsolutePath("/Sources") - let targetDir = sourcesDir.appending("Baz") - try customFS.createDirectory(targetDir, recursive: true) - try customFS.writeFileContents(targetDir.appending("file.swift"), bytes: "") - - let bazURL = SourceControlURL("https://example.com/baz") - let bazPackageReference = PackageReference( - identity: PackageIdentity(url: bazURL), - kind: .remoteSourceControl(bazURL) - ) - let bazContainer = MockPackageContainer( - package: bazPackageReference, - dependencies: ["1.0.0": []], - fileSystem: customFS, - customRetrievalPath: .root - ) - - let fooPath = AbsolutePath("/tmp/ws/Foo") - let fooPackageReference = PackageReference(identity: PackageIdentity(path: fooPath), kind: .root(fooPath)) - let fooContainer = MockPackageContainer(package: fooPackageReference) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "Foo", - targets: [ - MockTarget(name: "Foo", dependencies: ["Bar"]), - MockTarget(name: "Bar", dependencies: [.product(name: "Baz", package: "baz")]), - MockTarget(name: "BarTests", dependencies: ["Bar"], type: .test), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo", "Bar"]), - ], - dependencies: [ - .sourceControl(url: bazURL, requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Baz", - url: bazURL.absoluteString, - targets: [ - MockTarget(name: "Baz"), - ], - products: [ - MockProduct(name: "Baz", modules: ["Baz"]), - ], - versions: ["1.0.0"] - ), - ], - customPackageContainerProvider: MockPackageContainerProvider(containers: [fooContainer, bazContainer]) - ) - - let deps: [MockDependency] = [ - .sourceControl(url: bazURL, requirement: .exact("1.0.0")), - ] - try await workspace.checkPackageGraph(roots: ["Foo"], deps: deps) { graph, diagnostics in - PackageGraphTester(graph) { result in - result.check(roots: "Foo") - result.check(packages: "Baz", "Foo") - result.check(modules: "Bar", "Baz", "Foo") - result.check(testModules: "BarTests") - result.checkTarget("Foo") { result in result.check(dependencies: "Bar") } - result.checkTarget("Bar") { result in result.check(dependencies: "Baz") } - result.checkTarget("BarTests") { result in result.check(dependencies: "Bar") } - } - XCTAssertNoDiagnostics(diagnostics) - } - workspace.checkManagedDependencies { result in - result.check(dependency: "baz", at: .custom(Version(1, 0, 0), .root)) - } - } - - func testRegistryMissingConfigurationErrors() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - configuration: .init(), - fileSystem: fs - ) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0"] - ), - ], - registryClient: registryClient - ) - - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check(diagnostic: .equal("no registry configured for 'org' scope"), severity: .error) - } - } - } - - func testRegistryReleasesServerErrors() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0"] - ), - ] - ) - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - releasesRequestHandler: { _, _ in - throw StringError("boom") - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal("failed fetching org.foo releases list from http://localhost: boom"), - severity: .error - ) - } - } - } - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - releasesRequestHandler: { _, _ in - .serverError() - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal( - "failed fetching org.foo releases list from http://localhost: server error 500: Internal Server Error" - ), - severity: .error - ) - } - } - } - } - - func testRegistryReleaseChecksumServerErrors() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0"] - ), - ] - ) - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - versionMetadataRequestHandler: { _, _ in - throw StringError("boom") - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal( - "failed fetching org.foo version 1.0.0 release information from http://localhost: boom" - ), - severity: .error - ) - } - } - } - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - versionMetadataRequestHandler: { _, _ in - return .serverError() - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal( - "failed fetching org.foo version 1.0.0 release information from http://localhost: server error 500: Internal Server Error" - ), - severity: .error - ) - } - } - } - } - - func testRegistryManifestServerErrors() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0"] - ), - ] - ) - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - manifestRequestHandler: { _, _ in - throw StringError("boom") - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal("failed retrieving org.foo version 1.0.0 manifest from http://localhost: boom"), - severity: .error - ) - } - } - } - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - manifestRequestHandler: { _, _ in - return .serverError() - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal( - "failed retrieving org.foo version 1.0.0 manifest from http://localhost: server error 500: Internal Server Error" - ), - severity: .error - ) - } - } - } - } - - func testRegistryDownloadServerErrors() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0"] - ), - ] - ) - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - downloadArchiveRequestHandler: { _, _ in - throw StringError("boom") - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal( - "failed downloading org.foo version 1.0.0 source archive from http://localhost: boom" - ), - severity: .error - ) - } - } - } - - do { - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - downloadArchiveRequestHandler: { _, _ in - .serverError() - }, - fileSystem: fs - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .equal( - "failed downloading org.foo version 1.0.0 source archive from http://localhost: server error 500: Internal Server Error" - ), - severity: .error - ) - } - } - } - } - - func testRegistryArchiveErrors() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - archiver: MockArchiver(handler: { _, _, _, completion in - completion(.failure(StringError("boom"))) - }), - fileSystem: fs - ) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0"] - ), - ], - registryClient: registryClient - ) - - try workspace.closeWorkspace() - workspace.registryClient = registryClient - await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in - testDiagnostics(diagnostics) { result in - result.check( - diagnostic: .regex( - "failed extracting '.*[\\\\/]registry[\\\\/]downloads[\\\\/]org[\\\\/]foo[\\\\/]1.0.0.zip' to '.*[\\\\/]registry[\\\\/]downloads[\\\\/]org[\\\\/]foo[\\\\/]1.0.0': boom" - ), - severity: .error - ) - } - } - } - - func testRegistryMetadata() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - let registryURL = URL("https://packages.example.com") - var registryConfiguration = RegistryConfiguration() - registryConfiguration.defaultRegistry = Registry(url: registryURL, supportsAvailability: false) - registryConfiguration.security = RegistryConfiguration.Security() - registryConfiguration.security!.default.signing = RegistryConfiguration.Security.Signing() - registryConfiguration.security!.default.signing!.onUnsigned = .silentAllow - - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.5.1", - targets: ["Foo"], - configuration: registryConfiguration, - fileSystem: fs - ) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - products: [ - MockProduct(name: "MyProduct", modules: ["MyTarget"]), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - registryClient: registryClient - ) - - // for mock manifest loader to work with an actual registry download - // we populate the mock manifest with a pointer to the correct download location - let defaultLocations = try Workspace.Location(forRootPackage: sandbox, fileSystem: fs) - let packagePath = defaultLocations.registryDownloadDirectory.appending(components: ["org", "foo", "1.5.1"]) - workspace.manifestLoader.manifests[.init(url: "org.foo", version: "1.5.1")] = - Manifest.createManifest( - displayName: "Foo", - path: packagePath.appending(component: Manifest.filename), - packageKind: .registry("org.foo"), - packageLocation: "org.foo", - toolsVersion: .current, - products: [ - try .init(name: "Foo", type: .library(.automatic), targets: ["Foo"]), - ], - targets: [ - try .init(name: "Foo"), - ] - ) - - try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - guard let foo = result.find(package: "org.foo") else { - return XCTFail("missing package") - } - XCTAssertNotNil(foo.registryMetadata, "expecting registry metadata") - XCTAssertEqual(foo.registryMetadata?.source, .registry(registryURL)) - XCTAssertMatch(foo.registryMetadata?.metadata.description, .contains("org.foo")) - XCTAssertMatch(foo.registryMetadata?.metadata.readmeURL?.absoluteString, .contains("org.foo")) - XCTAssertMatch(foo.registryMetadata?.metadata.licenseURL?.absoluteString, .contains("org.foo")) - } - } - - workspace.checkManagedDependencies { result in - result.check(dependency: "org.foo", at: .registryDownload("1.5.1")) - } - } - - func testRegistryDefaultRegistryConfiguration() async throws { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - var configuration = RegistryConfiguration() - configuration.security = .testDefault - - let registryClient = try makeRegistryClient( - packageIdentity: .plain("org.foo"), - packageVersion: "1.0.0", - configuration: configuration, - fileSystem: fs - ) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0"] - ), - ], - registryClient: registryClient, - defaultRegistry: .init( - url: "http://some-registry.com", - supportsAvailability: false - ) - ) - - try await workspace.checkPackageGraph(roots: ["MyPackage"]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - XCTAssertNotNil(result.find(package: "org.foo"), "missing package") - } - } - } - - // MARK: - Expected signing entity verification - - func createBasicRegistryWorkspace(metadata: [String: RegistryReleaseMetadata], mirrors: DependencyMirrors? = nil) async throws -> MockWorkspace { - let sandbox = AbsolutePath("/tmp/ws/") - let fs = InMemoryFileSystem() - - return try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - MockPackage( - name: "MyPackage", - targets: [ - MockTarget( - name: "MyTarget1", - dependencies: [ - .product(name: "Foo", package: "org.foo"), - ] - ), - MockTarget( - name: "MyTarget2", - dependencies: [ - .product(name: "Bar", package: "org.bar"), - ] - ), - ], - products: [ - MockProduct(name: "MyProduct", modules: ["MyTarget1", "MyTarget2"]), - ], - dependencies: [ - .registry(identity: "org.foo", requirement: .upToNextMajor(from: "1.0.0")), - .registry(identity: "org.bar", requirement: .upToNextMajor(from: "2.0.0")), - ] - ), - ], - packages: [ - MockPackage( - name: "Foo", - identity: "org.foo", - metadata: metadata["org.foo"], - targets: [ - MockTarget(name: "Foo"), - ], - products: [ - MockProduct(name: "Foo", modules: ["Foo"]), - ], - versions: ["1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", "1.5.1"] - ), - MockPackage( - name: "Bar", - identity: "org.bar", - metadata: metadata["org.bar"], - targets: [ - MockTarget(name: "Bar"), - ], - products: [ - MockProduct(name: "Bar", modules: ["Bar"]), - ], - versions: ["2.0.0", "2.1.0", "2.2.0"] - ), - MockPackage( - name: "BarMirror", - url: "https://scm.com/org/bar-mirror", - targets: [ - MockTarget(name: "Bar"), - ], - products: [ - MockProduct(name: "Bar", modules: ["Bar"]), - ], - versions: ["2.0.0", "2.1.0", "2.2.0"] - ), - MockPackage( - name: "BarMirrorRegistry", - identity: "ecorp.bar", - metadata: metadata["ecorp.bar"], - targets: [ - MockTarget(name: "Bar"), - ], - products: [ - MockProduct(name: "Bar", modules: ["Bar"]), - ], - versions: ["2.0.0", "2.1.0", "2.2.0"] - ), - ], - mirrors: mirrors - ) - } - - func testSigningEntityVerification_SignedCorrectly() async throws { - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: ["org.bar": actualMetadata]) - - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): try XCTUnwrap(actualMetadata.signature?.signedBy), - ]) { _, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - } - } - - func testSigningEntityVerification_SignedIncorrectly() async throws { - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") - ) - let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( - type: "adp", - commonName: "John Doe", - organization: "Evil Corp", - identity: "ABC" - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: ["org.bar": actualMetadata]) - - do { - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): expectedSigningEntity, - ]) { _, _ in } - XCTFail("should not succeed") - } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual){ - XCTAssertEqual(actual, actualMetadata.signature?.signedBy) - XCTAssertEqual(expected, expectedSigningEntity) - } catch { - XCTFail("unexpected error: \(error)") - } - } - - func testSigningEntityVerification_Unsigned() async throws { - let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( - type: "adp", - commonName: "Jane Doe", - organization: "Example Corp", - identity: "XYZ" - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: [:]) - - do { - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): expectedSigningEntity, - ]) { _, _ in } - XCTFail("should not succeed") - } catch Workspace.SigningError.unsigned(_, let expected) { - XCTAssertEqual(expected, expectedSigningEntity) - } catch { - XCTFail("unexpected error: \(error)") - } - } - - func testSigningEntityVerification_NotFound() async throws { - let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( - type: "adp", - commonName: "Jane Doe", - organization: "Example Corp", - identity: "XYZ" - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: [:]) - - do { - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("foo.bar"): expectedSigningEntity, - ]) { _, _ in } - XCTFail("should not succeed") - } catch Workspace.SigningError.expectedIdentityNotFound(let package) { - XCTAssertEqual(package.description, "foo.bar") - } catch { - XCTFail("unexpected error: \(error)") - } - } - - func testSigningEntityVerification_MirroredSignedCorrectly() async throws { - let mirrors = try DependencyMirrors() - try mirrors.set(mirror: "ecorp.bar", for: "org.bar") - - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: ["ecorp.bar": actualMetadata], mirrors: mirrors) - - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): try XCTUnwrap(actualMetadata.signature?.signedBy), - ]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - XCTAssertNotNil(result.find(package: "ecorp.bar"), "missing package") - XCTAssertNil(result.find(package: "org.bar"), "unexpectedly present package") - } - } - } - - func testSigningEntityVerification_MirrorSignedIncorrectly() async throws { - let mirrors = try DependencyMirrors() - try mirrors.set(mirror: "ecorp.bar", for: "org.bar") - - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") - ) - let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( - type: "adp", - commonName: "John Doe", - organization: "Evil Corp", - identity: "ABC" - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: ["ecorp.bar": actualMetadata], mirrors: mirrors) - - do { - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): expectedSigningEntity, - ]) { _, _ in } - XCTFail("should not succeed") - } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual){ - XCTAssertEqual(actual, actualMetadata.signature?.signedBy) - XCTAssertEqual(expected, expectedSigningEntity) - } catch { - XCTFail("unexpected error: \(error)") - } - } - - func testSigningEntityVerification_MirroredUnsigned() async throws { - let mirrors = try DependencyMirrors() - try mirrors.set(mirror: "ecorp.bar", for: "org.bar") - - let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( - type: "adp", - commonName: "Jane Doe", - organization: "Example Corp", - identity: "XYZ" - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: [:], mirrors: mirrors) - - do { - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): expectedSigningEntity, - ]) { _, _ in } - XCTFail("should not succeed") - } catch Workspace.SigningError.unsigned(_, let expected) { - XCTAssertEqual(expected, expectedSigningEntity) - } catch { - XCTFail("unexpected error: \(error)") - } - } - - func testSigningEntityVerification_MirroredToSCM() async throws { - let mirrors = try DependencyMirrors() - try mirrors.set(mirror: "https://scm.com/org/bar-mirror", for: "org.bar") - - let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( - type: "adp", - commonName: "Jane Doe", - organization: "Example Corp", - identity: "XYZ" - ) - - let workspace = try await createBasicRegistryWorkspace(metadata: [:], mirrors: mirrors) - - do { - try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): expectedSigningEntity, - ]) { _, _ in } - XCTFail("should not succeed") - } catch Workspace.SigningError.expectedSignedMirroredToSourceControl(_, let expected) { - XCTAssertEqual(expected, expectedSigningEntity) - } catch { - XCTFail("unexpected error: \(error)") - } - } - - func makeRegistryClient( - packageIdentity: PackageIdentity, - packageVersion: Version, - targets: [String] = [], - configuration: PackageRegistry.RegistryConfiguration? = .none, - identityResolver: IdentityResolver? = .none, - fingerprintStorage: PackageFingerprintStorage? = .none, - fingerprintCheckingMode: FingerprintCheckingMode = .strict, - signingEntityStorage: PackageSigningEntityStorage? = .none, - signingEntityCheckingMode: SigningEntityCheckingMode = .strict, - authorizationProvider: AuthorizationProvider? = .none, - releasesRequestHandler: HTTPClient.Implementation? = .none, - versionMetadataRequestHandler: HTTPClient.Implementation? = .none, - manifestRequestHandler: HTTPClient.Implementation? = .none, - downloadArchiveRequestHandler: HTTPClient.Implementation? = .none, - archiver: Archiver? = .none, - fileSystem: FileSystem - ) throws -> RegistryClient { - let jsonEncoder = JSONEncoder.makeWithDefaults() - - guard let identity = packageIdentity.registry else { - throw StringError("Invalid package identifier: '\(packageIdentity)'") - } - - let configuration = configuration ?? { - var configuration = PackageRegistry.RegistryConfiguration() - configuration.defaultRegistry = .init(url: "http://localhost", supportsAvailability: false) - configuration.security = .testDefault - return configuration - }() - - let releasesRequestHandler = releasesRequestHandler ?? { _, _ in - let metadata = RegistryClient.Serialization.PackageMetadata( - releases: [packageVersion.description: .init(url: .none, problem: .none)] - ) - - return HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/json", - ], - body: try! jsonEncoder.encode(metadata) - ) - } - - let versionMetadataRequestHandler = versionMetadataRequestHandler ?? { _, _ in - let metadata = RegistryClient.Serialization.VersionMetadata( - id: packageIdentity.description, - version: packageVersion.description, - resources: [ - .init( - name: "source-archive", - type: "application/zip", - checksum: "", - signing: nil - ), - ], - metadata: .init( - description: "package \(identity) description", - licenseURL: "/\(identity)/license", - readmeURL: "/\(identity)/readme" - ), - publishedAt: nil - ) - - return HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/json", - ], - body: try! jsonEncoder.encode(metadata) - ) - } - - let manifestRequestHandler = manifestRequestHandler ?? { _, _ in - HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "text/x-swift", - ], - body: Data("// swift-tools-version:\(ToolsVersion.current)".utf8) - ) - } - - let downloadArchiveRequestHandler = downloadArchiveRequestHandler ?? { request, _ in - switch request.kind { - case .download(let fileSystem, let destination): - // creates a dummy zipfile which is required by the archiver step - try! fileSystem.createDirectory(destination.parentDirectory, recursive: true) - try! fileSystem.writeFileContents(destination, string: "") - default: - preconditionFailure("invalid request") - } - - return HTTPClientResponse( - statusCode: 200, - headers: [ - "Content-Version": "1", - "Content-Type": "application/zip", - ], - body: Data("".utf8) - ) - } - - let archiver = archiver ?? MockArchiver(handler: { _, _, to, completion in - do { - let packagePath = to.appending("top") - try fileSystem.createDirectory(packagePath, recursive: true) - try fileSystem.writeFileContents(packagePath.appending(component: Manifest.filename), bytes: []) - try ToolsVersionSpecificationWriter.rewriteSpecification( - manifestDirectory: packagePath, - toolsVersion: .current, - fileSystem: fileSystem - ) - for target in targets { - try fileSystem.createDirectory( - packagePath.appending(components: "Sources", target), - recursive: true - ) - try fileSystem.writeFileContents( - packagePath.appending(components: ["Sources", target, "file.swift"]), - bytes: [] - ) - } - completion(.success(())) - } catch { - completion(.failure(error)) - } - }) - let fingerprintStorage = fingerprintStorage ?? MockPackageFingerprintStorage() - let signingEntityStorage = signingEntityStorage ?? MockPackageSigningEntityStorage() - - return RegistryClient( - configuration: configuration, - fingerprintStorage: fingerprintStorage, - fingerprintCheckingMode: fingerprintCheckingMode, - skipSignatureValidation: false, - signingEntityStorage: signingEntityStorage, - signingEntityCheckingMode: signingEntityCheckingMode, - authorizationProvider: authorizationProvider, - customHTTPClient: HTTPClient(implementation: { request, progress in - switch request.url.path { - // request to get package releases - case "/\(identity.scope)/\(identity.name)": - try await releasesRequestHandler(request, progress) - // request to get package version metadata - case "/\(identity.scope)/\(identity.name)/\(packageVersion)": - try await versionMetadataRequestHandler(request, progress) - // request to get package manifest - case "/\(identity.scope)/\(identity.name)/\(packageVersion)/Package.swift": - try await manifestRequestHandler(request, progress) - // request to get download the version source archive - case "/\(identity.scope)/\(identity.name)/\(packageVersion).zip": - try await downloadArchiveRequestHandler(request, progress) - default: - throw StringError("unexpected url \(request.url)") - } - }), - customArchiverProvider: { _ in archiver }, - delegate: .none, - checksumAlgorithm: MockHashAlgorithm() - ) - } -} +} func createDummyXCFramework(fileSystem: FileSystem, path: AbsolutePath, name: String) throws { let path = path.appending("\(name).xcframework") @@ -15309,13 +12483,3 @@ func createDummyArtifactBundle(fileSystem: FileSystem, path: AbsolutePath, name: struct DummyError: LocalizedError, Equatable { public var errorDescription: String? { "dummy error" } } - -fileprivate extension RegistryReleaseMetadata { - static func createWithSigningEntity(_ entity: RegistryReleaseMetadata.SigningEntity) -> RegistryReleaseMetadata { - return self.init( - source: .registry(URL(string: "https://example.com")!), - metadata: .init(scmRepositoryURLs: nil), - signature: .init(signedBy: entity, format: "xyz", value: []) - ) - } -} From 4f443d05156ffb870b75cde1f9f782d5f1f16546 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 4 Nov 2024 12:53:39 +0000 Subject: [PATCH 19/33] Clean up formatting --- .../WorkspaceRegistryTests.swift | 115 +++++++++++------- 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/Tests/WorkspaceTests/WorkspaceRegistryTests.swift b/Tests/WorkspaceTests/WorkspaceRegistryTests.swift index 545bf95c285..5d784e31a68 100644 --- a/Tests/WorkspaceTests/WorkspaceRegistryTests.swift +++ b/Tests/WorkspaceTests/WorkspaceRegistryTests.swift @@ -597,10 +597,12 @@ final class WorkspaceRegistryTests: XCTestCase { do { workspace.sourceControlToRegistryDependencyTransformation = .disabled - await XCTAssertAsyncThrowsError(try await workspace.checkPackageGraph(roots: ["root"]) { _, _ in }) { error in - XCTAssertEqual((error as? PackageGraphError)?.description, "multiple packages (\'foo\' (from \'https://git/org/foo\'), \'org.foo\') declare products with a conflicting name: \'FooProduct’; product names need to be unique across the package graph") + XCTAssertEqual( + (error as? PackageGraphError)?.description, + "multiple packages (\'foo\' (from \'https://git/org/foo\'), \'org.foo\') declare products with a conflicting name: \'FooProduct’; product names need to be unique across the package graph" + ) } } @@ -2000,7 +2002,7 @@ final class WorkspaceRegistryTests: XCTestCase { packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", versionMetadataRequestHandler: { _, _ in - return .serverError() + .serverError() }, fileSystem: fs ) @@ -2073,7 +2075,9 @@ final class WorkspaceRegistryTests: XCTestCase { await workspace.checkPackageGraphFailure(roots: ["MyPackage"]) { diagnostics in testDiagnostics(diagnostics) { result in result.check( - diagnostic: .equal("failed retrieving org.foo version 1.0.0 manifest from http://localhost: boom"), + diagnostic: .equal( + "failed retrieving org.foo version 1.0.0 manifest from http://localhost: boom" + ), severity: .error ) } @@ -2085,7 +2089,7 @@ final class WorkspaceRegistryTests: XCTestCase { packageIdentity: .plain("org.foo"), packageVersion: "1.0.0", manifestRequestHandler: { _, _ in - return .serverError() + .serverError() }, fileSystem: fs ) @@ -2303,17 +2307,17 @@ final class WorkspaceRegistryTests: XCTestCase { let defaultLocations = try Workspace.Location(forRootPackage: sandbox, fileSystem: fs) let packagePath = defaultLocations.registryDownloadDirectory.appending(components: ["org", "foo", "1.5.1"]) workspace.manifestLoader.manifests[.init(url: "org.foo", version: "1.5.1")] = - Manifest.createManifest( + try Manifest.createManifest( displayName: "Foo", path: packagePath.appending(component: Manifest.filename), packageKind: .registry("org.foo"), packageLocation: "org.foo", toolsVersion: .current, products: [ - try .init(name: "Foo", type: .library(.automatic), targets: ["Foo"]), + .init(name: "Foo", type: .library(.automatic), targets: ["Foo"]), ], targets: [ - try .init(name: "Foo"), + .init(name: "Foo"), ] ) @@ -2399,7 +2403,10 @@ final class WorkspaceRegistryTests: XCTestCase { // MARK: - Expected signing entity verification - func createBasicRegistryWorkspace(metadata: [String: RegistryReleaseMetadata], mirrors: DependencyMirrors? = nil) async throws -> MockWorkspace { + func createBasicRegistryWorkspace( + metadata: [String: RegistryReleaseMetadata], + mirrors: DependencyMirrors? = nil + ) async throws -> MockWorkspace { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() @@ -2486,28 +2493,32 @@ final class WorkspaceRegistryTests: XCTestCase { } func testSigningEntityVerification_SignedCorrectly() async throws { - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity( + .recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ" + ) ) let workspace = try await createBasicRegistryWorkspace(metadata: ["org.bar": actualMetadata]) try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): try XCTUnwrap(actualMetadata.signature?.signedBy), - ]) { _, diagnostics in - XCTAssertNoDiagnostics(diagnostics) + PackageIdentity.plain("org.bar"): XCTUnwrap(actualMetadata.signature?.signedBy), + ]) { _, diagnostics in + XCTAssertNoDiagnostics(diagnostics) } } func testSigningEntityVerification_SignedIncorrectly() async throws { - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity( + .recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ" + ) ) let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( type: "adp", @@ -2523,7 +2534,7 @@ final class WorkspaceRegistryTests: XCTestCase { PackageIdentity.plain("org.bar"): expectedSigningEntity, ]) { _, _ in } XCTFail("should not succeed") - } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual){ + } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual) { XCTAssertEqual(actual, actualMetadata.signature?.signedBy) XCTAssertEqual(expected, expectedSigningEntity) } catch { @@ -2579,23 +2590,28 @@ final class WorkspaceRegistryTests: XCTestCase { let mirrors = try DependencyMirrors() try mirrors.set(mirror: "ecorp.bar", for: "org.bar") - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity( + .recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ" + ) ) - let workspace = try await createBasicRegistryWorkspace(metadata: ["ecorp.bar": actualMetadata], mirrors: mirrors) + let workspace = try await createBasicRegistryWorkspace( + metadata: ["ecorp.bar": actualMetadata], + mirrors: mirrors + ) try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ - PackageIdentity.plain("org.bar"): try XCTUnwrap(actualMetadata.signature?.signedBy), - ]) { graph, diagnostics in - XCTAssertNoDiagnostics(diagnostics) - PackageGraphTester(graph) { result in - XCTAssertNotNil(result.find(package: "ecorp.bar"), "missing package") - XCTAssertNil(result.find(package: "org.bar"), "unexpectedly present package") - } + PackageIdentity.plain("org.bar"): XCTUnwrap(actualMetadata.signature?.signedBy), + ]) { graph, diagnostics in + XCTAssertNoDiagnostics(diagnostics) + PackageGraphTester(graph) { result in + XCTAssertNotNil(result.find(package: "ecorp.bar"), "missing package") + XCTAssertNil(result.find(package: "org.bar"), "unexpectedly present package") + } } } @@ -2603,11 +2619,13 @@ final class WorkspaceRegistryTests: XCTestCase { let mirrors = try DependencyMirrors() try mirrors.set(mirror: "ecorp.bar", for: "org.bar") - let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity(.recognized( - type: "adp", - commonName: "John Doe", - organization: "Example Corp", - identity: "XYZ") + let actualMetadata = RegistryReleaseMetadata.createWithSigningEntity( + .recognized( + type: "adp", + commonName: "John Doe", + organization: "Example Corp", + identity: "XYZ" + ) ) let expectedSigningEntity: RegistryReleaseMetadata.SigningEntity = .recognized( type: "adp", @@ -2616,14 +2634,17 @@ final class WorkspaceRegistryTests: XCTestCase { identity: "ABC" ) - let workspace = try await createBasicRegistryWorkspace(metadata: ["ecorp.bar": actualMetadata], mirrors: mirrors) + let workspace = try await createBasicRegistryWorkspace( + metadata: ["ecorp.bar": actualMetadata], + mirrors: mirrors + ) do { try await workspace.checkPackageGraph(roots: ["MyPackage"], expectedSigningEntities: [ PackageIdentity.plain("org.bar"): expectedSigningEntity, ]) { _, _ in } XCTFail("should not succeed") - } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual){ + } catch Workspace.SigningError.mismatchedSigningEntity(_, let expected, let actual) { XCTAssertEqual(actual, actualMetadata.signature?.signedBy) XCTAssertEqual(expected, expectedSigningEntity) } catch { @@ -2849,9 +2870,11 @@ final class WorkspaceRegistryTests: XCTestCase { } } -fileprivate extension RegistryReleaseMetadata { - static func createWithSigningEntity(_ entity: RegistryReleaseMetadata.SigningEntity) -> RegistryReleaseMetadata { - return self.init( +extension RegistryReleaseMetadata { + fileprivate static func createWithSigningEntity( + _ entity: RegistryReleaseMetadata.SigningEntity + ) -> RegistryReleaseMetadata { + self.init( source: .registry(URL(string: "https://example.com")!), metadata: .init(scmRepositoryURLs: nil), signature: .init(signedBy: entity, format: "xyz", value: []) From c204af218c74c8428d38ac1e6ecb5605486fdc1b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 4 Nov 2024 14:35:13 +0000 Subject: [PATCH 20/33] Fix up formatting --- Sources/PackageRegistry/ChecksumTOFU.swift | 7 +++---- .../PackageRegistryTests/RegistryClientTests.swift | 14 ++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Sources/PackageRegistry/ChecksumTOFU.swift b/Sources/PackageRegistry/ChecksumTOFU.swift index 4d7c9cbc1c9..ed6c58422f6 100644 --- a/Sources/PackageRegistry/ChecksumTOFU.swift +++ b/Sources/PackageRegistry/ChecksumTOFU.swift @@ -145,10 +145,9 @@ struct PackageVersionChecksumTOFU { case .strict: throw RegistryError.invalidChecksum(expected: expectedChecksum, actual: checksum) case .warn: - observabilityScope - .emit( - warning: "the checksum \(checksum) for \(contentType) of \(package) \(version) does not match previously recorded value \(expectedChecksum)" - ) + observabilityScope.emit( + warning: "the checksum \(checksum) for \(contentType) of \(package) \(version) does not match previously recorded value \(expectedChecksum)" + ) } } } diff --git a/Tests/PackageRegistryTests/RegistryClientTests.swift b/Tests/PackageRegistryTests/RegistryClientTests.swift index b89ba36d5cd..1c19e616746 100644 --- a/Tests/PackageRegistryTests/RegistryClientTests.swift +++ b/Tests/PackageRegistryTests/RegistryClientTests.swift @@ -931,14 +931,12 @@ final class RegistryClientTests: XCTestCase { let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient) await XCTAssertAsyncThrowsError(try await registryClient.getAvailableManifests(package: identity, version: version)) { error in - guard case RegistryError - .failedRetrievingManifest( - registry: configuration.defaultRegistry!, - package: identity, - version: version, - error: RegistryError.packageVersionNotFound - ) = error - else { + guard case RegistryError.failedRetrievingManifest( + registry: configuration.defaultRegistry!, + package: identity, + version: version, + error: RegistryError.packageVersionNotFound + ) = error else { return XCTFail("unexpected error: '\(error)'") } } From 885f314e380193853e7906403976ee37965f3053 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 4 Nov 2024 17:23:17 +0000 Subject: [PATCH 21/33] Refactor some of `RegistryDownloadsManagerTests` for Swift concurrency --- .../HTTPClient/HTTPClientConfiguration.swift | 3 +- Sources/PackageRegistry/RegistryClient.swift | 275 +++++++++++------- .../RegistryClientTests.swift | 9 +- .../RegistryDownloadsManagerTests.swift | 236 +++++++-------- 4 files changed, 292 insertions(+), 231 deletions(-) diff --git a/Sources/Basics/HTTPClient/HTTPClientConfiguration.swift b/Sources/Basics/HTTPClient/HTTPClientConfiguration.swift index 1c44c99fd67..e3e192a9d05 100644 --- a/Sources/Basics/HTTPClient/HTTPClientConfiguration.swift +++ b/Sources/Basics/HTTPClient/HTTPClientConfiguration.swift @@ -14,8 +14,7 @@ import Foundation public struct HTTPClientConfiguration: Sendable { // FIXME: this should be unified with ``AuthorizationProvider`` protocol or renamed to avoid unintended shadowing. - public typealias AuthorizationProvider = @Sendable (URL) - -> String? + public typealias AuthorizationProvider = @Sendable (URL) -> String? public init( requestHeaders: HTTPClientHeaders? = nil, diff --git a/Sources/PackageRegistry/RegistryClient.swift b/Sources/PackageRegistry/RegistryClient.swift index bd9f760b4d0..b6fb6aad9fd 100644 --- a/Sources/PackageRegistry/RegistryClient.swift +++ b/Sources/PackageRegistry/RegistryClient.swift @@ -40,7 +40,7 @@ public final class RegistryClient { private var configuration: RegistryConfiguration private let archiverProvider: (FileSystem) -> Archiver private let httpClient: HTTPClient - private let authorizationProvider: LegacyHTTPClientConfiguration.AuthorizationProvider? + private let authorizationProvider: HTTPClientConfiguration.AuthorizationProvider? private let fingerprintStorage: PackageFingerprintStorage? private let fingerprintCheckingMode: FingerprintCheckingMode private let skipSignatureValidation: Bool @@ -515,55 +515,77 @@ public final class RegistryClient { ) let start = DispatchTime.now() - observabilityScope - .emit(info: "retrieving available manifests for \(package) \(version) from \(request.url)") + observabilityScope.emit(info: "retrieving available manifests for \(package) \(version) from \(request.url)") + let response: HTTPClientResponse do { - let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) observabilityScope.emit( debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" ) + } catch { + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, + version: version, + error: error + ) + } - switch response.statusCode { - case 200: + switch response.statusCode { + case 200: + let data: Data + let manifestContent: String + + do { try response.validateAPIVersion() try response.validateContentType(.swift) - guard let data = response.body else { + guard let responseBody = response.body else { throw RegistryError.invalidResponse } - let manifestContent = String(decoding: data, as: UTF8.self) + data = responseBody - _ = try await signatureValidation.validate( + manifestContent = String(decoding: data, as: UTF8.self) + } catch { + throw RegistryError.failedRetrievingManifest( registry: registry, - package: package, + package: package.underlying, version: version, - toolsVersion: .none, - manifestContent: manifestContent, - configuration: self.configuration.signing(for: package, registry: registry), - timeout: timeout, - fileSystem: localFileSystem, - observabilityScope: observabilityScope + error: error ) + } - // TODO: expose Data based API on checksumAlgorithm - let actualChecksum = self.checksumAlgorithm.hash(.init(data)) - .hexadecimalRepresentation + _ = try await signatureValidation.validate( + registry: registry, + package: package, + version: version, + toolsVersion: .none, + manifestContent: manifestContent, + configuration: self.configuration.signing(for: package, registry: registry), + timeout: timeout, + fileSystem: localFileSystem, + observabilityScope: observabilityScope + ) - try checksumTOFU.validateManifest( - registry: registry, - package: package, - version: version, - toolsVersion: .none, - checksum: actualChecksum, - timeout: timeout, - observabilityScope: observabilityScope - ) + // TODO: expose Data based API on checksumAlgorithm + let actualChecksum = self.checksumAlgorithm.hash(.init(data)) + .hexadecimalRepresentation + + try checksumTOFU.validateManifest( + registry: registry, + package: package, + version: version, + toolsVersion: .none, + checksum: actualChecksum, + timeout: timeout, + observabilityScope: observabilityScope + ) + do { var result = [String: (toolsVersion: ToolsVersion, content: String?)]() - let toolsVersion = try ToolsVersionParser - .parse(utf8String: manifestContent) + let toolsVersion = try ToolsVersionParser.parse(utf8String: manifestContent) result[Manifest.filename] = ( toolsVersion: toolsVersion, content: manifestContent @@ -577,19 +599,29 @@ public final class RegistryClient { ) } return result + } catch { + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, + version: version, + error: error + ) + } - case 404: - throw RegistryError.packageVersionNotFound + case 404: + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, + version: version, + error: RegistryError.packageVersionNotFound + ) - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - } - } catch { + default: throw RegistryError.failedRetrievingManifest( registry: registry, package: package.underlying, version: version, - error: error + error: self.unexpectedStatusError(response, expectedStatus: [200, 404]) ) } } @@ -701,22 +733,37 @@ public final class RegistryClient { let start = DispatchTime.now() observabilityScope.emit(info: "retrieving \(package) \(version) manifest from \(request.url)") + let response: HTTPClientResponse do { - let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) + response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + } catch { + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, + version: version, + error: error + ) + } + switch response.statusCode { case 200: - try response.validateAPIVersion(isOptional: true) - try response.validateContentType(.swift) + let data: Data - guard let data = response.body else { - throw RegistryError.invalidResponse + do { + try response.validateAPIVersion(isOptional: true) + try response.validateContentType(.swift) + + guard let responseBody = response.body else { + throw RegistryError.invalidResponse + } + + data = responseBody } - let manifestContent = String(decoding: data, as: UTF8.self) + let manifestContent = String(decoding: data, as: UTF8.self) _ = try await signatureValidation.validate( registry: registry, package: package, @@ -730,8 +777,7 @@ public final class RegistryClient { ) // TODO: expose Data based API on checksumAlgorithm - let actualChecksum = self.checksumAlgorithm.hash(.init(data)) - .hexadecimalRepresentation + let actualChecksum = self.checksumAlgorithm.hash(.init(data)).hexadecimalRepresentation try checksumTOFU.validateManifest( registry: registry, @@ -746,18 +792,21 @@ public final class RegistryClient { return manifestContent case 404: - throw RegistryError.packageVersionNotFound + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, + version: version, + error: RegistryError.packageVersionNotFound + ) default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) + throw RegistryError.failedRetrievingManifest( + registry: registry, + package: package.underlying, + version: version, + error: self.unexpectedStatusError(response, expectedStatus: [200, 404]) + ) } - } catch { - throw RegistryError.failedRetrievingManifest( - registry: registry, - package: package.underlying, - version: version, - error: error - ) - } + } public func downloadSourceArchive( @@ -878,45 +927,55 @@ public final class RegistryClient { let downloadStart = DispatchTime.now() observabilityScope.emit(info: "downloading \(package) \(version) source archive from \(request.url)") + + let response: HTTPClientResponse do { - let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: progressHandler) + response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: progressHandler) observabilityScope.emit( debug: "server response for \(request.url): \(response.statusCode) in \(downloadStart.distance(to: .now()).descriptionInSeconds)" ) + } catch { + throw RegistryError.failedDownloadingSourceArchive( + registry: registry, + package: package.underlying, + version: version, + error: error + ) + } - switch response.statusCode { - case 200: - try response.validateAPIVersion(isOptional: true) - try response.validateContentType(.zip) + switch response.statusCode { + case 200: + try response.validateAPIVersion(isOptional: true) + try response.validateContentType(.zip) - let archiveContent: Data = try fileSystem.readFileContents(downloadPath) - // TODO: expose Data based API on checksumAlgorithm - let actualChecksum = self.checksumAlgorithm.hash(.init(archiveContent)) - .hexadecimalRepresentation + let archiveContent: Data = try fileSystem.readFileContents(downloadPath) + // TODO: expose Data based API on checksumAlgorithm + let actualChecksum = self.checksumAlgorithm.hash(.init(archiveContent)).hexadecimalRepresentation - observabilityScope.emit( - debug: "performing TOFU checks on \(package) \(version) source archive (checksum: '\(actualChecksum)')" - ) - let signingEntity = try await signatureValidation.validate( - registry: registry, - package: package, - version: version, - content: archiveContent, - configuration: self.configuration.signing(for: package, registry: registry), - timeout: timeout, - fileSystem: fileSystem, - observabilityScope: observabilityScope - ) + observabilityScope.emit( + debug: "performing TOFU checks on \(package) \(version) source archive (checksum: '\(actualChecksum)')" + ) + let signingEntity = try await signatureValidation.validate( + registry: registry, + package: package, + version: version, + content: archiveContent, + configuration: self.configuration.signing(for: package, registry: registry), + timeout: timeout, + fileSystem: fileSystem, + observabilityScope: observabilityScope + ) - try await checksumTOFU.validateSourceArchive( - registry: registry, - package: package, - version: version, - checksum: actualChecksum, - timeout: timeout, - observabilityScope: observabilityScope - ) + try await checksumTOFU.validateSourceArchive( + registry: registry, + package: package, + version: version, + checksum: actualChecksum, + timeout: timeout, + observabilityScope: observabilityScope + ) + do { // validate that the destination does not already exist // (again, as this is async) guard !fileSystem.exists(destinationPath) else { @@ -928,18 +987,14 @@ public final class RegistryClient { ) // extract the content let extractStart = DispatchTime.now() - observabilityScope - .emit( - debug: "extracting \(package) \(version) source archive to '\(destinationPath)'" - ) + observabilityScope.emit( + debug: "extracting \(package) \(version) source archive to '\(destinationPath)'" + ) let archiver = self.archiverProvider(fileSystem) do { // TODO: Bail if archive contains relative paths or overlapping files - try await archiver.extract( - from: downloadPath, - to: destinationPath - ) + try await archiver.extract(from: downloadPath, to: destinationPath) defer { try? fileSystem.removeFileTree(downloadPath) } @@ -968,20 +1023,28 @@ public final class RegistryClient { "failed extracting '\(downloadPath)' to '\(destinationPath)': \(error.interpolationDescription)" ) } - - case 404: - throw RegistryError.packageVersionNotFound - - default: - throw self.unexpectedStatusError(response, expectedStatus: [200, 404]) - + } catch { + throw RegistryError.failedDownloadingSourceArchive( + registry: registry, + package: package.underlying, + version: version, + error: error + ) } - } catch { + case 404: throw RegistryError.failedDownloadingSourceArchive( registry: registry, package: package.underlying, version: version, - error: error + error: RegistryError.packageVersionNotFound + ) + + default: + throw RegistryError.failedDownloadingSourceArchive( + registry: registry, + package: package.underlying, + version: version, + error: self.unexpectedStatusError(response, expectedStatus: [200, 404]) ) } } diff --git a/Tests/PackageRegistryTests/RegistryClientTests.swift b/Tests/PackageRegistryTests/RegistryClientTests.swift index 1c19e616746..9cf843dba6e 100644 --- a/Tests/PackageRegistryTests/RegistryClientTests.swift +++ b/Tests/PackageRegistryTests/RegistryClientTests.swift @@ -1720,17 +1720,14 @@ final class RegistryClientTests: XCTestCase { let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient) await XCTAssertAsyncThrowsError( - try await registryClient - .getManifestContent(package: identity, version: version, customToolsVersion: nil) + try await registryClient.getManifestContent(package: identity, version: version, customToolsVersion: nil) ) { error in - guard case RegistryError - .failedRetrievingManifest( + guard case RegistryError.failedRetrievingManifest( registry: configuration.defaultRegistry!, package: identity, version: version, error: RegistryError.packageVersionNotFound - ) = error - else { + ) = error else { return XCTFail("unexpected error: '\(error)'") } } diff --git a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift index b0fb1d9ecd5..5999996a40a 100644 --- a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift @@ -57,20 +57,19 @@ final class RegistryDownloadsManagerTests: XCTestCase { // try to get a package do { - delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try delegate.wait(timeout: .now() + 2) - XCTAssertEqual(delegate.willFetch.count, 1) - XCTAssertEqual(delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(delegate.willFetch.first?.fetchDetails, .init(fromCache: false, updatedCache: false)) + await delegate.consume() + await XCTAssertAsyncEqual(await delegate.willFetch.count, 1) + await XCTAssertAsyncEqual(await delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(await delegate.willFetch.first?.fetchDetails, .init(fromCache: false, updatedCache: false)) - XCTAssertEqual(delegate.didFetch.count, 1) - XCTAssertEqual(delegate.didFetch.first?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(try! delegate.didFetch.first?.result.get(), .init(fromCache: false, updatedCache: false)) + await XCTAssertAsyncEqual(await delegate.didFetch.count, 1) + await XCTAssertAsyncEqual(await delegate.didFetch.first?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(try! await delegate.didFetch.first?.result.get(), .init(fromCache: false, updatedCache: false)) } // try to get a package that does not exist @@ -79,47 +78,45 @@ final class RegistryDownloadsManagerTests: XCTestCase { let unknownPackageVersion: Version = "1.0.0" do { - delegate.prepare(fetchExpected: true) await XCTAssertAsyncThrowsError(try await manager.lookup(package: unknownPackage, version: unknownPackageVersion, observabilityScope: observability.topScope)) { error in XCTAssertNotNil(error as? RegistryError) } - try delegate.wait(timeout: .now() + 2) - XCTAssertEqual(delegate.willFetch.map { ($0.packageVersion) }, - [ - (PackageVersion(package: package, version: packageVersion)), - (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) - ] + await delegate.consume() + await XCTAssertAsyncEqual(await delegate.willFetch.map { ($0.packageVersion) }, + [ + (PackageVersion(package: package, version: packageVersion)), + (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) + ] ) - XCTAssertEqual(delegate.didFetch.map { ($0.packageVersion) }, - [ - (PackageVersion(package: package, version: packageVersion)), - (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) - ] + await XCTAssertAsyncEqual(await delegate.didFetch.map { ($0.packageVersion) }, + [ + (PackageVersion(package: package, version: packageVersion)), + (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) + ] ) } // try to get the existing package again, no fetching expected this time do { - delegate.prepare(fetchExpected: false) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try delegate.wait(timeout: .now() + 2) - XCTAssertEqual(delegate.willFetch.map { ($0.packageVersion) }, - [ - (PackageVersion(package: package, version: packageVersion)), - (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) - ] + await delegate.consume() + await XCTAssertAsyncEqual(await delegate.willFetch.map { ($0.packageVersion) }, + [ + (PackageVersion(package: package, version: packageVersion)), + (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) + ] ) - XCTAssertEqual(delegate.didFetch.map { ($0.packageVersion) }, - [ - (PackageVersion(package: package, version: packageVersion)), - (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) - ] + await XCTAssertAsyncEqual(await delegate.didFetch.map { ($0.packageVersion) }, + [ + (PackageVersion(package: package, version: packageVersion)), + (PackageVersion(package: unknownPackage, version: unknownPackageVersion)) + ] ) } @@ -128,26 +125,27 @@ final class RegistryDownloadsManagerTests: XCTestCase { do { try manager.remove(package: package) - delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try delegate.wait(timeout: .now() + 2) - XCTAssertEqual(delegate.willFetch.map { ($0.packageVersion) }, - [ - (PackageVersion(package: package, version: packageVersion)), - (PackageVersion(package: unknownPackage, version: unknownPackageVersion)), - (PackageVersion(package: package, version: packageVersion)) - ] + await delegate.consume() + await XCTAssertAsyncEqual( + await delegate.willFetch.map { ($0.packageVersion) }, + [ + (PackageVersion(package: package, version: packageVersion)), + (PackageVersion(package: unknownPackage, version: unknownPackageVersion)), + (PackageVersion(package: package, version: packageVersion)) + ] ) - XCTAssertEqual(delegate.didFetch.map { ($0.packageVersion) }, - [ - (PackageVersion(package: package, version: packageVersion)), - (PackageVersion(package: unknownPackage, version: unknownPackageVersion)), - (PackageVersion(package: package, version: packageVersion)) - ] + await XCTAssertAsyncEqual( + await delegate.didFetch.map { ($0.packageVersion) }, + [ + (PackageVersion(package: package, version: packageVersion)), + (PackageVersion(package: unknownPackage, version: unknownPackageVersion)), + (PackageVersion(package: package, version: packageVersion)) + ] ) } } @@ -189,44 +187,42 @@ final class RegistryDownloadsManagerTests: XCTestCase { // try to get a package do { - delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) XCTAssertTrue(fs.isDirectory(cachePath.appending(components: package.registry!.scope.description, package.registry!.name.description, packageVersion.description))) - try delegate.wait(timeout: .now() + 2) + await delegate.consume() - XCTAssertEqual(delegate.willFetch.count, 1) - XCTAssertEqual(delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(delegate.willFetch.first?.fetchDetails, .init(fromCache: false, updatedCache: false)) + await XCTAssertAsyncEqual(await delegate.willFetch.count, 1) + await XCTAssertAsyncEqual(await delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(await delegate.willFetch.first?.fetchDetails, .init(fromCache: false, updatedCache: false)) - XCTAssertEqual(delegate.didFetch.count, 1) - XCTAssertEqual(delegate.didFetch.first?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(try! delegate.didFetch.first?.result.get(), .init(fromCache: true, updatedCache: true)) + await XCTAssertAsyncEqual(await delegate.didFetch.count, 1) + await XCTAssertAsyncEqual(await delegate.didFetch.first?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(try! await delegate.didFetch.first?.result.get(), .init(fromCache: true, updatedCache: true)) } // remove the "local" package, should come from cache do { - try await manager.remove(package: package) + try manager.remove(package: package) - delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try delegate.wait(timeout: .now() + 2) + await delegate.consume() - XCTAssertEqual(delegate.willFetch.count, 2) - XCTAssertEqual(delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(delegate.willFetch.last?.fetchDetails, .init(fromCache: true, updatedCache: false)) + await XCTAssertAsyncEqual(await delegate.willFetch.count, 2) + await XCTAssertAsyncEqual(await delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(await delegate.willFetch.last?.fetchDetails, .init(fromCache: true, updatedCache: false)) - XCTAssertEqual(delegate.didFetch.count, 2) - XCTAssertEqual(delegate.didFetch.last?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(try! delegate.didFetch.last?.result.get(), .init(fromCache: true, updatedCache: false)) + await XCTAssertAsyncEqual(await delegate.didFetch.count, 2) + await XCTAssertAsyncEqual(await delegate.didFetch.last?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(try! await delegate.didFetch.last?.result.get(), .init(fromCache: true, updatedCache: false)) } // remove the "local" package, and purge cache @@ -235,21 +231,20 @@ final class RegistryDownloadsManagerTests: XCTestCase { try manager.remove(package: package) manager.purgeCache(observabilityScope: observability.topScope) - delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try delegate.wait(timeout: .now() + 2) + await delegate.consume() - XCTAssertEqual(delegate.willFetch.count, 3) - XCTAssertEqual(delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(delegate.willFetch.last?.fetchDetails, .init(fromCache: false, updatedCache: false)) + await XCTAssertAsyncEqual(await delegate.willFetch.count, 3) + await XCTAssertAsyncEqual(await delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(await delegate.willFetch.last?.fetchDetails, .init(fromCache: false, updatedCache: false)) - XCTAssertEqual(delegate.didFetch.count, 3) - XCTAssertEqual(delegate.didFetch.last?.packageVersion, .init(package: package, version: packageVersion)) - XCTAssertEqual(try! delegate.didFetch.last?.result.get(), .init(fromCache: true, updatedCache: true)) + await XCTAssertAsyncEqual(await delegate.didFetch.count, 3) + await XCTAssertAsyncEqual(await delegate.didFetch.last?.packageVersion, .init(package: package, version: packageVersion)) + await XCTAssertAsyncEqual(try! await delegate.didFetch.last?.result.get(), .init(fromCache: true, updatedCache: true)) } } @@ -294,16 +289,20 @@ final class RegistryDownloadsManagerTests: XCTestCase { try await withThrowingTaskGroup(of: Void.self) { group in for packageVersion in packageVersions { group.addTask { - delegate.prepare(fetchExpected: true) - results[packageVersion] = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope, delegateQueue: .sharedConcurrent) + results[packageVersion] = try await manager.lookup( + package: package, + version: packageVersion, + observabilityScope: observability.topScope, + delegateQueue: .sharedConcurrent + ) } } try await group.waitForAll() } - try delegate.wait(timeout: .now() + 2) - XCTAssertEqual(delegate.willFetch.count, concurrency) - XCTAssertEqual(delegate.didFetch.count, concurrency) + await delegate.consume() + await XCTAssertAsyncEqual(await delegate.willFetch.count, concurrency) + await XCTAssertAsyncEqual(await delegate.didFetch.count, concurrency) XCTAssertEqual(results.count, concurrency) for packageVersion in packageVersions { @@ -328,22 +327,21 @@ final class RegistryDownloadsManagerTests: XCTestCase { source: packageSource ) - delegate.reset() + await delegate.reset() let results = ThreadSafeKeyValueStore() try await withThrowingTaskGroup(of: Void.self) { group in for index in 0 ..< concurrency { group.addTask { - delegate.prepare(fetchExpected: index < concurrency / repeatRatio) - let packageVersion = Version(index % (concurrency / repeatRatio), 0 , 0) + let packageVersion = Version(index % (concurrency / repeatRatio), 0, 0) results[packageVersion] = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope, delegateQueue: .sharedConcurrent) } } try await group.waitForAll() } - try delegate.wait(timeout: .now() + 2) - XCTAssertEqual(delegate.willFetch.count, concurrency / repeatRatio) - XCTAssertEqual(delegate.didFetch.count, concurrency / repeatRatio) + await delegate.consume() + await XCTAssertAsyncEqual(await delegate.willFetch.count, concurrency / repeatRatio) + await XCTAssertAsyncEqual(await delegate.didFetch.count, concurrency / repeatRatio) XCTAssertEqual(results.count, concurrency / repeatRatio) for packageVersion in packageVersions { @@ -354,58 +352,62 @@ final class RegistryDownloadsManagerTests: XCTestCase { } } -private class MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDelegate { - private var _willFetch = [(packageVersion: PackageVersion, fetchDetails: RegistryDownloadsManager.FetchDetails)]() - private var _didFetch = [(packageVersion: PackageVersion, result: Result)]() +private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDelegate { + typealias WillFetch = (packageVersion: PackageVersion, fetchDetails: RegistryDownloadsManager.FetchDetails) + typealias DidFetch = (packageVersion: PackageVersion, result: Result) + + private(set) var willFetch = [WillFetch]() + private(set) var didFetch = [DidFetch]() + + private var expectedFetches = 0 + + private nonisolated let willFetchContinuation: AsyncStream.Continuation + private var willFetchStream: AsyncStream + + private nonisolated let didFetchContinuation: AsyncStream.Continuation + private var didFetchStream: AsyncStream - private let lock = NSLock() - private var group = DispatchGroup() + init() { + (willFetchStream, willFetchContinuation) = AsyncStream.makeStream() + (didFetchStream, didFetchContinuation) = AsyncStream.makeStream() + } - public func prepare(fetchExpected: Bool) { + func prepare(fetchExpected: Bool) { if fetchExpected { - group.enter() // will fetch - group.enter() // did fetch + expectedFetches += 1 } } - public func reset() { - self.group = DispatchGroup() - self._willFetch = [] - self._didFetch = [] - } + func consume() async { + var elementsToFetch = expectedFetches + for await element in willFetchStream where elementsToFetch > 0 { + self.willFetch.append(element) + elementsToFetch -= 1 + } - public func wait(timeout: DispatchTime) throws { - switch group.wait(timeout: timeout) { - case .success: - return - case .timedOut: - throw StringError("timeout") + elementsToFetch = expectedFetches + for await element in didFetchStream where elementsToFetch > 0 { + self.didFetch.append(element) + elementsToFetch -= 1 } - } - var willFetch: [(packageVersion: PackageVersion, fetchDetails: RegistryDownloadsManager.FetchDetails)] { - return self.lock.withLock { _willFetch } + expectedFetches = 0 } - var didFetch: [(packageVersion: PackageVersion, result: Result)] { - return self.lock.withLock { _didFetch } + func reset() { + self.willFetch = [] + self.didFetch = [] } - func willFetch(package: PackageIdentity, version: Version, fetchDetails: RegistryDownloadsManager.FetchDetails) { - self.lock.withLock { - _willFetch += [(PackageVersion(package: package, version: version), fetchDetails: fetchDetails)] - } - self.group.leave() + nonisolated func willFetch(package: PackageIdentity, version: Version, fetchDetails: RegistryDownloadsManager.FetchDetails) { + willFetchContinuation.yield((PackageVersion(package: package, version: version), fetchDetails: fetchDetails)) } - func didFetch(package: PackageIdentity, version: Version, result: Result, duration: DispatchTimeInterval) { - self.lock.withLock { - _didFetch += [(PackageVersion(package: package, version: version), result: result)] - } - self.group.leave() + nonisolated func didFetch(package: PackageIdentity, version: Version, result: Result, duration: DispatchTimeInterval) { + didFetchContinuation.yield((PackageVersion(package: package, version: version), result: result)) } - func fetching(package: PackageIdentity, version: Version, bytesDownloaded downloaded: Int64, totalBytesToDownload total: Int64?) { + nonisolated func fetching(package: PackageIdentity, version: Version, bytesDownloaded downloaded: Int64, totalBytesToDownload total: Int64?) { } } From 83355bf6704b1f8e3f8d0774f4ead44ca6d12d4f Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 5 Nov 2024 16:37:28 +0000 Subject: [PATCH 22/33] Fix most of `RegistryDownloadsManagerTests` --- .../RegistryDownloadsManager.swift | 33 +++----- Sources/Workspace/Workspace+Registry.swift | 3 +- .../RegistryDownloadsManagerTests.swift | 84 ++++++++++++------- 3 files changed, 67 insertions(+), 53 deletions(-) diff --git a/Sources/PackageRegistry/RegistryDownloadsManager.swift b/Sources/PackageRegistry/RegistryDownloadsManager.swift index a2fcba77bdf..b3576edee0e 100644 --- a/Sources/PackageRegistry/RegistryDownloadsManager.swift +++ b/Sources/PackageRegistry/RegistryDownloadsManager.swift @@ -47,8 +47,7 @@ public actor RegistryDownloadsManager { public func lookup( package: PackageIdentity, version: Version, - observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws -> AbsolutePath { let packageRelativePath: RelativePath let packagePath: AbsolutePath @@ -75,10 +74,8 @@ public actor RegistryDownloadsManager { // calculate if cached (for delegate call) outside queue as it may change while queue is processing let isCached = self.cachePath.map { self.fileSystem.exists($0.appending(packageRelativePath)) } ?? false let delegate = self.delegate - delegateQueue.async { - let details = FetchDetails(fromCache: isCached, updatedCache: false) - delegate?.willFetch(package: package, version: version, fetchDetails: details) - } + let details = FetchDetails(fromCache: isCached, updatedCache: false) + delegate?.willFetch(package: package, version: version, fetchDetails: details) // make sure destination is free. try? self.fileSystem.removeFileTree(packagePath) @@ -92,8 +89,7 @@ public actor RegistryDownloadsManager { package: package, version: version, packagePath: packagePath, - observabilityScope: observabilityScope, - delegateQueue: delegateQueue + observabilityScope: observabilityScope )) } catch { result = .failure(error) @@ -101,9 +97,7 @@ public actor RegistryDownloadsManager { // inform delegate that we finished to fetch let duration = start.distance(to: .now()) - delegateQueue.async { - delegate?.didFetch(package: package, version: version, result: result, duration: duration) - } + delegate?.didFetch(package: package, version: version, result: result, duration: duration) // remove the pending lookup defer { @@ -126,8 +120,7 @@ public actor RegistryDownloadsManager { package: PackageIdentity, version: Version, packagePath: AbsolutePath, - observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue + observabilityScope: ObservabilityScope ) async throws -> FetchDetails { if let cachePath { do { @@ -213,14 +206,12 @@ public actor RegistryDownloadsManager { @Sendable func updateDownloadProgress(downloaded: Int64, total: Int64?) { - delegateQueue.async { - self.delegate?.fetching( - package: package, - version: version, - bytesDownloaded: downloaded, - totalBytesToDownload: total - ) - } + self.delegate?.fetching( + package: package, + version: version, + bytesDownloaded: downloaded, + totalBytesToDownload: total + ) } } diff --git a/Sources/Workspace/Workspace+Registry.swift b/Sources/Workspace/Workspace+Registry.swift index 7eec1c5edb8..571068546c1 100644 --- a/Sources/Workspace/Workspace+Registry.swift +++ b/Sources/Workspace/Workspace+Registry.swift @@ -374,8 +374,7 @@ extension Workspace { let downloadPath = try await self.registryDownloadsManager.lookup( package: package.identity, version: version, - observabilityScope: observabilityScope, - delegateQueue: .sharedConcurrent + observabilityScope: observabilityScope ) // Record the new state. diff --git a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift index 5999996a40a..8292086112f 100644 --- a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift @@ -20,6 +20,7 @@ import XCTest import struct TSCUtility.Version +@available(macOS 15.0, *) final class RegistryDownloadsManagerTests: XCTestCase { func testNoCache() async throws { let observability = ObservabilitySystem.makeForTesting() @@ -57,6 +58,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { // try to get a package do { + await delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) @@ -78,6 +80,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { let unknownPackageVersion: Version = "1.0.0" do { + await delegate.prepare(fetchExpected: true) await XCTAssertAsyncThrowsError(try await manager.lookup(package: unknownPackage, version: unknownPackageVersion, observabilityScope: observability.topScope)) { error in XCTAssertNotNil(error as? RegistryError) } @@ -100,6 +103,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { // try to get the existing package again, no fetching expected this time do { + await delegate.prepare(fetchExpected: false) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) @@ -125,6 +129,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { do { try manager.remove(package: package) + await delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) @@ -164,7 +169,10 @@ final class RegistryDownloadsManagerTests: XCTestCase { let package: PackageIdentity = .plain("test.\(UUID().uuidString)") let packageVersion: Version = "1.0.0" - let packageSource = InMemoryRegistryPackageSource(fileSystem: fs, path: .root.appending(components: "registry", "server", package.description)) + let packageSource = InMemoryRegistryPackageSource( + fileSystem: fs, + path: .root.appending(components: "registry", "server", package.description) + ) try packageSource.writePackageContent() registry.addPackage( @@ -187,11 +195,16 @@ final class RegistryDownloadsManagerTests: XCTestCase { // try to get a package do { + await delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - XCTAssertTrue(fs.isDirectory(cachePath.appending(components: package.registry!.scope.description, package.registry!.name.description, packageVersion.description))) + XCTAssertTrue( + fs.isDirectory( + cachePath.appending(components: package.registry!.scope.description, package.registry!.name.description, packageVersion.description) + ) + ) await delegate.consume() @@ -209,6 +222,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { do { try manager.remove(package: package) + await delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) @@ -231,6 +245,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { try manager.remove(package: package) manager.purgeCache(observabilityScope: observability.topScope) + await delegate.prepare(fetchExpected: true) let path = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope) XCTAssertNoDiagnostics(observability.diagnostics) XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) @@ -276,7 +291,10 @@ final class RegistryDownloadsManagerTests: XCTestCase { let concurrency = 100 let package: PackageIdentity = .plain("test.\(UUID().uuidString)") let packageVersions = (0 ..< concurrency).map { Version($0, 0 , 0) } - let packageSource = InMemoryRegistryPackageSource(fileSystem: fs, path: .root.appending(components: "registry", "server", package.description)) + let packageSource = InMemoryRegistryPackageSource( + fileSystem: fs, + path: .root.appending(components: "registry", "server", package.description) + ) try packageSource.writePackageContent() registry.addPackage( @@ -285,19 +303,21 @@ final class RegistryDownloadsManagerTests: XCTestCase { source: packageSource ) - let results = ThreadSafeKeyValueStore() - try await withThrowingTaskGroup(of: Void.self) { group in + let results = try await withThrowingTaskGroup(of: (Version, AbsolutePath).self) { group in for packageVersion in packageVersions { group.addTask { - results[packageVersion] = try await manager.lookup( + await delegate.prepare(fetchExpected: true) + return (packageVersion, try await manager.lookup( package: package, version: packageVersion, - observabilityScope: observability.topScope, - delegateQueue: .sharedConcurrent - ) + observabilityScope: observability.topScope + )) } } - try await group.waitForAll() + + return try await group.reduce(into: [:]) { + $0[$1.0] = $1.1 + } } await delegate.consume() @@ -318,7 +338,10 @@ final class RegistryDownloadsManagerTests: XCTestCase { let repeatRatio = 10 let package: PackageIdentity = .plain("test.\(UUID().uuidString)") let packageVersions = (0 ..< concurrency / 10).map { Version($0, 0 , 0) } - let packageSource = InMemoryRegistryPackageSource(fileSystem: fs, path: .root.appending(components: "registry", "server", package.description)) + let packageSource = InMemoryRegistryPackageSource( + fileSystem: fs, + path: .root.appending(components: "registry", "server", package.description) + ) try packageSource.writePackageContent() registry.addPackage( @@ -328,15 +351,20 @@ final class RegistryDownloadsManagerTests: XCTestCase { ) await delegate.reset() - let results = ThreadSafeKeyValueStore() - try await withThrowingTaskGroup(of: Void.self) { group in + let results = try await withThrowingTaskGroup(of: (Version, AbsolutePath).self) { group in for index in 0 ..< concurrency { group.addTask { + await delegate.prepare(fetchExpected: index < concurrency / repeatRatio) let packageVersion = Version(index % (concurrency / repeatRatio), 0, 0) - results[packageVersion] = try await manager.lookup(package: package, version: packageVersion, observabilityScope: observability.topScope, delegateQueue: .sharedConcurrent) + return (packageVersion, try await manager.lookup( + package: package, + version: packageVersion, + observabilityScope: observability.topScope + )) } } - try await group.waitForAll() + + return try await group.reduce(into: [:]) { $0[$1.0] = $1.1 } } await delegate.consume() @@ -352,6 +380,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { } } +@available(macOS 15.0, *) private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDelegate { typealias WillFetch = (packageVersion: PackageVersion, fetchDetails: RegistryDownloadsManager.FetchDetails) typealias DidFetch = (packageVersion: PackageVersion, result: Result) @@ -362,14 +391,19 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele private var expectedFetches = 0 private nonisolated let willFetchContinuation: AsyncStream.Continuation - private var willFetchStream: AsyncStream + private let willFetchStream: AsyncStream + private var willFetchIterator: any AsyncIteratorProtocol private nonisolated let didFetchContinuation: AsyncStream.Continuation - private var didFetchStream: AsyncStream + private let didFetchStream: AsyncStream + private var didFetchIterator: any AsyncIteratorProtocol init() { (willFetchStream, willFetchContinuation) = AsyncStream.makeStream() + self.willFetchIterator = willFetchStream.makeAsyncIterator() + (didFetchStream, didFetchContinuation) = AsyncStream.makeStream() + self.didFetchIterator = didFetchStream.makeAsyncIterator() } func prepare(fetchExpected: Bool) { @@ -380,13 +414,14 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele func consume() async { var elementsToFetch = expectedFetches - for await element in willFetchStream where elementsToFetch > 0 { + + while elementsToFetch > 0, let element = await self.willFetchIterator.next(isolation: #isolation) { self.willFetch.append(element) elementsToFetch -= 1 } elementsToFetch = expectedFetches - for await element in didFetchStream where elementsToFetch > 0 { + while elementsToFetch > 0, let element = await self.didFetchIterator.next(isolation: #isolation) { self.didFetch.append(element) elementsToFetch -= 1 } @@ -411,17 +446,6 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele } } -extension RegistryDownloadsManager { - fileprivate func lookup(package: PackageIdentity, version: Version, observabilityScope: ObservabilityScope) async throws -> AbsolutePath { - try await self.lookup( - package: package, - version: version, - observabilityScope: observabilityScope, - delegateQueue: .sharedConcurrent - ) - } -} - fileprivate struct PackageVersion: Hashable, Equatable { let package: PackageIdentity let version: Version From 0e159d9370ec26f9ebbb6d818e3bb6dab254b626 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 5 Nov 2024 16:50:39 +0000 Subject: [PATCH 23/33] Fix `RegistryDownloadsManagerTests.testConcurrency` --- .../RegistryDownloadsManager.swift | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Sources/PackageRegistry/RegistryDownloadsManager.swift b/Sources/PackageRegistry/RegistryDownloadsManager.swift index b3576edee0e..f155313a6ce 100644 --- a/Sources/PackageRegistry/RegistryDownloadsManager.swift +++ b/Sources/PackageRegistry/RegistryDownloadsManager.swift @@ -28,7 +28,12 @@ public actor RegistryDownloadsManager { private let registryClient: RegistryClient private let delegate: Delegate? - private var pendingLookups = [PackageIdentity: [CheckedContinuation]]() + private struct LookupKey: Hashable { + let id: PackageIdentity + let version: Version + } + + private var pendingLookups = [LookupKey: [CheckedContinuation]]() public init( fileSystem: FileSystem, @@ -62,13 +67,14 @@ public actor RegistryDownloadsManager { } // next we check if there is a pending lookup - if self.pendingLookups.keys.contains(package) { + let key = LookupKey(id: package, version: version) + if self.pendingLookups.keys.contains(key) { // chain onto the pending lookup return await withCheckedContinuation { - self.pendingLookups[package]!.append($0) + self.pendingLookups[key]!.append($0) } } else { - self.pendingLookups[package] = [] + self.pendingLookups[key] = [] // inform delegate that we are starting to fetch // calculate if cached (for delegate call) outside queue as it may change while queue is processing @@ -101,12 +107,12 @@ public actor RegistryDownloadsManager { // remove the pending lookup defer { - if let pendingLookups = self.pendingLookups[package] { + if let pendingLookups = self.pendingLookups[key] { for lookup in pendingLookups { lookup.resume(returning: packagePath) } - self.pendingLookups[package] = nil + self.pendingLookups[key] = nil } } From c5ee0de7ceafd02328ab84e9ef8716fdbe34ea1b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 12:43:14 +0000 Subject: [PATCH 24/33] Fix remaining `PackageRegistry` tests --- Sources/PackageRegistry/ChecksumTOFU.swift | 25 +- .../PackageRegistry/SignatureValidation.swift | 244 +++++++++--------- 2 files changed, 141 insertions(+), 128 deletions(-) diff --git a/Sources/PackageRegistry/ChecksumTOFU.swift b/Sources/PackageRegistry/ChecksumTOFU.swift index ed6c58422f6..b89cbf0bd4b 100644 --- a/Sources/PackageRegistry/ChecksumTOFU.swift +++ b/Sources/PackageRegistry/ChecksumTOFU.swift @@ -76,6 +76,7 @@ struct PackageVersionChecksumTOFU { return savedChecksum } + let checksum: String // Try fetching checksum from registry if: // - No storage available // - Checksum not found in storage @@ -85,7 +86,7 @@ struct PackageVersionChecksumTOFU { guard let sourceArchiveResource = versionMetadata.sourceArchive else { throw RegistryError.missingSourceArchive } - guard let checksum = sourceArchiveResource.checksum else { + guard let computedChecksum = sourceArchiveResource.checksum else { throw RegistryError.sourceArchiveMissingChecksum( registry: registry, package: package.underlying, @@ -93,16 +94,7 @@ struct PackageVersionChecksumTOFU { ) } - try self.writeToStorage( - registry: registry, - package: package, - version: version, - checksum: checksum, - contentType: .sourceCode, - observabilityScope: observabilityScope - ) - - return checksum + checksum = computedChecksum } catch { throw RegistryError.failedRetrievingReleaseChecksum( registry: registry, @@ -111,6 +103,17 @@ struct PackageVersionChecksumTOFU { error: error ) } + + try self.writeToStorage( + registry: registry, + package: package, + version: version, + checksum: checksum, + contentType: .sourceCode, + observabilityScope: observabilityScope + ) + + return checksum } func validateManifest( diff --git a/Sources/PackageRegistry/SignatureValidation.swift b/Sources/PackageRegistry/SignatureValidation.swift index 106c1674d6d..252554fa6de 100644 --- a/Sources/PackageRegistry/SignatureValidation.swift +++ b/Sources/PackageRegistry/SignatureValidation.swift @@ -101,6 +101,9 @@ struct SignatureValidation { fileSystem: FileSystem, observabilityScope: ObservabilityScope ) async throws -> SigningEntity? { + let signatureData: Data + let signatureFormat: SignatureFormat + do { let versionMetadata = try await self.versionMetadataProvider(package, version) @@ -115,27 +118,18 @@ struct SignatureValidation { ) } - guard let signatureData = Data(base64Encoded: signatureBase64Encoded) else { + guard let data = Data(base64Encoded: signatureBase64Encoded) else { throw RegistryError.failedLoadingSignature } + signatureData = data + guard let signatureFormatString = sourceArchiveResource.signing?.signatureFormat else { throw RegistryError.missingSignatureFormat } - guard let signatureFormat = SignatureFormat(rawValue: signatureFormatString) else { + guard let format = SignatureFormat(rawValue: signatureFormatString) else { throw RegistryError.unknownSignatureFormat(signatureFormatString) } - - return try await self.validateSourceArchiveSignature( - registry: registry, - package: package, - version: version, - signature: Array(signatureData), - signatureFormat: signatureFormat, - content: Array(content), - configuration: configuration, - fileSystem: fileSystem, - observabilityScope: observabilityScope - ) + signatureFormat = format } catch RegistryError.sourceArchiveNotSigned { observabilityScope.emit( info: "\(package) \(version) from \(registry) is unsigned", @@ -188,6 +182,18 @@ struct SignatureValidation { error: error ) } + + return try await self.validateSourceArchiveSignature( + registry: registry, + package: package, + version: version, + signature: Array(signatureData), + signatureFormat: signatureFormat, + content: Array(content), + configuration: configuration, + fileSystem: fileSystem, + observabilityScope: observabilityScope + ) } private func validateSourceArchiveSignature( @@ -201,71 +207,72 @@ struct SignatureValidation { fileSystem: FileSystem, observabilityScope: ObservabilityScope ) async throws -> SigningEntity? { + let signatureStatus: SignatureStatus do { - let signatureStatus = try await SignatureProvider.status( + signatureStatus = try await SignatureProvider.status( signature: signature, content: content, format: signatureFormat, verifierConfiguration: try VerifierConfiguration.from(configuration, fileSystem: fileSystem), observabilityScope: observabilityScope ) + } catch { + throw RegistryError.failedToValidateSignature(error) + } - switch signatureStatus { - case .valid(let signingEntity): - observabilityScope.emit( - info: "\(package) \(version) from \(registry) is signed with a valid entity '\(signingEntity)'" - ) - return signingEntity - - case .invalid(let reason): - throw RegistryError.invalidSignature(reason: reason) + switch signatureStatus { + case .valid(let signingEntity): + observabilityScope.emit( + info: "\(package) \(version) from \(registry) is signed with a valid entity '\(signingEntity)'" + ) + return signingEntity - case .certificateInvalid(let reason): - throw RegistryError.invalidSigningCertificate(reason: reason) + case .invalid(let reason): + throw RegistryError.invalidSignature(reason: reason) - case .certificateNotTrusted(let signingEntity): - observabilityScope.emit( - info: "\(package) \(version) from \(registry) signing entity '\(signingEntity)' is untrusted", - metadata: .registryPackageMetadata(identity: package) - ) + case .certificateInvalid(let reason): + throw RegistryError.invalidSigningCertificate(reason: reason) - guard let onUntrusted = configuration.onUntrustedCertificate else { - throw RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") - } + case .certificateNotTrusted(let signingEntity): + observabilityScope.emit( + info: "\(package) \(version) from \(registry) signing entity '\(signingEntity)' is untrusted", + metadata: .registryPackageMetadata(identity: package) + ) - let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) + guard let onUntrusted = configuration.onUntrustedCertificate else { + throw RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") + } - switch onUntrusted { - case .prompt: - let `continue` = await self.delegate.onUntrusted( - registry: registry, - package: package.underlying, - version: version - ) + let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) - if `continue` { - return nil - } else { - throw signerNotTrustedError - } + switch onUntrusted { + case .prompt: + let `continue` = await self.delegate.onUntrusted( + registry: registry, + package: package.underlying, + version: version + ) - case .error: + if `continue` { + return nil + } else { throw signerNotTrustedError + } - case .warn: - observabilityScope.emit( - warning: "\(signerNotTrustedError)", - metadata: .registryPackageMetadata(identity: package) - ) - return nil + case .error: + throw signerNotTrustedError - case .silentAllow: - // Continue without logging - return nil - } + case .warn: + observabilityScope.emit( + warning: "\(signerNotTrustedError)", + metadata: .registryPackageMetadata(identity: package) + ) + return nil + + case .silentAllow: + // Continue without logging + return nil } - } catch { - throw RegistryError.failedToValidateSignature(error) } } @@ -324,22 +331,30 @@ struct SignatureValidation { ) async throws -> SigningEntity? { let manifestName = toolsVersion.map { "Package@swift-\($0).swift" } ?? Manifest.filename do { - let versionMetadata = try await self.versionMetadataProvider(package, version) - - guard let sourceArchiveResource = versionMetadata.sourceArchive else { - observabilityScope - .emit( + do { + let versionMetadata = try await self.versionMetadataProvider(package, version) + + guard let sourceArchiveResource = versionMetadata.sourceArchive else { + observabilityScope.emit( debug: "cannot determine if \(manifestName) should be signed because source archive for \(package) \(version) is not found in \(registry)", metadata: .registryPackageMetadata(identity: package) ) - return nil - } - guard sourceArchiveResource.signing?.signatureBase64Encoded != nil else { - throw RegistryError.sourceArchiveNotSigned( - registry: registry, - package: package.underlying, - version: version + return nil + } + guard sourceArchiveResource.signing?.signatureBase64Encoded != nil else { + throw RegistryError.sourceArchiveNotSigned( + registry: registry, + package: package.underlying, + version: version + ) + } + } catch { + observabilityScope.emit( + debug: "cannot determine if \(manifestName) should be signed because retrieval of source archive signature for \(package) \(version) from \(registry) failed", + metadata: .registryPackageMetadata(identity: package), + underlyingError: error ) + return nil } // source archive is signed, so the manifest must also be signed @@ -399,13 +414,6 @@ struct SignatureValidation { } } catch ManifestSignatureParser.Error.malformedManifestSignature { throw RegistryError.invalidSignature(reason: "manifest signature is malformed") - } catch { - observabilityScope.emit( - debug: "cannot determine if \(manifestName) should be signed because retrieval of source archive signature for \(package) \(version) from \(registry) failed", - metadata: .registryPackageMetadata(identity: package), - underlyingError: error - ) - return nil } } @@ -421,62 +429,64 @@ struct SignatureValidation { fileSystem: FileSystem, observabilityScope: ObservabilityScope ) async throws -> SigningEntity? { + let signatureStatus: SignatureStatus + do { - let signatureStatus = try await SignatureProvider.status( + signatureStatus = try await SignatureProvider.status( signature: signature, content: content, format: signatureFormat, verifierConfiguration: try VerifierConfiguration.from(configuration, fileSystem: fileSystem), observabilityScope: observabilityScope ) + } catch { + throw RegistryError.failedToValidateSignature(error) + } - switch signatureStatus { - case .valid(let signingEntity): - observabilityScope.emit( - info: "\(package) \(version) \(manifestName) from \(registry) is signed with a valid entity '\(signingEntity)'" - ) - return signingEntity - - case .invalid(let reason): - throw RegistryError.invalidSignature(reason: reason) + switch signatureStatus { + case .valid(let signingEntity): + observabilityScope.emit( + info: "\(package) \(version) \(manifestName) from \(registry) is signed with a valid entity '\(signingEntity)'" + ) + return signingEntity - case .certificateInvalid(let reason): - throw RegistryError.invalidSigningCertificate(reason: reason) + case .invalid(let reason): + throw RegistryError.invalidSignature(reason: reason) - case .certificateNotTrusted(let signingEntity): - observabilityScope.emit( - debug: "the signer '\(signingEntity)' of \(package) \(version) \(manifestName) from \(registry) is not trusted", - metadata: .registryPackageMetadata(identity: package) - ) + case .certificateInvalid(let reason): + throw RegistryError.invalidSigningCertificate(reason: reason) - guard let onUntrusted = configuration.onUntrustedCertificate else { - throw RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") - } + case .certificateNotTrusted(let signingEntity): + observabilityScope.emit( + debug: "the signer '\(signingEntity)' of \(package) \(version) \(manifestName) from \(registry) is not trusted", + metadata: .registryPackageMetadata(identity: package) + ) - let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) + guard let onUntrusted = configuration.onUntrustedCertificate else { + throw RegistryError.missingConfiguration(details: "security.signing.onUntrustedCertificate") + } - // Prompt if configured, otherwise just continue (this differs - // from source archive to minimize duplicate loggings). - switch onUntrusted { - case .prompt: - let `continue` = await self.delegate.onUntrusted( - registry: registry, - package: package.underlying, - version: version - ) + let signerNotTrustedError = RegistryError.signerNotTrusted(package.underlying, signingEntity) - if `continue` { - return nil - } else { - throw signerNotTrustedError - } + // Prompt if configured, otherwise just continue (this differs + // from source archive to minimize duplicate loggings). + switch onUntrusted { + case .prompt: + let `continue` = await self.delegate.onUntrusted( + registry: registry, + package: package.underlying, + version: version + ) - default: + if `continue` { return nil + } else { + throw signerNotTrustedError } + + default: + return nil } - } catch { - throw RegistryError.failedToValidateSignature(error) } } From 9aa2ae07b877d44dc0d609bbb7be34b26db88863 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 12:54:28 +0000 Subject: [PATCH 25/33] Fix `SignatureValidationTests` --- .../PackageRegistry/SignatureValidation.swift | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/Sources/PackageRegistry/SignatureValidation.swift b/Sources/PackageRegistry/SignatureValidation.swift index 252554fa6de..4416606a8e8 100644 --- a/Sources/PackageRegistry/SignatureValidation.swift +++ b/Sources/PackageRegistry/SignatureValidation.swift @@ -331,24 +331,10 @@ struct SignatureValidation { ) async throws -> SigningEntity? { let manifestName = toolsVersion.map { "Package@swift-\($0).swift" } ?? Manifest.filename do { + let versionMetadata: RegistryClient.PackageVersionMetadata do { - let versionMetadata = try await self.versionMetadataProvider(package, version) - - guard let sourceArchiveResource = versionMetadata.sourceArchive else { - observabilityScope.emit( - debug: "cannot determine if \(manifestName) should be signed because source archive for \(package) \(version) is not found in \(registry)", - metadata: .registryPackageMetadata(identity: package) - ) - return nil - } - guard sourceArchiveResource.signing?.signatureBase64Encoded != nil else { - throw RegistryError.sourceArchiveNotSigned( - registry: registry, - package: package.underlying, - version: version - ) - } - } catch { + versionMetadata = try await self.versionMetadataProvider(package, version) + } catch { observabilityScope.emit( debug: "cannot determine if \(manifestName) should be signed because retrieval of source archive signature for \(package) \(version) from \(registry) failed", metadata: .registryPackageMetadata(identity: package), @@ -357,6 +343,21 @@ struct SignatureValidation { return nil } + guard let sourceArchiveResource = versionMetadata.sourceArchive else { + observabilityScope.emit( + debug: "cannot determine if \(manifestName) should be signed because source archive for \(package) \(version) is not found in \(registry)", + metadata: .registryPackageMetadata(identity: package) + ) + return nil + } + guard sourceArchiveResource.signing?.signatureBase64Encoded != nil else { + throw RegistryError.sourceArchiveNotSigned( + registry: registry, + package: package.underlying, + version: version + ) + } + // source archive is signed, so the manifest must also be signed guard let manifestSignature = try ManifestSignatureParser.parse(utf8String: manifestContent) else { throw RegistryError.manifestNotSigned( From d167aed69a157c01734b9c923ff17719ae3d0602 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 14:59:48 +0000 Subject: [PATCH 26/33] Fix `SigningEntityTOFU` tests --- Sources/PackageRegistry/SigningEntityTOFU.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/PackageRegistry/SigningEntityTOFU.swift b/Sources/PackageRegistry/SigningEntityTOFU.swift index f3c57d73f60..ebe8e41d6ba 100644 --- a/Sources/PackageRegistry/SigningEntityTOFU.swift +++ b/Sources/PackageRegistry/SigningEntityTOFU.swift @@ -111,7 +111,7 @@ struct PackageSigningEntityTOFU { existing: signingEntitiesForVersion.first!, // !-safe since signingEntitiesForVersion is non-empty observabilityScope: observabilityScope ) - return true + return false } // Signer remains the same for the version return false From b8f4c15da088304d462adfec49c4fb438f9c18a8 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 15:10:47 +0000 Subject: [PATCH 27/33] Clean up formatting --- .../FileSystem/FileSystem+Extensions.swift | 14 ++++++- .../HTTPClient/URLSessionHTTPClient.swift | 5 --- .../Commands/CommandWorkspaceDelegate.swift | 12 +++++- Sources/PackageRegistry/RegistryClient.swift | 37 +++++++++---------- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/Sources/Basics/FileSystem/FileSystem+Extensions.swift b/Sources/Basics/FileSystem/FileSystem+Extensions.swift index 0fc4c47911e..d1055482922 100644 --- a/Sources/Basics/FileSystem/FileSystem+Extensions.swift +++ b/Sources/Basics/FileSystem/FileSystem+Extensions.swift @@ -196,12 +196,22 @@ extension FileSystem { } /// Execute the given block while holding the lock. - public func withLock(on path: AbsolutePath, type: FileLock.LockType, blocking: Bool = true, _ body: () throws -> T) throws -> T { + public func withLock( + on path: AbsolutePath, + type: FileLock.LockType, + blocking: Bool = true, + _ body: () throws -> T + ) throws -> T { try self.withLock(on: path.underlying, type: type, blocking: blocking, body) } /// Execute the given block while holding the lock. - public func withLock(on path: AbsolutePath, type: FileLock.LockType, blocking: Bool = true, _ body: () async throws -> T) async throws -> T { + public func withLock( + on path: AbsolutePath, + type: FileLock.LockType, + blocking: Bool = true, + _ body: () async throws -> T + ) async throws -> T { try await self.withLock(on: path.underlying, type: type, blocking: blocking, body) } diff --git a/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift b/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift index 450c09370b3..7c27608749d 100644 --- a/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift +++ b/Sources/Basics/HTTPClient/URLSessionHTTPClient.swift @@ -18,11 +18,6 @@ import struct TSCUtility.Versioning import FoundationNetworking #endif -protocol HTTPClientImplementation: Sendable { - @Sendable - func execute(request: HTTPClient.Request, progressHandler: HTTPClient.ProgressHandler?) async throws -> HTTPClient.Response -} - final class URLSessionHTTPClient: Sendable { private let dataSession: URLSession private let downloadSession: URLSession diff --git a/Sources/Commands/CommandWorkspaceDelegate.swift b/Sources/Commands/CommandWorkspaceDelegate.swift index b720a015b12..9031622cff0 100644 --- a/Sources/Commands/CommandWorkspaceDelegate.swift +++ b/Sources/Commands/CommandWorkspaceDelegate.swift @@ -184,7 +184,11 @@ final class CommandWorkspaceDelegate: WorkspaceDelegate { // registry signature handlers - func onUnsignedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version) async -> Bool { + func onUnsignedRegistryPackage( + registryURL: URL, + package: PackageModel.PackageIdentity, + version: TSCUtility.Version + ) async -> Bool { await withCheckedContinuation { continuation in self.inputHandler("\(package) \(version) from \(registryURL) is unsigned. okay to proceed? (yes/no) ") { response in switch response?.lowercased() { @@ -200,7 +204,11 @@ final class CommandWorkspaceDelegate: WorkspaceDelegate { } } - func onUntrustedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version) async -> Bool { + func onUntrustedRegistryPackage( + registryURL: URL, + package: PackageModel.PackageIdentity, + version: TSCUtility.Version + ) async -> Bool { await withCheckedContinuation { continuation in self.inputHandler("\(package) \(version) from \(registryURL) is signed with an untrusted certificate. okay to proceed? (yes/no) ") { response in switch response?.lowercased() { diff --git a/Sources/PackageRegistry/RegistryClient.swift b/Sources/PackageRegistry/RegistryClient.swift index b6fb6aad9fd..50df037ae82 100644 --- a/Sources/PackageRegistry/RegistryClient.swift +++ b/Sources/PackageRegistry/RegistryClient.swift @@ -208,10 +208,10 @@ public final class RegistryClient { do { let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + switch response.statusCode { case 200: let packageMetadata = try response.parseJSON( @@ -392,10 +392,10 @@ public final class RegistryClient { do { let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) + switch response.statusCode { case 200: let metadata = try response.parseJSON( @@ -1118,10 +1118,9 @@ public final class RegistryClient { do { let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) switch response.statusCode { case 200: let packageIdentities = try response.parseJSON( @@ -1159,10 +1158,9 @@ public final class RegistryClient { do { let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) switch response.statusCode { case 200: return @@ -1364,10 +1362,9 @@ public final class RegistryClient { do { let response = try await self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) - observabilityScope - .emit( - debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" - ) + observabilityScope.emit( + debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)" + ) switch response.statusCode { case 200: return .available From 0b6bbd951fc7b921e6a36c5822a37a4154cf1716 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 15:58:01 +0000 Subject: [PATCH 28/33] Address redundant use of `try` warnings --- Sources/Commands/PackageCommands/DumpCommands.swift | 2 +- Sources/Commands/SwiftTestCommand.swift | 4 ++-- Sources/CoreCommands/SwiftCommandState.swift | 2 +- Sources/_InternalTestSupport/MockWorkspace.swift | 2 +- Tests/CommandsTests/PackageCommandTests.swift | 2 +- Tests/FunctionalTests/PluginTests.swift | 8 ++++---- Tests/SPMBuildCoreTests/PluginInvocationTests.swift | 12 ++++++------ 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Sources/Commands/PackageCommands/DumpCommands.swift b/Sources/Commands/PackageCommands/DumpCommands.swift index be0d9f18b19..896b493ec84 100644 --- a/Sources/Commands/PackageCommands/DumpCommands.swift +++ b/Sources/Commands/PackageCommands/DumpCommands.swift @@ -118,7 +118,7 @@ struct DumpPackage: AsyncSwiftCommand { let workspace = try swiftCommandState.getActiveWorkspace() let root = try swiftCommandState.getWorkspaceRoot() - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: root.packages, observabilityScope: swiftCommandState.observabilityScope ) diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 0fa37496892..aba1c9f777d 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -558,7 +558,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) async throws { let workspace = try swiftCommandState.getActiveWorkspace() let root = try swiftCommandState.getWorkspaceRoot() - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: root.packages, observabilityScope: swiftCommandState.observabilityScope ) @@ -674,7 +674,7 @@ extension SwiftTestCommand { func printCodeCovPath(_ swiftCommandState: SwiftCommandState) async throws { let workspace = try swiftCommandState.getActiveWorkspace() let root = try swiftCommandState.getWorkspaceRoot() - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: root.packages, observabilityScope: swiftCommandState.observabilityScope ) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 8a7f714164a..f429fb391bd 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -486,7 +486,7 @@ public final class SwiftCommandState { public func getRootPackageInformation() async throws -> (dependencies: [PackageIdentity: [PackageIdentity]], targets: [PackageIdentity: [String]]) { let workspace = try self.getActiveWorkspace() let root = try self.getWorkspaceRoot() - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: root.packages, observabilityScope: self.observabilityScope ) diff --git a/Sources/_InternalTestSupport/MockWorkspace.swift b/Sources/_InternalTestSupport/MockWorkspace.swift index ac9db25ab30..02e77f26ad2 100644 --- a/Sources/_InternalTestSupport/MockWorkspace.swift +++ b/Sources/_InternalTestSupport/MockWorkspace.swift @@ -791,7 +791,7 @@ public final class MockWorkspace { let rootInput = PackageGraphRootInput( packages: try rootPaths(for: roots), dependencies: dependencies ) - let rootManifests = try await workspace.loadRootManifests(packages: rootInput.packages, observabilityScope: observability.topScope) + let rootManifests = await workspace.loadRootManifests(packages: rootInput.packages, observabilityScope: observability.topScope) let graphRoot = PackageGraphRoot(input: rootInput, manifests: rootManifests, observabilityScope: observability.topScope) let manifests = try await workspace.loadDependencyManifests(root: graphRoot, observabilityScope: observability.topScope) result(manifests, observability.diagnostics) diff --git a/Tests/CommandsTests/PackageCommandTests.swift b/Tests/CommandsTests/PackageCommandTests.swift index 69341f740f2..59cac8b4879 100644 --- a/Tests/CommandsTests/PackageCommandTests.swift +++ b/Tests/CommandsTests/PackageCommandTests.swift @@ -3663,7 +3663,7 @@ final class PackageCommandTests: CommandsTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) diff --git a/Tests/FunctionalTests/PluginTests.swift b/Tests/FunctionalTests/PluginTests.swift index 9bc95ac77a4..d349adc2e44 100644 --- a/Tests/FunctionalTests/PluginTests.swift +++ b/Tests/FunctionalTests/PluginTests.swift @@ -439,7 +439,7 @@ final class PluginTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -630,7 +630,7 @@ final class PluginTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -727,7 +727,7 @@ final class PluginTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -1043,7 +1043,7 @@ final class PluginTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) diff --git a/Tests/SPMBuildCoreTests/PluginInvocationTests.swift b/Tests/SPMBuildCoreTests/PluginInvocationTests.swift index b262e28eb7c..d0e5035719f 100644 --- a/Tests/SPMBuildCoreTests/PluginInvocationTests.swift +++ b/Tests/SPMBuildCoreTests/PluginInvocationTests.swift @@ -328,7 +328,7 @@ final class PluginInvocationTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -708,7 +708,7 @@ final class PluginInvocationTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -787,7 +787,7 @@ final class PluginInvocationTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -897,7 +897,7 @@ final class PluginInvocationTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -1092,7 +1092,7 @@ final class PluginInvocationTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) @@ -1238,7 +1238,7 @@ final class PluginInvocationTests: XCTestCase { // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) - let rootManifests = try await workspace.loadRootManifests( + let rootManifests = await workspace.loadRootManifests( packages: rootInput.packages, observabilityScope: observability.topScope ) From 683dd13784c5fa513b419e8d3096417d59fc2aaa Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 16:00:24 +0000 Subject: [PATCH 29/33] Fix `AsyncIteratorProtocol` in `MockRegistryDownloadsManagerDelegate` --- .../RegistryDownloadsManagerTests.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift index 8292086112f..1c3154c9a40 100644 --- a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift @@ -392,11 +392,19 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele private nonisolated let willFetchContinuation: AsyncStream.Continuation private let willFetchStream: AsyncStream + #if compiler(>=6.0) private var willFetchIterator: any AsyncIteratorProtocol + #else + private var willFetchIterator: any AsyncIteratorProtocol + #endif private nonisolated let didFetchContinuation: AsyncStream.Continuation private let didFetchStream: AsyncStream + #if compiler(>=6.0) private var didFetchIterator: any AsyncIteratorProtocol + #else + private var didFetchIterator: any AsyncIteratorProtocol + #endif init() { (willFetchStream, willFetchContinuation) = AsyncStream.makeStream() From e90022c92a0580a20b35fa6f95b04d061fc85c10 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 16:41:11 +0000 Subject: [PATCH 30/33] Avoid Swift 6.0+ features in MockRegistryDownloadsManagerDelegate --- .../RegistryDownloadsManagerTests.swift | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift index 1c3154c9a40..25b0409b543 100644 --- a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift @@ -64,7 +64,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 1) await XCTAssertAsyncEqual(await delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) await XCTAssertAsyncEqual(await delegate.willFetch.first?.fetchDetails, .init(fromCache: false, updatedCache: false)) @@ -85,7 +85,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertNotNil(error as? RegistryError) } - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.map { ($0.packageVersion) }, [ (PackageVersion(package: package, version: packageVersion)), @@ -109,7 +109,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.map { ($0.packageVersion) }, [ (PackageVersion(package: package, version: packageVersion)), @@ -135,7 +135,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual( await delegate.willFetch.map { ($0.packageVersion) }, [ @@ -206,7 +206,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { ) ) - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 1) await XCTAssertAsyncEqual(await delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) @@ -228,7 +228,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 2) await XCTAssertAsyncEqual(await delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) @@ -251,7 +251,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 3) await XCTAssertAsyncEqual(await delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) @@ -320,7 +320,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { } } - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, concurrency) await XCTAssertAsyncEqual(await delegate.didFetch.count, concurrency) @@ -367,7 +367,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { return try await group.reduce(into: [:]) { $0[$1.0] = $1.1 } } - await delegate.consume() + try await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, concurrency / repeatRatio) await XCTAssertAsyncEqual(await delegate.didFetch.count, concurrency / repeatRatio) @@ -392,19 +392,11 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele private nonisolated let willFetchContinuation: AsyncStream.Continuation private let willFetchStream: AsyncStream - #if compiler(>=6.0) - private var willFetchIterator: any AsyncIteratorProtocol - #else - private var willFetchIterator: any AsyncIteratorProtocol - #endif + private var willFetchIterator: any AsyncIteratorProtocol private nonisolated let didFetchContinuation: AsyncStream.Continuation private let didFetchStream: AsyncStream - #if compiler(>=6.0) - private var didFetchIterator: any AsyncIteratorProtocol - #else - private var didFetchIterator: any AsyncIteratorProtocol - #endif + private var didFetchIterator: any AsyncIteratorProtocol init() { (willFetchStream, willFetchContinuation) = AsyncStream.makeStream() @@ -420,16 +412,16 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele } } - func consume() async { + func consume() async throws { var elementsToFetch = expectedFetches - while elementsToFetch > 0, let element = await self.willFetchIterator.next(isolation: #isolation) { + while elementsToFetch > 0, let element = try await self.willFetchIterator.next(isolation: #isolation) as? WillFetch { self.willFetch.append(element) elementsToFetch -= 1 } elementsToFetch = expectedFetches - while elementsToFetch > 0, let element = await self.didFetchIterator.next(isolation: #isolation) { + while elementsToFetch > 0, let element = try await self.didFetchIterator.next(isolation: #isolation) as? DidFetch { self.didFetch.append(element) elementsToFetch -= 1 } From 4840facf57ca04f5a6e5ce43c27fce225df005fe Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 16:51:25 +0000 Subject: [PATCH 31/33] Enable `RegistryDownloadsManagerTests` only with Swift 6.0+ --- .../RegistryDownloadsManagerTests.swift | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift index 25b0409b543..a2dafa35af6 100644 --- a/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryDownloadsManagerTests.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +#if compiler(>=6.0) + import Basics import _Concurrency import PackageModel @@ -64,7 +66,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 1) await XCTAssertAsyncEqual(await delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) await XCTAssertAsyncEqual(await delegate.willFetch.first?.fetchDetails, .init(fromCache: false, updatedCache: false)) @@ -85,7 +87,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertNotNil(error as? RegistryError) } - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.map { ($0.packageVersion) }, [ (PackageVersion(package: package, version: packageVersion)), @@ -109,7 +111,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.map { ($0.packageVersion) }, [ (PackageVersion(package: package, version: packageVersion)), @@ -135,7 +137,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual( await delegate.willFetch.map { ($0.packageVersion) }, [ @@ -206,7 +208,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { ) ) - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 1) await XCTAssertAsyncEqual(await delegate.willFetch.first?.packageVersion, .init(package: package, version: packageVersion)) @@ -228,7 +230,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 2) await XCTAssertAsyncEqual(await delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) @@ -251,7 +253,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { XCTAssertEqual(path, try downloadsPath.appending(package.downloadPath(version: packageVersion))) XCTAssertTrue(fs.isDirectory(path)) - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, 3) await XCTAssertAsyncEqual(await delegate.willFetch.last?.packageVersion, .init(package: package, version: packageVersion)) @@ -320,7 +322,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { } } - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, concurrency) await XCTAssertAsyncEqual(await delegate.didFetch.count, concurrency) @@ -367,7 +369,7 @@ final class RegistryDownloadsManagerTests: XCTestCase { return try await group.reduce(into: [:]) { $0[$1.0] = $1.1 } } - try await delegate.consume() + await delegate.consume() await XCTAssertAsyncEqual(await delegate.willFetch.count, concurrency / repeatRatio) await XCTAssertAsyncEqual(await delegate.didFetch.count, concurrency / repeatRatio) @@ -392,11 +394,11 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele private nonisolated let willFetchContinuation: AsyncStream.Continuation private let willFetchStream: AsyncStream - private var willFetchIterator: any AsyncIteratorProtocol + private var willFetchIterator: any AsyncIteratorProtocol private nonisolated let didFetchContinuation: AsyncStream.Continuation private let didFetchStream: AsyncStream - private var didFetchIterator: any AsyncIteratorProtocol + private var didFetchIterator: any AsyncIteratorProtocol init() { (willFetchStream, willFetchContinuation) = AsyncStream.makeStream() @@ -412,16 +414,16 @@ private actor MockRegistryDownloadsManagerDelegate: RegistryDownloadsManagerDele } } - func consume() async throws { + func consume() async { var elementsToFetch = expectedFetches - while elementsToFetch > 0, let element = try await self.willFetchIterator.next(isolation: #isolation) as? WillFetch { + while elementsToFetch > 0, let element = await self.willFetchIterator.next(isolation: #isolation) { self.willFetch.append(element) elementsToFetch -= 1 } elementsToFetch = expectedFetches - while elementsToFetch > 0, let element = try await self.didFetchIterator.next(isolation: #isolation) as? DidFetch { + while elementsToFetch > 0, let element = await self.didFetchIterator.next(isolation: #isolation) { self.didFetch.append(element) elementsToFetch -= 1 } @@ -450,3 +452,5 @@ fileprivate struct PackageVersion: Hashable, Equatable { let package: PackageIdentity let version: Version } + +#endif From 22d68f4ce2e68afd4038bfae85d58be0d61859e4 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 17:33:57 +0000 Subject: [PATCH 32/33] Convert `ManifestLoader.evaluateManifest` to `async` --- Sources/PackageLoading/ManifestLoader.swift | 483 ++++++++---------- .../FileSystemPackageContainer.swift | 3 +- .../RegistryPackageContainer.swift | 3 +- .../SourceControlPackageContainer.swift | 3 +- Sources/Workspace/Workspace+Manifests.swift | 3 +- Sources/Workspace/Workspace+Registry.swift | 6 +- .../MockManifestLoader.swift | 9 +- Sources/swift-bootstrap/main.swift | 3 +- .../ManifestSourceGenerationTests.swift | 6 +- .../RegistryPackageContainerTests.swift | 3 +- Tests/WorkspaceTests/WorkspaceTests.swift | 3 +- 11 files changed, 216 insertions(+), 309 deletions(-) diff --git a/Sources/PackageLoading/ManifestLoader.swift b/Sources/PackageLoading/ManifestLoader.swift index 327270f88c5..0b9037c6e45 100644 --- a/Sources/PackageLoading/ManifestLoader.swift +++ b/Sources/PackageLoading/ManifestLoader.swift @@ -112,7 +112,6 @@ public protocol ManifestLoaderProtocol { /// - dependencyMapper: A helper to map dependencies. /// - fileSystem: File system to load from. /// - observabilityScope: Observability scope to emit diagnostics. - /// - callbackQueue: The dispatch queue to perform completion handler on. /// - completion: The completion handler . func load( manifestPath: AbsolutePath, @@ -125,8 +124,7 @@ public protocol ManifestLoaderProtocol { dependencyMapper: DependencyMapper, fileSystem: FileSystem, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue ) async throws -> Manifest /// Reset any internal cache held by the manifest loader. @@ -199,8 +197,7 @@ extension ManifestLoaderProtocol { dependencyMapper: DependencyMapper, fileSystem: FileSystem, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue ) async throws -> Manifest { // find the manifest path and parse it's tools-version let manifestPath = try ManifestLoader.findManifest(packagePath: packagePath, fileSystem: fileSystem, currentToolsVersion: currentToolsVersion) @@ -219,8 +216,7 @@ extension ManifestLoaderProtocol { dependencyMapper: dependencyMapper, fileSystem: fileSystem, observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue + delegateQueue: delegateQueue ) } } @@ -251,11 +247,6 @@ public final class ManifestLoader: ManifestLoaderProtocol { private let useInMemoryCache: Bool private let memoryCache = ThreadSafeKeyValueStore() - /// DispatchSemaphore to restrict concurrent manifest evaluations - private let concurrencySemaphore: DispatchSemaphore - /// OperationQueue to park pending lookups - private let evaluationQueue: OperationQueue - private let tokenBucket = TokenBucket(tokens: Concurrency.maxOperations) public init( @@ -278,12 +269,6 @@ public final class ManifestLoader: ManifestLoaderProtocol { self.useInMemoryCache = useInMemoryCache self.databaseCacheDir = try? cacheDir.map(resolveSymlinks) - - // this queue and semaphore is used to limit the amount of concurrent manifest loading taking place - self.evaluationQueue = OperationQueue() - self.evaluationQueue.name = "org.swift.swiftpm.manifest-loader" - self.evaluationQueue.maxConcurrentOperationCount = Concurrency.maxOperations - self.concurrencySemaphore = DispatchSemaphore(value: Concurrency.maxOperations) } public func load( @@ -297,8 +282,7 @@ public final class ManifestLoader: ManifestLoaderProtocol { dependencyMapper: DependencyMapper, fileSystem: FileSystem, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue ) async throws -> Manifest { // Inform the delegate. let start = DispatchTime.now() @@ -327,8 +311,7 @@ public final class ManifestLoader: ManifestLoaderProtocol { fileSystem: fileSystem, observabilityScope: observabilityScope, delegate: delegate, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue + delegateQueue: delegateQueue ) // Convert legacy system packages to the current target‐based model. @@ -462,8 +445,7 @@ public final class ManifestLoader: ManifestLoaderProtocol { fileSystem: FileSystem, observabilityScope: ObservabilityScope, delegate: Delegate?, - delegateQueue: DispatchQueue?, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue? ) async throws -> ManifestJSONParser.Result { let key = try CacheKey( packageIdentity: packageIdentity, @@ -550,8 +532,7 @@ public final class ManifestLoader: ManifestLoaderProtocol { toolsVersion: key.toolsVersion, observabilityScope: observabilityScope, delegate: delegate, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue + delegateQueue: delegateQueue ) // only cache successfully parsed manifests @@ -620,8 +601,7 @@ public final class ManifestLoader: ManifestLoaderProtocol { toolsVersion: ToolsVersion, observabilityScope: ObservabilityScope, delegate: Delegate?, - delegateQueue: DispatchQueue?, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue? ) async throws -> EvaluationResult { let manifestPreamble: ByteString if toolsVersion >= .v5_8 { @@ -649,24 +629,16 @@ public final class ManifestLoader: ManifestLoaderProtocol { toolsVersion: toolsVersion ) - return try await withCheckedThrowingContinuation { continuation in - do { - try self.evaluateManifest( - at: manifestPath, - vfsOverlayPath: vfsOverlayTempFilePath, - packageIdentity: packageIdentity, - packageLocation: packageLocation, - toolsVersion: toolsVersion, - observabilityScope: observabilityScope, - delegate: delegate, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue, - completion: { continuation.resume(with: $0) } - ) - } catch { - continuation.resume(throwing: error) - } - } + return try await self.evaluateManifest( + at: manifestPath, + vfsOverlayPath: vfsOverlayTempFilePath, + packageIdentity: packageIdentity, + packageLocation: packageLocation, + toolsVersion: toolsVersion, + observabilityScope: observabilityScope, + delegate: delegate, + delegateQueue: delegateQueue + ) } } @@ -679,20 +651,14 @@ public final class ManifestLoader: ManifestLoaderProtocol { toolsVersion: ToolsVersion, observabilityScope: ObservabilityScope, delegate: Delegate?, - delegateQueue: DispatchQueue?, - callbackQueue: DispatchQueue, - completion: @escaping (Result) -> Void - ) throws { + delegateQueue: DispatchQueue? + ) async throws -> EvaluationResult { // The compiler has special meaning for files with extensions like .ll, .bc etc. // Assert that we only try to load files with extension .swift to avoid unexpected loading behavior. guard manifestPath.extension == "swift" else { - return callbackQueue.async { - completion(.failure(InternalError("Manifest files must contain .swift suffix in their name, given: \(manifestPath)."))) - } + throw InternalError("Manifest files must contain .swift suffix in their name, given: \(manifestPath).") } - var evaluationResult = EvaluationResult() - // For now, we load the manifest by having Swift interpret it directly. // Eventually, we should have two loading processes, one that loads only // the declarative package specification using the Swift compiler directly @@ -707,254 +673,209 @@ public final class ManifestLoader: ManifestLoaderProtocol { Environment.current["SWIFTPM_MODULECACHE_OVERRIDE"] ?? Environment.current["SWIFTPM_TESTS_MODULECACHE"]).flatMap { try AbsolutePath(validating: $0) } - var cmd: [String] = [] - cmd += [self.toolchain.swiftCompilerPathForManifests.pathString] + return try await tokenBucket.withToken { + var evaluationResult = EvaluationResult() - if let vfsOverlayPath { - cmd += ["-vfsoverlay", vfsOverlayPath.pathString] - } + var cmd: [String] = [] + cmd += [self.toolchain.swiftCompilerPathForManifests.pathString] - // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode - // which produces a framework for dynamic package products. - if runtimePath.extension == "framework" { - cmd += [ - "-F", runtimePath.parentDirectory.pathString, - "-framework", "PackageDescription", - "-Xlinker", "-rpath", "-Xlinker", runtimePath.parentDirectory.pathString, - ] - } else { - cmd += [ - "-L", runtimePath.pathString, - "-lPackageDescription", - ] -#if !os(Windows) - // -rpath argument is not supported on Windows, - // so we add runtimePath to PATH when executing the manifest instead - cmd += ["-Xlinker", "-rpath", "-Xlinker", runtimePath.pathString] -#endif - } + if let vfsOverlayPath { + cmd += ["-vfsoverlay", vfsOverlayPath.pathString] + } - // Use the same minimum deployment target as the PackageDescription library (with a fallback to the default host triple). -#if os(macOS) - if let version = self.toolchain.swiftPMLibrariesLocation.manifestLibraryMinimumDeploymentTarget?.versionString { - cmd += ["-target", "\(self.toolchain.targetTriple.tripleString(forPlatformVersion: version))"] - } else { - cmd += ["-target", self.toolchain.targetTriple.tripleString] - } -#endif + // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode + // which produces a framework for dynamic package products. + if runtimePath.extension == "framework" { + cmd += [ + "-F", runtimePath.parentDirectory.pathString, + "-framework", "PackageDescription", + "-Xlinker", "-rpath", "-Xlinker", runtimePath.parentDirectory.pathString, + ] + } else { + cmd += [ + "-L", runtimePath.pathString, + "-lPackageDescription", + ] + #if !os(Windows) + // -rpath argument is not supported on Windows, + // so we add runtimePath to PATH when executing the manifest instead + cmd += ["-Xlinker", "-rpath", "-Xlinker", runtimePath.pathString] + #endif + } - // Add any extra flags required as indicated by the ManifestLoader. - cmd += self.toolchain.swiftCompilerFlags + // Use the same minimum deployment target as the PackageDescription library (with a fallback to the default host triple). + #if os(macOS) + if let version = self.toolchain.swiftPMLibrariesLocation.manifestLibraryMinimumDeploymentTarget?.versionString { + cmd += ["-target", "\(self.toolchain.targetTriple.tripleString(forPlatformVersion: version))"] + } else { + cmd += ["-target", self.toolchain.targetTriple.tripleString] + } + #endif - cmd += self.interpreterFlags(for: toolsVersion) - if let moduleCachePath { - cmd += ["-module-cache-path", moduleCachePath.pathString] - } + // Add any extra flags required as indicated by the ManifestLoader. + cmd += self.toolchain.swiftCompilerFlags + + cmd += self.interpreterFlags(for: toolsVersion) + if let moduleCachePath { + cmd += ["-module-cache-path", moduleCachePath.pathString] + } + + // Add the arguments for emitting serialized diagnostics, if requested. + if self.serializedDiagnostics, let databaseCacheDir = self.databaseCacheDir { + let diaDir = databaseCacheDir.appending("ManifestLoading") + let diagnosticFile = diaDir.appending("\(packageIdentity).dia") - // Add the arguments for emitting serialized diagnostics, if requested. - if self.serializedDiagnostics, let databaseCacheDir = self.databaseCacheDir { - let diaDir = databaseCacheDir.appending("ManifestLoading") - let diagnosticFile = diaDir.appending("\(packageIdentity).dia") - do { try localFileSystem.createDirectory(diaDir, recursive: true) cmd += ["-Xfrontend", "-serialize-diagnostics-path", "-Xfrontend", diagnosticFile.pathString] evaluationResult.diagnosticFile = diagnosticFile - } catch { - return callbackQueue.async { - completion(.failure(error)) - } } - } - cmd += [manifestPath._normalized] + cmd += [manifestPath._normalized] - cmd += self.extraManifestFlags + cmd += self.extraManifestFlags - // wrap the completion to free concurrency control semaphore - let completion: (Result) -> Void = { result in - self.concurrencySemaphore.signal() - completion(result) - } + // run the evaluation + let compileStart = DispatchTime.now() + delegateQueue?.async { + delegate?.willCompile( + packageIdentity: packageIdentity, + packageLocation: packageLocation, + manifestPath: manifestPath + ) + } + + return try await withTemporaryDirectory(removeTreeOnDeinit: true) { tmpDir in + // Set path to compiled manifest executable. +#if os(Windows) + let executableSuffix = ".exe" +#else + let executableSuffix = "" +#endif + let compiledManifestFile = tmpDir.appending("\(packageIdentity)-manifest\(executableSuffix)") + cmd += ["-o", compiledManifestFile.pathString] + + evaluationResult.compilerCommandLine = cmd + + // Compile the manifest. + let compilerResult = try await AsyncProcess.popen( + arguments: cmd, + environment: self.toolchain.swiftCompilerEnvironment + ) + + evaluationResult.compilerOutput = try (compilerResult.utf8Output() + compilerResult.utf8stderrOutput()).spm_chuzzle() + + // Return now if there was an error. + if compilerResult.exitStatus != .terminated(code: 0) { + return evaluationResult + } + + // Pass an open file descriptor of a file to which the JSON representation of the manifest will be written. + let jsonOutputFile = tmpDir.appending("\(packageIdentity)-output.json") + guard let jsonOutputFileDesc = fopen(jsonOutputFile.pathString, "w") else { + throw StringError("couldn't create the manifest's JSON output file") + } + + defer { + fclose(jsonOutputFileDesc) + } + + cmd = [compiledManifestFile.pathString] +#if os(Windows) + // NOTE: `_get_osfhandle` returns a non-owning, unsafe, + // unretained HANDLE. DO NOT invoke `CloseHandle` on `hFile`. + let hFile: Int = _get_osfhandle(_fileno(jsonOutputFileDesc)) + cmd += ["-handle", "\(String(hFile, radix: 16))"] +#else + cmd += ["-fileno", "\(fileno(jsonOutputFileDesc))"] +#endif + + let packageDirectory = manifestPath.parentDirectory.pathString + + let gitInformation: ContextModel.GitInformation? + do { + let repo = GitRepository(path: manifestPath.parentDirectory) + gitInformation = ContextModel.GitInformation( + currentTag: repo.getCurrentTag(), + currentCommit: try repo.getCurrentRevision().identifier, + hasUncommittedChanges: repo.hasUncommittedChanges() + ) + } catch { + gitInformation = nil + } + + let contextModel = ContextModel( + packageDirectory: packageDirectory, + gitInformation: gitInformation + ) + cmd += ["-context", try contextModel.encode()] + + // If enabled, run command in a sandbox. + // This provides some safety against arbitrary code execution when parsing manifest files. + // We only allow the permissions which are absolutely necessary. + if self.isManifestSandboxEnabled { + let cacheDirectories = [self.databaseCacheDir?.appending("ManifestLoading"), moduleCachePath].compactMap{ $0 } + let strictness: Sandbox.Strictness = toolsVersion < .v5_3 ? .manifest_pre_53 : .default + cmd = try Sandbox.apply(command: cmd, fileSystem: localFileSystem, strictness: strictness, writableDirectories: cacheDirectories) + } - // we must not block the calling thread (for concurrency control) so nesting this in a queue - self.evaluationQueue.addOperation { - do { - // park the evaluation thread based on the max concurrency allowed - self.concurrencySemaphore.wait() - // run the evaluation - let compileStart = DispatchTime.now() delegateQueue?.async { - delegate?.willCompile( + delegate?.didCompile( + packageIdentity: packageIdentity, + packageLocation: packageLocation, + manifestPath: manifestPath, + duration: compileStart.distance(to: .now()) + ) + } + + // Run the compiled manifest. + + let evaluationStart = DispatchTime.now() + delegateQueue?.async { + delegate?.willEvaluate( packageIdentity: packageIdentity, packageLocation: packageLocation, manifestPath: manifestPath ) } - try withTemporaryDirectory { tmpDir, cleanupTmpDir in - // Set path to compiled manifest executable. - #if os(Windows) - let executableSuffix = ".exe" - #else - let executableSuffix = "" - #endif - let compiledManifestFile = tmpDir.appending("\(packageIdentity)-manifest\(executableSuffix)") - cmd += ["-o", compiledManifestFile.pathString] - - evaluationResult.compilerCommandLine = cmd - - // Compile the manifest. - AsyncProcess.popen( - arguments: cmd, - environment: self.toolchain.swiftCompilerEnvironment, - queue: callbackQueue - ) { result in - dispatchPrecondition(condition: .onQueue(callbackQueue)) - - var cleanupIfError = DelayableAction(target: tmpDir, action: cleanupTmpDir) - defer { cleanupIfError.perform() } - - let compilerResult: AsyncProcessResult - do { - compilerResult = try result.get() - evaluationResult.compilerOutput = try (compilerResult.utf8Output() + compilerResult.utf8stderrOutput()).spm_chuzzle() - } catch { - return completion(.failure(error)) - } - - // Return now if there was an error. - if compilerResult.exitStatus != .terminated(code: 0) { - return completion(.success(evaluationResult)) - } - - // Pass an open file descriptor of a file to which the JSON representation of the manifest will be written. - let jsonOutputFile = tmpDir.appending("\(packageIdentity)-output.json") - guard let jsonOutputFileDesc = fopen(jsonOutputFile.pathString, "w") else { - return completion(.failure(StringError("couldn't create the manifest's JSON output file"))) - } - - cmd = [compiledManifestFile.pathString] - #if os(Windows) - // NOTE: `_get_osfhandle` returns a non-owning, unsafe, - // unretained HANDLE. DO NOT invoke `CloseHandle` on `hFile`. - let hFile: Int = _get_osfhandle(_fileno(jsonOutputFileDesc)) - cmd += ["-handle", "\(String(hFile, radix: 16))"] - #else - cmd += ["-fileno", "\(fileno(jsonOutputFileDesc))"] - #endif - - do { - let packageDirectory = manifestPath.parentDirectory.pathString - - let gitInformation: ContextModel.GitInformation? - do { - let repo = GitRepository(path: manifestPath.parentDirectory) - gitInformation = ContextModel.GitInformation( - currentTag: repo.getCurrentTag(), - currentCommit: try repo.getCurrentRevision().identifier, - hasUncommittedChanges: repo.hasUncommittedChanges() - ) - } catch { - gitInformation = nil - } - - let contextModel = ContextModel( - packageDirectory: packageDirectory, - gitInformation: gitInformation - ) - cmd += ["-context", try contextModel.encode()] - } catch { - return completion(.failure(error)) - } - - // If enabled, run command in a sandbox. - // This provides some safety against arbitrary code execution when parsing manifest files. - // We only allow the permissions which are absolutely necessary. - if self.isManifestSandboxEnabled { - let cacheDirectories = [self.databaseCacheDir?.appending("ManifestLoading"), moduleCachePath].compactMap{ $0 } - let strictness: Sandbox.Strictness = toolsVersion < .v5_3 ? .manifest_pre_53 : .default - do { - cmd = try Sandbox.apply(command: cmd, fileSystem: localFileSystem, strictness: strictness, writableDirectories: cacheDirectories) - } catch { - return completion(.failure(error)) - } - } - - delegateQueue?.async { - delegate?.didCompile( - packageIdentity: packageIdentity, - packageLocation: packageLocation, - manifestPath: manifestPath, - duration: compileStart.distance(to: .now()) - ) - } - - // Run the compiled manifest. - - let evaluationStart = DispatchTime.now() - delegateQueue?.async { - delegate?.willEvaluate( - packageIdentity: packageIdentity, - packageLocation: packageLocation, - manifestPath: manifestPath - ) - } - - var environment = Environment.current - #if os(Windows) - let windowsPathComponent = runtimePath.pathString.replacingOccurrences(of: "/", with: "\\") - environment["Path"] = "\(windowsPathComponent);\(environment["Path"] ?? "")" - #endif - - let cleanupAfterRunning = cleanupIfError.delay() - AsyncProcess.popen( - arguments: cmd, - environment: environment, - queue: callbackQueue - ) { result in - dispatchPrecondition(condition: .onQueue(callbackQueue)) - - defer { cleanupAfterRunning.perform() } - fclose(jsonOutputFileDesc) - - do { - let runResult = try result.get() - if let runOutput = try (runResult.utf8Output() + runResult.utf8stderrOutput()).spm_chuzzle() { - // Append the runtime output to any compiler output we've received. - evaluationResult.compilerOutput = (evaluationResult.compilerOutput ?? "") + runOutput - } - - // Return now if there was an error. - if runResult.exitStatus != .terminated(code: 0) { - // TODO: should this simply be an error? - // return completion(.failure(AsyncProcessResult.Error.nonZeroExit(runResult))) - evaluationResult.errorOutput = evaluationResult.compilerOutput - return completion(.success(evaluationResult)) - } - - // Read the JSON output that was emitted by libPackageDescription. - let jsonOutput: String = try localFileSystem.readFileContents(jsonOutputFile) - evaluationResult.manifestJSON = jsonOutput - - delegateQueue?.async { - delegate?.didEvaluate( - packageIdentity: packageIdentity, - packageLocation: packageLocation, - manifestPath: manifestPath, - duration: evaluationStart.distance(to: .now()) - ) - } - - completion(.success(evaluationResult)) - } catch { - completion(.failure(error)) - } - } - } + + var environment = Environment.current +#if os(Windows) + let windowsPathComponent = runtimePath.pathString.replacingOccurrences(of: "/", with: "\\") + environment["Path"] = "\(windowsPathComponent);\(environment["Path"] ?? "")" +#endif + + let runResult = try await AsyncProcess.popen( + arguments: cmd, + environment: environment + ) + + if let runOutput = try (runResult.utf8Output() + runResult.utf8stderrOutput()).spm_chuzzle() { + // Append the runtime output to any compiler output we've received. + evaluationResult.compilerOutput = (evaluationResult.compilerOutput ?? "") + runOutput } - } catch { - return callbackQueue.async { - completion(.failure(error)) + + // Return now if there was an error. + if runResult.exitStatus != .terminated(code: 0) { + // TODO: should this simply be an error? + // return completion(.failure(AsyncProcessResult.Error.nonZeroExit(runResult))) + evaluationResult.errorOutput = evaluationResult.compilerOutput + return evaluationResult + } + + // Read the JSON output that was emitted by libPackageDescription. + let jsonOutput: String = try localFileSystem.readFileContents(jsonOutputFile) + evaluationResult.manifestJSON = jsonOutput + + delegateQueue?.async { + delegate?.didEvaluate( + packageIdentity: packageIdentity, + packageLocation: packageLocation, + manifestPath: manifestPath, + duration: evaluationStart.distance(to: .now()) + ) } + + return evaluationResult } } } diff --git a/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift b/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift index 15d61bd95d9..9d9c728745d 100644 --- a/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift +++ b/Sources/Workspace/PackageContainer/FileSystemPackageContainer.swift @@ -89,8 +89,7 @@ public struct FileSystemPackageContainer: PackageContainer { dependencyMapper: self.dependencyMapper, fileSystem: self.fileSystem, observabilityScope: self.observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) } } diff --git a/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift b/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift index 4d4a6f99115..acd18e20ed9 100644 --- a/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift +++ b/Sources/Workspace/PackageContainer/RegistryPackageContainer.swift @@ -152,8 +152,7 @@ public class RegistryPackageContainer: PackageContainer { dependencyMapper: self.dependencyMapper, fileSystem: result.fileSystem, observabilityScope: self.observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) } diff --git a/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift b/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift index da1f43b7245..21790a6ef5a 100644 --- a/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift +++ b/Sources/Workspace/PackageContainer/SourceControlPackageContainer.swift @@ -405,8 +405,7 @@ internal final class SourceControlPackageContainer: PackageContainer, CustomStri dependencyMapper: self.dependencyMapper, fileSystem: fileSystem, observabilityScope: self.observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) } diff --git a/Sources/Workspace/Workspace+Manifests.swift b/Sources/Workspace/Workspace+Manifests.swift index 780a957e90d..37a8b79825d 100644 --- a/Sources/Workspace/Workspace+Manifests.swift +++ b/Sources/Workspace/Workspace+Manifests.swift @@ -722,8 +722,7 @@ extension Workspace { dependencyMapper: self.dependencyMapper, fileSystem: fileSystem, observabilityScope: manifestLoadingScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) let duration = start.distance(to: .now()) diff --git a/Sources/Workspace/Workspace+Registry.swift b/Sources/Workspace/Workspace+Registry.swift index 571068546c1..71f0a44cb35 100644 --- a/Sources/Workspace/Workspace+Registry.swift +++ b/Sources/Workspace/Workspace+Registry.swift @@ -84,8 +84,7 @@ extension Workspace { dependencyMapper: any DependencyMapper, fileSystem: any FileSystem, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue ) async throws -> Manifest { let manifest = try await self.underlying.load( manifestPath: manifestPath, @@ -98,8 +97,7 @@ extension Workspace { dependencyMapper: dependencyMapper, fileSystem: fileSystem, observabilityScope: observabilityScope, - delegateQueue: delegateQueue, - callbackQueue: callbackQueue + delegateQueue: delegateQueue ) return try await self.transformSourceControlDependenciesToRegistry( diff --git a/Sources/_InternalTestSupport/MockManifestLoader.swift b/Sources/_InternalTestSupport/MockManifestLoader.swift index 43fa9fd82c4..69999d76095 100644 --- a/Sources/_InternalTestSupport/MockManifestLoader.swift +++ b/Sources/_InternalTestSupport/MockManifestLoader.swift @@ -61,8 +61,7 @@ public final class MockManifestLoader: ManifestLoaderProtocol { dependencyMapper: DependencyMapper, fileSystem: FileSystem, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue ) throws -> Manifest { let key = Key(url: packageLocation, version: packageVersion?.version) if let result = self.manifests[key] { @@ -117,8 +116,7 @@ extension ManifestLoader { dependencyMapper: dependencyMapper ?? DefaultDependencyMapper(identityResolver: identityResolver), fileSystem: fileSystem, observabilityScope: observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) } } @@ -164,8 +162,7 @@ extension ManifestLoader { dependencyMapper: dependencyMapper ?? DefaultDependencyMapper(identityResolver: identityResolver), fileSystem: fileSystem, observabilityScope: observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) } } diff --git a/Sources/swift-bootstrap/main.swift b/Sources/swift-bootstrap/main.swift index 4bc8d04ba54..1d1c4e45ce7 100644 --- a/Sources/swift-bootstrap/main.swift +++ b/Sources/swift-bootstrap/main.swift @@ -435,8 +435,7 @@ struct SwiftBootstrapBuildTool: AsyncParsableCommand { dependencyMapper: dependencyMapper, fileSystem: fileSystem, observabilityScope: observabilityScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) } } diff --git a/Tests/WorkspaceTests/ManifestSourceGenerationTests.swift b/Tests/WorkspaceTests/ManifestSourceGenerationTests.swift index 9b121165fcd..55f643e7ba7 100644 --- a/Tests/WorkspaceTests/ManifestSourceGenerationTests.swift +++ b/Tests/WorkspaceTests/ManifestSourceGenerationTests.swift @@ -63,8 +63,7 @@ final class ManifestSourceGenerationTests: XCTestCase { dependencyMapper: dependencyMapper, fileSystem: fs, observabilityScope: observability.topScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) XCTAssertNoDiagnostics(observability.diagnostics) @@ -92,8 +91,7 @@ final class ManifestSourceGenerationTests: XCTestCase { dependencyMapper: dependencyMapper, fileSystem: fs, observabilityScope: observability.topScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) XCTAssertNoDiagnostics(observability.diagnostics) diff --git a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift index 91178942302..d4d89d6e516 100644 --- a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift +++ b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift @@ -271,8 +271,7 @@ final class RegistryPackageContainerTests: XCTestCase { dependencyMapper: DependencyMapper, fileSystem: FileSystem, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue ) async throws -> Manifest { Manifest.createManifest( displayName: packageIdentity.description, diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 2e054a6e503..4cf3d23fdff 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -12027,8 +12027,7 @@ final class WorkspaceTests: XCTestCase { dependencyMapper: DependencyMapper, fileSystem: FileSystem, observabilityScope: ObservabilityScope, - delegateQueue: DispatchQueue, - callbackQueue: DispatchQueue + delegateQueue: DispatchQueue ) throws -> Manifest { if let error { throw error From a8a995b9edac6d9c20aa979f9fbfe878b6cd104f Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 6 Nov 2024 17:41:44 +0000 Subject: [PATCH 33/33] Fix build issues in tests --- Tests/PackageLoadingTests/PD_4_2_LoadingTests.swift | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Tests/PackageLoadingTests/PD_4_2_LoadingTests.swift b/Tests/PackageLoadingTests/PD_4_2_LoadingTests.swift index 801a92bf543..3436f113f76 100644 --- a/Tests/PackageLoadingTests/PD_4_2_LoadingTests.swift +++ b/Tests/PackageLoadingTests/PD_4_2_LoadingTests.swift @@ -638,8 +638,7 @@ final class PackageDescription4_2LoadingTests: PackageDescriptionLoadingTests { dependencyMapper: dependencyMapper, fileSystem: localFileSystem, observabilityScope: observability.topScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) XCTAssertNoDiagnostics(observability.diagnostics) @@ -658,8 +657,7 @@ final class PackageDescription4_2LoadingTests: PackageDescriptionLoadingTests { dependencyMapper: dependencyMapper, fileSystem: localFileSystem, observabilityScope: observability.topScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) XCTAssertNoDiagnostics(observability.diagnostics) @@ -722,8 +720,7 @@ final class PackageDescription4_2LoadingTests: PackageDescriptionLoadingTests { dependencyMapper: dependencyMapper, fileSystem: localFileSystem, observabilityScope: observability.topScope, - delegateQueue: .sharedConcurrent, - callbackQueue: .sharedConcurrent + delegateQueue: .sharedConcurrent ) XCTAssertEqual(manifest.displayName, "Trivial-\(random)")