From ea3e8245233856cfc22901fa6f9c28101ace5e54 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 29 Aug 2024 13:44:42 +0100 Subject: [PATCH] Enable Swift 6 mode when available, sync `QueryEngine` sources (#136) As SwiftPM is more actively developed, let's consider it to be the source of truth and vendor `QueryEngine` back into `swift-sdk-generator` together with more fresh `AsyncFileSystem` code, storing all those files in `Source/Helpers/Vendor` subdirectory. All this allows us to fix concurrency errors that prevent it from building in Swift 6 mode. --- Package.swift | 24 +- Sources/AsyncProcess/ProcessExecutor.swift | 6 +- .../Cache/FileCacheRecord.swift | 38 --- .../FileSystem/FileSystem.swift | 36 --- .../FileSystem/OpenReadableFile.swift | 53 ---- .../FileSystem/OpenWritableFile.swift | 31 -- .../FileSystem/ReadableFileStream.swift | 103 ------- .../FileSystem/VirtualFileSystem.swift | 45 --- Sources/GeneratorEngine/Query.swift | 282 ----------------- .../Vendor/QueryEngine}/CacheKey.swift | 0 .../Vendor/QueryEngine/FileCacheRecord.swift | 49 +++ .../Vendor/QueryEngine}/FileLock.swift | 4 +- .../Helpers/Vendor/QueryEngine/Query.swift | 284 ++++++++++++++++++ .../Vendor/QueryEngine/QueryEngine.swift} | 46 +-- .../Vendor/QueryEngine}/SQLite.swift | 2 +- .../QueryEngine}/SQLiteBackedCache.swift | 2 +- Sources/Helpers/Vendor/README.md | 8 +- Sources/Helpers/Vendor/Triple.swift | 93 ++++-- .../_AsyncFileSystem/AsyncFileSystem.swift | 77 +++++ .../_AsyncFileSystem/ConcurrencySupport.swift | 32 ++ .../_AsyncFileSystem/MockFileSystem.swift | 72 +++++ .../_AsyncFileSystem/OSFileSystem.swift} | 24 +- .../_AsyncFileSystem/OpenReadableFile.swift | 51 ++++ .../_AsyncFileSystem/OpenWritableFile.swift | 79 +++++ .../_AsyncFileSystem/ReadableFileStream.swift | 126 ++++++++ .../_AsyncFileSystem/WritableStream.swift | 32 ++ .../Artifacts/DownloadableArtifacts.swift | 2 +- .../SwiftSDKGenerator+Download.swift | 12 +- .../SwiftSDKGenerator+Entrypoint.swift | 3 +- .../Generator/SwiftSDKGenerator+Unpack.swift | 4 +- .../Generator/SwiftSDKGenerator.swift | 3 +- .../Queries/CMakeBuildQuery.swift | 4 +- .../Queries/DownloadArtifactQuery.swift | 7 +- .../Queries/DownloadFileQuery.swift | 4 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 7 +- .../SwiftSDKRecipes/SwiftSDKRecipe.swift | 4 +- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 4 +- .../AsyncProcessTests/IntegrationTests.swift | 14 +- Tests/GeneratorEngineTests/EngineTests.swift | 25 +- .../EndToEndTests.swift | 4 +- 40 files changed, 986 insertions(+), 710 deletions(-) delete mode 100644 Sources/GeneratorEngine/Cache/FileCacheRecord.swift delete mode 100644 Sources/GeneratorEngine/FileSystem/FileSystem.swift delete mode 100644 Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift delete mode 100644 Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift delete mode 100644 Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift delete mode 100644 Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift delete mode 100644 Sources/GeneratorEngine/Query.swift rename Sources/{GeneratorEngine/Cache => Helpers/Vendor/QueryEngine}/CacheKey.swift (100%) create mode 100644 Sources/Helpers/Vendor/QueryEngine/FileCacheRecord.swift rename Sources/{GeneratorEngine/FileSystem => Helpers/Vendor/QueryEngine}/FileLock.swift (98%) create mode 100644 Sources/Helpers/Vendor/QueryEngine/Query.swift rename Sources/{GeneratorEngine/Engine.swift => Helpers/Vendor/QueryEngine/QueryEngine.swift} (80%) rename Sources/{GeneratorEngine/Cache => Helpers/Vendor/QueryEngine}/SQLite.swift (99%) rename Sources/{GeneratorEngine/Cache => Helpers/Vendor/QueryEngine}/SQLiteBackedCache.swift (99%) create mode 100644 Sources/Helpers/Vendor/_AsyncFileSystem/AsyncFileSystem.swift create mode 100644 Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift create mode 100644 Sources/Helpers/Vendor/_AsyncFileSystem/MockFileSystem.swift rename Sources/{GeneratorEngine/FileSystem/LocalFileSystem.swift => Helpers/Vendor/_AsyncFileSystem/OSFileSystem.swift} (63%) create mode 100644 Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift create mode 100644 Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift create mode 100644 Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift create mode 100644 Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift diff --git a/Package.swift b/Package.swift index 7acaaed..8094ef8 100644 --- a/Package.swift +++ b/Package.swift @@ -34,7 +34,6 @@ let package = Package( .product(name: "NIOHTTP1", package: "swift-nio"), .product(name: "Logging", package: "swift-log"), .product(name: "SystemPackage", package: "swift-system"), - "GeneratorEngine", "Helpers", ], exclude: ["Dockerfiles"], @@ -51,24 +50,20 @@ let package = Package( .enableExperimentalFeature("StrictConcurrency=complete"), ] ), - .target( - name: "GeneratorEngine", - dependencies: [ - .product(name: "Crypto", package: "swift-crypto"), - .product(name: "Logging", package: "swift-log"), - .product(name: "SystemPackage", package: "swift-system"), - "Helpers", - "SystemSQLite", - ] - ), .testTarget( name: "GeneratorEngineTests", dependencies: [ - "GeneratorEngine", + "Helpers", ] ), .target( name: "Helpers", + dependencies: [ + "SystemSQLite", + .product(name: "Crypto", package: "swift-crypto"), + .product(name: "Logging", package: "swift-log"), + .product(name: "SystemPackage", package: "swift-system"), + ], exclude: ["Vendor/README.md"], swiftSettings: [ .enableExperimentalFeature("StrictConcurrency=complete"), @@ -103,7 +98,8 @@ let package = Package( .product(name: "Logging", package: "swift-log"), ] ), - ] + ], + swiftLanguageVersions: [.v5, .version("6")] ) struct Configuration { @@ -124,7 +120,7 @@ if configuration.useAsyncHttpClient { package.dependencies.append( .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.19.0") ) - let targetsToAppend: Set = ["SwiftSDKGenerator"] + let targetsToAppend: Set = ["SwiftSDKGenerator", "Helpers"] for target in package.targets.filter({ targetsToAppend.contains($0.name) }) { target.dependencies.append( .product(name: "AsyncHTTPClient", package: "async-http-client") diff --git a/Sources/AsyncProcess/ProcessExecutor.swift b/Sources/AsyncProcess/ProcessExecutor.swift index 9bbb9c1..59f1d0a 100644 --- a/Sources/AsyncProcess/ProcessExecutor.swift +++ b/Sources/AsyncProcess/ProcessExecutor.swift @@ -119,10 +119,10 @@ private struct OutputConsumptionState: OptionSet { } /// Type-erasing type analogous to `AnySequence` from the Swift standard library. -private struct AnyAsyncSequence: AsyncSequence { - private let iteratorFactory: () -> AsyncIterator +private struct AnyAsyncSequence: AsyncSequence & Sendable where Element: Sendable { + private let iteratorFactory: @Sendable () -> AsyncIterator - init(_ asyncSequence: S) where S.Element == Element { + init(_ asyncSequence: S) where S.Element == Element { self.iteratorFactory = { var iterator = asyncSequence.makeAsyncIterator() return AsyncIterator { try await iterator.next() } diff --git a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift deleted file mode 100644 index fdef551..0000000 --- a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift +++ /dev/null @@ -1,38 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import struct SystemPackage.FilePath - -public struct FileCacheRecord: Sendable { - public let path: FilePath - public let hash: String -} - -extension FileCacheRecord: Codable { - enum CodingKeys: CodingKey { - case path - case hash - } - - // FIXME: `Codable` on `FilePath` is broken, thus all `Codable` types with `FilePath` properties need a custom impl. - public init(from decoder: any Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - self.path = try FilePath(container.decode(String.self, forKey: .path)) - self.hash = try container.decode(String.self, forKey: .hash) - } - - public func encode(to encoder: any Encoder) throws { - var container = encoder.container(keyedBy: CodingKeys.self) - try container.encode(self.path.string, forKey: .path) - try container.encode(self.hash, forKey: .hash) - } -} diff --git a/Sources/GeneratorEngine/FileSystem/FileSystem.swift b/Sources/GeneratorEngine/FileSystem/FileSystem.swift deleted file mode 100644 index 90c70c2..0000000 --- a/Sources/GeneratorEngine/FileSystem/FileSystem.swift +++ /dev/null @@ -1,36 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import protocol Crypto.HashFunction -import struct SystemPackage.Errno -import struct SystemPackage.FilePath - -public protocol FileSystem: Actor { - func withOpenReadableFile(_ path: FilePath, _ body: (OpenReadableFile) async throws -> T) async throws -> T - func withOpenWritableFile(_ path: FilePath, _ body: (OpenWritableFile) async throws -> T) async throws -> T -} - -enum FileSystemError: Error { - case fileDoesNotExist(FilePath) - case bufferLimitExceeded(FilePath) - case systemError(FilePath, Errno) -} - -extension Error { - func attach(path: FilePath) -> any Error { - if let error = self as? Errno { - return FileSystemError.systemError(path, error) - } else { - return self - } - } -} diff --git a/Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift b/Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift deleted file mode 100644 index 05b441a..0000000 --- a/Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift +++ /dev/null @@ -1,53 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import struct SystemPackage.FileDescriptor - -public struct OpenReadableFile { - let readChunkSize: Int - - enum FileHandle { - case local(FileDescriptor) - case virtual([UInt8]) - } - - let fileHandle: FileHandle - - func read() async throws -> ReadableFileStream { - switch self.fileHandle { - case let .local(fileDescriptor): - return ReadableFileStream.local(.init(fileDescriptor: fileDescriptor, readChunkSize: self.readChunkSize)) - case let .virtual(array): - return ReadableFileStream.virtual(.init(bytes: array)) - } - } - - func hash(with hashFunction: inout some HashFunction) async throws { - switch self.fileHandle { - case let .local(fileDescriptor): - var buffer = [UInt8](repeating: 0, count: readChunkSize) - var bytesRead = 0 - repeat { - bytesRead = try buffer.withUnsafeMutableBytes { - try fileDescriptor.read(into: $0) - } - - if bytesRead > 0 { - hashFunction.update(data: buffer[0.. 0 - case let .virtual(array): - hashFunction.update(data: array) - } - } -} diff --git a/Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift b/Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift deleted file mode 100644 index 0131baa..0000000 --- a/Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import struct SystemPackage.FileDescriptor - -public struct OpenWritableFile { - enum FileHandle { - case local(FileDescriptor) - case virtual(VirtualFileSystem.Storage, FilePath) - } - - let fileHandle: FileHandle - - func write(_ bytes: some Sequence) async throws { - switch self.fileHandle { - case let .local(fileDescriptor): - _ = try fileDescriptor.writeAll(bytes) - case let .virtual(storage, path): - storage.content[path, default: []].append(contentsOf: bytes) - } - } -} diff --git a/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift b/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift deleted file mode 100644 index c19160e..0000000 --- a/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift +++ /dev/null @@ -1,103 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SystemPackage - -public enum ReadableFileStream: AsyncSequence { - public typealias Element = [UInt8] - - case local(LocalReadableFileStream) - case virtual(VirtualReadableFileStream) - - public enum Iterator: AsyncIteratorProtocol { - case local(LocalReadableFileStream.Iterator) - case virtual(VirtualReadableFileStream.Iterator) - - public func next() async throws -> [UInt8]? { - switch self { - case let .local(local): - return try await local.next() - case let .virtual(virtual): - return try await virtual.next() - } - } - } - - public func makeAsyncIterator() -> Iterator { - switch self { - case let .local(local): - return .local(local.makeAsyncIterator()) - case let .virtual(virtual): - return .virtual(virtual.makeAsyncIterator()) - } - } -} - -public struct LocalReadableFileStream: AsyncSequence { - public typealias Element = [UInt8] - - let fileDescriptor: FileDescriptor - let readChunkSize: Int - - public final class Iterator: AsyncIteratorProtocol { - init(_ fileDescriptor: FileDescriptor, readChunkSize: Int) { - self.fileDescriptor = fileDescriptor - self.readChunkSize = readChunkSize - } - - private let fileDescriptor: FileDescriptor - private let readChunkSize: Int - - public func next() async throws -> [UInt8]? { - var buffer = [UInt8](repeating: 0, count: readChunkSize) - - let bytesRead = try buffer.withUnsafeMutableBytes { - try self.fileDescriptor.read(into: $0) - } - - guard bytesRead > 0 else { - return nil - } - - buffer.removeLast(self.readChunkSize - bytesRead) - return buffer - } - } - - public func makeAsyncIterator() -> Iterator { - Iterator(self.fileDescriptor, readChunkSize: self.readChunkSize) - } -} - -public struct VirtualReadableFileStream: AsyncSequence { - public typealias Element = [UInt8] - - public final class Iterator: AsyncIteratorProtocol { - init(bytes: [UInt8]? = nil) { - self.bytes = bytes - } - - var bytes: [UInt8]? - - public func next() async throws -> [UInt8]? { - defer { bytes = nil } - - return self.bytes - } - } - - let bytes: [UInt8] - - public func makeAsyncIterator() -> Iterator { - Iterator(bytes: self.bytes) - } -} diff --git a/Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift b/Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift deleted file mode 100644 index 4216a12..0000000 --- a/Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift +++ /dev/null @@ -1,45 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import struct SystemPackage.FilePath - -actor VirtualFileSystem: FileSystem { - public static let defaultChunkSize = 512 * 1024 - - let readChunkSize: Int - - final class Storage { - init(_ content: [FilePath: [UInt8]]) { - self.content = content - } - - var content: [FilePath: [UInt8]] - } - - private let storage: Storage - - init(content: [FilePath: [UInt8]] = [:], readChunkSize: Int = defaultChunkSize) { - self.storage = .init(content) - self.readChunkSize = readChunkSize - } - - func withOpenReadableFile(_ path: FilePath, _ body: (OpenReadableFile) async throws -> T) async throws -> T { - guard let bytes = storage.content[path] else { - throw FileSystemError.fileDoesNotExist(path) - } - return try await body(.init(readChunkSize: self.readChunkSize, fileHandle: .virtual(bytes))) - } - - func withOpenWritableFile(_ path: FilePath, _ body: (OpenWritableFile) async throws -> T) async throws -> T { - try await body(.init(fileHandle: .virtual(self.storage, path))) - } -} diff --git a/Sources/GeneratorEngine/Query.swift b/Sources/GeneratorEngine/Query.swift deleted file mode 100644 index 557d557..0000000 --- a/Sources/GeneratorEngine/Query.swift +++ /dev/null @@ -1,282 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import struct SystemPackage.FilePath - -public protocol Query: Sendable { - associatedtype Key: CacheKey - var cacheKey: Key { get } - func run(engine: Engine) async throws -> FilePath -} - -public protocol CachingQuery: Query, CacheKey where Self.Key == Self {} -public extension CachingQuery { - var cacheKey: Key { self } -} - -final class HashEncoder: Encoder { - enum Error: Swift.Error { - case noCacheKeyConformance(Encodable.Type) - } - - var codingPath: [any CodingKey] - - var userInfo: [CodingUserInfoKey: Any] - - func container(keyedBy type: Key.Type) -> KeyedEncodingContainer where Key: CodingKey { - .init(KeyedContainer(encoder: self)) - } - - func unkeyedContainer() -> any UnkeyedEncodingContainer { - self - } - - func singleValueContainer() -> any SingleValueEncodingContainer { - self - } - - init() { - self.hashFunction = Hash() - self.codingPath = [] - self.userInfo = [:] - } - - fileprivate var hashFunction = Hash() - - func finalize() -> Hash.Digest { - self.hashFunction.finalize() - } -} - -extension HashEncoder: SingleValueEncodingContainer { - func encodeNil() throws { - // FIXME: this doesn't encode the name of the underlying optional type, - // but `Encoder` protocol is limited and can't provide this for us. - var str = "nil" - str.withUTF8 { - self.hashFunction.update(data: $0) - } - } - - func encode(_ value: Bool) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: String) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: Double) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: Float) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: Int) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: Int8) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: Int16) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: Int32) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: Int64) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: UInt) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: UInt8) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: UInt16) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: UInt32) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: UInt64) throws { - value.hash(with: &self.hashFunction) - } - - func encode(_ value: T) throws where T: Encodable { - if let leaf = value as? LeafCacheKey { - leaf.hash(with: &self.hashFunction) - return - } - - guard value is CacheKey else { - throw Error.noCacheKeyConformance(T.self) - } - - try String(describing: T.self).encode(to: self) - try value.encode(to: self) - } -} - -extension HashEncoder: UnkeyedEncodingContainer { - var count: Int { - 0 - } - - func nestedContainer(keyedBy keyType: NestedKey.Type) -> KeyedEncodingContainer - where NestedKey: CodingKey - { - KeyedEncodingContainer(KeyedContainer(encoder: self)) - } - - func nestedUnkeyedContainer() -> any UnkeyedEncodingContainer { - self - } - - func superEncoder() -> any Encoder { - fatalError() - } -} - -extension HashEncoder { - struct KeyedContainer: KeyedEncodingContainerProtocol { - var encoder: HashEncoder - var codingPath: [any CodingKey] { self.encoder.codingPath } - - mutating func encodeNil(forKey key: K) throws { - // FIXME: this doesn't encode the name of the underlying optional type, - // but `Encoder` protocol is limited and can't provide this for us. - var str = "nil" - str.withUTF8 { - self.encoder.hashFunction.update(data: $0) - } - } - - mutating func encode(_ value: Bool, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: String, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: Double, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: Float, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: Int, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: Int8, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: Int16, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: Int32, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: Int64, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: UInt, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: UInt8, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: UInt16, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: UInt32, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: UInt64, forKey key: K) throws { - key.stringValue.hash(with: &self.encoder.hashFunction) - value.hash(with: &self.encoder.hashFunction) - } - - mutating func encode(_ value: T, forKey key: K) throws where T: Encodable { - if let leaf = value as? LeafCacheKey { - leaf.hash(with: &self.encoder.hashFunction) - return - } - guard value is CacheKey else { - throw Error.noCacheKeyConformance(T.self) - } - - try String(reflecting: T.self).encode(to: self.encoder) - key.stringValue.hash(with: &self.encoder.hashFunction) - try value.encode(to: self.encoder) - } - - mutating func nestedContainer( - keyedBy keyType: NestedKey.Type, - forKey key: K - ) -> KeyedEncodingContainer where NestedKey: CodingKey { - key.stringValue.hash(with: &self.encoder.hashFunction) - return self.encoder.nestedContainer(keyedBy: keyType) - } - - mutating func nestedUnkeyedContainer(forKey key: K) -> any UnkeyedEncodingContainer { - key.stringValue.hash(with: &self.encoder.hashFunction) - return self.encoder - } - - mutating func superEncoder() -> any Encoder { - fatalError() - } - - mutating func superEncoder(forKey key: K) -> any Encoder { - fatalError() - } - - typealias Key = K - } -} diff --git a/Sources/GeneratorEngine/Cache/CacheKey.swift b/Sources/Helpers/Vendor/QueryEngine/CacheKey.swift similarity index 100% rename from Sources/GeneratorEngine/Cache/CacheKey.swift rename to Sources/Helpers/Vendor/QueryEngine/CacheKey.swift diff --git a/Sources/Helpers/Vendor/QueryEngine/FileCacheRecord.swift b/Sources/Helpers/Vendor/QueryEngine/FileCacheRecord.swift new file mode 100644 index 0000000..a5aa195 --- /dev/null +++ b/Sources/Helpers/Vendor/QueryEngine/FileCacheRecord.swift @@ -0,0 +1,49 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +// FIXME: need a new swift-system tag to remove `@preconcurrency` +@preconcurrency import struct SystemPackage.FilePath + +public struct FileCacheRecord: Sendable { + public let path: FilePath + public let hash: String +} + +extension FileCacheRecord: Codable { + enum CodingKeys: CodingKey { + case path + case hash + } + + // FIXME: `Codable` on `FilePath` is broken, thus all `Codable` types with `FilePath` properties need a custom impl. + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.path = try FilePath(container.decode(String.self, forKey: .path)) + self.hash = try container.decode(String.self, forKey: .hash) + } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.path.string, forKey: .path) + try container.encode(self.hash, forKey: .hash) + } +} + +extension OpenReadableFile { + func hash(with hashFunction: inout some HashFunction) async throws { + let stream = try await self.read() + + for try await bytes in stream { + hashFunction.update(data: bytes) + } + } +} diff --git a/Sources/GeneratorEngine/FileSystem/FileLock.swift b/Sources/Helpers/Vendor/QueryEngine/FileLock.swift similarity index 98% rename from Sources/GeneratorEngine/FileSystem/FileLock.swift rename to Sources/Helpers/Vendor/QueryEngine/FileLock.swift index b765027..574718a 100644 --- a/Sources/GeneratorEngine/FileSystem/FileLock.swift +++ b/Sources/Helpers/Vendor/QueryEngine/FileLock.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -115,7 +115,7 @@ final class FileLock { permissions: [.groupReadWrite, .ownerReadWrite, .otherReadWrite] ).rawValue } catch { - throw error.attach(path: self.lockFile) + throw error.attach(self.lockFile) } } // Acquire lock on the file. diff --git a/Sources/Helpers/Vendor/QueryEngine/Query.swift b/Sources/Helpers/Vendor/QueryEngine/Query.swift new file mode 100644 index 0000000..c3d2751 --- /dev/null +++ b/Sources/Helpers/Vendor/QueryEngine/Query.swift @@ -0,0 +1,284 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct SystemPackage.FilePath + +public protocol Query: Sendable { + associatedtype Key: CacheKey + var cacheKey: Key { get } + func run(engine: QueryEngine) async throws -> FilePath +} + +public protocol CachingQuery: Query, CacheKey where Self.Key == Self {} +extension CachingQuery { + public var cacheKey: Key { self } +} + +// SwiftPM has to be built with Swift 5.8 on CI and also needs to support CMake for bootstrapping on Windows. +// This means we can't implement persistable hashing with macros (unavailable in Swift 5.8 and additional effort to +// set up with CMake when Swift 5.9 is available for all CI jobs) and have to stick to `Encodable` for now. +final class HashEncoder: Encoder { + enum Error: Swift.Error { + case noCacheKeyConformance(Encodable.Type) + } + + var codingPath: [any CodingKey] + + var userInfo: [CodingUserInfoKey: Any] + + func container(keyedBy type: Key.Type) -> KeyedEncodingContainer where Key: CodingKey { + .init(KeyedContainer(encoder: self)) + } + + func unkeyedContainer() -> any UnkeyedEncodingContainer { + self + } + + func singleValueContainer() -> any SingleValueEncodingContainer { + self + } + + init() { + self.hashFunction = Hash() + self.codingPath = [] + self.userInfo = [:] + } + + fileprivate var hashFunction = Hash() + + func finalize() -> Hash.Digest { + self.hashFunction.finalize() + } +} + +extension HashEncoder: SingleValueEncodingContainer { + func encodeNil() throws { + // FIXME: this doesn't encode the name of the underlying optional type, + // but `Encoder` protocol is limited and can't provide this for us. + var str = "nil" + str.withUTF8 { + self.hashFunction.update(data: $0) + } + } + + func encode(_ value: Bool) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: String) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Double) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Float) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int8) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int16) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int32) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: Int64) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt8) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt16) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt32) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: UInt64) throws { + value.hash(with: &self.hashFunction) + } + + func encode(_ value: T) throws where T: Encodable { + if let leaf = value as? LeafCacheKey { + leaf.hash(with: &self.hashFunction) + return + } + + guard value is CacheKey else { + throw Error.noCacheKeyConformance(T.self) + } + + try String(describing: T.self).encode(to: self) + try value.encode(to: self) + } +} + +extension HashEncoder: UnkeyedEncodingContainer { + var count: Int { + 0 + } + + func nestedContainer(keyedBy keyType: NestedKey.Type) -> KeyedEncodingContainer + where NestedKey: CodingKey { + KeyedEncodingContainer(KeyedContainer(encoder: self)) + } + + func nestedUnkeyedContainer() -> any UnkeyedEncodingContainer { + self + } + + func superEncoder() -> any Encoder { + fatalError() + } +} + +extension HashEncoder { + struct KeyedContainer: KeyedEncodingContainerProtocol { + var encoder: HashEncoder + var codingPath: [any CodingKey] { self.encoder.codingPath } + + mutating func encodeNil(forKey key: K) throws { + // FIXME: this doesn't encode the name of the underlying optional type, + // but `Encoder` protocol is limited and can't provide this for us. + var str = "nil" + str.withUTF8 { + self.encoder.hashFunction.update(data: $0) + } + } + + mutating func encode(_ value: Bool, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: String, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Double, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Float, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int8, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int16, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int32, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: Int64, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt8, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt16, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt32, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: UInt64, forKey key: K) throws { + key.stringValue.hash(with: &self.encoder.hashFunction) + value.hash(with: &self.encoder.hashFunction) + } + + mutating func encode(_ value: T, forKey key: K) throws where T: Encodable { + if let leaf = value as? LeafCacheKey { + leaf.hash(with: &self.encoder.hashFunction) + return + } + guard value is CacheKey else { + throw Error.noCacheKeyConformance(T.self) + } + + try String(reflecting: T.self).encode(to: self.encoder) + key.stringValue.hash(with: &self.encoder.hashFunction) + try value.encode(to: self.encoder) + } + + mutating func nestedContainer( + keyedBy keyType: NestedKey.Type, + forKey key: K + ) -> KeyedEncodingContainer where NestedKey: CodingKey { + key.stringValue.hash(with: &self.encoder.hashFunction) + return self.encoder.nestedContainer(keyedBy: keyType) + } + + mutating func nestedUnkeyedContainer(forKey key: K) -> any UnkeyedEncodingContainer { + key.stringValue.hash(with: &self.encoder.hashFunction) + return self.encoder + } + + mutating func superEncoder() -> any Encoder { + fatalError() + } + + mutating func superEncoder(forKey key: K) -> any Encoder { + fatalError() + } + + typealias Key = K + } +} diff --git a/Sources/GeneratorEngine/Engine.swift b/Sources/Helpers/Vendor/QueryEngine/QueryEngine.swift similarity index 80% rename from Sources/GeneratorEngine/Engine.swift rename to Sources/Helpers/Vendor/QueryEngine/QueryEngine.swift index 5d4cbfc..eb560e5 100644 --- a/Sources/GeneratorEngine/Engine.swift +++ b/Sources/Helpers/Vendor/QueryEngine/QueryEngine.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,18 +10,21 @@ // //===----------------------------------------------------------------------===// -@_exported import Crypto -import Helpers -import struct Logging.Logger -@_exported import struct SystemPackage.FilePath +import Crypto +import Logging +@preconcurrency import SystemPackage -public func withEngine( - _ fileSystem: any FileSystem, +#if canImport(AsyncHTTPClient) + import AsyncHTTPClient +#endif + +public func withQueryEngine( + _ fileSystem: some AsyncFileSystem, _ logger: Logger, cacheLocation: SQLite.Location, - _ body: @Sendable (Engine) async throws -> () + _ body: @Sendable (QueryEngine) async throws -> Void ) async throws { - let engine = Engine( + let engine = QueryEngine( fileSystem, logger, cacheLocation: cacheLocation @@ -36,12 +39,17 @@ public func withEngine( /// Cacheable computations engine. Currently the engine makes an assumption that computations produce same results for /// the same query values and write results to a single file path. -public actor Engine { +public actor QueryEngine { private(set) var cacheHits = 0 private(set) var cacheMisses = 0 - public let fileSystem: any FileSystem - public let logger: Logger + public let fileSystem: any AsyncFileSystem + + #if canImport(AsyncHTTPClient) + public let httpClient = HTTPClient() + #endif + + public let observabilityScope: Logger private let resultsCache: SQLiteBackedCache private var isShutDown = false @@ -52,19 +60,23 @@ public actor Engine { /// - Parameter cacheLocation: Location of cache storage used by the engine. /// - Parameter logger: Logger to use during queries execution. init( - _ fileSystem: any FileSystem, - _ logger: Logger, - cacheLocation: SQLite.Location + _ fileSystem: any AsyncFileSystem, + _ observabilityScope: Logger, + cacheLocation: SQLite.Location = .memory ) { self.fileSystem = fileSystem - self.logger = logger - self.resultsCache = SQLiteBackedCache(tableName: "cache_table", location: cacheLocation, logger) + self.observabilityScope = observabilityScope + self.resultsCache = SQLiteBackedCache(tableName: "cache_table", location: cacheLocation, observabilityScope) } public func shutDown() async throws { precondition(!self.isShutDown, "`QueryEngine/shutDown` should be called only once") try self.resultsCache.close() + #if canImport(AsyncHTTPClient) + try await httpClient.shutdown() + #endif + self.isShutDown = true } diff --git a/Sources/GeneratorEngine/Cache/SQLite.swift b/Sources/Helpers/Vendor/QueryEngine/SQLite.swift similarity index 99% rename from Sources/GeneratorEngine/Cache/SQLite.swift rename to Sources/Helpers/Vendor/QueryEngine/SQLite.swift index 9828db6..f8700c8 100644 --- a/Sources/GeneratorEngine/Cache/SQLite.swift +++ b/Sources/Helpers/Vendor/QueryEngine/SQLite.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information diff --git a/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift b/Sources/Helpers/Vendor/QueryEngine/SQLiteBackedCache.swift similarity index 99% rename from Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift rename to Sources/Helpers/Vendor/QueryEngine/SQLiteBackedCache.swift index 50a53c1..435a76d 100644 --- a/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift +++ b/Sources/Helpers/Vendor/QueryEngine/SQLiteBackedCache.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors +// Copyright (c) 2021-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information diff --git a/Sources/Helpers/Vendor/README.md b/Sources/Helpers/Vendor/README.md index 34febc1..306d5d0 100644 --- a/Sources/Helpers/Vendor/README.md +++ b/Sources/Helpers/Vendor/README.md @@ -1,3 +1,7 @@ The `Triple.swift` file in this directory has been vendored from the -[swift-driver](https://github.com/apple/swift-driver) project. The purpose of -this vendoring is not to bring the entire swift-driver package as a dependency. +[swift-driver](https://github.com/swiftlang/swift-driver) project. + +The `_AsyncFileSystem` and `QueryEngine` code in this directory has been vendored from the +[swift-package-manager](https://github.com/swiftlang/swift-package-manager) project. + +The purpose of this vendoring is not to bring these entire packages as dependencies. diff --git a/Sources/Helpers/Vendor/Triple.swift b/Sources/Helpers/Vendor/Triple.swift index 88c6ad0..714babc 100644 --- a/Sources/Helpers/Vendor/Triple.swift +++ b/Sources/Helpers/Vendor/Triple.swift @@ -32,7 +32,7 @@ /// /// This is a port of https://github.com/apple/swift-llvm/blob/stable/include/llvm/ADT/Triple.h @dynamicMemberLookup -public struct Triple { +public struct Triple: Sendable { /// `Triple` proxies predicates from `Triple.OS`, returning `false` for an unknown OS. public subscript(dynamicMember predicate: KeyPath) -> Bool { os?[keyPath: predicate] ?? false @@ -60,7 +60,7 @@ public struct Triple { public let objectFormat: ObjectFormat? /// Represents a version that may be present in the target triple. - public struct Version: Equatable, Comparable, CustomStringConvertible { + public struct Version: Equatable, Comparable, CustomStringConvertible, Sendable { public static let zero = Version(0, 0, 0) public var major: Int @@ -126,7 +126,7 @@ public struct Triple { parser.components.resize(toCount: 4, paddingWith: "") parser.components[2] = "windows" if parsedEnv?.value.environment == nil { - if let objectFormat = parsedEnv?.value.objectFormat, objectFormat != .coff { + if let objectFormat = parsedEnv?.value.objectFormat { parser.components[3] = Substring(objectFormat.name) } else { parser.components[3] = "msvc" @@ -415,7 +415,7 @@ extension Triple { } } - public enum Arch: String, CaseIterable, Decodable { + public enum Arch: String, CaseIterable, Decodable, Sendable { /// ARM (little endian): arm, armv.*, xscale case arm // ARM (big endian): armeb @@ -520,6 +520,8 @@ extension Triple { case renderscript32 // 64-bit RenderScript case renderscript64 + // Xtensa instruction set + case xtensa static func parse(_ archName: Substring) -> Triple.Arch? { switch archName { @@ -633,6 +635,8 @@ extension Triple { return .renderscript32 case "renderscript64": return .renderscript64 + case "xtensa": + return .xtensa case _ where archName.hasPrefix("arm") || archName.hasPrefix("thumb") || archName.hasPrefix("aarch64"): return parseARMArch(archName) @@ -822,7 +826,7 @@ extension Triple { case .arc, .arm, .armeb, .hexagon, .le32, .mips, .mipsel, .nvptx, .ppc, .r600, .riscv32, .sparc, .sparcel, .tce, .tcele, .thumb, .thumbeb, .x86, .xcore, .amdil, .hsail, .spir, .kalimba,.lanai, - .shave, .wasm32, .renderscript32, .aarch64_32, .m68k: + .shave, .wasm32, .renderscript32, .aarch64_32, .m68k, .xtensa: return 32 case .aarch64, .aarch64e, .aarch64_be, .amdgcn, .bpfel, .bpfeb, .le64, .mips64, @@ -837,11 +841,11 @@ extension Triple { // MARK: - Parse SubArch extension Triple { - public enum SubArch: Hashable { + public enum SubArch: Hashable, Sendable { - public enum ARM { + public enum ARM: Sendable { - public enum Profile { + public enum Profile: Sendable { case a, r, m } @@ -909,13 +913,13 @@ extension Triple { } } - public enum Kalimba { + public enum Kalimba: Sendable { case v3 case v4 case v5 } - public enum MIPS { + public enum MIPS: Sendable { case r6 } @@ -1015,7 +1019,7 @@ extension Triple { // MARK: - Parse Vendor extension Triple { - public enum Vendor: String, CaseIterable, TripleComponent { + public enum Vendor: String, CaseIterable, TripleComponent, Sendable { case apple case pc case scei @@ -1032,6 +1036,7 @@ extension Triple { case mesa case suse case openEmbedded = "oe" + case swift fileprivate static func parse(_ component: Substring) -> Triple.Vendor? { switch component { @@ -1067,6 +1072,8 @@ extension Triple { return .suse case "oe": return .openEmbedded + case "swift": + return .swift default: return nil } @@ -1077,7 +1084,7 @@ extension Triple { // MARK: - Parse OS extension Triple { - public enum OS: String, CaseIterable, TripleComponent { + public enum OS: String, CaseIterable, TripleComponent, Sendable { case ananas case cloudABI = "cloudabi" case darwin @@ -1113,6 +1120,7 @@ extension Triple { case hurd case wasi case emscripten + case visionos = "xros" case noneOS // 'OS' suffix purely to avoid name clash with Optional.none var name: String { @@ -1195,6 +1203,8 @@ extension Triple { return .emscripten case _ where os.hasPrefix("none"): return .noneOS + case _ where os.hasPrefix("xros"): + return .visionos default: return nil } @@ -1251,7 +1261,7 @@ extension Triple { } } - public enum Environment: String, CaseIterable, Equatable { + public enum Environment: String, CaseIterable, Equatable, Sendable { case eabihf case eabi case elfv1 @@ -1347,7 +1357,7 @@ extension Triple { // MARK: - Parse Object Format extension Triple { - public enum ObjectFormat { + public enum ObjectFormat: Sendable { case coff case elf case macho @@ -1422,7 +1432,8 @@ extension Triple { case .tce: fallthrough case .tcele: fallthrough case .thumbeb: fallthrough - case .xcore: + case .xcore: fallthrough + case .xtensa: return .elf case .ppc, .ppc64: @@ -1487,9 +1498,14 @@ extension Triple.OS { self == .watchos } + public var isVisionOS: Bool { + self == .visionos + } + + /// isOSDarwin - Is this a "Darwin" OS (OS X, iOS, or watchOS). public var isDarwin: Bool { - isMacOSX || isiOS || isWatchOS + isMacOSX || isiOS || isWatchOS || isVisionOS } } @@ -1598,8 +1614,7 @@ extension Triple { if version.major < 10 { return nil } - - case .ios, .tvos, .watchos: + case .ios, .tvos, .watchos, .visionos: // Ignore the version from the triple. This is only handled because the // the clang driver combines OS X and IOS support into a common Darwin // toolchain that wants to know the OS X version number even when targeting @@ -1632,6 +1647,8 @@ extension Triple { version.major = arch == .aarch64 ? 7 : 5 } return version + case .visionos: + return Version(15, 0, 0) case .watchos: fatalError("conflicting triple info") default: @@ -1664,6 +1681,24 @@ extension Triple { fatalError("unexpected OS for Darwin triple") } } + + public var _visionOSVersion: Version { + switch os { + case .darwin, .macosx: + return Version(1, 0, 0) + case .visionos, .ios: + var version = self.osVersion + // Default to 1.0 + if version.major == 0 { + version.major = 1 + } + return version + case .watchos: + fatalError("conflicting triple info") + default: + fatalError("unexpected OS for Darwin triple") + } + } } // MARK: - Catalyst @@ -1711,13 +1746,21 @@ fileprivate extension Array { } } -// MARK: - Linker support +// MARK: - Fully static Linux support extension Triple { - /// Returns `true` if a given triple supports producing fully statically linked executables by providing `-static` - /// flag to the linker. This implies statically linking platform's libc, and of those that Swift supports currently - /// only Musl allows that reliably. - var supportsStaticExecutables: Bool { - self.environment == .musl - } + /// Returns `true` if this is the triple for Swift's fully statically + /// linked Linux target. + var isFullyStaticLinux: Bool { + self.vendor == .swift && self.environment == .musl + } + + /// Returns `true` if a given triple supports producing fully + /// statically linked executables by providing `-static` flag to + /// the linker. This implies statically linking platform's libc, + /// and of those that Swift supports currently only Musl allows + /// that reliably. + var supportsStaticExecutables: Bool { + self.isFullyStaticLinux + } } diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/AsyncFileSystem.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/AsyncFileSystem.swift new file mode 100644 index 0000000..ef28016 --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/AsyncFileSystem.swift @@ -0,0 +1,77 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import _Concurrency +@preconcurrency import struct SystemPackage.Errno +@preconcurrency import struct SystemPackage.FilePath + +/// An abstract file system protocol with first-class support for Swift Concurrency. +public protocol AsyncFileSystem: Actor { + /// Whether a file exists on the file system. + /// - Parameter path: Absolute path to the file to check. + /// - Returns: `true` if the file exists, `false` otherwise. + func exists(_ path: FilePath) async -> Bool + + /// Temporarily opens a read-only file within a scope defined by a given closure. + /// - Parameters: + /// - path: Absolute path to a readable file on this file system. + /// - body: Closure that has temporary read-only access to the open file. The underlying file handle is closed + /// when this closure returns, thus in the current absence of `~Escapable` types in Swift 6.0, users should take + /// care not to allow this handle to escape the closure. + /// - Returns: Result of the `body` closure. + func withOpenReadableFile( + _ path: FilePath, + _ body: @Sendable (_ fileHandle: OpenReadableFile) async throws -> T + ) async throws -> T + + /// Temporarily opens a write-only file within a scope defined by a given closure. + /// - Parameters: + /// - path: Absolute path to a writable file on this file system. + /// - body: Closure that has temporary write-only access to the open file. The underlying file handle is closed + /// when this closure returns, thus in the current absence of `~Escapable` types in Swift 6.0, users should take + /// care not to allow this handle to escape the closure. + /// - Returns: Result of the `body` closure. + func withOpenWritableFile( + _ path: FilePath, + _ body: @Sendable (_ fileHandle: OpenWritableFile) async throws -> T + ) async throws -> T +} + +/// Errors that can be thrown by the ``AsyncFileSystem`` type. +public enum AsyncFileSystemError: Error { + /// A file with the associated ``FilePath`` value does not exist. + case fileDoesNotExist(FilePath) + + /// A wrapper for the underlying `SystemPackage` error types that attaches a ``FilePath`` value to it. + case systemError(FilePath, Errno) +} + +extension Error { + /// Makes a system error value more actionable and readable by the end user. + /// - Parameter path: absolute path to the file, operations on which caused this error. + /// - Returns: An ``AsyncFileSystemError`` value augmented by the given file path. + func attach(_ path: FilePath) -> any Error { + if let error = self as? Errno { + return AsyncFileSystemError.systemError(path, error) + } else { + return self + } + } +} + +extension AsyncFileSystem { + public func write(_ path: FilePath, bytes: some Collection & Sendable) async throws { + try await self.withOpenWritableFile(path) { fileHandle in + try await fileHandle.write(bytes) + } + } +} diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift new file mode 100644 index 0000000..b62b190 --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift @@ -0,0 +1,32 @@ + +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2020-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import _Concurrency +import class Dispatch.DispatchQueue + +extension DispatchQueue { + /// Schedules blocking synchronous work item on this ``DispatchQueue`` instance. + /// - Parameter work: Blocking synchronous closure that should be scheduled on this queue. + /// - Returns: Result of the `work` closure. + func scheduleOnQueue(work: @escaping @Sendable () throws -> T) async throws -> T { + try await withCheckedThrowingContinuation { continuation in + self.async { + do { + continuation.resume(returning: try work()) + } catch { + continuation.resume(throwing: error) + } + } + } + } +} diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/MockFileSystem.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/MockFileSystem.swift new file mode 100644 index 0000000..8ce5da2 --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/MockFileSystem.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@preconcurrency import struct SystemPackage.FilePath + +/// In-memory implementation of `AsyncFileSystem` for mocking and testing purposes. +public actor MockFileSystem: AsyncFileSystem { + /// The default size of chunks in bytes read by this file system. + public static let defaultChunkSize = 512 * 1024 + + /// Maximum size of chunks in bytes read by this instance of file system. + let readChunkSize: Int + + /// Underlying in-memory dictionary-based storage for this mock file system. + final class Storage { + init(_ content: [FilePath: [UInt8]]) { + self.content = content + } + + var content: [FilePath: [UInt8]] + } + + /// Concrete instance of the underlying storage used by this file system. + private let storage: Storage + + /// Creates a new instance of the mock file system. + /// - Parameters: + /// - content: Dictionary of paths to their in-memory contents to use for seeding the file system. + /// - readChunkSize: Size of chunks in bytes produced by this file system when reading files. + public init(content: [FilePath: [UInt8]] = [:], readChunkSize: Int = defaultChunkSize) { + self.storage = .init(content) + self.readChunkSize = readChunkSize + } + + public func exists(_ path: FilePath) -> Bool { + self.storage.content.keys.contains(path) + } + + /// Appends a sequence of bytes to a file. + /// - Parameters: + /// - path: absolute path of the file to append bytes to. + /// - bytes: sequence of bytes to append to file's contents. + func append(path: FilePath, bytes: some Sequence) { + storage.content[path, default: []].append(contentsOf: bytes) + } + + public func withOpenReadableFile( + _ path: FilePath, + _ body: (OpenReadableFile) async throws -> T + ) async throws -> T { + guard let bytes = storage.content[path] else { + throw AsyncFileSystemError.fileDoesNotExist(path) + } + return try await body(.init(chunkSize: self.readChunkSize, fileHandle: .mock(bytes))) + } + + public func withOpenWritableFile( + _ path: FilePath, + _ body: (OpenWritableFile) async throws -> T + ) async throws -> T { + try await body(.init(storage: .mock(self), path: path)) + } +} diff --git a/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/OSFileSystem.swift similarity index 63% rename from Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift rename to Sources/Helpers/Vendor/_AsyncFileSystem/OSFileSystem.swift index 6fe1a2b..789df6e 100644 --- a/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/OSFileSystem.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,45 +10,51 @@ // //===----------------------------------------------------------------------===// +import Foundation import SystemPackage -public actor LocalFileSystem: FileSystem { +public actor OSFileSystem: AsyncFileSystem { public static let defaultChunkSize = 512 * 1024 let readChunkSize: Int + private let ioQueue = DispatchQueue(label: "org.swift.sdk-generator-io") public init(readChunkSize: Int = defaultChunkSize) { self.readChunkSize = readChunkSize } - public func withOpenReadableFile( + public func withOpenReadableFile( _ path: FilePath, _ body: (OpenReadableFile) async throws -> T ) async throws -> T { let fd = try FileDescriptor.open(path, .readOnly) // Can't use ``FileDescriptor//closeAfter` here, as that doesn't support async closures. do { - let result = try await body(.init(readChunkSize: readChunkSize, fileHandle: .local(fd))) + let result = try await body(.init(chunkSize: readChunkSize, fileHandle: .real(fd, self.ioQueue))) try fd.close() return result } catch { try fd.close() - throw error.attach(path: path) + throw error.attach(path) } } - public func withOpenWritableFile( - _ path: SystemPackage.FilePath, + public func withOpenWritableFile( + _ path: FilePath, _ body: (OpenWritableFile) async throws -> T ) async throws -> T { let fd = try FileDescriptor.open(path, .writeOnly) do { - let result = try await body(.init(fileHandle: .local(fd))) + let result = try await body(.init(storage: .real(fd, self.ioQueue), path: path)) try fd.close() return result } catch { try fd.close() - throw error.attach(path: path) + throw error.attach(path) } } + + public func exists(_ path: SystemPackage.FilePath) async -> Bool { + FileManager.default.fileExists(atPath: path.string) + } } diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift new file mode 100644 index 0000000..6965689 --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift @@ -0,0 +1,51 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import class Dispatch.DispatchQueue +import struct SystemPackage.FileDescriptor + +/// A read-only thread-safe handle to an open file. +public struct OpenReadableFile: Sendable { + /// Maximum size of chunks in bytes produced when reading this file handle. + let chunkSize: Int + + /// Underlying storage for this file handle, dependent on the file system type that produced it. + enum Storage { + /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking + /// the Swift Concurrency thread pool. + case real(FileDescriptor, DispatchQueue) + + /// Complete contents of the file represented by this handle stored in memory as an array of bytes. + case mock([UInt8]) + } + + /// Concrete instance of underlying file storage. + let fileHandle: Storage + + /// Creates a readable ``AsyncSequence`` that can be iterated on to read from this file handle. + /// - Returns: `ReadableFileStream` value conforming to ``AsyncSequence``, ready for asynchronous iteration. + public func read() async throws -> ReadableFileStream { + switch self.fileHandle { + case let .real(fileDescriptor, ioQueue): + return ReadableFileStream.real( + .init( + fileDescriptor: fileDescriptor, + ioQueue: ioQueue, + readChunkSize: self.chunkSize + ) + ) + + case .mock(let array): + return ReadableFileStream.mock(.init(bytes: array, chunkSize: self.chunkSize)) + } + } +} diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift new file mode 100644 index 0000000..442f5d0 --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import class Dispatch.DispatchQueue +@preconcurrency import struct SystemPackage.FileDescriptor +import struct SystemPackage.FilePath + +/// A write-only thread-safe handle to an open file. +public actor OpenWritableFile: WritableStream { + /// Underlying storage for this file handle, dependent on the file system type that produced it. + enum Storage { + /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking + /// the Swift Concurrency thread pool. + case real(FileDescriptor, DispatchQueue) + + /// Reference to the ``MockFileSystem`` actor that provides file storage. + case mock(MockFileSystem) + } + + /// Concrete instance of underlying storage. + let storage: Storage + + /// Absolute path to the file represented by this file handle. + let path: FilePath + + /// Whether the underlying file descriptor has been closed. + private var isClosed = false + + /// Creates a new write-only file handle. + /// - Parameters: + /// - storage: Underlying storage for the file. + /// - path: Absolute path to the file on the file system that provides `storage`. + init(storage: OpenWritableFile.Storage, path: FilePath) { + self.storage = storage + self.path = path + } + + /// Writes a sequence of bytes to the buffer. + public func write(_ bytes: some Collection & Sendable) async throws { + assert(!isClosed) + switch self.storage { + case let .real(fileDescriptor, queue): + let path = self.path + try await queue.scheduleOnQueue { + do { + let writtenBytesCount = try fileDescriptor.writeAll(bytes) + assert(bytes.count == writtenBytesCount) + } catch { + throw error.attach(path) + } + } + case let .mock(storage): + await storage.append(path: self.path, bytes: bytes) + } + } + + /// Closes the underlying stream handle. It is a programmer error to write to a stream after it's closed. + public func close() async throws { + isClosed = true + + guard case let .real(fileDescriptor, queue) = self.storage else { + return + } + + try await queue.scheduleOnQueue { + try fileDescriptor.close() + } + } +} + diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift new file mode 100644 index 0000000..980ff2b --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift @@ -0,0 +1,126 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import _Concurrency +import SystemPackage +import class Dispatch.DispatchQueue + +/// Type-erasure wrapper over underlying file system readable streams. +public enum ReadableFileStream: AsyncSequence { + public typealias Element = ArraySlice + + case real(RealReadableFileStream) + case mock(MockReadableFileStream) + + public enum Iterator: AsyncIteratorProtocol { + case real(RealReadableFileStream.Iterator) + case mock(MockReadableFileStream.Iterator) + + public func next() async throws -> ArraySlice? { + switch self { + case .real(let local): + return try await local.next() + case .mock(let virtual): + return try await virtual.next() + } + } + } + + public func makeAsyncIterator() -> Iterator { + switch self { + case .real(let real): + return .real(real.makeAsyncIterator()) + case .mock(let mock): + return .mock(mock.makeAsyncIterator()) + } + } +} + +/// A stream of file contents from the real file system provided by the OS. +public struct RealReadableFileStream: AsyncSequence { + public typealias Element = ArraySlice + + let fileDescriptor: FileDescriptor + let ioQueue: DispatchQueue + let readChunkSize: Int + + public final class Iterator: AsyncIteratorProtocol { + init(_ fileDescriptor: FileDescriptor, ioQueue: DispatchQueue, readChunkSize: Int) { + self.fileDescriptor = fileDescriptor + self.ioQueue = ioQueue + self.chunkSize = readChunkSize + } + + private let fileDescriptor: FileDescriptor + private let ioQueue: DispatchQueue + private let chunkSize: Int + + public func next() async throws -> ArraySlice? { + let chunkSize = self.chunkSize + let fileDescriptor = self.fileDescriptor + + return try await ioQueue.scheduleOnQueue { + var buffer = [UInt8](repeating: 0, count: chunkSize) + + let bytesRead = try buffer.withUnsafeMutableBytes { + try fileDescriptor.read(into: $0) + } + + guard bytesRead > 0 else { + return nil + } + + buffer.removeLast(chunkSize - bytesRead) + return buffer[...] + } + } + } + + public func makeAsyncIterator() -> Iterator { + Iterator(self.fileDescriptor, ioQueue: ioQueue, readChunkSize: self.readChunkSize) + } +} + + +/// A stream of file contents backed by an in-memory array of bytes. +public struct MockReadableFileStream: AsyncSequence { + public typealias Element = ArraySlice + + public final class Iterator: AsyncIteratorProtocol { + init(bytes: [UInt8], chunkSize: Int) { + self.bytes = bytes + self.chunkSize = chunkSize + } + + private let chunkSize: Int + var bytes: [UInt8] + private var position = 0 + + public func next() async throws -> ArraySlice? { + let nextPosition = Swift.min(bytes.count, position + chunkSize) + + guard nextPosition > position else { + return nil + } + + defer { self.position = nextPosition } + return self.bytes[position.. Iterator { + Iterator(bytes: self.bytes, chunkSize: self.chunkSize) + } +} diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift new file mode 100644 index 0000000..844ba04 --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift @@ -0,0 +1,32 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors +*/ + +import _Concurrency + +/// An asynchronous output byte stream. +/// +/// This protocol is designed to be able to support efficient streaming to +/// different output destinations, e.g., a file or an in memory buffer. +/// +/// The stream is generally used in conjunction with the ``WritableStream/write(_:)`` function. +/// For example: +/// ```swift +/// let stream = fileSystem.withOpenWritableFile { stream in +/// stream.write("Hello, world!".utf8) +/// } +/// ``` +/// would write the UTF8 encoding of "Hello, world!" to the stream. +public protocol WritableStream: Actor { + /// Writes a sequence of bytes to the buffer. + func write(_ bytes: some Collection & Sendable) async throws + + /// Closes the underlying stream handle. It is a programmer error to write to a stream after it's closed. + func close() async throws +} diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index 2dea010..beee4b1 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// import struct Foundation.URL -import GeneratorEngine +import Helpers import struct SystemPackage.FilePath private extension Triple { diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 2657eb3..d26c661 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// import AsyncAlgorithms -import GeneratorEngine +import Helpers import RegexBuilder import class Foundation.ByteCountFormatter @@ -23,11 +23,9 @@ import struct SystemPackage.FilePath private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" -let byteCountFormatter = ByteCountFormatter() - extension SwiftSDKGenerator { func downloadArtifacts( - _ client: some HTTPClientProtocol, _ engine: Engine, + _ client: some HTTPClientProtocol, _ engine: QueryEngine, downloadableArtifacts: inout DownloadableArtifacts, itemsToDownload: @Sendable (DownloadableArtifacts) -> [DownloadableArtifacts.Item] ) async throws { @@ -67,7 +65,7 @@ extension SwiftSDKGenerator { func downloadUbuntuPackages( _ client: some HTTPClientProtocol, - _ engine: Engine, + _ engine: QueryEngine, requiredPackages: [String], versionsConfiguration: VersionsConfiguration, sdkDirPath: FilePath @@ -129,7 +127,7 @@ extension SwiftSDKGenerator { from urls: [URL], to directory: FilePath, _ client: some HTTPClientProtocol, - _ engine: Engine + _ engine: QueryEngine ) async throws -> [(URL, UInt64)] { try await withThrowingTaskGroup(of: (URL, UInt64).self) { for url in urls { @@ -157,6 +155,8 @@ extension SwiftSDKGenerator { } private func report(downloadedFiles: [(URL, UInt64)]) { + let byteCountFormatter = ByteCountFormatter() + for (url, bytes) in downloadedFiles { print("\(url) – \(byteCountFormatter.string(fromByteCount: Int64(bytes)))") } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index b323b76..41d9bd9 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -15,7 +15,6 @@ import AsyncAlgorithms import AsyncHTTPClient #endif import Foundation -import GeneratorEngine import Helpers import RegexBuilder import SystemPackage @@ -34,7 +33,7 @@ public extension Triple.Arch { public extension SwiftSDKGenerator { func run(recipe: SwiftSDKRecipe) async throws { - try await withEngine(LocalFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in + try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in let httpClientType: HTTPClientProtocol.Type #if canImport(AsyncHTTPClient) httpClientType = HTTPClient.self diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index 536b741..a478a47 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// -import GeneratorEngine +import Helpers import struct SystemPackage.FilePath let unusedDarwinPlatforms = [ @@ -82,7 +82,7 @@ extension SwiftSDKGenerator { } } - func prepareLLDLinker(_ engine: Engine, llvmArtifact: DownloadableArtifacts.Item) async throws { + func prepareLLDLinker(_ engine: QueryEngine, llvmArtifact: DownloadableArtifacts.Item) async throws { logGenerationStep("Unpacking and copying `lld` linker...") let pathsConfiguration = self.pathsConfiguration let targetOS = self.targetTriple.os diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 2dc9962..c05c9cf 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// import Foundation -import GeneratorEngine import Helpers import Logging import SystemPackage @@ -37,7 +36,7 @@ public actor SwiftSDKGenerator { ) async throws { logGenerationStep("Looking up configuration values...") - let sourceRoot = FilePath(#file) + let sourceRoot = FilePath(#filePath) .removingLastComponent() .removingLastComponent() .removingLastComponent() diff --git a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift index be66279..a5d5e20 100644 --- a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// -import GeneratorEngine +import Helpers import struct SystemPackage.FilePath struct CMakeBuildQuery: CachingQuery { @@ -19,7 +19,7 @@ struct CMakeBuildQuery: CachingQuery { let outputBinarySubpath: [FilePath.Component] let options: String - func run(engine: Engine) async throws -> FilePath { + func run(engine: QueryEngine) async throws -> FilePath { try await Shell.run( """ cmake -S "\(self.sourcesDirectory)"/llvm -B "\( diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift index 39ce819..b95fb26 100644 --- a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift @@ -10,7 +10,8 @@ // //===----------------------------------------------------------------------===// -import GeneratorEngine +import class Foundation.ByteCountFormatter +import Helpers import struct SystemPackage.FilePath struct DownloadArtifactQuery: Query { @@ -18,7 +19,7 @@ struct DownloadArtifactQuery: Query { let artifact: DownloadableArtifacts.Item let httpClient: any HTTPClientProtocol - func run(engine: Engine) async throws -> FilePath { + func run(engine: QueryEngine) async throws -> FilePath { print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)") let stream = self.httpClient.streamDownloadProgress( from: self.artifact.remoteURL, to: self.artifact.localPath @@ -53,6 +54,8 @@ private func didProgressChangeSignificantly( } private func report(progress: DownloadProgress, for artifact: DownloadableArtifacts.Item) { + let byteCountFormatter = ByteCountFormatter() + if let total = progress.totalBytes { print(""" \(artifact.remoteURL.lastPathComponent) \( diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift index 69b03b5..41d9a1a 100644 --- a/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// import struct Foundation.URL -import GeneratorEngine +import Helpers import struct SystemPackage.FilePath struct DownloadFileQuery: Query { @@ -28,7 +28,7 @@ struct DownloadFileQuery: Query { let localDirectory: FilePath let httpClient: any HTTPClientProtocol - func run(engine: Engine) async throws -> FilePath { + func run(engine: QueryEngine) async throws -> FilePath { let downloadedFilePath = self.localDirectory.appending(self.remoteURL.lastPathComponent) _ = try await self.httpClient.downloadFile(from: self.remoteURL, to: downloadedFilePath) return downloadedFilePath diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index c587d75..f92c4b9 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// import Foundation -import GeneratorEngine +import Helpers import struct SystemPackage.FilePath public struct LinuxRecipe: SwiftSDKRecipe { @@ -133,7 +133,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { public func makeSwiftSDK( generator: SwiftSDKGenerator, - engine: Engine, + engine: QueryEngine, httpClient client: some HTTPClientProtocol ) async throws -> SwiftSDKProduct { let sdkDirPath = self.sdkDirPath(paths: generator.pathsConfiguration) @@ -225,7 +225,8 @@ public struct LinuxRecipe: SwiftSDKRecipe { hostTriple: self.mainHostTriple ) - if versionsConfiguration.swiftVersion.hasPrefix("5.9") || versionsConfiguration.swiftVersion.hasPrefix("5.10") { + if self.versionsConfiguration.swiftVersion.hasPrefix("5.9") || + self.versionsConfiguration.swiftVersion .hasPrefix("5.10") { try await generator.symlinkClangHeaders() } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift index 197852b..baeda00 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// -import GeneratorEngine +import Helpers import struct SystemPackage.FilePath public struct SwiftSDKProduct { @@ -35,7 +35,7 @@ public protocol SwiftSDKRecipe: Sendable { var defaultArtifactID: String { get } /// The main entrypoint of the recipe to make a Swift SDK - func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient: some HTTPClientProtocol) async throws + func makeSwiftSDK(generator: SwiftSDKGenerator, engine: QueryEngine, httpClient: some HTTPClientProtocol) async throws -> SwiftSDKProduct } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index 224ab96..3385866 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// -import GeneratorEngine +import Helpers import struct SystemPackage.FilePath public struct WebAssemblyRecipe: SwiftSDKRecipe { @@ -90,7 +90,7 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { public func makeSwiftSDK( generator: SwiftSDKGenerator, - engine: Engine, + engine: QueryEngine, httpClient: some HTTPClientProtocol ) async throws -> SwiftSDKProduct { let pathsConfiguration = generator.pathsConfiguration diff --git a/Tests/AsyncProcessTests/IntegrationTests.swift b/Tests/AsyncProcessTests/IntegrationTests.swift index 97073d1..7b5a4ee 100644 --- a/Tests/AsyncProcessTests/IntegrationTests.swift +++ b/Tests/AsyncProcessTests/IntegrationTests.swift @@ -278,15 +278,15 @@ final class IntegrationTests: XCTestCase { group.addTask { try await echo.run().throwIfNonZero() } - group.addTask { + group.addTask { [elg = self.group!, logger = self.logger!] in let echoOutput = await echo.standardOutput let sed = ProcessExecutor( - group: self.group, + group: elg, executable: "/usr/bin/tr", ["[:lower:]", "[:upper:]"], standardInput: echoOutput, - logger: self.logger + logger: logger ) let output = try await sed.runGetAllOutput() XCTAssertEqual(String(buffer: output.standardOutput), "FOO\n") @@ -957,15 +957,15 @@ final class IntegrationTests: XCTestCase { of: ProcessExitReason?.self, returning: ProcessExitReason.self ) { group in - group.addTask { + group.addTask { [logger = self.logger!] in try await ProcessExecutor.run( executable: "/bin/sleep", ["100000"], - logger: self.logger + logger: logger ) } - group.addTask { + group.addTask { [logger = self.logger!] in let waitNS = UInt64.random(in: 0..<10_000_000) - self.logger.info("waiting", metadata: ["wait-ns": "\(waitNS)"]) + logger.info("waiting", metadata: ["wait-ns": "\(waitNS)"]) try? await Task.sleep(nanoseconds: waitNS) return nil } diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift index a9c2fd7..2e1a90b 100644 --- a/Tests/GeneratorEngineTests/EngineTests.swift +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -11,7 +11,8 @@ //===----------------------------------------------------------------------===// import struct Foundation.Data -@testable import GeneratorEngine +import Crypto +@testable import Helpers import struct Logging.Logger import struct SystemPackage.FilePath import XCTest @@ -19,16 +20,14 @@ import XCTest private let encoder = JSONEncoder() private let decoder = JSONDecoder() -private extension FileSystem { +private extension AsyncFileSystem { func read(_ path: FilePath, bufferLimit: Int = 10 * 1024 * 1024, as: V.Type) async throws -> V { let data = try await self.withOpenReadableFile(path) { var data = Data() for try await chunk in try await $0.read() { data.append(contentsOf: chunk) - guard data.count < bufferLimit else { - throw FileSystemError.bufferLimitExceeded(path) - } + assert(data.count < bufferLimit) } return data } @@ -47,7 +46,7 @@ private extension FileSystem { struct Const: CachingQuery { let x: Int - func run(engine: Engine) async throws -> FilePath { + func run(engine: QueryEngine) async throws -> FilePath { let resultPath = FilePath("/Const-\(x)") try await engine.fileSystem.write(resultPath, self.x) return resultPath @@ -57,7 +56,7 @@ struct Const: CachingQuery { struct MultiplyByTwo: CachingQuery { let x: Int - func run(engine: Engine) async throws -> FilePath { + func run(engine: QueryEngine) async throws -> FilePath { let constPath = try await engine[Const(x: self.x)].path let constResult = try await engine.fileSystem.read(constPath, as: Int.self) @@ -70,7 +69,7 @@ struct MultiplyByTwo: CachingQuery { struct AddThirty: CachingQuery { let x: Int - func run(engine: Engine) async throws -> FilePath { + func run(engine: QueryEngine) async throws -> FilePath { let constPath = try await engine[Const(x: self.x)].path let constResult = try await engine.fileSystem.read(constPath, as: Int.self) @@ -84,7 +83,7 @@ struct Expression: CachingQuery { let x: Int let y: Int - func run(engine: Engine) async throws -> FilePath { + func run(engine: QueryEngine) async throws -> FilePath { let multiplyPath = try await engine[MultiplyByTwo(x: self.x)].path let addThirtyPath = try await engine[AddThirty(x: self.y)].path @@ -114,10 +113,10 @@ final class EngineTests: XCTestCase { } func testSimpleCaching() async throws { - let engine = Engine( - VirtualFileSystem(), - Logger(label: "engine-tests"), - cacheLocation: .memory + let engine = QueryEngine( + MockFileSystem(), + Logger(label: "engine-tests") +// cacheLocation: .memory ) var resultPath = try await engine[Expression(x: 1, y: 2)].path diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 83bba9e..513ac98 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -41,7 +41,7 @@ final class EndToEndTests: XCTestCase { func testPackageInitExecutable() async throws { let fm = FileManager.default - var packageDirectory = FilePath(#file) + var packageDirectory = FilePath(#filePath) packageDirectory.removeLastComponent() packageDirectory.removeLastComponent() @@ -104,7 +104,7 @@ final class EndToEndTests: XCTestCase { func testRepeatedSDKBuilds() async throws { let fm = FileManager.default - var packageDirectory = FilePath(#file) + var packageDirectory = FilePath(#filePath) packageDirectory.removeLastComponent() packageDirectory.removeLastComponent()