Forward port fixes for macro build prebuilts

Sources/CoreCommands/Options.swift

@@ -188,10 +188,19 @@ public struct CachingOptions: ParsableArguments {
     }
 
     /// Whether to use macro prebuilts or not
-    @Flag(name: .customLong("experimental-prebuilts"),
-          inversion: .prefixedEnableDisable,
-          help: "Whether to use prebuilt swift-syntax libraries for macros")
+    @Flag(
+        name: .customLong("experimental-prebuilts"),
+        inversion: .prefixedEnableDisable,
+        help: "Whether to use prebuilt swift-syntax libraries for macros"
+    )
     public var usePrebuilts: Bool = false
+
+    /// Hidden option to override the prebuilts download location for testing
+    @Option(
+        name: .customLong("experimental-prebuilts-download-url"),
+        help: .hidden
+    )
+    public var prebuiltsDownloadURL: String?
 }
 
 public struct LoggingOptions: ParsableArguments {

Sources/CoreCommands/SwiftCommandState.swift

@@ -472,7 +472,8 @@ public final class SwiftCommandState {
                     .init(url: $0, supportsAvailability: true)
                 },
                 manifestImportRestrictions: .none,
-                usePrebuilts: options.caching.usePrebuilts
+                usePrebuilts: options.caching.usePrebuilts,
+                prebuiltsDownloadURL: options.caching.prebuiltsDownloadURL
             ),
             cancellator: self.cancellator,
             initializationWarningHandler: { self.observabilityScope.emit(warning: $0) },

Sources/PackageGraph/ModulesGraph+Loading.swift

@@ -260,7 +260,12 @@ extension ModulesGraph {
         )
 
         let rootPackages = resolvedPackages.filter { root.manifests.values.contains($0.manifest) }
-        checkAllDependenciesAreUsed(packages: resolvedPackages, rootPackages, observabilityScope: observabilityScope)
+        checkAllDependenciesAreUsed(
+            packages: resolvedPackages,
+            rootPackages,
+            prebuilts: prebuilts,
+            observabilityScope: observabilityScope
+        )
 
         return try ModulesGraph(
             rootPackages: rootPackages,
@@ -275,6 +280,7 @@ extension ModulesGraph {
 private func checkAllDependenciesAreUsed(
     packages: IdentifiableSet<ResolvedPackage>,
     _ rootPackages: [ResolvedPackage],
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
     observabilityScope: ObservabilityScope
 ) {
     for package in rootPackages {
@@ -352,9 +358,10 @@ private func checkAllDependenciesAreUsed(
                 let usedByPackage = productDependencies.contains { $0.name == product.name }
                 // We check if any of the products of this dependency is guarded by a trait.
                 let traitGuarded = traitGuardedProductDependencies.contains(product.name)
+                // Consider prebuilts as used
+                let prebuilt = prebuilts[dependency.identity]?.keys.contains(product.name) ?? false
 
-                // If the product is either used directly or guarded by a trait we consider it as used
-                return usedByPackage || traitGuarded
+                return usedByPackage || traitGuarded || prebuilt
             }
 
             if !dependencyIsUsed && !observabilityScope.errorsReportedInAnyScope {
@@ -672,8 +679,21 @@ private func createResolvedPackages(
                 }
 
                 if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
-                    // using a prebuilt instead.
-                    continue
+                    // See if we're using a prebuilt instead
+                    if moduleBuilder.module.type == .macro {
+                        continue
+                    } else if moduleBuilder.module.type == .test {
+                        // use prebuilt if this is a test that depends a macro target
+                        // these are guaranteed built for host
+                        if moduleBuilder.module.dependencies.contains(where: { dep in
+                            guard let module = dep.module else {
+                                return false
+                            }
+                            return module.type == .macro
+                        }) {
+                            continue
+                        }
+                    }
                 }
 
                 // Find the product in this package's dependency products.

Sources/PackageLoading/PackageBuilder.swift

@@ -270,8 +270,8 @@ public struct BinaryArtifact {
 
 /// A structure representing a prebuilt library to be used instead of a source dependency
 public struct PrebuiltLibrary {
-    /// The package reference.
-    public let packageRef: PackageReference
+    /// The package identity.
+    public let identity: PackageIdentity
 
     /// The name of the binary target the artifact corresponds to.
     public let libraryName: String
@@ -285,8 +285,8 @@ public struct PrebuiltLibrary {
     /// The C modules that need their includes directory added to the include path
     public let cModules: [String]
 
-    public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
-        self.packageRef = packageRef
+    public init(identity: PackageIdentity, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
+        self.identity = identity
         self.libraryName = libraryName
         self.path = path
         self.products = products
@@ -1242,31 +1242,34 @@ public final class PackageBuilder {
             table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS)
         }
 
-        // Add in flags for prebuilts
-        let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
-            guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
-                  let package = package,
-                  let prebuilt = prebuilts[.plain(package)]?[name]
-            else {
-                return
-            }
+        // Add in flags for prebuilts if the target is a macro or a macro test.
+        // Currently we only support prebuilts for macros.
+        if target.type == .macro || target.isMacroTest(in: manifest) {
+            let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
+                guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
+                      let package = package,
+                      let prebuilt = prebuilts[.plain(package)]?[name]
+                else {
+                    return
+                }
 
-            $0[prebuilt.libraryName] = prebuilt
-        }
+                $0[prebuilt.libraryName] = prebuilt
+            }
 
-        for prebuilt in prebuiltLibraries.values {
-            let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
-            var ldFlagsAssignment = BuildSettings.Assignment()
-            ldFlagsAssignment.values = [lib]
-            table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
+            for prebuilt in prebuiltLibraries.values {
+                let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
+                var ldFlagsAssignment = BuildSettings.Assignment()
+                ldFlagsAssignment.values = [lib]
+                table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
 
-            var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
-            for cModule in prebuilt.cModules {
-                includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+                var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
+                for cModule in prebuilt.cModules {
+                    includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+                }
+                var includeAssignment = BuildSettings.Assignment()
+                includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
+                table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
             }
-            var includeAssignment = BuildSettings.Assignment()
-            includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
-            table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
         }
 
         return table
@@ -1853,4 +1856,26 @@ extension TargetDescription {
     fileprivate var usesUnsafeFlags: Bool {
         settings.filter(\.kind.isUnsafeFlags).isEmpty == false
     }
+
+    fileprivate func isMacroTest(in manifest: Manifest) -> Bool {
+        guard self.type == .test else { return false }
+
+        return self.dependencies.contains(where: {
+            let name: String
+            switch $0 {
+            case .byName(name: let n, condition: _):
+                name = n
+            case .target(name: let n, condition: _):
+                name = n
+            default:
+                return false
+            }
+
+            guard let target = manifest.targetMap[name] else {
+                return false
+            }
+
+            return target.type == .macro
+        })
+    }
 }

Sources/Workspace/ManagedPrebuilt.swift

@@ -10,14 +10,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+import struct TSCUtility.Version
+
 import Basics
 import PackageModel
 
 extension Workspace {
     /// A downloaded prebuilt managed by the workspace.
     public struct ManagedPrebuilt {
-        /// The package reference.
-        public let packageRef: PackageReference
+        /// The package identity
+        public let identity: PackageIdentity
+
+        /// The package version
+        public let version: Version
 
         /// The name of the binary target the artifact corresponds to.
         public let libraryName: String
@@ -35,7 +40,7 @@ extension Workspace {
 
 extension Workspace.ManagedPrebuilt: CustomStringConvertible {
     public var description: String {
-        return "<ManagedArtifact: \(self.packageRef.identity).\(self.libraryName)>"
+        return "<ManagedArtifact: \(self.identity).\(self.libraryName)>"
     }
 }
 
@@ -45,60 +50,60 @@ extension Workspace {
     /// A collection of managed artifacts which have been downloaded.
     public final class ManagedPrebuilts {
         /// A mapping from package identity, to target name, to ManagedArtifact.
-        private var artifactMap: [PackageIdentity: [String: ManagedPrebuilt]]
+        private var prebuiltMap: [PackageIdentity: [String: ManagedPrebuilt]]
 
-        internal var artifacts: AnyCollection<ManagedPrebuilt> {
-            AnyCollection(self.artifactMap.values.lazy.flatMap{ $0.values })
+        internal var prebuilts: AnyCollection<ManagedPrebuilt> {
+            AnyCollection(self.prebuiltMap.values.lazy.flatMap{ $0.values })
         }
 
         init() {
-            self.artifactMap = [:]
+            self.prebuiltMap = [:]
         }
 
-        init(_ artifacts: [ManagedPrebuilt]) throws {
-            let artifactsByPackagePath = Dictionary(grouping: artifacts, by: { $0.packageRef.identity })
-            self.artifactMap = try artifactsByPackagePath.mapValues{ artifacts in
-                try Dictionary(artifacts.map { ($0.libraryName, $0) }, uniquingKeysWith: { _, _ in
+        init(_ prebuilts: [ManagedPrebuilt]) throws {
+            let prebuiltsByPackagePath = Dictionary(grouping: prebuilts, by: { $0.identity })
+            self.prebuiltMap = try prebuiltsByPackagePath.mapValues{ prebuilt in
+                try Dictionary(prebuilt.map { ($0.libraryName, $0) }, uniquingKeysWith: { _, _ in
                     // should be unique
-                    throw StringError("binary artifact already exists in managed artifacts")
+                    throw StringError("prebuilt already exists in managed prebuilts")
                 })
             }
         }
 
         public subscript(packageIdentity packageIdentity: PackageIdentity, targetName targetName: String) -> ManagedPrebuilt? {
-            self.artifactMap[packageIdentity]?[targetName]
+            self.prebuiltMap[packageIdentity]?[targetName]
         }
 
-        public func add(_ artifact: ManagedPrebuilt) {
-            self.artifactMap[artifact.packageRef.identity, default: [:]][artifact.libraryName] = artifact
+        public func add(_ prebuilt: ManagedPrebuilt) {
+            self.prebuiltMap[prebuilt.identity, default: [:]][prebuilt.libraryName] = prebuilt
         }
 
         public func remove(packageIdentity: PackageIdentity, targetName: String) {
-            self.artifactMap[packageIdentity]?[targetName] = nil
+            self.prebuiltMap[packageIdentity]?[targetName] = nil
         }
     }
 }
 
 extension Workspace.ManagedPrebuilts: Collection {
     public var startIndex: AnyIndex {
-        self.artifacts.startIndex
+        self.prebuilts.startIndex
     }
 
     public var endIndex: AnyIndex {
-        self.artifacts.endIndex
+        self.prebuilts.endIndex
     }
 
     public subscript(index: AnyIndex) -> Workspace.ManagedPrebuilt {
-        self.artifacts[index]
+        self.prebuilts[index]
     }
 
     public func index(after index: AnyIndex) -> AnyIndex {
-        self.artifacts.index(after: index)
+        self.prebuilts.index(after: index)
     }
 }
 
 extension Workspace.ManagedPrebuilts: CustomStringConvertible {
     public var description: String {
-        "<ManagedArtifacts: \(Array(self.artifacts))>"
+        "<ManagedArtifacts: \(Array(self.prebuilts))>"
     }
 }

Sources/Workspace/Workspace+Configuration.swift

@@ -792,6 +792,9 @@ public struct WorkspaceConfiguration {
     /// Whether or not to use prebuilt swift-syntax for macros
     public var usePrebuilts: Bool
 
+    /// String URL to allow override of the prebuilts download location
+    public var prebuiltsDownloadURL: String?
+
     public init(
         skipDependenciesUpdates: Bool,
         prefetchBasedOnResolvedFile: Bool,
@@ -805,7 +808,8 @@ public struct WorkspaceConfiguration {
         sourceControlToRegistryDependencyTransformation: SourceControlToRegistryDependencyTransformation,
         defaultRegistry: Registry?,
         manifestImportRestrictions: (startingToolsVersion: ToolsVersion, allowedImports: [String])?,
-        usePrebuilts: Bool
+        usePrebuilts: Bool,
+        prebuiltsDownloadURL: String?
     ) {
         self.skipDependenciesUpdates = skipDependenciesUpdates
         self.prefetchBasedOnResolvedFile = prefetchBasedOnResolvedFile
@@ -820,6 +824,7 @@ public struct WorkspaceConfiguration {
         self.defaultRegistry = defaultRegistry
         self.manifestImportRestrictions = manifestImportRestrictions
         self.usePrebuilts = usePrebuilts
+        self.prebuiltsDownloadURL = prebuiltsDownloadURL
     }
 
     /// Default instance of WorkspaceConfiguration
@@ -837,7 +842,8 @@ public struct WorkspaceConfiguration {
             sourceControlToRegistryDependencyTransformation: .disabled,
             defaultRegistry: .none,
             manifestImportRestrictions: .none,
-            usePrebuilts: false
+            usePrebuilts: false,
+            prebuiltsDownloadURL: nil
         )
     }