Merge pull request #208 from polac24/support-xcode14

Refactor existing implementation for swift-driver change

Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift

@@ -21,6 +21,11 @@ import Foundation
 
 typealias BuildSettings = [String: Any]
 
+struct BuildSettingsIntegrateAppenderOption: OptionSet {
+    let rawValue: Int
+
+    static let disableSwiftDriverIntegration = BuildSettingsIntegrateAppenderOption(rawValue: 1 << 0)
+}
 // Manages Xcode build settings
 protocol BuildSettingsIntegrateAppender {
     /// Appends XCRemoteCache-specific build settings
@@ -35,18 +40,28 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
     private let repoRoot: URL
     private let fakeSrcRoot: URL
     private let sdksExclude: [String]
+    private let options: BuildSettingsIntegrateAppenderOption
 
-    init(mode: Mode, repoRoot: URL, fakeSrcRoot: URL, sdksExclude: [String]) {
+    init(
+        mode: Mode,
+        repoRoot: URL,
+        fakeSrcRoot: URL,
+        sdksExclude: [String],
+        options: BuildSettingsIntegrateAppenderOption
+    ) {
         self.mode = mode
         self.repoRoot = repoRoot
         self.fakeSrcRoot = fakeSrcRoot
         self.sdksExclude = sdksExclude
+        self.options = options
     }
 
     func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings {
         var result = buildSettings
         setBuildSetting(buildSettings: &result, key: "SWIFT_EXEC", value: wrappers.swiftc.path )
-        setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" )
+        if options.contains(.disableSwiftDriverIntegration) {
+            setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" )
+        }
         // When generating artifacts, no need to shell-out all compilation commands to our wrappers
         if case .consumer = mode {
             setBuildSetting(buildSettings: &result, key: "CC", value: wrappers.cc.path )

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -98,11 +98,15 @@ public class XCIntegrate {
                 excludes: targetsExclude.integrateArrayArguments,
                 includes: targetsInclude.integrateArrayArguments
             )
+            let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption = [
+                .disableSwiftDriverIntegration,
+            ]
             let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender(
                 mode: context.mode,
                 repoRoot: context.repoRoot,
                 fakeSrcRoot: context.fakeSrcRoot,
-                sdksExclude: sdksExclude.integrateArrayArguments
+                sdksExclude: sdksExclude.integrateArrayArguments,
+                options: buildSettingsAppenderOptions
             )
             let lldbPatcher: LLDBInitPatcher
             switch lldbMode {

Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift

@@ -0,0 +1,38 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where
+/// only individual .o files are created and not .swiftmodule of -Swift.h
+/// (which is part of swift-frontend -emit-module invocation)
+class NoopSwiftcProductsGenerator: SwiftcProductsGenerator {
+    func generateFrom(
+        artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
+        artifactSwiftModuleObjCFile: URL
+    ) throws -> SwiftcProductsGeneratorOutput {
+        infoLog("""
+        Invoking module generation from NoopSwiftcProductsGenerator does nothing. \
+        It might be a side-effect of a plugin asking to generate a module.
+        """)
+        // NoopSwiftcProductsGenerator is intended only for the swift-frontend
+        let trivialURL = URL(fileURLWithPath: "/non-existing")
+        return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL)
+    }
+}

Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift

@@ -0,0 +1,46 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+class StaticSwiftcInputReader: SwiftcInputReader {
+    private let moduleDependencies: URL?
+    private let swiftDependencies: URL?
+    private let compilationFiles: [SwiftFileCompilationInfo]
+
+    init(
+        moduleDependencies: URL?,
+        swiftDependencies: URL?,
+        compilationFiles: [SwiftFileCompilationInfo]
+    ) {
+        self.moduleDependencies = moduleDependencies
+        self.swiftDependencies = swiftDependencies
+        self.compilationFiles = compilationFiles
+    }
+
+    func read() throws -> SwiftCompilationInfo {
+        return .init(
+            info: .init(
+                dependencies: moduleDependencies,
+                swiftDependencies: swiftDependencies
+            ),
+            files: compilationFiles
+        )
+    }
+}

Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift

@@ -132,7 +132,7 @@ class Swiftc: SwiftcProtocol {
 
         // Read swiftmodule location from XCRemoteCache
         // arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo}
-        let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent
+        let moduleName = context.moduleName
         let allCompilations = try inputFilesReader.read()
         let artifactSwiftmoduleDir = artifactLocation
             .appendingPathComponent("swiftmodule")
@@ -145,20 +145,24 @@ class Swiftc: SwiftcProtocol {
                 }
         )
 
-        // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h
-        let artifactSwiftModuleObjCDir = artifactLocation
-            .appendingPathComponent("include")
-            .appendingPathComponent(context.arch)
-            .appendingPathComponent(context.moduleName)
-        // Move cached xxxx-Swift.h to the location passed in arglist
-        // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir
-        let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir
-            .appendingPathComponent(context.objcHeaderOutput.lastPathComponent)
+        // emit module (if requested)
+        if let emitModule = context.steps.emitModule {
+            // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h
+            let artifactSwiftModuleObjCDir = artifactLocation
+                .appendingPathComponent("include")
+                .appendingPathComponent(context.arch)
+                .appendingPathComponent(context.moduleName)
+            // Move cached xxxx-Swift.h to the location passed in arglist
+            // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h
+            // file in artifactSwiftModuleObjCDir
+            let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir
+                .appendingPathComponent(emitModule.objcHeaderOutput.lastPathComponent)
 
-        _ = try productsGenerator.generateFrom(
-            artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
-            artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
-        )
+            _ = try productsGenerator.generateFrom(
+                artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
+                artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
+            )
+        }
 
         try plugins.forEach {
             try $0.generate(for: allCompilations)
@@ -176,8 +180,10 @@ class Swiftc: SwiftcProtocol {
                 try cachedDependenciesWriterFactory.generate(output: individualDeps)
             }
         }
-        // Save .d for the entire module
-        try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies)
+        // Save .d for the entire module (might not be required in the `swift-frontend -c` mode)
+        if let swiftDependencies = allCompilations.info.swiftDependencies {
+            try cachedDependenciesWriterFactory.generate(output: swiftDependencies)
+        }
         // Generate .d file with all deps in the "-master.d" (e.g. for WMO)
         if let wmoDeps = allCompilations.info.dependencies {
             try cachedDependenciesWriterFactory.generate(output: wmoDeps)

Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift

@@ -20,6 +20,53 @@
 import Foundation
 
 public struct SwiftcContext {
+    /// Describes the action if the module emit should happen
+    /// that generates .swiftmodule and/or -Swift.h
+    public struct SwiftcStepEmitModule: Equatable {
+        // where the -Swift.h should be placed
+        let objcHeaderOutput: URL
+        // where should the .swiftmodule be placed
+        let modulePathOutput: URL
+        // might be passed as an explicit argument in the swiftc
+        // -emit-dependencies-path
+        let dependencies: URL?
+    }
+
+    /// Which files (from the list of all files in the module)
+    /// should be compiled in this process
+    public enum SwiftcStepCompileFilesScope: Equatable {
+        /// used if only emit module should be done
+        case none
+        case all
+        case subset([URL])
+    }
+
+    /// Describes which steps should be done as a part of this process
+    public struct SwiftcSteps: Equatable {
+        /// which files should be compiled
+        let compileFilesScope: SwiftcStepCompileFilesScope
+        /// if a module should be generated
+        let emitModule: SwiftcStepEmitModule?
+    }
+
+    /// Defines how a list of input files (*.swift) is passed to the invocation
+    public enum CompilationFilesSource: Equatable {
+        /// defined in a separate file (via @/.../*.SwiftFileList)
+        case fileList(String)
+        /// explicitly passed a list of files
+        case list([String])
+    }
+
+    /// Defines how a list of output files (*.d, *.o etc.) is passed to the invocation
+    public enum CompilationFilesInputs: Equatable {
+        /// defined in a separate file (via -output-file-map)
+        case fileMap(String)
+        /// defined in a separate file (via -supplementary-output-file-map)
+        case supplementaryFileMap(String)
+        /// explicitly passed in the invocation
+        case map([String: SwiftFileCompilationInfo])
+    }
+
     enum SwiftcMode: Equatable {
         case producer
         /// Commit sha of the commit to use during remote cache
@@ -28,44 +75,43 @@ public struct SwiftcContext {
         case producerFast
     }
 
-    let objcHeaderOutput: URL
+    let steps: SwiftcSteps
     let moduleName: String
-    let modulePathOutput: URL
-    /// File that defines output files locations (.d, .swiftmodule etc.)
-    let filemap: URL
+    /// A source that defines output files locations (.d, .swiftmodule etc.)
+    let inputs: CompilationFilesInputs
     let target: String
-    /// File that contains input files for the swift module compilation
-    let fileList: URL
+    /// A source that contains all input files for the swift module compilation
+    let compilationFiles: CompilationFilesSource
     let tempDir: URL
     let arch: String
     let prebuildDependenciesPath: String
     let mode: SwiftcMode
     /// File that stores all compilation invocation arguments
     let invocationHistoryFile: URL
 
-
     public init(
         config: XCRemoteCacheConfig,
-        objcHeaderOutput: String,
         moduleName: String,
-        modulePathOutput: String,
-        filemap: String,
+        steps: SwiftcSteps,
+        inputs: CompilationFilesInputs,
         target: String,
-        fileList: String
+        compilationFiles: CompilationFilesSource,
+        /// any workspace file path - all other intermediate files for this compilation
+        /// are placed next to it. This path is used to infer the arch and TARGET_TEMP_DIR
+        exampleWorkspaceFilePath: String
     ) throws {
-        self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput)
         self.moduleName = moduleName
-        self.modulePathOutput = URL(fileURLWithPath: modulePathOutput)
-        self.filemap = URL(fileURLWithPath: filemap)
+        self.steps = steps
+        self.inputs = inputs
         self.target = target
-        self.fileList = URL(fileURLWithPath: fileList)
-        // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule
+        self.compilationFiles = compilationFiles
+        // exampleWorkspaceFilePath has a format $TARGET_TEMP_DIR/Objects-normal/$ARCH/some.file
         // That may be subject to change for other Xcode versions
-        tempDir = URL(fileURLWithPath: modulePathOutput)
+        tempDir = URL(fileURLWithPath: exampleWorkspaceFilePath)
             .deletingLastPathComponent()
             .deletingLastPathComponent()
             .deletingLastPathComponent()
-        arch = URL(fileURLWithPath: modulePathOutput).deletingLastPathComponent().lastPathComponent
+        arch = URL(fileURLWithPath: exampleWorkspaceFilePath).deletingLastPathComponent().lastPathComponent
 
         let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot)
         let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot)
@@ -92,14 +138,25 @@ public struct SwiftcContext {
         config: XCRemoteCacheConfig,
         input: SwiftcArgInput
     ) throws {
+        let steps = SwiftcSteps(
+            compileFilesScope: .all,
+            emitModule: SwiftcStepEmitModule(
+                objcHeaderOutput: URL(fileURLWithPath: (input.objcHeaderOutput)),
+                modulePathOutput: URL(fileURLWithPath: input.modulePathOutput),
+                // in `swiftc`, .d dependencies are pass in the output filemap
+                dependencies: nil
+            )
+        )
+        let inputs = CompilationFilesInputs.fileMap(input.filemap)
+        let compilationFiles = CompilationFilesSource.fileList(input.fileList)
         try self.init(
             config: config,
-            objcHeaderOutput: input.objcHeaderOutput,
             moduleName: input.moduleName,
-            modulePathOutput: input.modulePathOutput,
-            filemap: input.filemap,
+            steps: steps,
+            inputs: inputs,
             target: input.target,
-            fileList: input.fileList
+            compilationFiles: compilationFiles,
+            exampleWorkspaceFilePath: input.modulePathOutput
         )
     }
 }