[WIP] Add support for Xcode 14 swift-driver integration

Package.swift

@@ -32,6 +32,10 @@ let package = Package(
             name: "xcswiftc",
             dependencies: ["XCRemoteCache"]
         ),
+        .target(
+            name: "xcswift-frontend",
+            dependencies: ["XCRemoteCache"]
+        ),
         .target(
             name: "xclibtoolSupport",
             dependencies: ["XCRemoteCache"]
@@ -69,6 +73,7 @@ let package = Package(
             dependencies: [
                 "xcprebuild",
                 "xcswiftc",
+                "xcswift-frontend",
                 "xclibtool",
                 "xcpostbuild",
                 "xcprepare",

README.md

@@ -359,6 +359,7 @@ Note: This step is not required if at least one of these is true:
 | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
 | `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ |
 | `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ |
+| `enable_swift_driver_integration` | Enable experimental integration with swift driver, added in Xcode 13 | `false` | ⬜️ |
 
 ## Backend cache server
 

Rakefile

@@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze
 RELEASES_ROOT_DIR = File.join('releases').freeze
 
 EXECUTABLE_NAME = 'XCRemoteCache'
-EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld', 'xcldplusplus', 'xclipo']
+EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo']
 PROJECT_NAME = 'XCRemoteCache'
 
 SWIFTLINT_ENABLED = true
@@ -59,6 +59,10 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args|
 
     # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache`
     build_path_base = File.join(DERIVED_DATA_DIR, args.configuration)
+    # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create
+    # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend
+    system("cd #{build_path_base} && ln -s xcswiftc swiftc")
+    system("cd #{build_path_base} && ln -s xcswift-frontend swift-frontend")
     sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)}
 
     build_paths.push(sdk_build_paths)
@@ -130,7 +134,9 @@ def create_release_zip(build_paths)
   # Create and move files into the release directory
   mkdir_p release_dir
   build_paths.each {|p|
-    cp_r p, release_dir
+    # -r for recursive
+    # -P for copying symbolic link as is
+    system("cp -rP #{p} #{release_dir}")
   }
 
   output_artifact_basename = "#{PROJECT_NAME}.zip"
@@ -139,7 +145,8 @@ def create_release_zip(build_paths)
     # -X: no extras (uid, gid, file times, ...)
     # -x: exclude .DS_Store
     # -r: recursive
-    system("zip -X -x '*.DS_Store' -r #{output_artifact_basename} .") or abort "zip failure"
+    # -y: to store symbolic links (used for swiftc -> xcswiftc)
+    system("zip -X -x '*.DS_Store' -r -y #{output_artifact_basename} .") or abort "zip failure"
     # List contents of zip file
     system("unzip -l #{output_artifact_basename}") or abort "unzip failure"
   end

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -89,6 +89,9 @@ public struct PostbuildContext {
     var publicHeadersFolderPath: URL?
     /// XCRemoteCache is explicitly disabled
     let disabled: Bool
+    /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier
+    /// it is used in the swift-frontend flow
+    let llbuildIdLockFile: URL
 }
 
 extension PostbuildContext {
@@ -149,5 +152,7 @@ extension PostbuildContext {
             publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath)
         }
         disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
+        let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
+        llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: targetTempDir)
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -60,6 +60,7 @@ public class XCPostbuild {
             dependenciesWriter: FileDependenciesWriter.init,
             dependenciesReader: FileDependenciesReader.init,
             markerWriter: NoopMarkerWriter.init,
+            llbuildLockFile: context.llbuildIdLockFile,
             fileManager: fileManager
         )
 

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -48,6 +48,9 @@ public struct PrebuildContext {
     let overlayHeadersPath: URL
     /// XCRemoteCache is explicitly disabled
     let disabled: Bool
+    /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier
+    /// it is used in the swift-frontend flow
+    let llbuildIdLockFile: URL
 }
 
 extension PrebuildContext {
@@ -72,5 +75,7 @@ extension PrebuildContext {
         /// Note: The file has yaml extension, even it is in the json format
         overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
         disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
+        let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
+        llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: targetTempDir)
     }
 }

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -55,6 +55,7 @@ public class XCPrebuild {
             dependenciesWriter: FileDependenciesWriter.init,
             dependenciesReader: FileDependenciesReader.init,
             markerWriter: lazyMarkerWriterFactory,
+            llbuildLockFile: context.llbuildIdLockFile,
             fileManager: fileManager
         )
 

Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift

@@ -21,6 +21,11 @@ import Foundation
 
 typealias BuildSettings = [String: Any]
 
+struct BuildSettingsIntegrateAppenderOption: OptionSet {
+    let rawValue: Int
+
+    static let disableSwiftDriverIntegration = BuildSettingsIntegrateAppenderOption(rawValue: 1 << 0)
+}
 // Manages Xcode build settings
 protocol BuildSettingsIntegrateAppender {
     /// Appends XCRemoteCache-specific build settings
@@ -35,18 +40,28 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
     private let repoRoot: URL
     private let fakeSrcRoot: URL
     private let sdksExclude: [String]
+    private let options: BuildSettingsIntegrateAppenderOption
 
-    init(mode: Mode, repoRoot: URL, fakeSrcRoot: URL, sdksExclude: [String]) {
+    init(
+        mode: Mode,
+        repoRoot: URL,
+        fakeSrcRoot: URL,
+        sdksExclude: [String],
+        options: BuildSettingsIntegrateAppenderOption
+    ) {
         self.mode = mode
         self.repoRoot = repoRoot
         self.fakeSrcRoot = fakeSrcRoot
         self.sdksExclude = sdksExclude
+        self.options = options
     }
 
     func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings {
         var result = buildSettings
         setBuildSetting(buildSettings: &result, key: "SWIFT_EXEC", value: wrappers.swiftc.path )
-        setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" )
+        if options.contains(.disableSwiftDriverIntegration) {
+            setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" )
+        }
         // When generating artifacts, no need to shell-out all compilation commands to our wrappers
         if case .consumer = mode {
             setBuildSetting(buildSettings: &result, key: "CC", value: wrappers.cc.path )

Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift

@@ -38,7 +38,8 @@ extension IntegrateContext {
         env: [String: String],
         binariesDir: URL,
         fakeSrcRoot: String,
-        outputPath: String?
+        outputPath: String?,
+        useFontendIntegration: Bool
     ) throws {
         projectPath = URL(fileURLWithPath: input)
         let srcRoot = projectPath.deletingLastPathComponent()
@@ -47,10 +48,11 @@ extension IntegrateContext {
         configOverride = URL(fileURLWithPath: configOverridePath, relativeTo: srcRoot)
         output = outputPath.flatMap(URL.init(fileURLWithPath:))
         self.fakeSrcRoot = URL(fileURLWithPath: fakeSrcRoot)
+        let swiftcBinaryName = useFontendIntegration ? "swiftc" : "xcswiftc"
         binaries = XCRCBinariesPaths(
             prepare: binariesDir.appendingPathComponent("xcprepare"),
             cc: binariesDir.appendingPathComponent("xccc"),
-            swiftc: binariesDir.appendingPathComponent("xcswiftc"),
+            swiftc: binariesDir.appendingPathComponent(swiftcBinaryName),
             libtool: binariesDir.appendingPathComponent("xclibtool"),
             lipo: binariesDir.appendingPathComponent("xclipo"),
             ld: binariesDir.appendingPathComponent("xcld"),

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -88,7 +88,8 @@ public class XCIntegrate {
                 env: env,
                 binariesDir: binariesDir,
                 fakeSrcRoot: fakeSrcRoot,
-                outputPath: output
+                outputPath: output,
+                useFontendIntegration: config.enableSwifDriverIntegration
             )
             let configurationOracle = IncludeExcludeOracle(
                 excludes: configurationsExclude.integrateArrayArguments,
@@ -98,11 +99,18 @@ public class XCIntegrate {
                 excludes: targetsExclude.integrateArrayArguments,
                 includes: targetsInclude.integrateArrayArguments
             )
+            let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption
+            if config.enableSwifDriverIntegration {
+                buildSettingsAppenderOptions = []
+            } else {
+                buildSettingsAppenderOptions = [.disableSwiftDriverIntegration]
+            }
             let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender(
                 mode: context.mode,
                 repoRoot: context.repoRoot,
                 fakeSrcRoot: context.fakeSrcRoot,
-                sdksExclude: sdksExclude.integrateArrayArguments
+                sdksExclude: sdksExclude.integrateArrayArguments,
+                options: buildSettingsAppenderOptions
             )
             let lldbPatcher: LLDBInitPatcher
             switch lldbMode {