diff --git a/Package.swift b/Package.swift index 426a40d9..00c88e43 100644 --- a/Package.swift +++ b/Package.swift @@ -32,6 +32,10 @@ let package = Package( name: "xcswiftc", dependencies: ["XCRemoteCache"] ), + .target( + name: "xcswift-frontend", + dependencies: ["XCRemoteCache"] + ), .target( name: "xclibtoolSupport", dependencies: ["XCRemoteCache"] @@ -69,6 +73,7 @@ let package = Package( dependencies: [ "xcprebuild", "xcswiftc", + "xcswift-frontend", "xclibtool", "xcpostbuild", "xcprepare", diff --git a/README.md b/README.md index 3e2dd1e2..9d861e4c 100755 --- a/README.md +++ b/README.md @@ -359,6 +359,7 @@ Note: This step is not required if at least one of these is true: | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | | `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ | | `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ | +| `enable_swift_driver_integration` | Enable experimental integration with swift driver, added in Xcode 13 | `false` | ⬜️ | ## Backend cache server diff --git a/Rakefile b/Rakefile index 3934c77f..45a4e867 100644 --- a/Rakefile +++ b/Rakefile @@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze RELEASES_ROOT_DIR = File.join('releases').freeze EXECUTABLE_NAME = 'XCRemoteCache' -EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld', 'xcldplusplus', 'xclipo'] +EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo'] PROJECT_NAME = 'XCRemoteCache' SWIFTLINT_ENABLED = true @@ -59,6 +59,10 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args| # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache` build_path_base = File.join(DERIVED_DATA_DIR, args.configuration) + # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create + # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend + system("cd #{build_path_base} && ln -s xcswiftc swiftc") + system("cd #{build_path_base} && ln -s xcswift-frontend swift-frontend") sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)} build_paths.push(sdk_build_paths) @@ -130,7 +134,9 @@ def create_release_zip(build_paths) # Create and move files into the release directory mkdir_p release_dir build_paths.each {|p| - cp_r p, release_dir + # -r for recursive + # -P for copying symbolic link as is + system("cp -rP #{p} #{release_dir}") } output_artifact_basename = "#{PROJECT_NAME}.zip" @@ -139,7 +145,8 @@ def create_release_zip(build_paths) # -X: no extras (uid, gid, file times, ...) # -x: exclude .DS_Store # -r: recursive - system("zip -X -x '*.DS_Store' -r #{output_artifact_basename} .") or abort "zip failure" + # -y: to store symbolic links (used for swiftc -> xcswiftc) + system("zip -X -x '*.DS_Store' -r -y #{output_artifact_basename} .") or abort "zip failure" # List contents of zip file system("unzip -l #{output_artifact_basename}") or abort "unzip failure" end diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index d71ea9ae..77b0c337 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -89,6 +89,9 @@ public struct PostbuildContext { var publicHeadersFolderPath: URL? /// XCRemoteCache is explicitly disabled let disabled: Bool + /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier + /// it is used in the swift-frontend flow + let llbuildIdLockFile: URL } extension PostbuildContext { @@ -149,5 +152,7 @@ extension PostbuildContext { publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath) } disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: targetTempDir) } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index cc597bf5..235f56c1 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -60,6 +60,7 @@ public class XCPostbuild { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: FileDependenciesReader.init, markerWriter: NoopMarkerWriter.init, + llbuildLockFile: context.llbuildIdLockFile, fileManager: fileManager ) diff --git a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift index d36bcfaf..0a144c61 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift @@ -48,6 +48,9 @@ public struct PrebuildContext { let overlayHeadersPath: URL /// XCRemoteCache is explicitly disabled let disabled: Bool + /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier + /// it is used in the swift-frontend flow + let llbuildIdLockFile: URL } extension PrebuildContext { @@ -72,5 +75,7 @@ extension PrebuildContext { /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: targetTempDir) } } diff --git a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift index 97fc759a..b4c8dd8c 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift @@ -55,6 +55,7 @@ public class XCPrebuild { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: FileDependenciesReader.init, markerWriter: lazyMarkerWriterFactory, + llbuildLockFile: context.llbuildIdLockFile, fileManager: fileManager ) diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift index 70c9c0ea..ec4f4b1c 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift @@ -21,6 +21,11 @@ import Foundation typealias BuildSettings = [String: Any] +struct BuildSettingsIntegrateAppenderOption: OptionSet { + let rawValue: Int + + static let disableSwiftDriverIntegration = BuildSettingsIntegrateAppenderOption(rawValue: 1 << 0) +} // Manages Xcode build settings protocol BuildSettingsIntegrateAppender { /// Appends XCRemoteCache-specific build settings @@ -35,18 +40,28 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender { private let repoRoot: URL private let fakeSrcRoot: URL private let sdksExclude: [String] + private let options: BuildSettingsIntegrateAppenderOption - init(mode: Mode, repoRoot: URL, fakeSrcRoot: URL, sdksExclude: [String]) { + init( + mode: Mode, + repoRoot: URL, + fakeSrcRoot: URL, + sdksExclude: [String], + options: BuildSettingsIntegrateAppenderOption + ) { self.mode = mode self.repoRoot = repoRoot self.fakeSrcRoot = fakeSrcRoot self.sdksExclude = sdksExclude + self.options = options } func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings { var result = buildSettings setBuildSetting(buildSettings: &result, key: "SWIFT_EXEC", value: wrappers.swiftc.path ) - setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" ) + if options.contains(.disableSwiftDriverIntegration) { + setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" ) + } // When generating artifacts, no need to shell-out all compilation commands to our wrappers if case .consumer = mode { setBuildSetting(buildSettings: &result, key: "CC", value: wrappers.cc.path ) diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift index aec0450c..6f0d0176 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift @@ -38,7 +38,8 @@ extension IntegrateContext { env: [String: String], binariesDir: URL, fakeSrcRoot: String, - outputPath: String? + outputPath: String?, + useFontendIntegration: Bool ) throws { projectPath = URL(fileURLWithPath: input) let srcRoot = projectPath.deletingLastPathComponent() @@ -47,10 +48,11 @@ extension IntegrateContext { configOverride = URL(fileURLWithPath: configOverridePath, relativeTo: srcRoot) output = outputPath.flatMap(URL.init(fileURLWithPath:)) self.fakeSrcRoot = URL(fileURLWithPath: fakeSrcRoot) + let swiftcBinaryName = useFontendIntegration ? "swiftc" : "xcswiftc" binaries = XCRCBinariesPaths( prepare: binariesDir.appendingPathComponent("xcprepare"), cc: binariesDir.appendingPathComponent("xccc"), - swiftc: binariesDir.appendingPathComponent("xcswiftc"), + swiftc: binariesDir.appendingPathComponent(swiftcBinaryName), libtool: binariesDir.appendingPathComponent("xclibtool"), lipo: binariesDir.appendingPathComponent("xclipo"), ld: binariesDir.appendingPathComponent("xcld"), diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift index acb52aae..2c81af8a 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift @@ -88,7 +88,8 @@ public class XCIntegrate { env: env, binariesDir: binariesDir, fakeSrcRoot: fakeSrcRoot, - outputPath: output + outputPath: output, + useFontendIntegration: config.enableSwifDriverIntegration ) let configurationOracle = IncludeExcludeOracle( excludes: configurationsExclude.integrateArrayArguments, @@ -98,11 +99,18 @@ public class XCIntegrate { excludes: targetsExclude.integrateArrayArguments, includes: targetsInclude.integrateArrayArguments ) + let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption + if config.enableSwifDriverIntegration { + buildSettingsAppenderOptions = [] + } else { + buildSettingsAppenderOptions = [.disableSwiftDriverIntegration] + } let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender( mode: context.mode, repoRoot: context.repoRoot, fakeSrcRoot: context.fakeSrcRoot, - sdksExclude: sdksExclude.integrateArrayArguments + sdksExclude: sdksExclude.integrateArrayArguments, + options: buildSettingsAppenderOptions ) let lldbPatcher: LLDBInitPatcher switch lldbMode { diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift new file mode 100644 index 00000000..dda0b23d --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift @@ -0,0 +1,212 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +enum SwiftFrontendArgInputError: Error { + // swift-frontend should either be compling or emiting a module + case bothCompilationAndEmitAction + // no .swift files have been passed as input files + case noCompilationInputs + // no -primary-file .swift files have been passed as input files + case noPrimaryFileCompilationInputs + // number of -emit-dependencies-path doesn't match compilation inputs + case dependenciesOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -serialize-diagnostics-path doesn't match compilation inputs + case diagnosticsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o doesn't match compilation inputs + case outputsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o for emit-module can be only 1 + case emitModulOuputCountIsNot1(parsed: Int) + // number of -emit-dependencies-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDependenciesOuputCountIsHigherThan1(parsed: Int) + // number of -serialize-diagnostics-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDiagnosticsOuputCountIsHigherThan1(parsed: Int) + // emit-module requires -emit-objc-header-path + case emitModuleMissingObjcHeaderPath + // -target is required + case emitMissingTarget + // -moduleName is required + case emiMissingModuleName +} + +public struct SwiftFrontendArgInput { + let compile: Bool + let emitModule: Bool + let objcHeaderOutput: String? + let moduleName: String? + let target: String? + let primaryInputPaths: [String] + let inputPaths: [String] + var outputPaths: [String] + var dependenciesPaths: [String] + // Extra params + // Diagnostics are not supported yet in the XCRemoteCache (cached artifacts assumes no warnings) + var diagnosticsPaths: [String] + // Unsed for now: + // .swiftsourceinfo and .swiftdoc will be placed next to the .swiftmodule + let sourceInfoPath: String? + let docPath: String? + let supplementaryOutputFileMap: String? + + /// Manual initializer implementation required to be public + public init( + compile: Bool, + emitModule: Bool, + objcHeaderOutput: String?, + moduleName: String?, + target: String?, + primaryInputPaths: [String], + inputPaths: [String], + outputPaths: [String], + dependenciesPaths: [String], + diagnosticsPaths: [String], + sourceInfoPath: String?, + docPath: String?, + supplementaryOutputFileMap: String? + ) { + self.compile = compile + self.emitModule = emitModule + self.objcHeaderOutput = objcHeaderOutput + self.moduleName = moduleName + self.target = target + self.primaryInputPaths = primaryInputPaths + self.inputPaths = inputPaths + self.outputPaths = outputPaths + self.dependenciesPaths = dependenciesPaths + self.diagnosticsPaths = diagnosticsPaths + self.sourceInfoPath = sourceInfoPath + self.docPath = docPath + self.supplementaryOutputFileMap = supplementaryOutputFileMap + } + + // swiftlint:disable:next cyclomatic_complexity function_body_length + func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext { + guard compile != emitModule else { + throw SwiftFrontendArgInputError.bothCompilationAndEmitAction + } + let inputPathsCount = inputPaths.count + let primaryInputsCount = primaryInputPaths.count + guard inputPathsCount > 0 else { + throw SwiftFrontendArgInputError.noCompilationInputs + } + guard let target = target else { + throw SwiftFrontendArgInputError.emitMissingTarget + } + guard let moduleName = moduleName else { + throw SwiftFrontendArgInputError.emiMissingModuleName + } + + if compile { + guard primaryInputsCount > 0 else { + throw SwiftFrontendArgInputError.noPrimaryFileCompilationInputs + } + guard [primaryInputsCount, 0].contains(dependenciesPaths.count) else { + throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: dependenciesPaths.count + ) + } + guard [primaryInputsCount, 0].contains(diagnosticsPaths.count) else { + throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: diagnosticsPaths.count + ) + } + guard outputPaths.count == primaryInputsCount else { + throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: outputPaths.count + ) + } + let primaryInputFilesURLs: [URL] = primaryInputPaths.map(URL.init(fileURLWithPath:)) + + let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps( + compileFilesScope: .subset(primaryInputFilesURLs), + emitModule: nil + ) + + let compilationFilesOutputs: SwiftcContext.CompilationFilesOutputs + if let compimentaryFileMa = supplementaryOutputFileMap { + compilationFilesOutputs = .supplementaryFileMap(compimentaryFileMa) + } else { + compilationFilesOutputs = .map((0.. ()) throws +} + +/// The default orchestrator that manages the order or swift-frontend invocations. +/// For emit-module (the "first" process) action, it locks a shared file between all swift-frontend invcations, +/// verifies that the mocking can be done and continues the mocking/fallbacking along the lock release +/// For the compilation action, tries to ackquire a lock and waits until the "emit-module" makes a decision +/// if the compilation should be skipped and a "mocking" should used instead +class CommonSwiftFrontendOrchestrator { + /// Content saved to the shared file + /// Safe to use forced unwrapping + private static let emitModuleContent = "done".data(using: .utf8)! + + enum Action { + case emitModule + case compile + } + private let mode: SwiftcContext.SwiftcMode + private let action: Action + private let lockAccessor: ExclusiveFileAccessor + private let maxLockTimeout: TimeInterval + + init( + mode: SwiftcContext.SwiftcMode, + action: Action, + lockAccessor: ExclusiveFileAccessor, + maxLockTimeout: TimeInterval + ) { + self.mode = mode + self.action = action + self.lockAccessor = lockAccessor + self.maxLockTimeout = maxLockTimeout + } + + func run(criticalSection: () throws -> ()) throws { + guard case .consumer(commit: .available) = mode else { + // no need to lock anything - just allow fallbacking to the `swiftc or swift-frontend` + // if we face producer or a consumer where RC is disabled (we have already caught the + // cache miss) + try criticalSection() + return + } + try executeMockAttemp(criticalSection: criticalSection) + } + + private func executeMockAttemp(criticalSection: () throws -> ()) throws { + switch action { + case .emitModule: + try validateEmitModuleStep(criticalSection: criticalSection) + case .compile: + try waitForEmitModuleLock(criticalSection: criticalSection) + } + } + + + /// Fro emit-module, wraps the critical section with the shared lock so other processes (compilation) + /// have to wait until it finishes. + /// Once the emit-module is done, the "magical" string is saved to the file and the lock is released + /// + /// Note: The design of wrapping the entire "emit-module" has a small performance downside if inside + /// the critical section, the code realizes that remote cache cannot be used (in practice - a new file has been added) + /// None of compilation process (so with '-c' args) can continue until the entire emit-module logic finishes. + /// Because it is expected to happen no that often and emit-module is usually quite fast, this makes the + /// implementation way simpler. If we ever want to optimize it, we should release the lock as early + /// as we know, the remote cache cannot be used. Then all other compilation process (-c) can run + /// in parallel with emit-module + private func validateEmitModuleStep(criticalSection: () throws -> ()) throws { + try lockAccessor.exclusiveAccess { handle in + defer { + handle.write(Self.self.emitModuleContent) + } + do { + try criticalSection() + } + } + } + + /// Locks a shared file in a loop until its content non-empty, which means the "parent" emit-module has finished + private func waitForEmitModuleLock(criticalSection: () throws -> ()) throws { + // emit-module process should really quickly retain a lock (it is always invoked + // by Xcode as a first process) + var executed = false + let startingDate = Date() + while !executed { + try lockAccessor.exclusiveAccess { handle in + if !handle.availableData.isEmpty { + // the file is not empty so the emit-module process is done with the "check" + try criticalSection() + executed = true + } + } + // When a max locking time is achieved, execute anyway + if !executed && Date().timeIntervalSince(startingDate) > self.maxLockTimeout { + errorLog(""" + Executing command \(action) without lock synchronization. That may be cause by the\ + crashed or extremly long emit-module. Contact XCRemoteCache authors about this error. + """) + try criticalSection() + executed = true + } + } + } +} diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift new file mode 100644 index 00000000..1fd671ab --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -0,0 +1,89 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class XCSwiftFrontend: XCSwiftAbstract { + // don't lock individual compilation invocations for more than 10s + private static let MaxLockingTimeout: TimeInterval = 10 + private let env: [String: String] + + public init( + command: String, + inputArgs: SwiftFrontendArgInput, + env: [String: String], + dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, + touchFactory: @escaping (URL, FileManager) -> Touch + ) throws { + self.env = env + super.init( + command: command, + inputArgs: inputArgs, + dependenciesWriter: dependenciesWriter, + touchFactory: touchFactory + ) + } + + override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { + let fileManager = FileManager.default + let config: XCRemoteCacheConfig + let context: SwiftcContext + + let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) + .readConfiguration() + context = try SwiftcContext(config: config, input: inputArgs) + // do not cache this context, as it is subject to change when + // the emit-module finds that the cached artifact cannot be used + return (config, context) + } + + override public func run() throws { + do { + /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + let (_, context) = try buildContext() + + let sharedLockFileURL = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: context.tempDir) + let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) + + let action: CommonSwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile + let swiftFrontendOrchestrator = CommonSwiftFrontendOrchestrator( + mode: context.mode, + action: action, + lockAccessor: sharedLock, + maxLockTimeout: Self.self.MaxLockingTimeout + ) + + try swiftFrontendOrchestrator.run(criticalSection: super.run) + } catch { + // Splitting into 2 invocations as os_log truncates a massage + defaultLog("Cannot correctly orchestrate the \(command) with params \(inputArgs)") + defaultLog("Cannot correctly orchestrate error: \(error)") + throw error + } + } +} + +extension XCSwiftFrontend { + /// The file is used to sycnhronize mutliple swift-frontend invocations + static func generateLlbuildIdSharedLock(llbuildId: String, tmpDir: URL) -> URL { + return tmpDir.appendingPathComponent(llbuildId).appendingPathExtension("lock") + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift new file mode 100644 index 00000000..e6d4026f --- /dev/null +++ b/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift @@ -0,0 +1,39 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where +/// only individual .o files are created and not .swiftmodule of -Swift.h +/// (which is part of swift-frontend -emit-module invocation) +class NoopSwiftcProductsGenerator: SwiftcProductsGenerator { + func generateFrom( + artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], + artifactSwiftModuleObjCFile: URL + ) throws -> SwiftcProductsGeneratorOutput { + printWarning(""" + Invoking module generation from NoopSwiftcProductsGenerator which does nothing. \ + It might be an error of the swift-frontend mocking. + """) + // TODO: Refactor API: this url is never used: + // NoopSwiftcProductsGenerator is intended only for the swift-frontend + let trivialURL = URL(fileURLWithPath: "/non-existing") + return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL) + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift b/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift new file mode 100644 index 00000000..fd3d3fa4 --- /dev/null +++ b/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift @@ -0,0 +1,46 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +class StaticSwiftcInputReader: SwiftcInputReader { + private let moduleDependencies: URL? + private let swiftDependencies: URL? + private let compilationFiles: [SwiftFileCompilationInfo] + + init( + moduleDependencies: URL?, + swiftDependencies: URL?, + compilationFiles: [SwiftFileCompilationInfo] + ) { + self.moduleDependencies = moduleDependencies + self.swiftDependencies = swiftDependencies + self.compilationFiles = compilationFiles + } + + func read() throws -> SwiftCompilationInfo { + return .init( + info: .init( + dependencies: moduleDependencies, + swiftDependencies: swiftDependencies + ), + files: compilationFiles + ) + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift index ef30547a..9bc70687 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift @@ -132,7 +132,7 @@ class Swiftc: SwiftcProtocol { // Read swiftmodule location from XCRemoteCache // arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo} - let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent + let moduleName = context.moduleName let allCompilations = try inputFilesReader.read() let artifactSwiftmoduleDir = artifactLocation .appendingPathComponent("swiftmodule") @@ -145,20 +145,24 @@ class Swiftc: SwiftcProtocol { } ) - // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h - let artifactSwiftModuleObjCDir = artifactLocation - .appendingPathComponent("include") - .appendingPathComponent(context.arch) - .appendingPathComponent(context.moduleName) - // Move cached xxxx-Swift.h to the location passed in arglist - // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir - let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir - .appendingPathComponent(context.objcHeaderOutput.lastPathComponent) + // emit module (if requested) + if let emitModule = context.steps.emitModule { + // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h + let artifactSwiftModuleObjCDir = artifactLocation + .appendingPathComponent("include") + .appendingPathComponent(context.arch) + .appendingPathComponent(context.moduleName) + // Move cached xxxx-Swift.h to the location passed in arglist + // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h + // file in artifactSwiftModuleObjCDir + let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir + .appendingPathComponent(emitModule.objcHeaderOutput.lastPathComponent) - _ = try productsGenerator.generateFrom( - artifactSwiftModuleFiles: artifactSwiftmoduleFiles, - artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile - ) + _ = try productsGenerator.generateFrom( + artifactSwiftModuleFiles: artifactSwiftmoduleFiles, + artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile + ) + } try plugins.forEach { try $0.generate(for: allCompilations) @@ -176,8 +180,10 @@ class Swiftc: SwiftcProtocol { try cachedDependenciesWriterFactory.generate(output: individualDeps) } } - // Save .d for the entire module - try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies) + // Save .d for the entire module (might not be required in the `swift-frontend -c` mode) + if let swiftDependencies = allCompilations.info.swiftDependencies { + try cachedDependenciesWriterFactory.generate(output: swiftDependencies) + } // Generate .d file with all deps in the "-master.d" (e.g. for WMO) if let wmoDeps = allCompilations.info.dependencies { try cachedDependenciesWriterFactory.generate(output: wmoDeps) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 1e58ff4f..47c2a964 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -20,6 +20,40 @@ import Foundation public struct SwiftcContext { + public struct SwiftcStepEmitModule { + let objcHeaderOutput: URL + let modulePathOutput: URL + let dependencies: URL? + } + public enum SwiftcStepCompileFilesScope { + case none + case all + case subset([URL]) + } + + public struct SwiftcSteps { + let compileFilesScope: SwiftcStepCompileFilesScope + let emitModule: SwiftcStepEmitModule? + } + + /// Defines how a list of input files (*.swift) is passed to the invocation + public enum CompilationFilesSource { + /// defined in a separate file (via @/.../*.SwiftFileList) + case fileList(String) + /// explicitly passed a list of files + case list([String]) + } + + /// Defines how a list of output files (*.d, *.o etc.) is passed to the invocation + public enum CompilationFilesOutputs { + /// defined in a separate file (via -output-file-map) + case fileMap(String) + /// defined in a separate file (via -supplementary-output-file-map) + case supplementaryFileMap(String) + /// explicitly passed in the invocation + case map([String: SwiftFileCompilationInfo]) + } + enum SwiftcMode: Equatable { case producer /// Commit sha of the commit to use during remote cache @@ -28,14 +62,13 @@ public struct SwiftcContext { case producerFast } - let objcHeaderOutput: URL + let steps: SwiftcSteps let moduleName: String - let modulePathOutput: URL - /// File that defines output files locations (.d, .swiftmodule etc.) - let filemap: URL + /// A source that defines output files locations (.d, .swiftmodule etc.) + let outputs: CompilationFilesOutputs let target: String - /// File that contains input files for the swift module compilation - let fileList: URL + /// A source that contains all input files for the swift module compilation + let inputs: CompilationFilesSource let tempDir: URL let arch: String let prebuildDependenciesPath: String @@ -43,29 +76,29 @@ public struct SwiftcContext { /// File that stores all compilation invocation arguments let invocationHistoryFile: URL - public init( config: XCRemoteCacheConfig, - objcHeaderOutput: String, moduleName: String, - modulePathOutput: String, - filemap: String, + steps: SwiftcSteps, + outputs: CompilationFilesOutputs, target: String, - fileList: String + inputs: CompilationFilesSource, + /// any workspace file path - all other intermediate files for this compilation + /// are placed next to it. This path is used to infer the arch and TARGET_TEMP_DIR + exampleWorkspaceFilePath: String ) throws { - self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput) self.moduleName = moduleName - self.modulePathOutput = URL(fileURLWithPath: modulePathOutput) - self.filemap = URL(fileURLWithPath: filemap) + self.steps = steps + self.outputs = outputs self.target = target - self.fileList = URL(fileURLWithPath: fileList) + self.inputs = inputs // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule // That may be subject to change for other Xcode versions - tempDir = URL(fileURLWithPath: modulePathOutput) + tempDir = URL(fileURLWithPath: exampleWorkspaceFilePath) .deletingLastPathComponent() .deletingLastPathComponent() .deletingLastPathComponent() - arch = URL(fileURLWithPath: modulePathOutput).deletingLastPathComponent().lastPathComponent + arch = URL(fileURLWithPath: exampleWorkspaceFilePath).deletingLastPathComponent().lastPathComponent let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot) let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot) @@ -92,14 +125,31 @@ public struct SwiftcContext { config: XCRemoteCacheConfig, input: SwiftcArgInput ) throws { + let steps = SwiftcSteps( + compileFilesScope: .all, + emitModule: SwiftcStepEmitModule( + objcHeaderOutput: URL(fileURLWithPath: (input.objcHeaderOutput)), + modulePathOutput: URL(fileURLWithPath: input.modulePathOutput), + dependencies: nil + ) + ) + let outputs = CompilationFilesOutputs.fileMap(input.filemap) + let inputs = CompilationFilesSource.fileList(input.fileList) try self.init( config: config, - objcHeaderOutput: input.objcHeaderOutput, moduleName: input.moduleName, - modulePathOutput: input.modulePathOutput, - filemap: input.filemap, + steps: steps, + outputs: outputs, target: input.target, - fileList: input.fileList + inputs: inputs, + exampleWorkspaceFilePath: input.modulePathOutput ) } + + init( + config: XCRemoteCacheConfig, + input: SwiftFrontendArgInput + ) throws { + self = try input.generateSwiftcContext(config: config) + } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift index 7a803b78..d4c23cd3 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift @@ -18,11 +18,13 @@ // under the License. import Foundation +import Yams /// Errors with reading swiftc inputs enum SwiftcInputReaderError: Error { case readingFailed case invalidFormat + case invalidYamlFormat case missingField(String) } @@ -45,10 +47,11 @@ struct SwiftCompilationInfo: Encodable, Equatable { struct SwiftModuleCompilationInfo: Encodable, Equatable { // not present for incremental builds let dependencies: URL? - let swiftDependencies: URL + // not present for the swift-frontend '-c' invocation + let swiftDependencies: URL? } -struct SwiftFileCompilationInfo: Encodable, Equatable { +public struct SwiftFileCompilationInfo: Encodable, Hashable { let file: URL // not present for WMO builds let dependencies: URL? @@ -60,11 +63,18 @@ struct SwiftFileCompilationInfo: Encodable, Equatable { class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { + enum Format { + case json + case yaml + } + private let file: URL + private let fileFormat: Format private let fileManager: FileManager - init(_ file: URL, fileManager: FileManager) { + init(_ file: URL, fileFormat: Format, fileManager: FileManager) { self.file = file + self.fileFormat = fileFormat self.fileManager = fileManager } @@ -72,7 +82,7 @@ class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { guard let content = fileManager.contents(atPath: file.path) else { throw SwiftcInputReaderError.readingFailed } - guard let representation = try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any] else { + guard let representation = try decodeFile(content: content) else { throw SwiftcInputReaderError.invalidFormat } return try SwiftCompilationInfo(from: representation) @@ -82,11 +92,20 @@ class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { let data = try JSONSerialization.data(withJSONObject: info.dump(), options: [.prettyPrinted]) fileManager.createFile(atPath: file.path, contents: data, attributes: nil) } + + private func decodeFile(content: Data) throws -> [String: Any]? { + switch fileFormat { + case .json: + return try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any] + case .yaml: + return try Yams.load(yaml: String(data: content, encoding: .utf8)!) as? [String: Any] + } + } } extension SwiftCompilationInfo { init(from object: [String: Any]) throws { - info = try SwiftModuleCompilationInfo(from: object[""]) + info = try SwiftModuleCompilationInfo(from: object["", default: [:]]) files = try object.reduce([]) { prev, new in let (key, value) = new if key.isEmpty { @@ -111,14 +130,14 @@ extension SwiftModuleCompilationInfo { guard let dict = object as? [String: String] else { throw SwiftcInputReaderError.invalidFormat } - swiftDependencies = try dict.readURL(key: "swift-dependencies") + swiftDependencies = dict.readURL(key: "swift-dependencies") dependencies = dict.readURL(key: "dependencies") } func dump() -> [String: String] { return [ "dependencies": dependencies?.path, - "swift-dependencies": swiftDependencies.path, + "swift-dependencies": swiftDependencies?.path, ].compactMapValues { $0 } } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift index 86fbc5a3..b1a1794c 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift @@ -27,8 +27,10 @@ class SwiftcOrchestrator { private let mode: SwiftcContext.SwiftcMode // swiftc command that should be called to generate artifacts private let swiftcCommand: String - private let objcHeaderOutput: URL - private let moduleOutput: URL + // That is nil if invoking from frontend compilation: `swift-frontend -c` + private let objcHeaderOutput: URL? + // That is nil if invoking from frontend compilation: `swift-frontend -c` + private let moduleOutput: URL? private let arch: String private let artifactBuilder: ArtifactSwiftProductsBuilder private let shellOut: ShellOut @@ -39,8 +41,8 @@ class SwiftcOrchestrator { mode: SwiftcContext.SwiftcMode, swiftc: SwiftcProtocol, swiftcCommand: String, - objcHeaderOutput: URL, - moduleOutput: URL, + objcHeaderOutput: URL?, + moduleOutput: URL?, arch: String, artifactBuilder: ArtifactSwiftProductsBuilder, producerFallbackCommandProcessors: [ShellCommandsProcessor], @@ -128,10 +130,14 @@ class SwiftcOrchestrator { try processor.applyArgsRewrite(args) } try fallbackToDefaultAndWait(command: swiftcCommand, args: swiftcArgs) - // move generated .h to the location where artifact creator expects it - try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput) - // move generated .swiftmodule to the location where artifact creator expects it - try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput) + if let objcHeaderOutput = objcHeaderOutput { + // move generated .h to the location where artifact creator expects it + try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput) + } + if let moduleOutput = moduleOutput { + // move generated .swiftmodule to the location where artifact creator expects it + try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput) + } try producerFallbackCommandProcessors.forEach { try $0.postCommandProcessing() diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 7ea2aa20..a071f93c 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -45,15 +45,15 @@ public struct SwiftcArgInput { } } -public class XCSwiftc { - private let command: String - private let inputArgs: SwiftcArgInput +public class XCSwiftAbstract { + let command: String + let inputArgs: InputArgs private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter private let touchFactory: (URL, FileManager) -> Touch public init( command: String, - inputArgs: SwiftcArgInput, + inputArgs: InputArgs, dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, touchFactory: @escaping (URL, FileManager) -> Touch ) { @@ -63,26 +63,46 @@ public class XCSwiftc { self.touchFactory = touchFactory } + func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { + fatalError("Need to override in \(Self.self)") + } + // swiftlint:disable:next function_body_length - public func run() { + public func run() throws { let fileManager = FileManager.default - let config: XCRemoteCacheConfig - let context: SwiftcContext - do { - let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) - .readConfiguration() - context = try SwiftcContext(config: config, input: inputArgs) - } catch { - exit(1, "FATAL: Swiftc initialization failed with error: \(error)") - } + let (config, context) = try buildContext() + let swiftcCommand = config.swiftcCommand let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager) - let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) - let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) + let inputReader: SwiftcInputReader + // TODO: consider renaming + // outputs and outputs of the swift step are described in a map together + // and the file/list is named "output" + switch context.outputs { + case .fileMap(let path): + inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileFormat: .json, fileManager: fileManager) + case .supplementaryFileMap(let path): + inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileFormat: .yaml, fileManager: fileManager) + case .map(let map): + // static - passed via the arguments list + // TODO: check if first 2 ars can always be `nil` + // with Xcode 13, inputs via cmd are only used for compilations + inputReader = StaticSwiftcInputReader( + moduleDependencies: context.steps.emitModule?.dependencies, + swiftDependencies: nil, + compilationFiles: Array(map.values) + ) + } + let fileListReader: ListReader + switch context.inputs { + case .fileList(let path): + fileListReader = FileListEditor(URL(fileURLWithPath: path), fileManager: fileManager) + case .list(let paths): + fileListReader = StaticFileListReader(list: paths.map(URL.init(fileURLWithPath:))) + } let artifactOrganizer = ZipArtifactOrganizer( targetTempDir: context.tempDir, // xcswiftc doesn't call artifact preprocessing @@ -101,11 +121,16 @@ public class XCSwiftc { moduleName: context.moduleName, fileManager: fileManager ) - let productsGenerator = DiskSwiftcProductsGenerator( - modulePathOutput: context.modulePathOutput, - objcHeaderOutput: context.objcHeaderOutput, - diskCopier: HardLinkDiskCopier(fileManager: fileManager) - ) + let productsGenerator: SwiftcProductsGenerator + if let emitModule = context.steps.emitModule { + productsGenerator = DiskSwiftcProductsGenerator( + modulePathOutput: emitModule.modulePathOutput, + objcHeaderOutput: emitModule.objcHeaderOutput, + diskCopier: HardLinkDiskCopier(fileManager: fileManager) + ) + } else { + productsGenerator = NoopSwiftcProductsGenerator() + } let allInvocationsStorage = ExistingFileStorage( storageFile: context.invocationHistoryFile, command: swiftcCommand @@ -119,7 +144,7 @@ public class XCSwiftc { let shellOut = ProcessShellOut() let swiftc = Swiftc( - inputFileListReader: fileListEditor, + inputFileListReader: fileListReader, markerReader: markerReader, allowedFilesListScanner: allowedFilesListScanner, artifactOrganizer: artifactOrganizer, @@ -136,18 +161,28 @@ public class XCSwiftc { mode: context.mode, swiftc: swiftc, swiftcCommand: swiftcCommand, - objcHeaderOutput: context.objcHeaderOutput, - moduleOutput: context.modulePathOutput, + objcHeaderOutput: context.steps.emitModule?.objcHeaderOutput, + moduleOutput: context.steps.emitModule?.modulePathOutput, arch: context.arch, artifactBuilder: artifactBuilder, producerFallbackCommandProcessors: [], invocationStorage: invocationStorage, shellOut: shellOut ) - do { - try orchestrator.run() - } catch { - exit(1, "Swiftc failed with error: \(error)") - } + try orchestrator.run() + } +} + +public class XCSwiftc: XCSwiftAbstract { + override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { + let fileReader = FileManager.default + let config: XCRemoteCacheConfig + let context: SwiftcContext + let srcRoot: URL = URL(fileURLWithPath: fileReader.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileReader) + .readConfiguration() + context = try SwiftcContext(config: config, input: inputArgs) + + return (config, context) } } diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index 4ef7b77f..cad41e63 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -57,6 +57,8 @@ public struct XCRemoteCacheConfig: Encodable { var clangCommand: String = "clang" /// Command for a standard Swift compilation (swiftc) var swiftcCommand: String = "swiftc" + /// Command for a standard Swift frontend compilation (swift-frontend) + var swiftFrontendCommand: String = "swift-frontend" /// Path of the primary repository that produces cache artifacts var primaryRepo: String = "" /// Main (primary) branch that produces cache artifacts (default to 'master') @@ -151,6 +153,8 @@ public struct XCRemoteCacheConfig: Encodable { /// If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch /// That might useful on CI, where a shallow clone is used var gracefullyHandleMissingCommonSha: Bool = false + /// Enable experimental integration with swift driver, added in Xcode 13 + var enableSwifDriverIntegration: Bool = false } extension XCRemoteCacheConfig { @@ -211,6 +215,7 @@ extension XCRemoteCacheConfig { merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths merge.gracefullyHandleMissingCommonSha = scheme.gracefullyHandleMissingCommonSha ?? gracefullyHandleMissingCommonSha + merge.enableSwifDriverIntegration = scheme.enableSwifDriverIntegration ?? enableSwifDriverIntegration return merge } @@ -279,6 +284,7 @@ struct ConfigFileScheme: Decodable { let customRewriteEnvs: [String]? let irrelevantDependenciesPaths: [String]? let gracefullyHandleMissingCommonSha: Bool? + let enableSwifDriverIntegration: Bool? // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84 enum CodingKeys: String, CodingKey { @@ -330,6 +336,7 @@ struct ConfigFileScheme: Decodable { case customRewriteEnvs = "custom_rewrite_envs" case irrelevantDependenciesPaths = "irrelevant_dependencies_paths" case gracefullyHandleMissingCommonSha = "gracefully_handle_missing_common_sha" + case enableSwifDriverIntegration = "enable_swift_driver_integration" } } diff --git a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift index db8871c3..175fc952 100644 --- a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift +++ b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift @@ -48,6 +48,7 @@ class PhaseCacheModeController: CacheModeController { private let dependenciesWriter: DependenciesWriter private let dependenciesReader: DependenciesReader private let markerWriter: MarkerWriter + private let llbuildLockFile: URL private let fileManager: FileManager init( @@ -59,6 +60,7 @@ class PhaseCacheModeController: CacheModeController { dependenciesWriter: (URL, FileManager) -> DependenciesWriter, dependenciesReader: (URL, FileManager) -> DependenciesReader, markerWriter: (URL, FileManager) -> MarkerWriter, + llbuildLockFile: URL, fileManager: FileManager ) { @@ -69,10 +71,12 @@ class PhaseCacheModeController: CacheModeController { let discoveryURL = tempDir.appendingPathComponent(phaseDependencyPath) self.dependenciesWriter = dependenciesWriter(discoveryURL, fileManager) self.dependenciesReader = dependenciesReader(discoveryURL, fileManager) + self.llbuildLockFile = llbuildLockFile self.markerWriter = markerWriter(modeMarker, fileManager) } func enable(allowedInputFiles: [URL], dependencies: [URL]) throws { + try cleanupLlBuildLock() // marker file contains filepaths that contribute to the build products // and should invalidate all other target steps (swiftc,libtool etc.) let targetSensitiveFiles = dependencies + [modeMarker, Self.xcodeSelectLink] @@ -84,6 +88,7 @@ class PhaseCacheModeController: CacheModeController { } func disable() throws { + try cleanupLlBuildLock() guard !forceCached else { throw PhaseCacheModeControllerError.cannotUseRemoteCacheForForcedCacheMode } @@ -114,4 +119,14 @@ class PhaseCacheModeController: CacheModeController { } return false } + + private func cleanupLlBuildLock() throws { + if fileManager.fileExists(atPath: llbuildLockFile.path) { + do { + try fileManager.removeItem(at: llbuildLockFile) + } catch { + printWarning("Removing llbuild lock at \(llbuildLockFile.path) failed. Error: \(error)") + } + } + } } diff --git a/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift b/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift new file mode 100644 index 00000000..a4985241 --- /dev/null +++ b/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift @@ -0,0 +1,34 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +class StaticFileListReader: ListReader { + private let list: [URL] + init(list: [URL]) { + self.list = list + } + func listFilesURLs() throws -> [URL] { + list + } + + func canRead() -> Bool { + true + } +} diff --git a/Sources/XCRemoteCache/Utils/Array+Utils.swift b/Sources/XCRemoteCache/Utils/Array+Utils.swift new file mode 100644 index 00000000..67e2e75a --- /dev/null +++ b/Sources/XCRemoteCache/Utils/Array+Utils.swift @@ -0,0 +1,29 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public extension Array { + func get(_ i: Index) -> Element? { + guard count > i else { + return nil + } + return self[i] + } +} diff --git a/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift b/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift index 001d6d7e..11d5ffef 100644 --- a/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift +++ b/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift @@ -35,3 +35,12 @@ public extension URL { throw URLError.invalidURLFormat(string) } } + +extension URL { + init?(_ string: String?) throws { + guard let string = string else { + return nil + } + self = URL(fileURLWithPath: string) + } +} diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift new file mode 100644 index 00000000..05141f49 --- /dev/null +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -0,0 +1,135 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import XCRemoteCache + +/// Wrapper for a `swift-frontend` that skips compilation and +/// produces empty output files (.o). As a compilation dependencies +/// (.d) file, it copies all dependency files from the prebuild marker file +/// Fallbacks to a standard `swift-frontend` when the ramote cache is not applicable (e.g. modified sources) +public class XCSwiftcFrontendMain { + // swiftlint:disable:next function_body_length cyclomatic_complexity + public func main() { + let env = ProcessInfo.processInfo.environment + let command = ProcessInfo().processName + let args = ProcessInfo().arguments + var compile = false + var emitModule = false + var objcHeaderOutput: String? + var moduleName: String? + var target: String? + var inputPaths: [String] = [] + var primaryInputPaths: [String] = [] + var outputPaths: [String] = [] + var dependenciesPaths: [String] = [] + var diagnosticsPaths: [String] = [] + var sourceInfoPath: String? + var docPath: String? + var supplementaryOutputFileMap: String? + + for i in 0.. Never { + let developerDir = env["DEVELOPER_DIR"]! + // limitation: always using the Xcode's toolchain + let swiftFrontendCommand = "\(developerDir)/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend" + + let args = ProcessInfo().arguments + let paramList = [swiftFrontendCommand] + args.dropFirst() + let cargs = paramList.map { strdup($0) } + [nil] + execvp(swiftFrontendCommand, cargs) + + /// C-function `execv` returns only when the command fails + exit(1) + } +} diff --git a/Sources/xcswift-frontend/main.swift b/Sources/xcswift-frontend/main.swift new file mode 100644 index 00000000..0bdf4a1a --- /dev/null +++ b/Sources/xcswift-frontend/main.swift @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import XCRemoteCache + +XCSwiftcFrontendMain().main() diff --git a/Sources/xcswiftc/XCSwiftcMain.swift b/Sources/xcswiftc/XCSwiftcMain.swift index acb04868..ceadab6b 100644 --- a/Sources/xcswiftc/XCSwiftcMain.swift +++ b/Sources/xcswiftc/XCSwiftcMain.swift @@ -60,30 +60,37 @@ public class XCSwiftcMain { let targetInputInput = target, let swiftFileListInput = swiftFileList else { - let swiftcCommand = "swiftc" - print("Fallbacking to compilation using \(swiftcCommand).") + executeFallback() + } + do { + let swiftcArgsInput = SwiftcArgInput( + objcHeaderOutput: objcHeaderOutputInput, + moduleName: moduleNameInput, + modulePathOutput: modulePathOutputInput, + filemap: filemapInput, + target: targetInputInput, + fileList: swiftFileListInput + ) + try XCSwiftc( + command: command, + inputArgs: swiftcArgsInput, + dependenciesWriter: FileDependenciesWriter.init, + touchFactory: FileTouch.init + ).run() + } catch { + executeFallback() + } + } + private func executeFallback() -> Never { + let swiftcCommand = "swiftc" + print("Fallbacking to compilation using \(swiftcCommand).") - let args = ProcessInfo().arguments - let paramList = [swiftcCommand] + args.dropFirst() - let cargs = paramList.map { strdup($0) } + [nil] - execvp(swiftcCommand, cargs) + let args = ProcessInfo().arguments + let paramList = [swiftcCommand] + args.dropFirst() + let cargs = paramList.map { strdup($0) } + [nil] + execvp(swiftcCommand, cargs) - /// C-function `execv` returns only when the command fails - exit(1) - } - let swiftcArgsInput = SwiftcArgInput( - objcHeaderOutput: objcHeaderOutputInput, - moduleName: moduleNameInput, - modulePathOutput: modulePathOutputInput, - filemap: filemapInput, - target: targetInputInput, - fileList: swiftFileListInput - ) - XCSwiftc( - command: command, - inputArgs: swiftcArgsInput, - dependenciesWriter: FileDependenciesWriter.init, - touchFactory: FileTouch.init - ).run() + /// C-function `execv` returns only when the command fails + exit(1) } } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift index be78500b..9c6f4eda 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift @@ -28,7 +28,7 @@ class PostbuildContextTests: FileXCTestCase { "TARGET_TEMP_DIR": "TARGET_TEMP_DIR", "DERIVED_FILE_DIR": "DERIVED_FILE_DIR", "ARCHS": "x86_64", - "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal" , + "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal", "CONFIGURATION": "CONFIGURATION", "PLATFORM_NAME": "PLATFORM_NAME", "XCODE_PRODUCT_BUILD_VERSION": "XCODE_PRODUCT_BUILD_VERSION", @@ -45,6 +45,7 @@ class PostbuildContextTests: FileXCTestCase { "DERIVED_SOURCES_DIR": "DERIVED_SOURCES_DIR", "CURRENT_VARIANT": "normal", "PUBLIC_HEADERS_FOLDER_PATH": "/usr/local/include", + "LLBUILD_BUILD_ID": "1" ] override func setUpWithError() throws { diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index 31c34595..9c9f39b8 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -57,7 +57,8 @@ class PostbuildTests: FileXCTestCase { overlayHeadersPath: "", irrelevantDependenciesPaths: [], publicHeadersFolderPath: nil, - disabled: false + disabled: false, + llbuildIdLockFile: "/file" ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), diff --git a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift index 84bcea4b..aa5e1131 100644 --- a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift @@ -64,7 +64,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) contextCached = PrebuildContext( targetTempDir: sampleURL, @@ -78,7 +79,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) organizer = ArtifactOrganizerFake(artifactRoot: artifactsRoot, unzippedExtension: "unzip") globalCacheSwitcher = InMemoryGlobalCacheSwitcher() @@ -244,7 +246,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) let prebuild = Prebuild( @@ -276,7 +279,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) metaContent = try generateMeta(fingerprint: generator.generate(), filekey: "1") let downloadedArtifactPackage = artifactsRoot.appendingPathComponent("1") @@ -340,7 +344,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: false, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) try globalCacheSwitcher.enable(sha: "1") let prebuild = Prebuild( @@ -372,7 +377,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: true + disabled: true, + llbuildIdLockFile: "/tmp/lock" ) let prebuild = Prebuild( diff --git a/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift index 66da9aa7..819323c7 100644 --- a/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift @@ -49,7 +49,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let resultURL = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String) @@ -64,7 +65,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let resultURL: String = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String) @@ -79,7 +81,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlus: String = try XCTUnwrap(result["LDPLUSPLUS"] as? String) @@ -93,7 +96,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*"] + sdksExclude: ["watchOS*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlusWatchOS: String = try XCTUnwrap(result["LDPLUSPLUS[sdk=watchOS*]"] as? String) @@ -107,7 +111,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*"] + sdksExclude: ["watchOS*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let libtoolWatchOS: String = try XCTUnwrap(result["LIBTOOL[sdk=watchOS*]"] as? String) @@ -121,7 +126,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*", "watchsimulator*"] + sdksExclude: ["watchOS*", "watchsimulator*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlusWatchOS: String = try XCTUnwrap(result["LDPLUSPLUS[sdk=watchOS*]"] as? String) @@ -137,7 +143,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*", "watchsimulator*"] + sdksExclude: ["watchOS*", "watchsimulator*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let disabledWatchOS: String = try XCTUnwrap(result["XCRC_DISABLED[sdk=watchOS*]"] as? String) @@ -146,4 +153,34 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { XCTAssertEqual(disabledWatchOS, "YES") XCTAssertEqual(disabledWatchSimulator, "YES") } + + func testExcludesSwiftFrontendIntegrationForSpecificOption() throws { + let mode: Mode = .consumer + let appender = XcodeProjBuildSettingsIntegrateAppender( + mode: mode, + repoRoot: rootURL, + fakeSrcRoot: "/", + sdksExclude: [], + options: [.disableSwiftDriverIntegration] + ) + let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) + let useSwiftIntegrationDriver: String = try XCTUnwrap(result["SWIFT_USE_INTEGRATED_DRIVER"] as? String) + + XCTAssertEqual(useSwiftIntegrationDriver, "NO") + } + + func testDoesntExcludesSwiftFrontendIntegrationForEmptyOptions() throws { + let mode: Mode = .consumer + let appender = XcodeProjBuildSettingsIntegrateAppender( + mode: mode, + repoRoot: rootURL, + fakeSrcRoot: "/", + sdksExclude: [], + options: [] + ) + let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) + let useSwiftIntegrationDriver: String? = result["SWIFT_USE_INTEGRATED_DRIVER"] as? String + + XCTAssertNil(useSwiftIntegrationDriver) + } } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift new file mode 100644 index 00000000..18c07f11 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift @@ -0,0 +1,155 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + + +final class SwiftFrontendOrchestratorTests: FileXCTestCase { + private let prohibitedAccessor = DisallowedExclusiveFileAccessor() + private var nonEmptyFile: URL! + private let maxLocking: TimeInterval = 10 + + override func setUp() async throws { + nonEmptyFile = try prepareTempDir().appendingPathComponent("lock.lock") + try fileManager.write(toPath: nonEmptyFile.path, contents: "Done".data(using: .utf8)) + } + func testRunsCriticalSectionImmediatellyForProducer() throws { + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .producer, + action: .compile, + lockAccessor: prohibitedAccessor, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + invoked = true + } + XCTAssertTrue(invoked) + } + + func testRunsCriticalSectionImmediatellyForDisabledConsumer() throws { + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .unavailable), + action: .compile, + lockAccessor: prohibitedAccessor, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + invoked = true + } + XCTAssertTrue(invoked) + } + + func testRunsEmitModuleLogicInAnExclusiveLock() throws { + let lock = FakeExclusiveFileAccessor() + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .emitModule, + lockAccessor: lock, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + XCTAssertTrue(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } + + func testCompilationInvokesCriticalSectionOnlyForNonEmptyLockFile() throws { + let lock = FakeExclusiveFileAccessor(pattern: [.empty, .nonEmpty(nonEmptyFile)]) + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .compile, + lockAccessor: lock, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + XCTAssertTrue(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } + + func testExecutesActionWithoutLockIfLockingFileIsEmptyForALongTime() throws { + let lock = FakeExclusiveFileAccessor(pattern: []) + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .compile, + lockAccessor: lock, + maxLockTimeout: 0 + ) + + var invoked = false + try orchestrator.run { + XCTAssertFalse(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } +} + +private class DisallowedExclusiveFileAccessor: ExclusiveFileAccessor { + func exclusiveAccess(block: (FileHandle) throws -> (T)) throws -> T { + throw "Invoked ProhibitedExclusiveFileAccessor" + } +} + +// Thread-unsafe, in-memory lock +private class FakeExclusiveFileAccessor: ExclusiveFileAccessor { + private(set) var isLocked = false + private var pattern: [LockFileContent] + + enum LockFileContent { + case empty + case nonEmpty(URL) + + func fileHandle() throws -> FileHandle { + switch self { + case .empty: return FileHandle.nullDevice + case .nonEmpty(let url): return try FileHandle(forReadingFrom: url) + } + } + } + + init(pattern: [LockFileContent] = []) { + // keep in the reversed order to always pop + self.pattern = pattern.reversed() + } + + func exclusiveAccess(block: (FileHandle) throws -> (T)) throws -> T { + if isLocked { + throw "FakeExclusiveFileAccessor lock is already locked" + } + defer { + isLocked = false + } + isLocked = true + let fileHandle = try (pattern.popLast() ?? .empty).fileHandle() + return try block(fileHandle) + } + +} diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift index ed9e08d5..30020e11 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift @@ -34,7 +34,7 @@ class SwiftcContextTests: FileXCTestCase { config = XCRemoteCacheConfig(remoteCommitFile: remoteCommitFile.path, sourceRoot: workingDir.path) input = SwiftcArgInput( objcHeaderOutput: "Target-Swift.h", - moduleName: "", + moduleName: "Target", modulePathOutput: modulePathOutput.path, filemap: "", target: "", diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift index 80b2eb15..8bd688ba 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift @@ -34,19 +34,21 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { ) private let sampleInfoContentData = #"{"":{"swift-dependencies":"/"}}"#.data(using: .utf8)! private var inputFile: URL! - private var editor: SwiftcFilemapInputEditor! + private var editorJson: SwiftcFilemapInputEditor! + private var editorYaml: SwiftcFilemapInputEditor! override func setUpWithError() throws { try super.setUpWithError() try prepareTempDir() inputFile = workingDirectory!.appendingPathComponent("swift.json") - editor = SwiftcFilemapInputEditor(inputFile, fileManager: fileManager) + editorJson = SwiftcFilemapInputEditor(inputFile, fileFormat: .json, fileManager: fileManager) + editorYaml = SwiftcFilemapInputEditor(inputFile, fileFormat: .yaml, fileManager: fileManager) } func testReading() throws { try fileManager.spt_writeToFile(atPath: inputFile.path, contents: sampleInfoContentData) - let readInfo = try editor.read() + let readInfo = try editorJson.read() XCTAssertEqual(readInfo, sampleInfo) } @@ -80,13 +82,13 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { ]) try fileManager.spt_writeToFile(atPath: inputFile.path, contents: infoContentData) - let readInfo = try editor.read() + let readInfo = try editorJson.read() XCTAssertEqual(readInfo, expectedInfo) } func testWritingSavesContent() throws { - try editor.write(sampleInfo) + try editorJson.write(sampleInfo) let savedContent = try Data(contentsOf: inputFile) let content = try JSONSerialization.jsonObject(with: savedContent, options: []) as? [String: Any] @@ -108,7 +110,7 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { ), ]) - try editor.write(extendedInfo) + try editorJson.write(extendedInfo) let savedContent = try Data(contentsOf: inputFile) let content = try JSONSerialization.jsonObject(with: savedContent, options: []) as? [String: Any] @@ -119,12 +121,50 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { func testModifyingFileCompilationInfo() throws { try fileManager.spt_writeToFile(atPath: inputFile.path, contents: sampleInfoContentData) - let originalInfo = try editor.read() + let originalInfo = try editorJson.read() var modifiedInfo = originalInfo modifiedInfo.files = [file] - try editor.write(modifiedInfo) - let finalInfo = try editor.read() + try editorJson.write(modifiedInfo) + let finalInfo = try editorJson.read() XCTAssertEqual(finalInfo, modifiedInfo) } + + func testReadingSupplementaryInfoWithOptionalProperties() throws { + let infoContentData = #""" + "/file1.swift": + swift-dependencies: "/file1.swiftdeps" + dependencies: "/file1.d" + "/file2.swift": + dependencies: "/file2.d" + object: "/file2.o" + swift-dependencies: "/file2.swiftdeps" + """#.data(using: .utf8)! + let expectedInfo = SwiftCompilationInfo( + info: SwiftModuleCompilationInfo( + dependencies: nil, + swiftDependencies: nil + ), + files: [ + SwiftFileCompilationInfo( + file: "/file1.swift", + dependencies: "/file1.d", + object: nil, + swiftDependencies: "/file1.swiftdeps" + ), + SwiftFileCompilationInfo( + file: "/file2.swift", + dependencies: "/file2.d", + object: "/file2.o", + swiftDependencies: "/file2.swiftdeps" + ), + ]) + try fileManager.spt_writeToFile(atPath: inputFile.path, contents: infoContentData) + + let readInfo = try editorYaml.read() + + // `files` order doesn't match + XCTAssertEqual(readInfo.info, expectedInfo.info) + XCTAssertEqual(Set(readInfo.files), Set(expectedInfo.files)) + } } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift index 5df313ff..50bf67a2 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift @@ -62,7 +62,7 @@ class SwiftcTests: FileXCTestCase { input = SwiftcArgInput( objcHeaderOutput: "Target-Swift.h", - moduleName: "", + moduleName: "Target", modulePathOutput: modulePathOutput.path, filemap: "", target: "", @@ -276,7 +276,7 @@ class SwiftcTests: FileXCTestCase { func testCompilationUsesArchSpecificSwiftmoduleFiles() throws { let artifactRoot = URL(fileURLWithPath: "/cachedArtifact") - let artifactObjCHeader = URL(fileURLWithPath: "/cachedArtifact/include/archTest/Target-Swift.h") + let artifactObjCHeader = URL(fileURLWithPath: "/cachedArtifact/include/archTest/Target/Target-Swift.h") let artifactSwiftmodule = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftmodule") let artifactSwiftdoc = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftdoc") let artifactSwiftSourceInfo = URL( diff --git a/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift b/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift index 3ad44a4b..8a14b71e 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift @@ -34,6 +34,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/file", fileManager: FileManager.default ) @@ -51,6 +52,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -68,6 +70,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -85,6 +88,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -105,6 +109,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -125,6 +130,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in DependenciesWriterSpy() }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in markerWriter }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -142,6 +148,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in DependenciesWriterSpy() }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -163,6 +170,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in markerWriterSpy }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) diff --git a/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb b/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb index 7a270f77..3c1c3110 100644 --- a/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb +++ b/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb @@ -123,7 +123,8 @@ def self.enable_xcremotecache( exclude_build_configurations, final_target, fake_src_root, - exclude_sdks_configurations + exclude_sdks_configurations, + enable_swift_driver_integration ) srcroot_relative_xc_location = parent_dir(xc_location, repo_distance) # location of the entrite CocoaPods project, relative to SRCROOT @@ -137,14 +138,15 @@ def self.enable_xcremotecache( elsif mode == 'producer' || mode == 'producer-fast' config.build_settings.delete('CC') if config.build_settings.key?('CC') end - reset_build_setting(config.build_settings, 'SWIFT_EXEC', "$SRCROOT/#{srcroot_relative_xc_location}/xcswiftc", exclude_sdks_configurations) + swiftc_name = enable_swift_driver_integration ? 'swiftc' : 'xcswiftc' + reset_build_setting(config.build_settings, 'SWIFT_EXEC', "$SRCROOT/#{srcroot_relative_xc_location}/#{swiftc_name}", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LIBTOOL', "$SRCROOT/#{srcroot_relative_xc_location}/xclibtool", exclude_sdks_configurations) # Setting LIBTOOL to '' breaks SwiftDriver intengration so resetting it to the original value 'libtool' for all excluded configurations add_build_setting_for_sdks(config.build_settings, 'LIBTOOL', 'libtool', exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LD', "$SRCROOT/#{srcroot_relative_xc_location}/xcld", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LDPLUSPLUS', "$SRCROOT/#{srcroot_relative_xc_location}/xcldplusplus", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LIPO', "$SRCROOT/#{srcroot_relative_xc_location}/xclipo", exclude_sdks_configurations) - reset_build_setting(config.build_settings, 'SWIFT_USE_INTEGRATED_DRIVER', 'NO', exclude_sdks_configurations) + reset_build_setting(config.build_settings, 'SWIFT_USE_INTEGRATED_DRIVER', 'NO', exclude_sdks_configurations) unless enable_swift_driver_integration reset_build_setting(config.build_settings, 'XCREMOTE_CACHE_FAKE_SRCROOT', fake_src_root, exclude_sdks_configurations) reset_build_setting(config.build_settings, 'XCRC_PLATFORM_PREFERRED_ARCH', "$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)", exclude_sdks_configurations) @@ -498,6 +500,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) check_platform = @@configuration['check_platform'] fake_src_root = @@configuration['fake_src_root'] exclude_sdks_configurations = @@configuration['exclude_sdks_configurations'] || [] + enable_swift_driver_integration = @@configuration['enable_swift_driver_integration'] || false xccc_location_absolute = "#{user_proj_directory}/#{xccc_location}" xcrc_location_absolute = "#{user_proj_directory}/#{xcrc_location}" @@ -521,7 +524,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) next if target.name.start_with?("Pods-") next if target.name.end_with?("Tests") next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end # Create .rcinfo into `Pods` directory as that .xcodeproj reads configuration from .xcodeproj location @@ -534,7 +537,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) next if target.source_build_phase.files_references.empty? next if target.name.end_with?("Tests") next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end generated_project.save() end @@ -575,7 +578,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) # Attach XCRC to the app targets user_project.targets.each do |target| next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 0, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 0, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end # Set Target sourcemap diff --git a/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb b/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb index ddf31972..b2acb8e4 100644 --- a/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb +++ b/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb @@ -13,5 +13,5 @@ # limitations under the License. module CocoapodsXcremotecache - VERSION = "0.0.16" + VERSION = "0.0.17" end diff --git a/e2eTests/StandaloneSampleApp/.rcinfo b/e2eTests/StandaloneSampleApp/.rcinfo index 60030012..3cd0f26b 100644 --- a/e2eTests/StandaloneSampleApp/.rcinfo +++ b/e2eTests/StandaloneSampleApp/.rcinfo @@ -1,8 +1,9 @@ --- -cache_addresses: +cache_addresses: - 'http://localhost:8080/cache/pods' primary_repo: '.' primary_branch: 'e2e-test-branch' mode: 'consumer' final_target': XCRemoteCacheSample' artifact_maximum_age: 0 # do not use local cache in ~/Library/Caches/XCRemoteCache +enable_swift_driver_integration: true diff --git a/tasks/e2e.rb b/tasks/e2e.rb index 9aa9dbd6..029299f2 100644 --- a/tasks/e2e.rb +++ b/tasks/e2e.rb @@ -25,7 +25,8 @@ 'primary_branch' => GIT_BRANCH, 'mode' => 'consumer', 'final_target' => 'XCRemoteCacheSample', - 'artifact_maximum_age' => 0 + 'artifact_maximum_age' => 0, + 'enable_swift_driver_integration' => true }.freeze DEFAULT_EXPECTATIONS = { 'misses' => 0,