From db0f1da423a12d11b9eb6384eea6bfa5a539b4f3 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sat, 20 May 2023 20:14:27 -0700 Subject: [PATCH 01/33] Scaffold for swift-frontend wrapper --- Package.swift | 5 + .../SwiftFrontend/XCSwiftFrontend.swift | 257 ++++++++++++++++++ .../Utils/URL+ThrowingInitializer.swift | 9 + .../xcswift-frontend/XCSwiftFrontend.swift | 122 +++++++++ Sources/xcswift-frontend/main.swift | 22 ++ .../StandaloneApp.xcodeproj/project.pbxproj | 18 ++ 6 files changed, 433 insertions(+) create mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift create mode 100644 Sources/xcswift-frontend/XCSwiftFrontend.swift create mode 100644 Sources/xcswift-frontend/main.swift diff --git a/Package.swift b/Package.swift index 426a40d9..00c88e43 100644 --- a/Package.swift +++ b/Package.swift @@ -32,6 +32,10 @@ let package = Package( name: "xcswiftc", dependencies: ["XCRemoteCache"] ), + .target( + name: "xcswift-frontend", + dependencies: ["XCRemoteCache"] + ), .target( name: "xclibtoolSupport", dependencies: ["XCRemoteCache"] @@ -69,6 +73,7 @@ let package = Package( dependencies: [ "xcprebuild", "xcswiftc", + "xcswift-frontend", "xclibtool", "xcpostbuild", "xcprepare", diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift new file mode 100644 index 00000000..f00f4905 --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -0,0 +1,257 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +enum SwiftFrontendAction { + case emitModule(emitModuleInfo: SwiftFrontendEmitModuleInfo, inputFiles: [URL]) + case compile([SwiftFileCompilationInfo]) +} + +struct SwiftFrontendEmitModuleInfo: Equatable { + let inputs: [URL] + let objcHeader: URL? + let diagnostics: URL? + let dependencies: URL? + let output: URL + let target: String + let sourceInfo: URL? + let doc: URL? +} + +enum SwiftFrontendArgInputError: Error { + // swift-frontend should either be compling or emiting a module + case bothCompilationAndEmitAction + // no .swift files have been passed as input files + case noCompilationInputs + // number of -emit-dependencies-path doesn't match compilation inputs + case dependenciesOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -serialize-diagnostics-path doesn't match compilation inputs + case diagnosticsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o doesn't match compilation inputs + case outputsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o for emit-module can be only 1 + case emitModulOuputCountIsNot1(parsed: Int) + // number of -emit-dependencies-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDependenciesOuputCountIsHigherThan1(parsed: Int) + // number of -serialize-diagnostics-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDiagnosticsOuputCountIsHigherThan1(parsed: Int) + // -target is required for -emit-module + case emitModuleMissingTarget +} + +public struct SwiftFrontendArgInput { + let compile: Bool + let emitModule: Bool + let objcHeaderOutput: String? + let moduleName: String? + let target: String? + let inputPaths: [String] + var outputPaths: [String] + var dependenciesPaths: [String] + var diagnosticsPaths: [String] + let sourceInfoPath: String? + let docPath: String? + + /// Manual initializer implementation required to be public + public init(compile: Bool, emitModule: Bool, objcHeaderOutput: String?, moduleName: String?, target: String?, inputPaths: [String], outputPaths: [String], dependenciesPaths: [String], diagnosticsPaths: [String], + sourceInfoPath: String?, docPath: String?) { + self.compile = compile + self.emitModule = emitModule + self.objcHeaderOutput = objcHeaderOutput + self.moduleName = moduleName + self.target = target + self.inputPaths = inputPaths + self.outputPaths = outputPaths + self.dependenciesPaths = dependenciesPaths + self.diagnosticsPaths = diagnosticsPaths + self.sourceInfoPath = sourceInfoPath + self.docPath = docPath + } + + func generateAction() throws -> SwiftFrontendAction { + guard compile != emitModule else { + throw SwiftFrontendArgInputError.bothCompilationAndEmitAction + } + let inputPathsCount = inputPaths.count + guard inputPathsCount > 0 else { + throw SwiftFrontendArgInputError.noCompilationInputs + } + let inputURLs: [URL] = inputPaths.map(URL.init(fileURLWithPath:)) + + if compile { + guard [inputPathsCount, 0].contains(dependenciesPaths.count) else { + throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch(expected: inputPathsCount, parsed: dependenciesPaths.count) + } + guard [inputPathsCount, 0].contains(diagnosticsPaths.count) else { + throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch(expected: inputPathsCount, parsed: diagnosticsPaths.count) + } + guard outputPaths.count == inputPathsCount else { + throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(expected: inputPathsCount, parsed: outputPaths.count) + } + let compilationInfos: [SwiftFileCompilationInfo] = (0.. DependenciesWriter + private let touchFactory: (URL, FileManager) -> Touch + + public init( + command: String, + inputArgs: SwiftFrontendArgInput, + dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, + touchFactory: @escaping (URL, FileManager) -> Touch + ) throws { + self.command = command + self.inputArgs = inputArgs + dependenciesWriterFactory = dependenciesWriter + self.touchFactory = touchFactory + + self.action = try inputArgs.generateAction() + } + + // swiftlint:disable:next function_body_length + public func run() { + let fileManager = FileManager.default + let config: XCRemoteCacheConfig + + /* + do { + let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) + .readConfiguration() + context = try SwiftFrontendContext(config: config, input: inputArgs) + } catch { + exit(1, "FATAL: Swiftc initialization failed with error: \(error)") + } + let swiftcCommand = config.swiftcCommand + let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) + let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) + let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager) + + let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) + let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) + let artifactOrganizer = ZipArtifactOrganizer( + targetTempDir: context.tempDir, + // xcswiftc doesn't call artifact preprocessing + artifactProcessors: [], + fileManager: fileManager + ) + // TODO: check for allowedFile comparing a list of all inputfiles, not dependencies from a marker + let makerReferencedFilesListScanner = FileListScannerImpl(markerReader, caseSensitive: false) + let allowedFilesListScanner = ExceptionsFilteredFileListScanner( + allowedFilenames: ["\(config.thinTargetMockFilename).swift"], + disallowedFilenames: [], + scanner: makerReferencedFilesListScanner + ) + let artifactBuilder: ArtifactSwiftProductsBuilder = ArtifactSwiftProductsBuilderImpl( + workingDir: context.tempDir, + moduleName: context.moduleName, + fileManager: fileManager + ) + let productsGenerator = DiskSwiftcProductsGenerator( + modulePathOutput: context.modulePathOutput, + objcHeaderOutput: context.objcHeaderOutput, + diskCopier: HardLinkDiskCopier(fileManager: fileManager) + ) + let allInvocationsStorage = ExistingFileStorage( + storageFile: context.invocationHistoryFile, + command: swiftcCommand + ) + // When fallbacking to local compilation do not call historical `swiftc` invocations + // The current fallback invocation already compiles all files in a target + let invocationStorage = FilteredInvocationStorage( + storage: allInvocationsStorage, + retrieveIgnoredCommands: [swiftcCommand] + ) + let shellOut = ProcessShellOut() + + let swiftc = Swiftc( + inputFileListReader: fileListEditor, + markerReader: markerReader, + allowedFilesListScanner: allowedFilesListScanner, + artifactOrganizer: artifactOrganizer, + inputReader: inputReader, + context: context, + markerWriter: markerWriter, + productsGenerator: productsGenerator, + fileManager: fileManager, + dependenciesWriterFactory: dependenciesWriterFactory, + touchFactory: touchFactory, + plugins: [] + ) + let orchestrator = SwiftcOrchestrator( + mode: context.mode, + swiftc: swiftc, + swiftcCommand: swiftcCommand, + objcHeaderOutput: context.objcHeaderOutput, + moduleOutput: context.modulePathOutput, + arch: context.arch, + artifactBuilder: artifactBuilder, + producerFallbackCommandProcessors: [], + invocationStorage: invocationStorage, + shellOut: shellOut + ) + do { + try orchestrator.run() + } catch { + exit(1, "Swiftc failed with error: \(error)") + } + */ + } +} diff --git a/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift b/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift index 001d6d7e..11d5ffef 100644 --- a/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift +++ b/Sources/XCRemoteCache/Utils/URL+ThrowingInitializer.swift @@ -35,3 +35,12 @@ public extension URL { throw URLError.invalidURLFormat(string) } } + +extension URL { + init?(_ string: String?) throws { + guard let string = string else { + return nil + } + self = URL(fileURLWithPath: string) + } +} diff --git a/Sources/xcswift-frontend/XCSwiftFrontend.swift b/Sources/xcswift-frontend/XCSwiftFrontend.swift new file mode 100644 index 00000000..9d58f892 --- /dev/null +++ b/Sources/xcswift-frontend/XCSwiftFrontend.swift @@ -0,0 +1,122 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import XCRemoteCache + +/// Wrapper for a `swift-frontend` that skips compilation and produces empty output files (.o). As a compilation dependencies +/// (.d) file, it copies all dependency files from the prebuild marker file +/// Fallbacks to a standard `swiftc` when the Ramote cache is not applicable (e.g. modified sources) +public class XCSwiftcFrontendMain { + // swiftlint:disable:next function_body_length + public func main() { + let command = ProcessInfo().processName + let args = ProcessInfo().arguments + + + + var compile = false + var emitModule = false + var objcHeaderOutput: String? + var moduleName: String? +// var modulePathOutput: String? +// var filemap: String? + var target: String? +// var swiftFileList: String? + var inputPaths: [String] = [] + var outputPaths: [String] = [] + var dependenciesPaths: [String] = [] + var diagnosticsPaths: [String] = [] + var sourceInfoPath: String? + var docPath: String? + + for i in 0.. Date: Sun, 21 May 2023 12:58:50 -0700 Subject: [PATCH 02/33] Add Context --- .../SwiftFrontend/SwiftFrontend.swift | 188 ++++++++++++++++++ .../SwiftFrontend/SwiftFrontendContext.swift | 95 +++++++++ .../SwiftFrontendProductsGenerator.swift | 118 +++++++++++ .../SwiftFrontend/XCSwiftFrontend.swift | 133 ++++++++++--- .../Config/XCRemoteCacheConfig.swift | 2 + .../xcswift-frontend/XCSwiftFrontend.swift | 2 + 6 files changed, 508 insertions(+), 30 deletions(-) create mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift create mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift create mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift new file mode 100644 index 00000000..53f9880f --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift @@ -0,0 +1,188 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +enum SwiftFrontendResult { + /// Swiftc mock cannot be used and fallback to the compilation is required + case forceFallback + /// All compilation steps were mocked correctly + case success +} + +/// Swiftc mocking compilation +protocol SwiftFrontendProtocol { + /// Tries to performs mocked compilation (moving all cached files to the expected location) + /// If cached compilation products are not valid or incompatible, fallbacks to build-from-source + /// - Returns: `.forceFallback` if the cached compilation products are incompatible and fallback + /// to a standard 'swiftc' is required, `.success` otherwise + /// - Throws: An error if there was an unrecoverable, serious error (e.g. IO error) + func mockCompilation() throws -> SwiftFrontendResult +} + +/// Swiftc wrapper that mocks compilation with noop and moves all expected products from cache location +class SwiftFrontend: SwiftFrontendProtocol { + /// Reader of all input files of the compilation + private let inputFileListReader: ListReader + /// Reader of the marker file lists - list of dependencies to set for swiftc compilation + private let markerReader: ListReader + /// Checks if the input file exists in the file list + private let allowedFilesListScanner: FileListScanner + /// Manager of the downloaded artifact package + private let artifactOrganizer: ArtifactOrganizer + /// Reads all input and output files for the compilation from an input filemap + private let inputFilesReader: SwiftcInputReader + /// Write manager of the marker file + private let markerWriter: MarkerWriter + /// Generates products at the desired destination + private let productsGenerator: SwiftcProductsGenerator + private let context: SwiftcContext + private let fileManager: FileManager + private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter + private let touchFactory: (URL, FileManager) -> Touch + private let plugins: [SwiftcProductGenerationPlugin] + + init( + inputFileListReader: ListReader, + markerReader: ListReader, + allowedFilesListScanner: FileListScanner, + artifactOrganizer: ArtifactOrganizer, + inputReader: SwiftcInputReader, + context: SwiftcContext, + markerWriter: MarkerWriter, + productsGenerator: SwiftcProductsGenerator, + fileManager: FileManager, + dependenciesWriterFactory: @escaping (URL, FileManager) -> DependenciesWriter, + touchFactory: @escaping (URL, FileManager) -> Touch, + plugins: [SwiftcProductGenerationPlugin] + ) { + self.inputFileListReader = inputFileListReader + self.markerReader = markerReader + self.allowedFilesListScanner = allowedFilesListScanner + self.artifactOrganizer = artifactOrganizer + inputFilesReader = inputReader + self.context = context + self.markerWriter = markerWriter + self.productsGenerator = productsGenerator + self.fileManager = fileManager + self.dependenciesWriterFactory = dependenciesWriterFactory + self.touchFactory = touchFactory + self.plugins = plugins + } + + // swiftlint:disable:next function_body_length + func mockCompilation() throws -> SwiftFrontendResult { + let rcModeEnabled = markerReader.canRead() + guard rcModeEnabled else { + infoLog("Swiftc marker doesn't exist") + return .forceFallback + } + + let inputFilesInputs = try inputFileListReader.listFilesURLs() + let markerAllowedFiles = try markerReader.listFilesURLs() + let cachedDependenciesWriterFactory = CachedFileDependenciesWriterFactory( + dependencies: markerAllowedFiles, + fileManager: fileManager, + writerFactory: dependenciesWriterFactory + ) + // Verify all input files to be present in a marker fileList + let disallowedInputs = try inputFilesInputs.filter { try !allowedFilesListScanner.contains($0) } + + if !disallowedInputs.isEmpty { + // New file (disallowedFile) added without modifying the rest of the feature. Fallback to swiftc and + // ensure that compilation from source will be forced up until next merge/rebase with "primary" branch + infoLog("Swiftc new input file \(disallowedInputs)") + // Deleting marker to indicate that the remote cached artifact cannot be used + try markerWriter.disable() + + // Save custom prebuild discovery content to make sure that the following prebuild + // phase will not try to reuse cached artifact (if present) + // In other words: let prebuild know that it should not try to reenable cache + // until the next merge with primary + switch context.mode { + case .consumer(commit: .available(let remoteCommit)): + let prebuildDiscoveryURL = context.tempDir.appendingPathComponent(context.prebuildDependenciesPath) + let prebuildDiscoverWriter = dependenciesWriterFactory(prebuildDiscoveryURL, fileManager) + try prebuildDiscoverWriter.write(skipForSha: remoteCommit) + case .consumer, .producer, .producerFast: + // Never skip prebuild phase and fallback to the swiftc compilation for: + // 1) Not enabled remote cache, 2) producer(s) + break + } + return .forceFallback + } + + let artifactLocation = artifactOrganizer.getActiveArtifactLocation() + + // Read swiftmodule location from XCRemoteCache + // arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo} + let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent + let allCompilations = try inputFilesReader.read() + let artifactSwiftmoduleDir = artifactLocation + .appendingPathComponent("swiftmodule") + .appendingPathComponent(context.arch) + let artifactSwiftmoduleBase = artifactSwiftmoduleDir.appendingPathComponent(moduleName) + let artifactSwiftmoduleFiles = Dictionary( + uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions + .map { ext, _ in + (ext, artifactSwiftmoduleBase.appendingPathExtension(ext.rawValue)) + } + ) + + // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h + let artifactSwiftModuleObjCDir = artifactLocation + .appendingPathComponent("include") + .appendingPathComponent(context.arch) + .appendingPathComponent(context.moduleName) + // Move cached xxxx-Swift.h to the location passed in arglist + // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir + let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir + .appendingPathComponent(context.objcHeaderOutput.lastPathComponent) + + _ = try productsGenerator.generateFrom( + artifactSwiftModuleFiles: artifactSwiftmoduleFiles, + artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile + ) + + try plugins.forEach { + try $0.generate(for: allCompilations) + } + + // Save individual .d and touch .o for each .swift file + for compilation in allCompilations.files { + if let object = compilation.object { + // Touching .o is required to invalidate already existing .a or linked library + let touch = touchFactory(object, fileManager) + try touch.touch() + } + if let individualDeps = compilation.dependencies { + // swiftc product should be invalidated if any of dependencies file has changed + try cachedDependenciesWriterFactory.generate(output: individualDeps) + } + } + // Save .d for the entire module + try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies) + // Generate .d file with all deps in the "-master.d" (e.g. for WMO) + if let wmoDeps = allCompilations.info.dependencies { + try cachedDependenciesWriterFactory.generate(output: wmoDeps) + } + infoLog("Swiftc noop for \(context.target)") + return .success + } +} diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift new file mode 100644 index 00000000..99687196 --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift @@ -0,0 +1,95 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +struct SwiftFrontendContext { + enum SwiftFrontendMode: Equatable { + case producer + /// Commit sha of the commit to use during remote cache + case consumer(commit: RemoteCommitInfo) + /// Remote artifact exists and can be optimistically used in place of a local compilation + case producerFast + } + + let moduleName: String + let target: String + let tempDir: URL + let arch: String + let prebuildDependenciesPath: String + let mode: SwiftFrontendMode + /// File that stores all compilation invocation arguments + let invocationHistoryFile: URL + let action: SwiftFrontendAction + /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation + let llbuildId: String + + + private init( + config: XCRemoteCacheConfig, + env: [String: String], + moduleName: String, + target: String, + action: SwiftFrontendAction + ) throws { + self.moduleName = moduleName + self.target = target + self.action = action + llbuildId = try env.readEnv(key: "LLBUILD_BUILD_ID") + + tempDir = action.tmpDir + arch = action.arch + + let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot) + let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot) + prebuildDependenciesPath = config.prebuildDiscoveryPath + switch config.mode { + case .consumer: + let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim()) + mode = .consumer(commit: remoteCommit) + case .producer: + mode = .producer + case .producerFast: + let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim()) + switch remoteCommit { + case .unavailable: + mode = .producer + case .available: + mode = .producerFast + } + } + invocationHistoryFile = URL(fileURLWithPath: config.compilationHistoryFile, relativeTo: tempDir) + } + + init( + config: XCRemoteCacheConfig, + env: [String: String], + input: SwiftFrontendArgInput, + action: SwiftFrontendAction + ) throws { + try self.init( + config: config, + env: env, + moduleName: action.moduleName, + target: action.target, + action: action + ) + } +} + diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift new file mode 100644 index 00000000..0abe0969 --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift @@ -0,0 +1,118 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +enum DiskSwiftFrontendProductsGeneratorError: Error { + /// Emittiing module is available only from emit-module action + case requestedEmitingModuleForInvalidAction(SwiftFrontendAction) + /// When a generator was asked to generate unknown swiftmodule extension file + /// Probably a programmer error: asking to generate excessive extensions, not listed in + /// `SwiftmoduleFileExtension.SwiftmoduleExtensions` + case unknownSwiftmoduleFile +} + +struct SwiftFrontendEmitModuleProductsGeneratorOutput { + let swiftmoduleDir: URL + let objcHeaderFile: URL +} + +struct SwiftFrontendCompilationProductsGeneratorOutput { + // TODO: +} + +/// Generates SwiftFrontend product to the expected location +protocol SwiftFrontendProductsGenerator { + /// Generates products for the emit-module invocation from given files + /// - Returns: location dir where .swiftmodule and ObjC header files have been placed + func generateEmitModuleFrom( + artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], + artifactSwiftModuleObjCFile: URL + ) throws -> SwiftFrontendEmitModuleProductsGeneratorOutput + + /// Generates products for the compilation(s) invocation from given file(s) + /// - Returns: location dir where .swiftmodule and ObjC header files have been placed + func generateCompilationFrom( + // TODO: + ) throws -> SwiftFrontendCompilationProductsGeneratorOutput +} + +/// Generator that produces all products in the locations where Xcode expects it, using provided disk copier +class DiskSwiftFrontendProductsGenerator: SwiftFrontendProductsGenerator { + private let action: SwiftFrontendAction + private let diskCopier: DiskCopier + + init( + action: SwiftFrontendAction, + diskCopier: DiskCopier + ) { + self.action = action + self.diskCopier = diskCopier + } + + func generateEmitModuleFrom( + artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], + artifactSwiftModuleObjCFile: URL + ) throws -> SwiftFrontendEmitModuleProductsGeneratorOutput { + guard case .emitModule(emitModuleInfo: let info, inputFiles: let files) = action else { + throw DiskSwiftFrontendProductsGeneratorError.requestedEmitingModuleForInvalidAction(action) + } + + let modulePathOutput = info.output + let objcHeaderOutput = info.objcHeader + let modulePathBasename = modulePathOutput.deletingPathExtension() + // all swiftmodule-related should be located next to the ".swiftmodule" + let destinationSwiftmodulePaths = Dictionary( + uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions + .map { ext, _ in + (ext, modulePathBasename.appendingPathExtension(ext.rawValue)) + } + ) + + // Move cached -Swift.h file to the expected location + try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput) + for (ext, url) in sourceAtifactSwiftModuleFiles { + let dest = destinationSwiftmodulePaths[ext] + guard let destination = dest else { + throw DiskSwiftFrontendProductsGeneratorError.unknownSwiftmoduleFile + } + do { + // Move cached .swiftmodule to the expected location + try diskCopier.copy(file: url, destination: destination) + } catch { + if case .required = SwiftmoduleFileExtension.SwiftmoduleExtensions[ext] { + throw error + } else { + infoLog("Optional .\(ext) file not found in the artifact at: \(destination.path)") + } + } + } + + // Build parent dir of the .swiftmodule file that contains a module + return .init( + swiftmoduleDir: modulePathOutput.deletingLastPathComponent(), + objcHeaderFile: objcHeaderOutput + ) + } + + func generateCompilationFrom() throws -> SwiftFrontendCompilationProductsGeneratorOutput { + // TODO: + return .init() + } +} diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index f00f4905..292e576d 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -19,22 +19,75 @@ import Foundation -enum SwiftFrontendAction { - case emitModule(emitModuleInfo: SwiftFrontendEmitModuleInfo, inputFiles: [URL]) - case compile([SwiftFileCompilationInfo]) -} - struct SwiftFrontendEmitModuleInfo: Equatable { let inputs: [URL] - let objcHeader: URL? + let objcHeader: URL let diagnostics: URL? let dependencies: URL? let output: URL let target: String + let moduleName: String let sourceInfo: URL? let doc: URL? } +struct SwiftFrontendCompilationInfo: Equatable { + let target: String + let moduleName: String +} + +enum SwiftFrontendAction { + case emitModule(emitModuleInfo: SwiftFrontendEmitModuleInfo, inputFiles: [URL]) + case compile(compilationInfo: SwiftFrontendCompilationInfo, compilationFiles: [SwiftFileCompilationInfo]) +} + +extension SwiftFrontendAction { + var tmpDir: URL { + // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule + // That may be subject to change for other Xcode versions (or other variants) + return outputWorkspaceDir + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + } + var arch: String { + return outputWorkspaceDir.deletingLastPathComponent().lastPathComponent + } + + var target: String { + switch self { + case .emitModule(emitModuleInfo: let info, inputFiles: _): + return info.target + case .compile(compilationInfo: let info, compilationFiles: _): + return info.target + } + } + + var moduleName: String { + switch self { + case .emitModule(emitModuleInfo: let info, inputFiles: _): + return info.moduleName + case .compile(compilationInfo: let info, compilationFiles: _): + return info.moduleName + } + } + + + // The workspace where Xcode asks to put all compilation-relates + // files (like .d or .swiftmodule) + // This location is used to infere the tmpDir and arch + private var outputWorkspaceDir: URL { + switch self { + case .emitModule(emitModuleInfo: let info, inputFiles: _): + return info.output + case .compile(_, let files): + // if invoked compilation via swift-frontend, the .d file is always defined + return files[0].dependencies! + } + } +} + + enum SwiftFrontendArgInputError: Error { // swift-frontend should either be compling or emiting a module case bothCompilationAndEmitAction @@ -52,8 +105,12 @@ enum SwiftFrontendArgInputError: Error { case emitModuleDependenciesOuputCountIsHigherThan1(parsed: Int) // number of -serialize-diagnostics-path for emit-module can be 0 or 1 (generate or not) case emitModuleDiagnosticsOuputCountIsHigherThan1(parsed: Int) - // -target is required for -emit-module - case emitModuleMissingTarget + // emit-module requires -emit-objc-header-path + case emitModuleMissingObjcHeaderPath + // -target is required + case emitMissingTarget + // -moduleName is required + case emiMissingModuleName } public struct SwiftFrontendArgInput { @@ -93,6 +150,12 @@ public struct SwiftFrontendArgInput { guard inputPathsCount > 0 else { throw SwiftFrontendArgInputError.noCompilationInputs } + guard let target = target else { + throw SwiftFrontendArgInputError.emitMissingTarget + } + guard let moduleName = moduleName else { + throw SwiftFrontendArgInputError.emiMissingModuleName + } let inputURLs: [URL] = inputPaths.map(URL.init(fileURLWithPath:)) if compile { @@ -105,7 +168,7 @@ public struct SwiftFrontendArgInput { guard outputPaths.count == inputPathsCount else { throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(expected: inputPathsCount, parsed: outputPaths.count) } - let compilationInfos: [SwiftFileCompilationInfo] = (0.. DependenciesWriter @@ -154,11 +223,13 @@ public class XCSwiftFrontend { public init( command: String, inputArgs: SwiftFrontendArgInput, + env: [String: String], dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, touchFactory: @escaping (URL, FileManager) -> Touch ) throws { self.command = command self.inputArgs = inputArgs + self.env = env dependenciesWriterFactory = dependenciesWriter self.touchFactory = touchFactory @@ -169,23 +240,28 @@ public class XCSwiftFrontend { public func run() { let fileManager = FileManager.default let config: XCRemoteCacheConfig - - /* + let context: SwiftFrontendContext + do { let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) .readConfiguration() - context = try SwiftFrontendContext(config: config, input: inputArgs) + context = try SwiftFrontendContext(config: config, env: env, input: inputArgs, action: action) } catch { - exit(1, "FATAL: Swiftc initialization failed with error: \(error)") + exit(1, "FATAL: XCSwiftFrontend initialization failed with error: \(error)") } - let swiftcCommand = config.swiftcCommand + + + let swiftFrontendCommand = config.swiftFrontendCommand let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) + + let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager) - let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) - let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) + +// let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) +// let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) let artifactOrganizer = ZipArtifactOrganizer( targetTempDir: context.tempDir, // xcswiftc doesn't call artifact preprocessing @@ -204,29 +280,26 @@ public class XCSwiftFrontend { moduleName: context.moduleName, fileManager: fileManager ) - let productsGenerator = DiskSwiftcProductsGenerator( - modulePathOutput: context.modulePathOutput, - objcHeaderOutput: context.objcHeaderOutput, + let productsGenerator = DiskSwiftFrontendProductsGenerator( + action: action, diskCopier: HardLinkDiskCopier(fileManager: fileManager) ) let allInvocationsStorage = ExistingFileStorage( storageFile: context.invocationHistoryFile, - command: swiftcCommand + command: swiftFrontendCommand ) // When fallbacking to local compilation do not call historical `swiftc` invocations // The current fallback invocation already compiles all files in a target let invocationStorage = FilteredInvocationStorage( storage: allInvocationsStorage, - retrieveIgnoredCommands: [swiftcCommand] + retrieveIgnoredCommands: [swiftFrontendCommand] ) let shellOut = ProcessShellOut() - let swiftc = Swiftc( - inputFileListReader: fileListEditor, + let swiftc = SwiftFrontend( markerReader: markerReader, allowedFilesListScanner: allowedFilesListScanner, artifactOrganizer: artifactOrganizer, - inputReader: inputReader, context: context, markerWriter: markerWriter, productsGenerator: productsGenerator, @@ -235,7 +308,7 @@ public class XCSwiftFrontend { touchFactory: touchFactory, plugins: [] ) - let orchestrator = SwiftcOrchestrator( + /*let orchestrator = SwiftcOrchestrator( mode: context.mode, swiftc: swiftc, swiftcCommand: swiftcCommand, diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index 4ef7b77f..4b79827c 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -57,6 +57,8 @@ public struct XCRemoteCacheConfig: Encodable { var clangCommand: String = "clang" /// Command for a standard Swift compilation (swiftc) var swiftcCommand: String = "swiftc" + /// Command for a standard Swift frontend compilation (swift-frontend) + var swiftFrontendCommand: String = "swift-frontend" /// Path of the primary repository that produces cache artifacts var primaryRepo: String = "" /// Main (primary) branch that produces cache artifacts (default to 'master') diff --git a/Sources/xcswift-frontend/XCSwiftFrontend.swift b/Sources/xcswift-frontend/XCSwiftFrontend.swift index 9d58f892..227330cc 100644 --- a/Sources/xcswift-frontend/XCSwiftFrontend.swift +++ b/Sources/xcswift-frontend/XCSwiftFrontend.swift @@ -26,6 +26,7 @@ import XCRemoteCache public class XCSwiftcFrontendMain { // swiftlint:disable:next function_body_length public func main() { + let env = ProcessInfo.processInfo.environment let command = ProcessInfo().processName let args = ProcessInfo().arguments @@ -103,6 +104,7 @@ public class XCSwiftcFrontendMain { let frontend = try XCSwiftFrontend( command: command, inputArgs: argInput, + env: env, dependenciesWriter: FileDependenciesWriter.init, touchFactory: FileTouch.init) frontend.run() From 69fe92ede33890ba471fb71a01cdf9809b756851 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 21 May 2023 20:43:58 -0700 Subject: [PATCH 03/33] Generalize Swiftc --- .../SwiftFrontendOrchestrator.swift | 89 +++++++++++++++++++ .../SwiftFrontend/XCSwiftFrontend.swift | 2 +- .../Commands/Swiftc/Swiftc.swift | 37 ++++---- .../Commands/Swiftc/SwiftcContext.swift | 85 ++++++++++++++---- .../Swiftc/SwiftcFilemapInputEditor.swift | 31 ++++++- .../Commands/Swiftc/XCSwiftc.swift | 24 ++++- .../Dependencies/ListEditor.swift | 14 +++ 7 files changed, 245 insertions(+), 37 deletions(-) create mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift new file mode 100644 index 00000000..d3ce580c --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift @@ -0,0 +1,89 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Performs the `swift-frontend` logic: +/// For emit-module (the "first" process) action, it locks a shared file between all swift-frontend invcations, +/// verifies that the mocking can be done and continues the mocking/fallbacking along the lock release +/// For the compilation action, tries to ackquire a lock and waits until the "emit-module" makes a decision +/// if the compilation should be skipped and a "mocking" should used instead +class SwiftFrontendOrchestrator { + /// Content saved to the shared file + /// Safe to use forced unwrapping + private static let emitModuleContent = "done".data(using: .utf8)! + + private let mode: SwiftFrontendContext.SwiftFrontendMode + private let action: SwiftFrontendAction + private let lockAccessor: ExclusiveFileAccessor + + init( + mode: SwiftFrontendContext.SwiftFrontendMode, + action: SwiftFrontendAction, + lockAccessor: ExclusiveFileAccessor + ) { + self.mode = mode + self.action = action + self.lockAccessor = lockAccessor + } + func run() throws { + switch mode { + case .consumer(commit: .available): + /// attemp to mock + try executeMockAttemp() + case .consumer: + // TODO: fallback + case .producerFast: + // TODO: not supported yet + case .producer: + + } + + } + private func executeMockAttemp() throws { + switch action { + case .emitModule: + try validateEmitModuleStep() + case .compile: + try waitForEmitModuleLock() + } + } + + private func validateEmitModuleStep() throws { + try lockAccessor.exclusiveAccess { handle in + // TODO: check if the mocking compilation can happen (make sure + // all input files are listed in the list of dependencies) + + handle.write(SwiftFrontendOrchestrator.emitModuleContent) + } + } + + /// Locks a shared file in a loop until its content non-empty, which means the "parent" emit-module has finished + private func waitForEmitModuleLock() throws { + while (true) { + // TODO: add a max timeout + try lockAccessor.exclusiveAccess { handle in + if !handle.availableData.isEmpty { + // the file is not empty so emit-module is done with the "check" + return + } + } + } + } +} diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index 292e576d..7de75c2f 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -296,7 +296,7 @@ public class XCSwiftFrontend { ) let shellOut = ProcessShellOut() - let swiftc = SwiftFrontend( + let swiftFrontend = SwiftFrontend( markerReader: markerReader, allowedFilesListScanner: allowedFilesListScanner, artifactOrganizer: artifactOrganizer, diff --git a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift index ef30547a..18b0d8a2 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift @@ -132,7 +132,7 @@ class Swiftc: SwiftcProtocol { // Read swiftmodule location from XCRemoteCache // arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo} - let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent + let moduleName = context.moduleName let allCompilations = try inputFilesReader.read() let artifactSwiftmoduleDir = artifactLocation .appendingPathComponent("swiftmodule") @@ -145,20 +145,27 @@ class Swiftc: SwiftcProtocol { } ) - // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h - let artifactSwiftModuleObjCDir = artifactLocation - .appendingPathComponent("include") - .appendingPathComponent(context.arch) - .appendingPathComponent(context.moduleName) - // Move cached xxxx-Swift.h to the location passed in arglist - // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir - let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir - .appendingPathComponent(context.objcHeaderOutput.lastPathComponent) - - _ = try productsGenerator.generateFrom( - artifactSwiftModuleFiles: artifactSwiftmoduleFiles, - artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile - ) + // emit module (if requested) + for step in context.steps { + guard case .emitModule(let objcHeaderOutput, _) = step else { + break + } + + // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h + let artifactSwiftModuleObjCDir = artifactLocation + .appendingPathComponent("include") + .appendingPathComponent(context.arch) + .appendingPathComponent(context.moduleName) + // Move cached xxxx-Swift.h to the location passed in arglist + // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir + let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir + .appendingPathComponent(objcHeaderOutput.lastPathComponent) + + _ = try productsGenerator.generateFrom( + artifactSwiftModuleFiles: artifactSwiftmoduleFiles, + artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile + ) + } try plugins.forEach { try $0.generate(for: allCompilations) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 1e58ff4f..657cb703 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -19,7 +19,29 @@ import Foundation + public struct SwiftcContext { + public enum SwiftcStep { + case compileFiles + case emitModule(objcHeaderOutput: URL, modulePathOutput: URL) + } + + /// Defines how a list of input files (*.swift) is passed to the invocation + public enum CompilationFilesSource { + /// defined in a separate file (via @/.../*.SwiftFileList) + case fileList(String) + /// explicitly passed a list of files + case list([String]) + } + + /// Defines how a list of output files (*.d, *.o etc.) is passed to the invocation + public enum CompilationFilesOutputs { + /// defined in a separate file (via -output-file-map) + case fileMap(String) + /// explicitly passed in the invocation + case map([String: SwiftFileCompilationInfo]) + } + enum SwiftcMode: Equatable { case producer /// Commit sha of the commit to use during remote cache @@ -28,14 +50,15 @@ public struct SwiftcContext { case producerFast } - let objcHeaderOutput: URL + let steps: [SwiftcStep] +// let objcHeaderOutput: URL let moduleName: String - let modulePathOutput: URL - /// File that defines output files locations (.d, .swiftmodule etc.) - let filemap: URL +// let modulePathOutput: URL + /// A source that defines output files locations (.d, .swiftmodule etc.) + let outputs: CompilationFilesOutputs let target: String - /// File that contains input files for the swift module compilation - let fileList: URL + /// A source that contains all input files for the swift module compilation + let inputs: CompilationFilesSource let tempDir: URL let arch: String let prebuildDependenciesPath: String @@ -46,26 +69,32 @@ public struct SwiftcContext { public init( config: XCRemoteCacheConfig, - objcHeaderOutput: String, moduleName: String, - modulePathOutput: String, - filemap: String, + steps: [SwiftcStep], + outputs: CompilationFilesOutputs, target: String, - fileList: String + inputs: CompilationFilesSource, + // TODO: make sure it is required + /// any workspace file path - all other intermediate files for this compilation + /// are placed next to it. This path is used to infere the arch and TARGET_TEMP_DIR + exampleWorkspaceFilePath: String ) throws { - self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput) +// self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput) self.moduleName = moduleName - self.modulePathOutput = URL(fileURLWithPath: modulePathOutput) - self.filemap = URL(fileURLWithPath: filemap) +// self.modulePathOutput = URL(fileURLWithPath: modulePathOutput) + self.steps = steps + self.outputs = outputs +// self.filemap = URL(fileURLWithPath: filemap) self.target = target - self.fileList = URL(fileURLWithPath: fileList) +// self.fileList = URL(fileURLWithPath: fileList) + self.inputs = inputs // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule // That may be subject to change for other Xcode versions - tempDir = URL(fileURLWithPath: modulePathOutput) + tempDir = URL(fileURLWithPath: exampleWorkspaceFilePath) .deletingLastPathComponent() .deletingLastPathComponent() .deletingLastPathComponent() - arch = URL(fileURLWithPath: modulePathOutput).deletingLastPathComponent().lastPathComponent + arch = URL(fileURLWithPath: exampleWorkspaceFilePath).deletingLastPathComponent().lastPathComponent let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot) let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot) @@ -91,6 +120,30 @@ public struct SwiftcContext { init( config: XCRemoteCacheConfig, input: SwiftcArgInput + ) throws { + let steps: [SwiftcStep] = [ + .compileFiles, + .emitModule( + objcHeaderOutput: URL(fileURLWithPath: (input.objcHeaderOutput)), + modulePathOutput: URL(fileURLWithPath: input.modulePathOutput) + ) + ] + let outputs = CompilationFilesOutputs.fileMap(input.filemap) + let inputs = CompilationFilesSource.fileList(input.fileList) + try self.init( + config: config, + moduleName: input.moduleName, + steps: steps, + outputs: outputs, + target: input.target, + inputs: inputs, + exampleWorkspaceFilePath: input.modulePathOutput + ) + } + + init( + config: XCRemoteCacheConfig, + input: SwiftFrontendArgInput ) throws { try self.init( config: config, diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift index 7a803b78..fc59c35d 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift @@ -45,10 +45,11 @@ struct SwiftCompilationInfo: Encodable, Equatable { struct SwiftModuleCompilationInfo: Encodable, Equatable { // not present for incremental builds let dependencies: URL? - let swiftDependencies: URL + // not present for the swift-frontend '-c' invocation + let swiftDependencies: URL? } -struct SwiftFileCompilationInfo: Encodable, Equatable { +public struct SwiftFileCompilationInfo: Encodable, Equatable { let file: URL // not present for WMO builds let dependencies: URL? @@ -58,6 +59,32 @@ struct SwiftFileCompilationInfo: Encodable, Equatable { let swiftDependencies: URL? } +class StaticSwiftcInputReader: SwiftcInputReader { + private let moduleDependencies: URL? + private let swiftDependencies: URL? + private let compilationFiles: [SwiftFileCompilationInfo] + + init( + moduleDependencies: URL?, + swiftDependencies: URL?, + compilationFiles: [SwiftFileCompilationInfo] + ) { + self.moduleDependencies = moduleDependencies + self.swiftDependencies = swiftDependencies + self.compilationFiles = compilationFiles + } + + func read() throws -> SwiftCompilationInfo { + return .init( + info: .init( + dependencies: moduleDependencies, + swiftDependencies: swiftDependencies + ), + files: compilationFiles + ) + } +} + class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { private let file: URL diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 7ea2aa20..f622d038 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -81,8 +81,26 @@ public class XCSwiftc { let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager) - let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) - let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) + let inputReader: SwiftcInputReader + switch context.outputs { + case .fileMap(let path): + inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileManager: fileManager) + case .map(let map): + // static + // TODO: check if first 2 ars can always be `nil` + inputReader = StaticSwiftcInputReader( + moduleDependencies: nil, + swiftDependencies: nil, + compilationFiles: Array(map.values) + ) + } + let fileListReader: ListReader + switch context.inputs { + case .fileList(let path): + fileListReader = FileListEditor(URL(fileURLWithPath: path), fileManager: fileManager) + case .list(let paths): + fileListReader = StaticFileListReader(list: paths.map( URL(fileURLWithPath:))) + } let artifactOrganizer = ZipArtifactOrganizer( targetTempDir: context.tempDir, // xcswiftc doesn't call artifact preprocessing @@ -119,7 +137,7 @@ public class XCSwiftc { let shellOut = ProcessShellOut() let swiftc = Swiftc( - inputFileListReader: fileListEditor, + inputFileListReader: fileListReader, markerReader: markerReader, allowedFilesListScanner: allowedFilesListScanner, artifactOrganizer: artifactOrganizer, diff --git a/Sources/XCRemoteCache/Dependencies/ListEditor.swift b/Sources/XCRemoteCache/Dependencies/ListEditor.swift index fb2276e6..0a89fd1a 100644 --- a/Sources/XCRemoteCache/Dependencies/ListEditor.swift +++ b/Sources/XCRemoteCache/Dependencies/ListEditor.swift @@ -41,6 +41,20 @@ protocol ListWriter { func writerListFilesURLs(_ list: [URL]) throws } +class StaticFileListReader: ListReader { + private let list: [URL] + init(list: [URL]){ + self.list = list + } + func listFilesURLs() throws -> [URL] { + list + } + + func canRead() -> Bool { + true + } +} + /// Reads&Writes files that list files using one-file-per-line format class FileListEditor: ListReader, ListWriter { private let file: URL From 6dcfb9dc2afd09bf6b4f4b4f368b2998b49cc1c6 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Mon, 22 May 2023 22:03:10 -0700 Subject: [PATCH 04/33] Refactoring to an abstract swiftc --- ...iftFrontend.swift => SwiftFrontend.swift_} | 0 ...text.swift => SwiftFrontendContext.swift_} | 0 .../SwiftFrontendOrchestrator.swift | 28 ++- ... => SwiftFrontendProductsGenerator.swift_} | 0 .../SwiftFrontend/XCSwiftFrontend.swift | 210 ++++++++---------- .../Commands/Swiftc/Swiftc.swift | 14 +- .../Commands/Swiftc/SwiftcContext.swift | 39 ++-- .../Swiftc/SwiftcFilemapInputEditor.swift | 4 +- .../Commands/Swiftc/SwiftcOrchestrator.swift | 22 +- .../Swiftc/SwiftcProductsGenerator.swift | 10 + .../Commands/Swiftc/XCSwiftc.swift | 66 ++++-- Sources/XCRemoteCache/Utils/Array+Utils.swift | 29 +++ .../xcswift-frontend/XCSwiftFrontend.swift | 6 + 13 files changed, 235 insertions(+), 193 deletions(-) rename Sources/XCRemoteCache/Commands/SwiftFrontend/{SwiftFrontend.swift => SwiftFrontend.swift_} (100%) rename Sources/XCRemoteCache/Commands/SwiftFrontend/{SwiftFrontendContext.swift => SwiftFrontendContext.swift_} (100%) rename Sources/XCRemoteCache/Commands/SwiftFrontend/{SwiftFrontendProductsGenerator.swift => SwiftFrontendProductsGenerator.swift_} (100%) create mode 100644 Sources/XCRemoteCache/Utils/Array+Utils.swift diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift_ similarity index 100% rename from Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift rename to Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift_ diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift_ similarity index 100% rename from Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift rename to Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift_ diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift index d3ce580c..be031147 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift @@ -29,32 +29,30 @@ class SwiftFrontendOrchestrator { /// Safe to use forced unwrapping private static let emitModuleContent = "done".data(using: .utf8)! - private let mode: SwiftFrontendContext.SwiftFrontendMode - private let action: SwiftFrontendAction + enum Action { + case emitModule + case compile + } + private let mode: SwiftcContext.SwiftcMode + private let action: Action private let lockAccessor: ExclusiveFileAccessor init( - mode: SwiftFrontendContext.SwiftFrontendMode, - action: SwiftFrontendAction, + mode: SwiftcContext.SwiftcMode, + action: Action, lockAccessor: ExclusiveFileAccessor ) { self.mode = mode self.action = action self.lockAccessor = lockAccessor } + func run() throws { - switch mode { - case .consumer(commit: .available): - /// attemp to mock - try executeMockAttemp() - case .consumer: - // TODO: fallback - case .producerFast: - // TODO: not supported yet - case .producer: - + guard case .consumer(commit: .available) = mode else { + // no need to lock anything - just allow fallbacking to the `swiftc or swift-frontend` + return } - + try waitForEmitModuleLock() } private func executeMockAttemp() throws { switch action { diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift_ similarity index 100% rename from Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift rename to Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift_ diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index 7de75c2f..4fcf351b 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -93,6 +93,8 @@ enum SwiftFrontendArgInputError: Error { case bothCompilationAndEmitAction // no .swift files have been passed as input files case noCompilationInputs + // no -primary-file .swift files have been passed as input files + case noPrimaryFileCompilationInputs // number of -emit-dependencies-path doesn't match compilation inputs case dependenciesOuputCountDoesntMatch(expected: Int, parsed: Int) // number of -serialize-diagnostics-path doesn't match compilation inputs @@ -119,21 +121,27 @@ public struct SwiftFrontendArgInput { let objcHeaderOutput: String? let moduleName: String? let target: String? + let primaryInputPaths: [String] let inputPaths: [String] var outputPaths: [String] var dependenciesPaths: [String] + // Extra params + // Diagnostics are not supported yet in the XCRemoteCache (cached artifacts assumes no warnings) var diagnosticsPaths: [String] + // Unsed for now: + // .swiftsourceinfo and .swiftdoc will be placed next to the .swiftmodule let sourceInfoPath: String? let docPath: String? /// Manual initializer implementation required to be public - public init(compile: Bool, emitModule: Bool, objcHeaderOutput: String?, moduleName: String?, target: String?, inputPaths: [String], outputPaths: [String], dependenciesPaths: [String], diagnosticsPaths: [String], + public init(compile: Bool, emitModule: Bool, objcHeaderOutput: String?, moduleName: String?, target: String?, primaryInputPaths: [String], inputPaths: [String], outputPaths: [String], dependenciesPaths: [String], diagnosticsPaths: [String], sourceInfoPath: String?, docPath: String?) { self.compile = compile self.emitModule = emitModule self.objcHeaderOutput = objcHeaderOutput self.moduleName = moduleName self.target = target + self.primaryInputPaths = primaryInputPaths self.inputPaths = inputPaths self.outputPaths = outputPaths self.dependenciesPaths = dependenciesPaths @@ -142,11 +150,12 @@ public struct SwiftFrontendArgInput { self.docPath = docPath } - func generateAction() throws -> SwiftFrontendAction { + func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext { guard compile != emitModule else { throw SwiftFrontendArgInputError.bothCompilationAndEmitAction } let inputPathsCount = inputPaths.count + let primaryInputPathsCount = primaryInputPaths.count guard inputPathsCount > 0 else { throw SwiftFrontendArgInputError.noCompilationInputs } @@ -156,28 +165,47 @@ public struct SwiftFrontendArgInput { guard let moduleName = moduleName else { throw SwiftFrontendArgInputError.emiMissingModuleName } - let inputURLs: [URL] = inputPaths.map(URL.init(fileURLWithPath:)) - + if compile { - guard [inputPathsCount, 0].contains(dependenciesPaths.count) else { + guard primaryInputPathsCount > 0 else { + throw SwiftFrontendArgInputError.noPrimaryFileCompilationInputs + } + guard [primaryInputPathsCount, 0].contains(dependenciesPaths.count) else { throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch(expected: inputPathsCount, parsed: dependenciesPaths.count) } - guard [inputPathsCount, 0].contains(diagnosticsPaths.count) else { + guard [primaryInputPathsCount, 0].contains(diagnosticsPaths.count) else { throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch(expected: inputPathsCount, parsed: diagnosticsPaths.count) } - guard outputPaths.count == inputPathsCount else { + guard outputPaths.count == primaryInputPathsCount else { throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(expected: inputPathsCount, parsed: outputPaths.count) } - let compilationFileInfos: [SwiftFileCompilationInfo] = (0.. { private let env: [String: String] - // validated and frontend action - private let action: SwiftFrontendAction - private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter - private let touchFactory: (URL, FileManager) -> Touch - + private var cachedSwiftdContext: (XCRemoteCacheConfig, SwiftcContext)? + public init( command: String, inputArgs: SwiftFrontendArgInput, @@ -227,104 +253,54 @@ public class XCSwiftFrontend { dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, touchFactory: @escaping (URL, FileManager) -> Touch ) throws { - self.command = command - self.inputArgs = inputArgs self.env = env - dependenciesWriterFactory = dependenciesWriter - self.touchFactory = touchFactory - - self.action = try inputArgs.generateAction() + super.init( + command: command, + inputArgs: inputArgs, + dependenciesWriter: dependenciesWriter, + touchFactory: touchFactory + ) } - // swiftlint:disable:next function_body_length - public func run() { + override func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { + if let cachedSwiftdContext = cachedSwiftdContext { + return cachedSwiftdContext + } + let fileManager = FileManager.default let config: XCRemoteCacheConfig - let context: SwiftFrontendContext + let context: SwiftcContext do { let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) .readConfiguration() - context = try SwiftFrontendContext(config: config, env: env, input: inputArgs, action: action) + context = try SwiftcContext(config: config, input: inputArgs) } catch { exit(1, "FATAL: XCSwiftFrontend initialization failed with error: \(error)") } - - - let swiftFrontendCommand = config.swiftFrontendCommand - let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) - - - let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) - let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager) - - -// let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) -// let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) - let artifactOrganizer = ZipArtifactOrganizer( - targetTempDir: context.tempDir, - // xcswiftc doesn't call artifact preprocessing - artifactProcessors: [], - fileManager: fileManager - ) - // TODO: check for allowedFile comparing a list of all inputfiles, not dependencies from a marker - let makerReferencedFilesListScanner = FileListScannerImpl(markerReader, caseSensitive: false) - let allowedFilesListScanner = ExceptionsFilteredFileListScanner( - allowedFilenames: ["\(config.thinTargetMockFilename).swift"], - disallowedFilenames: [], - scanner: makerReferencedFilesListScanner - ) - let artifactBuilder: ArtifactSwiftProductsBuilder = ArtifactSwiftProductsBuilderImpl( - workingDir: context.tempDir, - moduleName: context.moduleName, - fileManager: fileManager - ) - let productsGenerator = DiskSwiftFrontendProductsGenerator( - action: action, - diskCopier: HardLinkDiskCopier(fileManager: fileManager) - ) - let allInvocationsStorage = ExistingFileStorage( - storageFile: context.invocationHistoryFile, - command: swiftFrontendCommand - ) - // When fallbacking to local compilation do not call historical `swiftc` invocations - // The current fallback invocation already compiles all files in a target - let invocationStorage = FilteredInvocationStorage( - storage: allInvocationsStorage, - retrieveIgnoredCommands: [swiftFrontendCommand] - ) - let shellOut = ProcessShellOut() - - let swiftFrontend = SwiftFrontend( - markerReader: markerReader, - allowedFilesListScanner: allowedFilesListScanner, - artifactOrganizer: artifactOrganizer, - context: context, - markerWriter: markerWriter, - productsGenerator: productsGenerator, - fileManager: fileManager, - dependenciesWriterFactory: dependenciesWriterFactory, - touchFactory: touchFactory, - plugins: [] - ) - /*let orchestrator = SwiftcOrchestrator( - mode: context.mode, - swiftc: swiftc, - swiftcCommand: swiftcCommand, - objcHeaderOutput: context.objcHeaderOutput, - moduleOutput: context.modulePathOutput, - arch: context.arch, - artifactBuilder: artifactBuilder, - producerFallbackCommandProcessors: [], - invocationStorage: invocationStorage, - shellOut: shellOut - ) + let builtContext = (config, context) + self.cachedSwiftdContext = builtContext + return builtContext + } + + override public func run() { do { + /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + let (_, context) = buildContext() + + let sharedLockFileURL = context.tempDir.appendingPathComponent(llbuildId).appendingPathExtension("lock") + let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) + + let action: SwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile + let orchestrator = SwiftFrontendOrchestrator(mode: context.mode, action: action, lockAccessor: sharedLock) + try orchestrator.run() } catch { - exit(1, "Swiftc failed with error: \(error)") + printWarning("Cannot correctly orchestrate the \(command) with params \(inputArgs): error: \(error)") } - */ + + super.run() } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift index 18b0d8a2..bf2de8aa 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift @@ -146,11 +146,7 @@ class Swiftc: SwiftcProtocol { ) // emit module (if requested) - for step in context.steps { - guard case .emitModule(let objcHeaderOutput, _) = step else { - break - } - + if let emitModule = context.steps.emitModule { // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h let artifactSwiftModuleObjCDir = artifactLocation .appendingPathComponent("include") @@ -159,7 +155,7 @@ class Swiftc: SwiftcProtocol { // Move cached xxxx-Swift.h to the location passed in arglist // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir - .appendingPathComponent(objcHeaderOutput.lastPathComponent) + .appendingPathComponent(emitModule.objcHeaderOutput.lastPathComponent) _ = try productsGenerator.generateFrom( artifactSwiftModuleFiles: artifactSwiftmoduleFiles, @@ -183,8 +179,10 @@ class Swiftc: SwiftcProtocol { try cachedDependenciesWriterFactory.generate(output: individualDeps) } } - // Save .d for the entire module - try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies) + // Save .d for the entire module (might not be required in the `swift-frontend -c` mode) + if let swiftDependencies = allCompilations.info.swiftDependencies { + try cachedDependenciesWriterFactory.generate(output: swiftDependencies) + } // Generate .d file with all deps in the "-master.d" (e.g. for WMO) if let wmoDeps = allCompilations.info.dependencies { try cachedDependenciesWriterFactory.generate(output: wmoDeps) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 657cb703..9f565005 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -21,9 +21,19 @@ import Foundation public struct SwiftcContext { - public enum SwiftcStep { - case compileFiles - case emitModule(objcHeaderOutput: URL, modulePathOutput: URL) + public struct SwiftcStepEmitModule { + let objcHeaderOutput: URL + let modulePathOutput: URL + } + public enum SwiftcStepCompileFilesScope { + case none + case all + case subset([URL]) + } + + public struct SwiftcSteps { + let compileFilesScope: SwiftcStepCompileFilesScope + let emitModule: SwiftcStepEmitModule? } /// Defines how a list of input files (*.swift) is passed to the invocation @@ -50,7 +60,7 @@ public struct SwiftcContext { case producerFast } - let steps: [SwiftcStep] + let steps: SwiftcSteps // let objcHeaderOutput: URL let moduleName: String // let modulePathOutput: URL @@ -66,11 +76,10 @@ public struct SwiftcContext { /// File that stores all compilation invocation arguments let invocationHistoryFile: URL - public init( config: XCRemoteCacheConfig, moduleName: String, - steps: [SwiftcStep], + steps: SwiftcSteps, outputs: CompilationFilesOutputs, target: String, inputs: CompilationFilesSource, @@ -121,13 +130,13 @@ public struct SwiftcContext { config: XCRemoteCacheConfig, input: SwiftcArgInput ) throws { - let steps: [SwiftcStep] = [ - .compileFiles, - .emitModule( + let steps = SwiftcSteps( + compileFilesScope: .all, + emitModule: SwiftcStepEmitModule( objcHeaderOutput: URL(fileURLWithPath: (input.objcHeaderOutput)), modulePathOutput: URL(fileURLWithPath: input.modulePathOutput) ) - ] + ) let outputs = CompilationFilesOutputs.fileMap(input.filemap) let inputs = CompilationFilesSource.fileList(input.fileList) try self.init( @@ -145,14 +154,6 @@ public struct SwiftcContext { config: XCRemoteCacheConfig, input: SwiftFrontendArgInput ) throws { - try self.init( - config: config, - objcHeaderOutput: input.objcHeaderOutput, - moduleName: input.moduleName, - modulePathOutput: input.modulePathOutput, - filemap: input.filemap, - target: input.target, - fileList: input.fileList - ) + self = try input.generateSwiftcContext(config: config) } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift index fc59c35d..a2e36d76 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift @@ -138,14 +138,14 @@ extension SwiftModuleCompilationInfo { guard let dict = object as? [String: String] else { throw SwiftcInputReaderError.invalidFormat } - swiftDependencies = try dict.readURL(key: "swift-dependencies") + swiftDependencies = dict.readURL(key: "swift-dependencies") dependencies = dict.readURL(key: "dependencies") } func dump() -> [String: String] { return [ "dependencies": dependencies?.path, - "swift-dependencies": swiftDependencies.path, + "swift-dependencies": swiftDependencies?.path, ].compactMapValues { $0 } } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift index 86fbc5a3..b1a1794c 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift @@ -27,8 +27,10 @@ class SwiftcOrchestrator { private let mode: SwiftcContext.SwiftcMode // swiftc command that should be called to generate artifacts private let swiftcCommand: String - private let objcHeaderOutput: URL - private let moduleOutput: URL + // That is nil if invoking from frontend compilation: `swift-frontend -c` + private let objcHeaderOutput: URL? + // That is nil if invoking from frontend compilation: `swift-frontend -c` + private let moduleOutput: URL? private let arch: String private let artifactBuilder: ArtifactSwiftProductsBuilder private let shellOut: ShellOut @@ -39,8 +41,8 @@ class SwiftcOrchestrator { mode: SwiftcContext.SwiftcMode, swiftc: SwiftcProtocol, swiftcCommand: String, - objcHeaderOutput: URL, - moduleOutput: URL, + objcHeaderOutput: URL?, + moduleOutput: URL?, arch: String, artifactBuilder: ArtifactSwiftProductsBuilder, producerFallbackCommandProcessors: [ShellCommandsProcessor], @@ -128,10 +130,14 @@ class SwiftcOrchestrator { try processor.applyArgsRewrite(args) } try fallbackToDefaultAndWait(command: swiftcCommand, args: swiftcArgs) - // move generated .h to the location where artifact creator expects it - try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput) - // move generated .swiftmodule to the location where artifact creator expects it - try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput) + if let objcHeaderOutput = objcHeaderOutput { + // move generated .h to the location where artifact creator expects it + try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput) + } + if let moduleOutput = moduleOutput { + // move generated .swiftmodule to the location where artifact creator expects it + try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput) + } try producerFallbackCommandProcessors.forEach { try $0.postCommandProcessing() diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift index 74da9ca4..61b4efa8 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift @@ -41,6 +41,16 @@ protocol SwiftcProductsGenerator { ) throws -> SwiftcProductsGeneratorOutput } +/// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where +/// only individual .o files are created and not .swiftmodule of -Swift.h (which is part of swift-frontend -emit-module invocation) +class NoopSwiftcProductsGenerator: SwiftcProductsGenerator { + func generateFrom(artifactSwiftModuleFiles: [SwiftmoduleFileExtension : URL], artifactSwiftModuleObjCFile: URL) throws -> SwiftcProductsGeneratorOutput { + printWarning("Invoking module generation from NoopSwiftcProductsGenerator which does nothing. It might be an error of the swift-frontend mocking.") + // TODO: Refactor API: this url is never used: NoopSwiftcProductsGenerator is intended only for the swift-frontend + let trivialURL = URL(fileURLWithPath: "/non-existing") + return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL) + } +} /// Generator that produces all products in the locations where Xcode expects it, using provided disk copier class DiskSwiftcProductsGenerator: SwiftcProductsGenerator { private let destinationSwiftmodulePaths: [SwiftmoduleFileExtension: URL] diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index f622d038..a15095a9 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -45,15 +45,15 @@ public struct SwiftcArgInput { } } -public class XCSwiftc { - private let command: String - private let inputArgs: SwiftcArgInput +public class XCSwiftAbstract { + let command: String + let inputArgs: T private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter private let touchFactory: (URL, FileManager) -> Touch - + public init( command: String, - inputArgs: SwiftcArgInput, + inputArgs: T, dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, touchFactory: @escaping (URL, FileManager) -> Touch ) { @@ -62,20 +62,16 @@ public class XCSwiftc { dependenciesWriterFactory = dependenciesWriter self.touchFactory = touchFactory } - + + func buildContext() -> (XCRemoteCacheConfig, SwiftcContext){ + fatalError("Override in \(Self.self)") + } + // swiftlint:disable:next function_body_length public func run() { let fileManager = FileManager.default - let config: XCRemoteCacheConfig - let context: SwiftcContext - do { - let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) - .readConfiguration() - context = try SwiftcContext(config: config, input: inputArgs) - } catch { - exit(1, "FATAL: Swiftc initialization failed with error: \(error)") - } + let (config, context) = buildContext() + let swiftcCommand = config.swiftcCommand let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) @@ -99,7 +95,7 @@ public class XCSwiftc { case .fileList(let path): fileListReader = FileListEditor(URL(fileURLWithPath: path), fileManager: fileManager) case .list(let paths): - fileListReader = StaticFileListReader(list: paths.map( URL(fileURLWithPath:))) + fileListReader = StaticFileListReader(list: paths.map(URL.init(fileURLWithPath:))) } let artifactOrganizer = ZipArtifactOrganizer( targetTempDir: context.tempDir, @@ -119,11 +115,16 @@ public class XCSwiftc { moduleName: context.moduleName, fileManager: fileManager ) - let productsGenerator = DiskSwiftcProductsGenerator( - modulePathOutput: context.modulePathOutput, - objcHeaderOutput: context.objcHeaderOutput, - diskCopier: HardLinkDiskCopier(fileManager: fileManager) - ) + let productsGenerator: SwiftcProductsGenerator + if let emitModule = context.steps.emitModule { + productsGenerator = DiskSwiftcProductsGenerator( + modulePathOutput: emitModule.modulePathOutput, + objcHeaderOutput: emitModule.objcHeaderOutput, + diskCopier: HardLinkDiskCopier(fileManager: fileManager) + ) + } else { + productsGenerator = NoopSwiftcProductsGenerator() + } let allInvocationsStorage = ExistingFileStorage( storageFile: context.invocationHistoryFile, command: swiftcCommand @@ -154,8 +155,8 @@ public class XCSwiftc { mode: context.mode, swiftc: swiftc, swiftcCommand: swiftcCommand, - objcHeaderOutput: context.objcHeaderOutput, - moduleOutput: context.modulePathOutput, + objcHeaderOutput: context.steps.emitModule?.objcHeaderOutput, + moduleOutput: context.steps.emitModule?.modulePathOutput, arch: context.arch, artifactBuilder: artifactBuilder, producerFallbackCommandProcessors: [], @@ -169,3 +170,20 @@ public class XCSwiftc { } } } + +public class XCSwiftc: XCSwiftAbstract { + override func buildContext() -> (XCRemoteCacheConfig, SwiftcContext){ + let fileReader = FileManager.default + let config: XCRemoteCacheConfig + let context: SwiftcContext + do { + let srcRoot: URL = URL(fileURLWithPath: fileReader.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileReader) + .readConfiguration() + context = try SwiftcContext(config: config, input: inputArgs) + } catch { + exit(1, "FATAL: Swiftc initialization failed with error: \(error)") + } + return (config, context) + } +} diff --git a/Sources/XCRemoteCache/Utils/Array+Utils.swift b/Sources/XCRemoteCache/Utils/Array+Utils.swift new file mode 100644 index 00000000..fcf0c444 --- /dev/null +++ b/Sources/XCRemoteCache/Utils/Array+Utils.swift @@ -0,0 +1,29 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public extension Array { + func get(_ i: Index) -> Element? { + guard count < i else { + return nil + } + return self[i] + } +} diff --git a/Sources/xcswift-frontend/XCSwiftFrontend.swift b/Sources/xcswift-frontend/XCSwiftFrontend.swift index 227330cc..00f4cfdb 100644 --- a/Sources/xcswift-frontend/XCSwiftFrontend.swift +++ b/Sources/xcswift-frontend/XCSwiftFrontend.swift @@ -41,6 +41,7 @@ public class XCSwiftcFrontendMain { var target: String? // var swiftFileList: String? var inputPaths: [String] = [] + var primaryInputPaths: [String] = [] var outputPaths: [String] = [] var dependenciesPaths: [String] = [] var diagnosticsPaths: [String] = [] @@ -78,6 +79,10 @@ public class XCSwiftcFrontendMain { case "-emit-module-doc-path": // .swiftsourceinfo docPath = args[i + 1] + case "-primary-file": + // .swift + primaryInputPaths.append(args[i + 1]) + inputPaths.append(args[i + 1]) default: if arg.hasPrefix(".swift") { inputPaths.append(arg) @@ -93,6 +98,7 @@ public class XCSwiftcFrontendMain { objcHeaderOutput: objcHeaderOutput, moduleName: moduleName, target: target, + primaryInputPaths: primaryInputPaths, inputPaths: inputPaths, outputPaths: outputPaths, dependenciesPaths: dependenciesPaths, From dd36876b2f05aec42d693f04f776e55ef2d83ca4 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Mon, 22 May 2023 22:03:30 -0700 Subject: [PATCH 05/33] Deleting unused files --- .../SwiftFrontend/SwiftFrontend.swift_ | 188 ------------------ .../SwiftFrontend/SwiftFrontendContext.swift_ | 95 --------- .../SwiftFrontendProductsGenerator.swift_ | 118 ----------- 3 files changed, 401 deletions(-) delete mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift_ delete mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift_ delete mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift_ diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift_ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift_ deleted file mode 100644 index 53f9880f..00000000 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontend.swift_ +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright (c) 2021 Spotify AB. -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import Foundation - -enum SwiftFrontendResult { - /// Swiftc mock cannot be used and fallback to the compilation is required - case forceFallback - /// All compilation steps were mocked correctly - case success -} - -/// Swiftc mocking compilation -protocol SwiftFrontendProtocol { - /// Tries to performs mocked compilation (moving all cached files to the expected location) - /// If cached compilation products are not valid or incompatible, fallbacks to build-from-source - /// - Returns: `.forceFallback` if the cached compilation products are incompatible and fallback - /// to a standard 'swiftc' is required, `.success` otherwise - /// - Throws: An error if there was an unrecoverable, serious error (e.g. IO error) - func mockCompilation() throws -> SwiftFrontendResult -} - -/// Swiftc wrapper that mocks compilation with noop and moves all expected products from cache location -class SwiftFrontend: SwiftFrontendProtocol { - /// Reader of all input files of the compilation - private let inputFileListReader: ListReader - /// Reader of the marker file lists - list of dependencies to set for swiftc compilation - private let markerReader: ListReader - /// Checks if the input file exists in the file list - private let allowedFilesListScanner: FileListScanner - /// Manager of the downloaded artifact package - private let artifactOrganizer: ArtifactOrganizer - /// Reads all input and output files for the compilation from an input filemap - private let inputFilesReader: SwiftcInputReader - /// Write manager of the marker file - private let markerWriter: MarkerWriter - /// Generates products at the desired destination - private let productsGenerator: SwiftcProductsGenerator - private let context: SwiftcContext - private let fileManager: FileManager - private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter - private let touchFactory: (URL, FileManager) -> Touch - private let plugins: [SwiftcProductGenerationPlugin] - - init( - inputFileListReader: ListReader, - markerReader: ListReader, - allowedFilesListScanner: FileListScanner, - artifactOrganizer: ArtifactOrganizer, - inputReader: SwiftcInputReader, - context: SwiftcContext, - markerWriter: MarkerWriter, - productsGenerator: SwiftcProductsGenerator, - fileManager: FileManager, - dependenciesWriterFactory: @escaping (URL, FileManager) -> DependenciesWriter, - touchFactory: @escaping (URL, FileManager) -> Touch, - plugins: [SwiftcProductGenerationPlugin] - ) { - self.inputFileListReader = inputFileListReader - self.markerReader = markerReader - self.allowedFilesListScanner = allowedFilesListScanner - self.artifactOrganizer = artifactOrganizer - inputFilesReader = inputReader - self.context = context - self.markerWriter = markerWriter - self.productsGenerator = productsGenerator - self.fileManager = fileManager - self.dependenciesWriterFactory = dependenciesWriterFactory - self.touchFactory = touchFactory - self.plugins = plugins - } - - // swiftlint:disable:next function_body_length - func mockCompilation() throws -> SwiftFrontendResult { - let rcModeEnabled = markerReader.canRead() - guard rcModeEnabled else { - infoLog("Swiftc marker doesn't exist") - return .forceFallback - } - - let inputFilesInputs = try inputFileListReader.listFilesURLs() - let markerAllowedFiles = try markerReader.listFilesURLs() - let cachedDependenciesWriterFactory = CachedFileDependenciesWriterFactory( - dependencies: markerAllowedFiles, - fileManager: fileManager, - writerFactory: dependenciesWriterFactory - ) - // Verify all input files to be present in a marker fileList - let disallowedInputs = try inputFilesInputs.filter { try !allowedFilesListScanner.contains($0) } - - if !disallowedInputs.isEmpty { - // New file (disallowedFile) added without modifying the rest of the feature. Fallback to swiftc and - // ensure that compilation from source will be forced up until next merge/rebase with "primary" branch - infoLog("Swiftc new input file \(disallowedInputs)") - // Deleting marker to indicate that the remote cached artifact cannot be used - try markerWriter.disable() - - // Save custom prebuild discovery content to make sure that the following prebuild - // phase will not try to reuse cached artifact (if present) - // In other words: let prebuild know that it should not try to reenable cache - // until the next merge with primary - switch context.mode { - case .consumer(commit: .available(let remoteCommit)): - let prebuildDiscoveryURL = context.tempDir.appendingPathComponent(context.prebuildDependenciesPath) - let prebuildDiscoverWriter = dependenciesWriterFactory(prebuildDiscoveryURL, fileManager) - try prebuildDiscoverWriter.write(skipForSha: remoteCommit) - case .consumer, .producer, .producerFast: - // Never skip prebuild phase and fallback to the swiftc compilation for: - // 1) Not enabled remote cache, 2) producer(s) - break - } - return .forceFallback - } - - let artifactLocation = artifactOrganizer.getActiveArtifactLocation() - - // Read swiftmodule location from XCRemoteCache - // arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo} - let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent - let allCompilations = try inputFilesReader.read() - let artifactSwiftmoduleDir = artifactLocation - .appendingPathComponent("swiftmodule") - .appendingPathComponent(context.arch) - let artifactSwiftmoduleBase = artifactSwiftmoduleDir.appendingPathComponent(moduleName) - let artifactSwiftmoduleFiles = Dictionary( - uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions - .map { ext, _ in - (ext, artifactSwiftmoduleBase.appendingPathExtension(ext.rawValue)) - } - ) - - // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h - let artifactSwiftModuleObjCDir = artifactLocation - .appendingPathComponent("include") - .appendingPathComponent(context.arch) - .appendingPathComponent(context.moduleName) - // Move cached xxxx-Swift.h to the location passed in arglist - // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir - let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir - .appendingPathComponent(context.objcHeaderOutput.lastPathComponent) - - _ = try productsGenerator.generateFrom( - artifactSwiftModuleFiles: artifactSwiftmoduleFiles, - artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile - ) - - try plugins.forEach { - try $0.generate(for: allCompilations) - } - - // Save individual .d and touch .o for each .swift file - for compilation in allCompilations.files { - if let object = compilation.object { - // Touching .o is required to invalidate already existing .a or linked library - let touch = touchFactory(object, fileManager) - try touch.touch() - } - if let individualDeps = compilation.dependencies { - // swiftc product should be invalidated if any of dependencies file has changed - try cachedDependenciesWriterFactory.generate(output: individualDeps) - } - } - // Save .d for the entire module - try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies) - // Generate .d file with all deps in the "-master.d" (e.g. for WMO) - if let wmoDeps = allCompilations.info.dependencies { - try cachedDependenciesWriterFactory.generate(output: wmoDeps) - } - infoLog("Swiftc noop for \(context.target)") - return .success - } -} diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift_ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift_ deleted file mode 100644 index 99687196..00000000 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendContext.swift_ +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) 2023 Spotify AB. -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import Foundation - -struct SwiftFrontendContext { - enum SwiftFrontendMode: Equatable { - case producer - /// Commit sha of the commit to use during remote cache - case consumer(commit: RemoteCommitInfo) - /// Remote artifact exists and can be optimistically used in place of a local compilation - case producerFast - } - - let moduleName: String - let target: String - let tempDir: URL - let arch: String - let prebuildDependenciesPath: String - let mode: SwiftFrontendMode - /// File that stores all compilation invocation arguments - let invocationHistoryFile: URL - let action: SwiftFrontendAction - /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation - let llbuildId: String - - - private init( - config: XCRemoteCacheConfig, - env: [String: String], - moduleName: String, - target: String, - action: SwiftFrontendAction - ) throws { - self.moduleName = moduleName - self.target = target - self.action = action - llbuildId = try env.readEnv(key: "LLBUILD_BUILD_ID") - - tempDir = action.tmpDir - arch = action.arch - - let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot) - let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot) - prebuildDependenciesPath = config.prebuildDiscoveryPath - switch config.mode { - case .consumer: - let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim()) - mode = .consumer(commit: remoteCommit) - case .producer: - mode = .producer - case .producerFast: - let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim()) - switch remoteCommit { - case .unavailable: - mode = .producer - case .available: - mode = .producerFast - } - } - invocationHistoryFile = URL(fileURLWithPath: config.compilationHistoryFile, relativeTo: tempDir) - } - - init( - config: XCRemoteCacheConfig, - env: [String: String], - input: SwiftFrontendArgInput, - action: SwiftFrontendAction - ) throws { - try self.init( - config: config, - env: env, - moduleName: action.moduleName, - target: action.target, - action: action - ) - } -} - diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift_ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift_ deleted file mode 100644 index 0abe0969..00000000 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendProductsGenerator.swift_ +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) 2021 Spotify AB. -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import Foundation - -enum DiskSwiftFrontendProductsGeneratorError: Error { - /// Emittiing module is available only from emit-module action - case requestedEmitingModuleForInvalidAction(SwiftFrontendAction) - /// When a generator was asked to generate unknown swiftmodule extension file - /// Probably a programmer error: asking to generate excessive extensions, not listed in - /// `SwiftmoduleFileExtension.SwiftmoduleExtensions` - case unknownSwiftmoduleFile -} - -struct SwiftFrontendEmitModuleProductsGeneratorOutput { - let swiftmoduleDir: URL - let objcHeaderFile: URL -} - -struct SwiftFrontendCompilationProductsGeneratorOutput { - // TODO: -} - -/// Generates SwiftFrontend product to the expected location -protocol SwiftFrontendProductsGenerator { - /// Generates products for the emit-module invocation from given files - /// - Returns: location dir where .swiftmodule and ObjC header files have been placed - func generateEmitModuleFrom( - artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], - artifactSwiftModuleObjCFile: URL - ) throws -> SwiftFrontendEmitModuleProductsGeneratorOutput - - /// Generates products for the compilation(s) invocation from given file(s) - /// - Returns: location dir where .swiftmodule and ObjC header files have been placed - func generateCompilationFrom( - // TODO: - ) throws -> SwiftFrontendCompilationProductsGeneratorOutput -} - -/// Generator that produces all products in the locations where Xcode expects it, using provided disk copier -class DiskSwiftFrontendProductsGenerator: SwiftFrontendProductsGenerator { - private let action: SwiftFrontendAction - private let diskCopier: DiskCopier - - init( - action: SwiftFrontendAction, - diskCopier: DiskCopier - ) { - self.action = action - self.diskCopier = diskCopier - } - - func generateEmitModuleFrom( - artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], - artifactSwiftModuleObjCFile: URL - ) throws -> SwiftFrontendEmitModuleProductsGeneratorOutput { - guard case .emitModule(emitModuleInfo: let info, inputFiles: let files) = action else { - throw DiskSwiftFrontendProductsGeneratorError.requestedEmitingModuleForInvalidAction(action) - } - - let modulePathOutput = info.output - let objcHeaderOutput = info.objcHeader - let modulePathBasename = modulePathOutput.deletingPathExtension() - // all swiftmodule-related should be located next to the ".swiftmodule" - let destinationSwiftmodulePaths = Dictionary( - uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions - .map { ext, _ in - (ext, modulePathBasename.appendingPathExtension(ext.rawValue)) - } - ) - - // Move cached -Swift.h file to the expected location - try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput) - for (ext, url) in sourceAtifactSwiftModuleFiles { - let dest = destinationSwiftmodulePaths[ext] - guard let destination = dest else { - throw DiskSwiftFrontendProductsGeneratorError.unknownSwiftmoduleFile - } - do { - // Move cached .swiftmodule to the expected location - try diskCopier.copy(file: url, destination: destination) - } catch { - if case .required = SwiftmoduleFileExtension.SwiftmoduleExtensions[ext] { - throw error - } else { - infoLog("Optional .\(ext) file not found in the artifact at: \(destination.path)") - } - } - } - - // Build parent dir of the .swiftmodule file that contains a module - return .init( - swiftmoduleDir: modulePathOutput.deletingLastPathComponent(), - objcHeaderFile: objcHeaderOutput - ) - } - - func generateCompilationFrom() throws -> SwiftFrontendCompilationProductsGeneratorOutput { - // TODO: - return .init() - } -} From 5b2e2e33e16c14122c09be318b8b4437ee2245e2 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Mon, 22 May 2023 22:04:46 -0700 Subject: [PATCH 06/33] --amend --- .../SwiftFrontend/XCSwiftFrontend.swift | 69 ------------------- .../Commands/Swiftc/XCSwiftc.swift | 2 +- 2 files changed, 1 insertion(+), 70 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index 4fcf351b..6c93fe55 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -19,75 +19,6 @@ import Foundation -struct SwiftFrontendEmitModuleInfo: Equatable { - let inputs: [URL] - let objcHeader: URL - let diagnostics: URL? - let dependencies: URL? - let output: URL - let target: String - let moduleName: String - let sourceInfo: URL? - let doc: URL? -} - -struct SwiftFrontendCompilationInfo: Equatable { - let target: String - let moduleName: String -} - -enum SwiftFrontendAction { - case emitModule(emitModuleInfo: SwiftFrontendEmitModuleInfo, inputFiles: [URL]) - case compile(compilationInfo: SwiftFrontendCompilationInfo, compilationFiles: [SwiftFileCompilationInfo]) -} - -extension SwiftFrontendAction { - var tmpDir: URL { - // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule - // That may be subject to change for other Xcode versions (or other variants) - return outputWorkspaceDir - .deletingLastPathComponent() - .deletingLastPathComponent() - .deletingLastPathComponent() - } - var arch: String { - return outputWorkspaceDir.deletingLastPathComponent().lastPathComponent - } - - var target: String { - switch self { - case .emitModule(emitModuleInfo: let info, inputFiles: _): - return info.target - case .compile(compilationInfo: let info, compilationFiles: _): - return info.target - } - } - - var moduleName: String { - switch self { - case .emitModule(emitModuleInfo: let info, inputFiles: _): - return info.moduleName - case .compile(compilationInfo: let info, compilationFiles: _): - return info.moduleName - } - } - - - // The workspace where Xcode asks to put all compilation-relates - // files (like .d or .swiftmodule) - // This location is used to infere the tmpDir and arch - private var outputWorkspaceDir: URL { - switch self { - case .emitModule(emitModuleInfo: let info, inputFiles: _): - return info.output - case .compile(_, let files): - // if invoked compilation via swift-frontend, the .d file is always defined - return files[0].dependencies! - } - } -} - - enum SwiftFrontendArgInputError: Error { // swift-frontend should either be compling or emiting a module case bothCompilationAndEmitAction diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index a15095a9..29a79c6e 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -64,7 +64,7 @@ public class XCSwiftAbstract { } func buildContext() -> (XCRemoteCacheConfig, SwiftcContext){ - fatalError("Override in \(Self.self)") + fatalError("Need to override in \(Self.self)") } // swiftlint:disable:next function_body_length From 69d4946869f3183a0fc6b4db194d29a012e6fd96 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Mon, 22 May 2023 22:09:09 -0700 Subject: [PATCH 07/33] Cleanup --- Rakefile | 2 +- .../Commands/Swiftc/SwiftcContext.swift | 7 ------- Sources/xcswift-frontend/XCSwiftFrontend.swift | 11 +---------- .../StandaloneApp.xcodeproj/project.pbxproj | 18 ------------------ 4 files changed, 2 insertions(+), 36 deletions(-) diff --git a/Rakefile b/Rakefile index 3934c77f..a5020248 100644 --- a/Rakefile +++ b/Rakefile @@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze RELEASES_ROOT_DIR = File.join('releases').freeze EXECUTABLE_NAME = 'XCRemoteCache' -EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld', 'xcldplusplus', 'xclipo'] +EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcswift-frontend', 'xcld', 'xcldplusplus', 'xclipo'] PROJECT_NAME = 'XCRemoteCache' SWIFTLINT_ENABLED = true diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 9f565005..6a5159e5 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -19,7 +19,6 @@ import Foundation - public struct SwiftcContext { public struct SwiftcStepEmitModule { let objcHeaderOutput: URL @@ -61,9 +60,7 @@ public struct SwiftcContext { } let steps: SwiftcSteps -// let objcHeaderOutput: URL let moduleName: String -// let modulePathOutput: URL /// A source that defines output files locations (.d, .swiftmodule etc.) let outputs: CompilationFilesOutputs let target: String @@ -88,14 +85,10 @@ public struct SwiftcContext { /// are placed next to it. This path is used to infere the arch and TARGET_TEMP_DIR exampleWorkspaceFilePath: String ) throws { -// self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput) self.moduleName = moduleName -// self.modulePathOutput = URL(fileURLWithPath: modulePathOutput) self.steps = steps self.outputs = outputs -// self.filemap = URL(fileURLWithPath: filemap) self.target = target -// self.fileList = URL(fileURLWithPath: fileList) self.inputs = inputs // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule // That may be subject to change for other Xcode versions diff --git a/Sources/xcswift-frontend/XCSwiftFrontend.swift b/Sources/xcswift-frontend/XCSwiftFrontend.swift index 00f4cfdb..7c2c8d84 100644 --- a/Sources/xcswift-frontend/XCSwiftFrontend.swift +++ b/Sources/xcswift-frontend/XCSwiftFrontend.swift @@ -22,7 +22,7 @@ import XCRemoteCache /// Wrapper for a `swift-frontend` that skips compilation and produces empty output files (.o). As a compilation dependencies /// (.d) file, it copies all dependency files from the prebuild marker file -/// Fallbacks to a standard `swiftc` when the Ramote cache is not applicable (e.g. modified sources) +/// Fallbacks to a standard `swift-frontend` when the ramote cache is not applicable (e.g. modified sources) public class XCSwiftcFrontendMain { // swiftlint:disable:next function_body_length public func main() { @@ -31,15 +31,11 @@ public class XCSwiftcFrontendMain { let args = ProcessInfo().arguments - var compile = false var emitModule = false var objcHeaderOutput: String? var moduleName: String? -// var modulePathOutput: String? -// var filemap: String? var target: String? -// var swiftFileList: String? var inputPaths: [String] = [] var primaryInputPaths: [String] = [] var outputPaths: [String] = [] @@ -61,10 +57,6 @@ public class XCSwiftcFrontendMain { objcHeaderOutput = args[i + 1] case "-module-name": moduleName = args[i + 1] -// case "-emit-module-path": -// modulePathOutput = args[i + 1] -// case "-output-file-map": -// filemap = args[i + 1] case "-target": target = args[i + 1] case "-serialize-diagnostics-path": @@ -116,7 +108,6 @@ public class XCSwiftcFrontendMain { frontend.run() } catch { let swiftFrontendCommand = "swift-frontend" -// print("Fallbacking to compilation using \(swiftFrontendCommand).") let args = ProcessInfo().arguments let paramList = [swiftFrontendCommand] + args.dropFirst() diff --git a/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj b/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj index cad5fb83..327cb466 100644 --- a/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj +++ b/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj @@ -234,7 +234,6 @@ 36201A242843B3D3002FF70F /* Frameworks */, 36201A252843B3D3002FF70F /* CopyFiles */, 36201A3A2843BE0E002FF70F /* Copy Swift Objective-C Interface Header */, - 4E7D74B02A109228002EF202 /* ShellScript */, ); buildRules = ( ); @@ -379,23 +378,6 @@ shellPath = /bin/sh; shellScript = "ditto \"${SCRIPT_INPUT_FILE_0}\" \"${SCRIPT_OUTPUT_FILE_0}\"\n[ -f \"${SCRIPT_INPUT_FILE_1}\" ] && ditto \"${SCRIPT_INPUT_FILE_1}\" \"${SCRIPT_OUTPUT_FILE_1}\" || rm \"${SCRIPT_OUTPUT_FILE_1}\"\n\n"; }; - 4E7D74B02A109228002EF202 /* ShellScript */ = { - isa = PBXShellScriptBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "# Type a script or drag a script file from your workspace to insert its path.\n"; - }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ From bcfeaa625a7719feec6fd0412e5dcc32caa83fa4 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 23 May 2023 19:17:31 -0700 Subject: [PATCH 08/33] Fix tests --- Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift | 2 +- Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift index ed9e08d5..30020e11 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift @@ -34,7 +34,7 @@ class SwiftcContextTests: FileXCTestCase { config = XCRemoteCacheConfig(remoteCommitFile: remoteCommitFile.path, sourceRoot: workingDir.path) input = SwiftcArgInput( objcHeaderOutput: "Target-Swift.h", - moduleName: "", + moduleName: "Target", modulePathOutput: modulePathOutput.path, filemap: "", target: "", diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift index 5df313ff..50bf67a2 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift @@ -62,7 +62,7 @@ class SwiftcTests: FileXCTestCase { input = SwiftcArgInput( objcHeaderOutput: "Target-Swift.h", - moduleName: "", + moduleName: "Target", modulePathOutput: modulePathOutput.path, filemap: "", target: "", @@ -276,7 +276,7 @@ class SwiftcTests: FileXCTestCase { func testCompilationUsesArchSpecificSwiftmoduleFiles() throws { let artifactRoot = URL(fileURLWithPath: "/cachedArtifact") - let artifactObjCHeader = URL(fileURLWithPath: "/cachedArtifact/include/archTest/Target-Swift.h") + let artifactObjCHeader = URL(fileURLWithPath: "/cachedArtifact/include/archTest/Target/Target-Swift.h") let artifactSwiftmodule = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftmodule") let artifactSwiftdoc = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftdoc") let artifactSwiftSourceInfo = URL( From 4b353184e7f5b03aff7b9abbbf1106b24e9453a4 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 23 May 2023 19:31:47 -0700 Subject: [PATCH 09/33] Fix linter --- .../Commands/Swiftc/SwiftcContext.swift | 8 ++++---- .../Swiftc/SwiftcProductsGenerator.swift | 16 ++++++++++++---- .../XCRemoteCache/Commands/Swiftc/XCSwiftc.swift | 12 ++++++------ Sources/xcswift-frontend/XCSwiftFrontend.swift | 9 ++++----- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 6a5159e5..57cba7ee 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -29,12 +29,12 @@ public struct SwiftcContext { case all case subset([URL]) } - + public struct SwiftcSteps { let compileFilesScope: SwiftcStepCompileFilesScope let emitModule: SwiftcStepEmitModule? } - + /// Defines how a list of input files (*.swift) is passed to the invocation public enum CompilationFilesSource { /// defined in a separate file (via @/.../*.SwiftFileList) @@ -42,7 +42,7 @@ public struct SwiftcContext { /// explicitly passed a list of files case list([String]) } - + /// Defines how a list of output files (*.d, *.o etc.) is passed to the invocation public enum CompilationFilesOutputs { /// defined in a separate file (via -output-file-map) @@ -50,7 +50,7 @@ public struct SwiftcContext { /// explicitly passed in the invocation case map([String: SwiftFileCompilationInfo]) } - + enum SwiftcMode: Equatable { case producer /// Commit sha of the commit to use during remote cache diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift index 61b4efa8..0d6496ce 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift @@ -42,11 +42,19 @@ protocol SwiftcProductsGenerator { } /// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where -/// only individual .o files are created and not .swiftmodule of -Swift.h (which is part of swift-frontend -emit-module invocation) +/// only individual .o files are created and not .swiftmodule of -Swift.h +/// (which is part of swift-frontend -emit-module invocation) class NoopSwiftcProductsGenerator: SwiftcProductsGenerator { - func generateFrom(artifactSwiftModuleFiles: [SwiftmoduleFileExtension : URL], artifactSwiftModuleObjCFile: URL) throws -> SwiftcProductsGeneratorOutput { - printWarning("Invoking module generation from NoopSwiftcProductsGenerator which does nothing. It might be an error of the swift-frontend mocking.") - // TODO: Refactor API: this url is never used: NoopSwiftcProductsGenerator is intended only for the swift-frontend + func generateFrom( + artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], + artifactSwiftModuleObjCFile: URL + ) throws -> SwiftcProductsGeneratorOutput { + printWarning(""" + Invoking module generation from NoopSwiftcProductsGenerator which does nothing. \ + It might be an error of the swift-frontend mocking. + """) + // TODO: Refactor API: this url is never used: + // NoopSwiftcProductsGenerator is intended only for the swift-frontend let trivialURL = URL(fileURLWithPath: "/non-existing") return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL) } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 29a79c6e..4ebf982f 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -63,7 +63,7 @@ public class XCSwiftAbstract { self.touchFactory = touchFactory } - func buildContext() -> (XCRemoteCacheConfig, SwiftcContext){ + func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { fatalError("Need to override in \(Self.self)") } @@ -92,10 +92,10 @@ public class XCSwiftAbstract { } let fileListReader: ListReader switch context.inputs { - case .fileList(let path): - fileListReader = FileListEditor(URL(fileURLWithPath: path), fileManager: fileManager) - case .list(let paths): - fileListReader = StaticFileListReader(list: paths.map(URL.init(fileURLWithPath:))) + case .fileList(let path): + fileListReader = FileListEditor(URL(fileURLWithPath: path), fileManager: fileManager) + case .list(let paths): + fileListReader = StaticFileListReader(list: paths.map(URL.init(fileURLWithPath:))) } let artifactOrganizer = ZipArtifactOrganizer( targetTempDir: context.tempDir, @@ -172,7 +172,7 @@ public class XCSwiftAbstract { } public class XCSwiftc: XCSwiftAbstract { - override func buildContext() -> (XCRemoteCacheConfig, SwiftcContext){ + override func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { let fileReader = FileManager.default let config: XCRemoteCacheConfig let context: SwiftcContext diff --git a/Sources/xcswift-frontend/XCSwiftFrontend.swift b/Sources/xcswift-frontend/XCSwiftFrontend.swift index 7c2c8d84..03dda52f 100644 --- a/Sources/xcswift-frontend/XCSwiftFrontend.swift +++ b/Sources/xcswift-frontend/XCSwiftFrontend.swift @@ -20,17 +20,16 @@ import Foundation import XCRemoteCache -/// Wrapper for a `swift-frontend` that skips compilation and produces empty output files (.o). As a compilation dependencies +/// Wrapper for a `swift-frontend` that skips compilation and +/// produces empty output files (.o). As a compilation dependencies /// (.d) file, it copies all dependency files from the prebuild marker file /// Fallbacks to a standard `swift-frontend` when the ramote cache is not applicable (e.g. modified sources) public class XCSwiftcFrontendMain { - // swiftlint:disable:next function_body_length + // swiftlint:disable:next function_body_length cyclomatic_complexity public func main() { let env = ProcessInfo.processInfo.environment let command = ProcessInfo().processName let args = ProcessInfo().arguments - - var compile = false var emitModule = false var objcHeaderOutput: String? @@ -43,7 +42,7 @@ public class XCSwiftcFrontendMain { var diagnosticsPaths: [String] = [] var sourceInfoPath: String? var docPath: String? - + for i in 0.. Date: Tue, 23 May 2023 19:34:28 -0700 Subject: [PATCH 10/33] Move XCSwiftcFrontendMain.swift --- .../{XCSwiftFrontend.swift => XCSwiftcFrontendMain.swift} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Sources/xcswift-frontend/{XCSwiftFrontend.swift => XCSwiftcFrontendMain.swift} (100%) diff --git a/Sources/xcswift-frontend/XCSwiftFrontend.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift similarity index 100% rename from Sources/xcswift-frontend/XCSwiftFrontend.swift rename to Sources/xcswift-frontend/XCSwiftcFrontendMain.swift From 6a78b4e424eeac94488a8ee49ab0087895f6f50c Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 23 May 2023 19:40:55 -0700 Subject: [PATCH 11/33] Fix linters --- Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift | 7 ++++--- Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift | 2 +- .../Commands/Swiftc/SwiftcFilemapInputEditor.swift | 4 ++-- Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift | 8 ++++---- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift index bf2de8aa..9bc70687 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift @@ -146,17 +146,18 @@ class Swiftc: SwiftcProtocol { ) // emit module (if requested) - if let emitModule = context.steps.emitModule { + if let emitModule = context.steps.emitModule { // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h let artifactSwiftModuleObjCDir = artifactLocation .appendingPathComponent("include") .appendingPathComponent(context.arch) .appendingPathComponent(context.moduleName) // Move cached xxxx-Swift.h to the location passed in arglist - // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir + // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h + // file in artifactSwiftModuleObjCDir let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir .appendingPathComponent(emitModule.objcHeaderOutput.lastPathComponent) - + _ = try productsGenerator.generateFrom( artifactSwiftModuleFiles: artifactSwiftmoduleFiles, artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 57cba7ee..d64340b9 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -142,7 +142,7 @@ public struct SwiftcContext { exampleWorkspaceFilePath: input.modulePathOutput ) } - + init( config: XCRemoteCacheConfig, input: SwiftFrontendArgInput diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift index a2e36d76..51f9e211 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift @@ -63,7 +63,7 @@ class StaticSwiftcInputReader: SwiftcInputReader { private let moduleDependencies: URL? private let swiftDependencies: URL? private let compilationFiles: [SwiftFileCompilationInfo] - + init( moduleDependencies: URL?, swiftDependencies: URL?, @@ -73,7 +73,7 @@ class StaticSwiftcInputReader: SwiftcInputReader { self.swiftDependencies = swiftDependencies self.compilationFiles = compilationFiles } - + func read() throws -> SwiftCompilationInfo { return .init( info: .init( diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 4ebf982f..1ecf08df 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -50,7 +50,7 @@ public class XCSwiftAbstract { let inputArgs: T private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter private let touchFactory: (URL, FileManager) -> Touch - + public init( command: String, inputArgs: T, @@ -62,16 +62,16 @@ public class XCSwiftAbstract { dependenciesWriterFactory = dependenciesWriter self.touchFactory = touchFactory } - + func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { fatalError("Need to override in \(Self.self)") } - + // swiftlint:disable:next function_body_length public func run() { let fileManager = FileManager.default let (config, context) = buildContext() - + let swiftcCommand = config.swiftcCommand let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) From 9b8c0de9d833f7a40defc06d5244aea8e2cca128 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 23 May 2023 19:53:07 -0700 Subject: [PATCH 12/33] Fix linters --- .../SwiftFrontendOrchestrator.swift | 13 +-- .../SwiftFrontend/XCSwiftFrontend.swift | 84 ++++++++++++------- .../Commands/PostbuildContextTests.swift | 2 +- 3 files changed, 64 insertions(+), 35 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift index be031147..c77621a6 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift @@ -28,7 +28,7 @@ class SwiftFrontendOrchestrator { /// Content saved to the shared file /// Safe to use forced unwrapping private static let emitModuleContent = "done".data(using: .utf8)! - + enum Action { case emitModule case compile @@ -46,7 +46,7 @@ class SwiftFrontendOrchestrator { self.action = action self.lockAccessor = lockAccessor } - + func run() throws { guard case .consumer(commit: .available) = mode else { // no need to lock anything - just allow fallbacking to the `swiftc or swift-frontend` @@ -54,6 +54,7 @@ class SwiftFrontendOrchestrator { } try waitForEmitModuleLock() } + private func executeMockAttemp() throws { switch action { case .emitModule: @@ -62,19 +63,19 @@ class SwiftFrontendOrchestrator { try waitForEmitModuleLock() } } - + private func validateEmitModuleStep() throws { try lockAccessor.exclusiveAccess { handle in // TODO: check if the mocking compilation can happen (make sure // all input files are listed in the list of dependencies) - + handle.write(SwiftFrontendOrchestrator.emitModuleContent) } } - + /// Locks a shared file in a loop until its content non-empty, which means the "parent" emit-module has finished private func waitForEmitModuleLock() throws { - while (true) { + while true { // TODO: add a max timeout try lockAccessor.exclusiveAccess { handle in if !handle.availableData.isEmpty { diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index 6c93fe55..2ec556bc 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -65,8 +65,20 @@ public struct SwiftFrontendArgInput { let docPath: String? /// Manual initializer implementation required to be public - public init(compile: Bool, emitModule: Bool, objcHeaderOutput: String?, moduleName: String?, target: String?, primaryInputPaths: [String], inputPaths: [String], outputPaths: [String], dependenciesPaths: [String], diagnosticsPaths: [String], - sourceInfoPath: String?, docPath: String?) { + public init( + compile: Bool, + emitModule: Bool, + objcHeaderOutput: String?, + moduleName: String?, + target: String?, + primaryInputPaths: [String], + inputPaths: [String], + outputPaths: [String], + dependenciesPaths: [String], + diagnosticsPaths: [String], + sourceInfoPath: String?, + docPath: String? + ) { self.compile = compile self.emitModule = emitModule self.objcHeaderOutput = objcHeaderOutput @@ -80,13 +92,14 @@ public struct SwiftFrontendArgInput { self.sourceInfoPath = sourceInfoPath self.docPath = docPath } - + + // swiftlint:disable:next cyclomatic_complexity function_body_length func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext { guard compile != emitModule else { throw SwiftFrontendArgInputError.bothCompilationAndEmitAction } let inputPathsCount = inputPaths.count - let primaryInputPathsCount = primaryInputPaths.count + let primaryInputsCount = primaryInputPaths.count guard inputPathsCount > 0 else { throw SwiftFrontendArgInputError.noCompilationInputs } @@ -98,26 +111,35 @@ public struct SwiftFrontendArgInput { } if compile { - guard primaryInputPathsCount > 0 else { + guard primaryInputsCount > 0 else { throw SwiftFrontendArgInputError.noPrimaryFileCompilationInputs } - guard [primaryInputPathsCount, 0].contains(dependenciesPaths.count) else { - throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch(expected: inputPathsCount, parsed: dependenciesPaths.count) + guard [primaryInputsCount, 0].contains(dependenciesPaths.count) else { + throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: dependenciesPaths.count + ) } - guard [primaryInputPathsCount, 0].contains(diagnosticsPaths.count) else { - throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch(expected: inputPathsCount, parsed: diagnosticsPaths.count) + guard [primaryInputsCount, 0].contains(diagnosticsPaths.count) else { + throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: diagnosticsPaths.count + ) } - guard outputPaths.count == primaryInputPathsCount else { - throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(expected: inputPathsCount, parsed: outputPaths.count) + guard outputPaths.count == primaryInputsCount else { + throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: outputPaths.count + ) } let primaryInputFilesURLs: [URL] = primaryInputPaths.map(URL.init(fileURLWithPath:)) - + let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps( compileFilesScope: .subset(primaryInputFilesURLs), emitModule: nil ) - - let compilationFileInfoMap: [String: SwiftFileCompilationInfo] = (0.. { private let env: [String: String] private var cachedSwiftdContext: (XCRemoteCacheConfig, SwiftcContext)? - + public init( command: String, inputArgs: SwiftFrontendArgInput, @@ -197,11 +225,11 @@ public class XCSwiftFrontend: XCSwiftAbstract { if let cachedSwiftdContext = cachedSwiftdContext { return cachedSwiftdContext } - + let fileManager = FileManager.default let config: XCRemoteCacheConfig let context: SwiftcContext - + do { let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) @@ -214,24 +242,24 @@ public class XCSwiftFrontend: XCSwiftAbstract { self.cachedSwiftdContext = builtContext return builtContext } - + override public func run() { do { /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") let (_, context) = buildContext() - + let sharedLockFileURL = context.tempDir.appendingPathComponent(llbuildId).appendingPathExtension("lock") let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) - + let action: SwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile let orchestrator = SwiftFrontendOrchestrator(mode: context.mode, action: action, lockAccessor: sharedLock) - + try orchestrator.run() } catch { printWarning("Cannot correctly orchestrate the \(command) with params \(inputArgs): error: \(error)") } - + super.run() } } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift index be78500b..aadb3249 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift @@ -28,7 +28,7 @@ class PostbuildContextTests: FileXCTestCase { "TARGET_TEMP_DIR": "TARGET_TEMP_DIR", "DERIVED_FILE_DIR": "DERIVED_FILE_DIR", "ARCHS": "x86_64", - "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal" , + "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal", "CONFIGURATION": "CONFIGURATION", "PLATFORM_NAME": "PLATFORM_NAME", "XCODE_PRODUCT_BUILD_VERSION": "XCODE_PRODUCT_BUILD_VERSION", From b24a252a0f87eeb8f2fcb82e6c498fc24466d0ed Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 23 May 2023 20:03:00 -0700 Subject: [PATCH 13/33] Fix linter --- Sources/XCRemoteCache/Dependencies/ListEditor.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/XCRemoteCache/Dependencies/ListEditor.swift b/Sources/XCRemoteCache/Dependencies/ListEditor.swift index 0a89fd1a..8a7bc35e 100644 --- a/Sources/XCRemoteCache/Dependencies/ListEditor.swift +++ b/Sources/XCRemoteCache/Dependencies/ListEditor.swift @@ -43,13 +43,13 @@ protocol ListWriter { class StaticFileListReader: ListReader { private let list: [URL] - init(list: [URL]){ + init(list: [URL]) { self.list = list } func listFilesURLs() throws -> [URL] { list } - + func canRead() -> Bool { true } From 501ef19c4c25ed6581c178ff106f82f120222b2f Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 23 May 2023 20:18:58 -0700 Subject: [PATCH 14/33] Add feature flag for frontent integration --- README.md | 1 + Rakefile | 5 ++++- .../Commands/Prepare/Integrate/IntegrateContext.swift | 6 ++++-- .../Commands/Prepare/Integrate/XCIntegrate.swift | 3 ++- Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift | 5 +++++ 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3e2dd1e2..b37fa738 100755 --- a/README.md +++ b/README.md @@ -359,6 +359,7 @@ Note: This step is not required if at least one of these is true: | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | | `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ | | `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ | +| `enable_swift_frontend_integration` | Enable experimental integration with swift-frontend added in Xcode 13 | `false` | ⬜️ | ## Backend cache server diff --git a/Rakefile b/Rakefile index a5020248..06b0de0e 100644 --- a/Rakefile +++ b/Rakefile @@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze RELEASES_ROOT_DIR = File.join('releases').freeze EXECUTABLE_NAME = 'XCRemoteCache' -EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcswift-frontend', 'xcld', 'xcldplusplus', 'xclipo'] +EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'xcld', 'xcldplusplus', 'xclipo'] PROJECT_NAME = 'XCRemoteCache' SWIFTLINT_ENABLED = true @@ -59,6 +59,9 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args| # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache` build_path_base = File.join(DERIVED_DATA_DIR, args.configuration) + # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create + # a symbolic link between swiftc->xcswiftc + system("cd #{build_path_base} && ln -s xcswiftc swiftc") sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)} build_paths.push(sdk_build_paths) diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift index aec0450c..6f0d0176 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift @@ -38,7 +38,8 @@ extension IntegrateContext { env: [String: String], binariesDir: URL, fakeSrcRoot: String, - outputPath: String? + outputPath: String?, + useFontendIntegration: Bool ) throws { projectPath = URL(fileURLWithPath: input) let srcRoot = projectPath.deletingLastPathComponent() @@ -47,10 +48,11 @@ extension IntegrateContext { configOverride = URL(fileURLWithPath: configOverridePath, relativeTo: srcRoot) output = outputPath.flatMap(URL.init(fileURLWithPath:)) self.fakeSrcRoot = URL(fileURLWithPath: fakeSrcRoot) + let swiftcBinaryName = useFontendIntegration ? "swiftc" : "xcswiftc" binaries = XCRCBinariesPaths( prepare: binariesDir.appendingPathComponent("xcprepare"), cc: binariesDir.appendingPathComponent("xccc"), - swiftc: binariesDir.appendingPathComponent("xcswiftc"), + swiftc: binariesDir.appendingPathComponent(swiftcBinaryName), libtool: binariesDir.appendingPathComponent("xclibtool"), lipo: binariesDir.appendingPathComponent("xclipo"), ld: binariesDir.appendingPathComponent("xcld"), diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift index acb52aae..7e61e3ca 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift @@ -88,7 +88,8 @@ public class XCIntegrate { env: env, binariesDir: binariesDir, fakeSrcRoot: fakeSrcRoot, - outputPath: output + outputPath: output, + useFontendIntegration: config.enableSwiftFrontendIntegration ) let configurationOracle = IncludeExcludeOracle( excludes: configurationsExclude.integrateArrayArguments, diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index 4b79827c..b8ff53ee 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -153,6 +153,8 @@ public struct XCRemoteCacheConfig: Encodable { /// If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch /// That might useful on CI, where a shallow clone is used var gracefullyHandleMissingCommonSha: Bool = false + /// Enable experimental integration with swift-frontend added in Xcode 13 + var enableSwiftFrontendIntegration: Bool = false } extension XCRemoteCacheConfig { @@ -213,6 +215,7 @@ extension XCRemoteCacheConfig { merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths merge.gracefullyHandleMissingCommonSha = scheme.gracefullyHandleMissingCommonSha ?? gracefullyHandleMissingCommonSha + merge.enableSwiftFrontendIntegration = scheme.enableSwiftFrontendIntegration ?? enableSwiftFrontendIntegration return merge } @@ -281,6 +284,7 @@ struct ConfigFileScheme: Decodable { let customRewriteEnvs: [String]? let irrelevantDependenciesPaths: [String]? let gracefullyHandleMissingCommonSha: Bool? + let enableSwiftFrontendIntegration: Bool? // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84 enum CodingKeys: String, CodingKey { @@ -332,6 +336,7 @@ struct ConfigFileScheme: Decodable { case customRewriteEnvs = "custom_rewrite_envs" case irrelevantDependenciesPaths = "irrelevant_dependencies_paths" case gracefullyHandleMissingCommonSha = "gracefully_handle_missing_common_sha" + case enableSwiftFrontendIntegration = "enable_swift_frontend_integration" } } From 62e518fb0eaf204127e9bb252e03675d5c1ed491 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 23 May 2023 22:05:35 -0700 Subject: [PATCH 15/33] Add pre/postbuild cleanup --- .../Commands/Postbuild/PostbuildContext.swift | 5 +++++ .../Commands/Postbuild/XCPostbuild.swift | 1 + .../Commands/Prebuild/PrebuildContext.swift | 5 +++++ .../Commands/Prebuild/XCPrebuild.swift | 1 + .../SwiftFrontend/XCSwiftFrontend.swift | 9 ++++++++- .../Dependencies/CacheModeController.swift | 15 +++++++++++++++ .../Commands/PostbuildContextTests.swift | 1 + .../Commands/PostbuildTests.swift | 3 ++- .../Commands/PrebuildTests.swift | 18 ++++++++++++------ .../PhaseCacheModeControllerTests.swift | 8 ++++++++ 10 files changed, 58 insertions(+), 8 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index d71ea9ae..77b0c337 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -89,6 +89,9 @@ public struct PostbuildContext { var publicHeadersFolderPath: URL? /// XCRemoteCache is explicitly disabled let disabled: Bool + /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier + /// it is used in the swift-frontend flow + let llbuildIdLockFile: URL } extension PostbuildContext { @@ -149,5 +152,7 @@ extension PostbuildContext { publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath) } disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: targetTempDir) } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index cc597bf5..235f56c1 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -60,6 +60,7 @@ public class XCPostbuild { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: FileDependenciesReader.init, markerWriter: NoopMarkerWriter.init, + llbuildLockFile: context.llbuildIdLockFile, fileManager: fileManager ) diff --git a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift index d36bcfaf..0a144c61 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift @@ -48,6 +48,9 @@ public struct PrebuildContext { let overlayHeadersPath: URL /// XCRemoteCache is explicitly disabled let disabled: Bool + /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier + /// it is used in the swift-frontend flow + let llbuildIdLockFile: URL } extension PrebuildContext { @@ -72,5 +75,7 @@ extension PrebuildContext { /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: targetTempDir) } } diff --git a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift index 97fc759a..b4c8dd8c 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift @@ -55,6 +55,7 @@ public class XCPrebuild { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: FileDependenciesReader.init, markerWriter: lazyMarkerWriterFactory, + llbuildLockFile: context.llbuildIdLockFile, fileManager: fileManager ) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index 2ec556bc..d8e45e93 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -249,7 +249,7 @@ public class XCSwiftFrontend: XCSwiftAbstract { let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") let (_, context) = buildContext() - let sharedLockFileURL = context.tempDir.appendingPathComponent(llbuildId).appendingPathExtension("lock") + let sharedLockFileURL = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: context.tempDir) let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) let action: SwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile @@ -263,3 +263,10 @@ public class XCSwiftFrontend: XCSwiftAbstract { super.run() } } + +extension XCSwiftFrontend { + /// The file is used to sycnhronize mutliple swift-frontend invocations + static func generateLlbuildIdSharedLock(llbuildId: String, tmpDir: URL) -> URL { + return tmpDir.appendingPathComponent(llbuildId).appendingPathExtension("lock") + } +} diff --git a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift index db8871c3..1600a3c0 100644 --- a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift +++ b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift @@ -48,6 +48,7 @@ class PhaseCacheModeController: CacheModeController { private let dependenciesWriter: DependenciesWriter private let dependenciesReader: DependenciesReader private let markerWriter: MarkerWriter + private let llbuildLockFile: URL private let fileManager: FileManager init( @@ -59,6 +60,7 @@ class PhaseCacheModeController: CacheModeController { dependenciesWriter: (URL, FileManager) -> DependenciesWriter, dependenciesReader: (URL, FileManager) -> DependenciesReader, markerWriter: (URL, FileManager) -> MarkerWriter, + llbuildLockFile: URL, fileManager: FileManager ) { @@ -69,10 +71,12 @@ class PhaseCacheModeController: CacheModeController { let discoveryURL = tempDir.appendingPathComponent(phaseDependencyPath) self.dependenciesWriter = dependenciesWriter(discoveryURL, fileManager) self.dependenciesReader = dependenciesReader(discoveryURL, fileManager) + self.llbuildLockFile = llbuildLockFile self.markerWriter = markerWriter(modeMarker, fileManager) } func enable(allowedInputFiles: [URL], dependencies: [URL]) throws { + try cleanupLlbuildLock() // marker file contains filepaths that contribute to the build products // and should invalidate all other target steps (swiftc,libtool etc.) let targetSensitiveFiles = dependencies + [modeMarker, Self.xcodeSelectLink] @@ -84,6 +88,7 @@ class PhaseCacheModeController: CacheModeController { } func disable() throws { + try cleanupLlbuildLock() guard !forceCached else { throw PhaseCacheModeControllerError.cannotUseRemoteCacheForForcedCacheMode } @@ -114,4 +119,14 @@ class PhaseCacheModeController: CacheModeController { } return false } + + private func cleanupLlbuildLock() throws { + if fileManager.fileExists(atPath: llbuildLockFile.path) { + do { + try fileManager.removeItem(at: llbuildLockFile) + } catch { + printWarning("Removing llbuild lock at \(llbuildLockFile.path) failed. Error: \(error)") + } + } + } } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift index aadb3249..9c6f4eda 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift @@ -45,6 +45,7 @@ class PostbuildContextTests: FileXCTestCase { "DERIVED_SOURCES_DIR": "DERIVED_SOURCES_DIR", "CURRENT_VARIANT": "normal", "PUBLIC_HEADERS_FOLDER_PATH": "/usr/local/include", + "LLBUILD_BUILD_ID": "1" ] override func setUpWithError() throws { diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index 31c34595..9c9f39b8 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -57,7 +57,8 @@ class PostbuildTests: FileXCTestCase { overlayHeadersPath: "", irrelevantDependenciesPaths: [], publicHeadersFolderPath: nil, - disabled: false + disabled: false, + llbuildIdLockFile: "/file" ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), diff --git a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift index 84bcea4b..aa5e1131 100644 --- a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift @@ -64,7 +64,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) contextCached = PrebuildContext( targetTempDir: sampleURL, @@ -78,7 +79,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) organizer = ArtifactOrganizerFake(artifactRoot: artifactsRoot, unzippedExtension: "unzip") globalCacheSwitcher = InMemoryGlobalCacheSwitcher() @@ -244,7 +246,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) let prebuild = Prebuild( @@ -276,7 +279,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) metaContent = try generateMeta(fingerprint: generator.generate(), filekey: "1") let downloadedArtifactPackage = artifactsRoot.appendingPathComponent("1") @@ -340,7 +344,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: false, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) try globalCacheSwitcher.enable(sha: "1") let prebuild = Prebuild( @@ -372,7 +377,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: true + disabled: true, + llbuildIdLockFile: "/tmp/lock" ) let prebuild = Prebuild( diff --git a/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift b/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift index 3ad44a4b..8a14b71e 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift @@ -34,6 +34,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/file", fileManager: FileManager.default ) @@ -51,6 +52,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -68,6 +70,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -85,6 +88,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -105,6 +109,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -125,6 +130,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in DependenciesWriterSpy() }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in markerWriter }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -142,6 +148,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in DependenciesWriterSpy() }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -163,6 +170,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in markerWriterSpy }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) From b6b63bd0dcf72de2a4f06dff745d9bc7b2ebfd98 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Wed, 24 May 2023 18:50:48 -0700 Subject: [PATCH 16/33] Wrap critical section in swiftFrontendOrchestrator --- .../SwiftFrontendOrchestrator.swift | 57 ++++++++++++++----- .../SwiftFrontend/XCSwiftFrontend.swift | 8 +-- .../Commands/Swiftc/XCSwiftc.swift | 6 +- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift index c77621a6..c4c1fa19 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift @@ -19,12 +19,19 @@ import Foundation -/// Performs the `swift-frontend` logic: +/// Manages the `swift-frontend` logic +protocol SwiftFrontendOrchestrator { + /// Executes the criticial secion according to the required order + /// - Parameter criticalSection: the block that should be synchronized + func run(criticalSection: () -> ()) throws +} + +/// The default orchestrator that manages the order or swift-frontend invocations. /// For emit-module (the "first" process) action, it locks a shared file between all swift-frontend invcations, /// verifies that the mocking can be done and continues the mocking/fallbacking along the lock release /// For the compilation action, tries to ackquire a lock and waits until the "emit-module" makes a decision /// if the compilation should be skipped and a "mocking" should used instead -class SwiftFrontendOrchestrator { +class CommonSwiftFrontendOrchestrator { /// Content saved to the shared file /// Safe to use forced unwrapping private static let emitModuleContent = "done".data(using: .utf8)! @@ -47,39 +54,59 @@ class SwiftFrontendOrchestrator { self.lockAccessor = lockAccessor } - func run() throws { + func run(criticalSection: () throws -> ()) throws { guard case .consumer(commit: .available) = mode else { // no need to lock anything - just allow fallbacking to the `swiftc or swift-frontend` + // if we face producer or a consumer where RC is disabled (we have already caught the + // cache miss) + try criticalSection() return } - try waitForEmitModuleLock() + try waitForEmitModuleLock(criticalSection: criticalSection) } - private func executeMockAttemp() throws { + private func executeMockAttemp(criticalSection: () throws -> ()) throws { switch action { case .emitModule: - try validateEmitModuleStep() + try validateEmitModuleStep(criticalSection: criticalSection) case .compile: - try waitForEmitModuleLock() + try waitForEmitModuleLock(criticalSection: criticalSection) } } - private func validateEmitModuleStep() throws { - try lockAccessor.exclusiveAccess { handle in - // TODO: check if the mocking compilation can happen (make sure - // all input files are listed in the list of dependencies) - handle.write(SwiftFrontendOrchestrator.emitModuleContent) + /// Fro emit-module, wraps the critical section with the shared lock so other processes (compilation) + /// have to wait until it finishes. + /// Once the emit-module is done, the "magical" string is saved to the file and the lock is released + /// + /// Note: The design of wrapping the entire "emit-module" has a small performance downside if inside + /// the critical section, the code realizes that remote cache cannot be used (in practice - a new file has been added) + /// None of compilation process (so with '-c' args) can continue until the entire emit-module logic finishes. + /// Because it is expected to happen no that often and emit-module is usually quite fast, this makes the + /// implementation way simpler. If we ever want to optimize it, we should release the lock as early + /// as we know, the remote cache cannot be used. Then all other compilation process (-c) can run + /// in parallel with emit-module + private func validateEmitModuleStep(criticalSection: () throws -> ()) throws { + try lockAccessor.exclusiveAccess { handle in + defer { + handle.write(Self.self.emitModuleContent) + } + do { + try criticalSection() + } } } /// Locks a shared file in a loop until its content non-empty, which means the "parent" emit-module has finished - private func waitForEmitModuleLock() throws { + private func waitForEmitModuleLock(criticalSection: () throws -> ()) throws { + // emit-module process should really quickly retain a lock (it is always invoked + // by Xcode as a first process) while true { - // TODO: add a max timeout + // TODO: consider adding a max timeout to cover a case when emit-module crashes try lockAccessor.exclusiveAccess { handle in if !handle.availableData.isEmpty { - // the file is not empty so emit-module is done with the "check" + // the file is not empty so the emit-module process is done with the "check" + try criticalSection() return } } diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index d8e45e93..a1955496 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -252,15 +252,13 @@ public class XCSwiftFrontend: XCSwiftAbstract { let sharedLockFileURL = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: context.tempDir) let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) - let action: SwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile - let orchestrator = SwiftFrontendOrchestrator(mode: context.mode, action: action, lockAccessor: sharedLock) + let action: CommonSwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile + let swiftFrontendOrchestrator = CommonSwiftFrontendOrchestrator(mode: context.mode, action: action, lockAccessor: sharedLock) - try orchestrator.run() + try swiftFrontendOrchestrator.run(criticalSection: super.run) } catch { printWarning("Cannot correctly orchestrate the \(command) with params \(inputArgs): error: \(error)") } - - super.run() } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 1ecf08df..9d906a4c 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -45,15 +45,15 @@ public struct SwiftcArgInput { } } -public class XCSwiftAbstract { +public class XCSwiftAbstract { let command: String - let inputArgs: T + let inputArgs: InputArgs private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter private let touchFactory: (URL, FileManager) -> Touch public init( command: String, - inputArgs: T, + inputArgs: InputArgs, dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, touchFactory: @escaping (URL, FileManager) -> Touch ) { From 623bbb68405c5ade824e08ea17d518cf7a92f9ef Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Wed, 24 May 2023 19:37:48 -0700 Subject: [PATCH 17/33] Add unitial tests --- .../SwiftFrontendOrchestrator.swift | 7 +- .../SwiftFrontendOrchestratorTests.swift | 133 ++++++++++++++++++ 2 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift index c4c1fa19..e354cda6 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift @@ -62,7 +62,7 @@ class CommonSwiftFrontendOrchestrator { try criticalSection() return } - try waitForEmitModuleLock(criticalSection: criticalSection) + try executeMockAttemp(criticalSection: criticalSection) } private func executeMockAttemp(criticalSection: () throws -> ()) throws { @@ -101,13 +101,14 @@ class CommonSwiftFrontendOrchestrator { private func waitForEmitModuleLock(criticalSection: () throws -> ()) throws { // emit-module process should really quickly retain a lock (it is always invoked // by Xcode as a first process) - while true { + var executed = false + while !executed { // TODO: consider adding a max timeout to cover a case when emit-module crashes try lockAccessor.exclusiveAccess { handle in if !handle.availableData.isEmpty { // the file is not empty so the emit-module process is done with the "check" try criticalSection() - return + executed = true } } } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift new file mode 100644 index 00000000..5fc128d4 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift @@ -0,0 +1,133 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + + +final class SwiftFrontendOrchestratorTests: FileXCTestCase { + private let prohibitedAccessor = DisallowedExclusiveFileAccessor() + private var nonEmptyFile: URL! + + override func setUp() async throws { + nonEmptyFile = try prepareTempDir().appendingPathComponent("lock.lock") + try fileManager.write(toPath: nonEmptyFile.path, contents: "Done".data(using: .utf8)) + } + func testRunsCriticalSectionImmediatellyForProducer() throws { + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .producer, + action: .compile, + lockAccessor: prohibitedAccessor + ) + + var invoked = false + try orchestrator.run { + invoked = true + } + XCTAssertTrue(invoked) + } + + func testRunsCriticalSectionImmediatellyForDisabledConsumer() throws { + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .unavailable), + action: .compile, + lockAccessor: prohibitedAccessor + ) + + var invoked = false + try orchestrator.run { + invoked = true + } + XCTAssertTrue(invoked) + } + + func testRunsEmitModuleLogicInAnExclusiveLock() throws { + let lock = FakeExclusiveFileAccessor() + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .emitModule, + lockAccessor: lock + ) + + var invoked = false + try orchestrator.run { + XCTAssertTrue(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } + + func testCompilationInvokesCriticalSectionOnlyForNonEmptyLockFile() throws { + let lock = FakeExclusiveFileAccessor(pattern: [.empty, .nonEmpty(nonEmptyFile)]) + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .compile, + lockAccessor: lock + ) + + var invoked = false + try orchestrator.run { + XCTAssertTrue(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } +} + +private class DisallowedExclusiveFileAccessor: ExclusiveFileAccessor { + func exclusiveAccess(block: (FileHandle) throws -> (T)) throws -> T { + throw "Invoked ProhibitedExclusiveFileAccessor" + } +} + +// Thread-unsafe, in-memory lock +private class FakeExclusiveFileAccessor: ExclusiveFileAccessor { + private(set) var isLocked = false + private var pattern: [LockFileContent] + + enum LockFileContent { + case empty + case nonEmpty(URL) + + func fileHandle() throws -> FileHandle { + switch self { + case .empty: return FileHandle.nullDevice + case .nonEmpty(let url): return try FileHandle(forReadingFrom: url) + } + } + } + + init(pattern: [LockFileContent] = []) { + // keep in the reversed order to always pop + self.pattern = pattern.reversed() + } + + func exclusiveAccess(block: (FileHandle) throws -> (T)) throws -> T { + if isLocked { + throw "FakeExclusiveFileAccessor lock is already locked" + } + defer { + isLocked = false + } + isLocked = true + let fileHandle = try (pattern.popLast() ?? .empty).fileHandle() + return try block(fileHandle) + } + +} From 6c3db27c21176cd22977a433cb4721e59d1a07b8 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Wed, 24 May 2023 21:19:40 -0700 Subject: [PATCH 18/33] Do not cache build context from swift-frontend --- .../Commands/SwiftFrontend/XCSwiftFrontend.swift | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index a1955496..b6409d3d 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -203,7 +203,6 @@ public struct SwiftFrontendArgInput { public class XCSwiftFrontend: XCSwiftAbstract { private let env: [String: String] - private var cachedSwiftdContext: (XCRemoteCacheConfig, SwiftcContext)? public init( command: String, @@ -222,10 +221,6 @@ public class XCSwiftFrontend: XCSwiftAbstract { } override func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { - if let cachedSwiftdContext = cachedSwiftdContext { - return cachedSwiftdContext - } - let fileManager = FileManager.default let config: XCRemoteCacheConfig let context: SwiftcContext @@ -238,9 +233,9 @@ public class XCSwiftFrontend: XCSwiftAbstract { } catch { exit(1, "FATAL: XCSwiftFrontend initialization failed with error: \(error)") } - let builtContext = (config, context) - self.cachedSwiftdContext = builtContext - return builtContext + // do not cache this context, as it is subject to change when + // the emit-module finds that the cached artifact cannot be used + return (config, context) } override public func run() { From 85bdb471ca82d9dfa62a012a7c12ac2c67ac238d Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 25 May 2023 20:55:52 -0700 Subject: [PATCH 19/33] Add max timeout for a locking --- .../SwiftFrontendOrchestrator.swift | 16 ++++++++-- .../SwiftFrontend/XCSwiftFrontend.swift | 9 +++++- .../SwiftFrontendOrchestratorTests.swift | 30 ++++++++++++++++--- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift index e354cda6..1255f038 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift @@ -43,15 +43,18 @@ class CommonSwiftFrontendOrchestrator { private let mode: SwiftcContext.SwiftcMode private let action: Action private let lockAccessor: ExclusiveFileAccessor + private let maxLockTimeout: TimeInterval init( mode: SwiftcContext.SwiftcMode, action: Action, - lockAccessor: ExclusiveFileAccessor + lockAccessor: ExclusiveFileAccessor, + maxLockTimeout: TimeInterval ) { self.mode = mode self.action = action self.lockAccessor = lockAccessor + self.maxLockTimeout = maxLockTimeout } func run(criticalSection: () throws -> ()) throws { @@ -102,8 +105,8 @@ class CommonSwiftFrontendOrchestrator { // emit-module process should really quickly retain a lock (it is always invoked // by Xcode as a first process) var executed = false + let startingDate = Date() while !executed { - // TODO: consider adding a max timeout to cover a case when emit-module crashes try lockAccessor.exclusiveAccess { handle in if !handle.availableData.isEmpty { // the file is not empty so the emit-module process is done with the "check" @@ -111,6 +114,15 @@ class CommonSwiftFrontendOrchestrator { executed = true } } + // When a max locking time is achieved, execute anyway + if !executed && Date().timeIntervalSince(startingDate) > self.maxLockTimeout { + errorLog(""" + Executing command \(action) without lock synchronization. That may be cause by the\ + crashed or extremly long emit-module. Contact XCRemoteCache authors about this error. + """) + try criticalSection() + executed = true + } } } } diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index b6409d3d..fd167639 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -202,6 +202,8 @@ public struct SwiftFrontendArgInput { } public class XCSwiftFrontend: XCSwiftAbstract { + // don't lock individual compilation invocations for more than 10s + private static let MaxLockingTimeout: TimeInterval = 10 private let env: [String: String] public init( @@ -248,7 +250,12 @@ public class XCSwiftFrontend: XCSwiftAbstract { let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) let action: CommonSwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile - let swiftFrontendOrchestrator = CommonSwiftFrontendOrchestrator(mode: context.mode, action: action, lockAccessor: sharedLock) + let swiftFrontendOrchestrator = CommonSwiftFrontendOrchestrator( + mode: context.mode, + action: action, + lockAccessor: sharedLock, + maxLockTimeout: Self.self.MaxLockingTimeout + ) try swiftFrontendOrchestrator.run(criticalSection: super.run) } catch { diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift index 5fc128d4..18c07f11 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift @@ -24,6 +24,7 @@ import XCTest final class SwiftFrontendOrchestratorTests: FileXCTestCase { private let prohibitedAccessor = DisallowedExclusiveFileAccessor() private var nonEmptyFile: URL! + private let maxLocking: TimeInterval = 10 override func setUp() async throws { nonEmptyFile = try prepareTempDir().appendingPathComponent("lock.lock") @@ -33,7 +34,8 @@ final class SwiftFrontendOrchestratorTests: FileXCTestCase { let orchestrator = CommonSwiftFrontendOrchestrator( mode: .producer, action: .compile, - lockAccessor: prohibitedAccessor + lockAccessor: prohibitedAccessor, + maxLockTimeout: maxLocking ) var invoked = false @@ -47,7 +49,8 @@ final class SwiftFrontendOrchestratorTests: FileXCTestCase { let orchestrator = CommonSwiftFrontendOrchestrator( mode: .consumer(commit: .unavailable), action: .compile, - lockAccessor: prohibitedAccessor + lockAccessor: prohibitedAccessor, + maxLockTimeout: maxLocking ) var invoked = false @@ -62,7 +65,8 @@ final class SwiftFrontendOrchestratorTests: FileXCTestCase { let orchestrator = CommonSwiftFrontendOrchestrator( mode: .consumer(commit: .available(commit: "")), action: .emitModule, - lockAccessor: lock + lockAccessor: lock, + maxLockTimeout: maxLocking ) var invoked = false @@ -78,7 +82,8 @@ final class SwiftFrontendOrchestratorTests: FileXCTestCase { let orchestrator = CommonSwiftFrontendOrchestrator( mode: .consumer(commit: .available(commit: "")), action: .compile, - lockAccessor: lock + lockAccessor: lock, + maxLockTimeout: maxLocking ) var invoked = false @@ -88,6 +93,23 @@ final class SwiftFrontendOrchestratorTests: FileXCTestCase { } XCTAssertTrue(invoked) } + + func testExecutesActionWithoutLockIfLockingFileIsEmptyForALongTime() throws { + let lock = FakeExclusiveFileAccessor(pattern: []) + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .compile, + lockAccessor: lock, + maxLockTimeout: 0 + ) + + var invoked = false + try orchestrator.run { + XCTAssertFalse(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } } private class DisallowedExclusiveFileAccessor: ExclusiveFileAccessor { From 33e82701e4a93d2d0271a662a0705dab5f1b270b Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 25 May 2023 20:59:26 -0700 Subject: [PATCH 20/33] Zip with symbolic links --- Rakefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index 06b0de0e..59b224f9 100644 --- a/Rakefile +++ b/Rakefile @@ -133,7 +133,9 @@ def create_release_zip(build_paths) # Create and move files into the release directory mkdir_p release_dir build_paths.each {|p| - cp_r p, release_dir + # -r for recursive + # -P for copying symbolic link as is + system("cp -rP #{p} #{release_dir}") } output_artifact_basename = "#{PROJECT_NAME}.zip" @@ -142,7 +144,8 @@ def create_release_zip(build_paths) # -X: no extras (uid, gid, file times, ...) # -x: exclude .DS_Store # -r: recursive - system("zip -X -x '*.DS_Store' -r #{output_artifact_basename} .") or abort "zip failure" + # -y: to store symbolic links (used for swiftc -> xcswiftc) + system("zip -X -x '*.DS_Store' -r -y #{output_artifact_basename} .") or abort "zip failure" # List contents of zip file system("unzip -l #{output_artifact_basename}") or abort "unzip failure" end From de5e25d5e0eb5181adcb2b967cea7af0185d0611 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 25 May 2023 21:17:55 -0700 Subject: [PATCH 21/33] Switch Standalone to the swift-frontend integration --- e2eTests/StandaloneSampleApp/.rcinfo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2eTests/StandaloneSampleApp/.rcinfo b/e2eTests/StandaloneSampleApp/.rcinfo index 60030012..75a50801 100644 --- a/e2eTests/StandaloneSampleApp/.rcinfo +++ b/e2eTests/StandaloneSampleApp/.rcinfo @@ -1,8 +1,9 @@ --- -cache_addresses: +cache_addresses: - 'http://localhost:8080/cache/pods' primary_repo: '.' primary_branch: 'e2e-test-branch' mode: 'consumer' final_target': XCRemoteCacheSample' artifact_maximum_age: 0 # do not use local cache in ~/Library/Caches/XCRemoteCache +enable_swift_frontend_integration: true From 623967cef399451723eea5a9f6cac4511a910e6f Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 25 May 2023 21:44:04 -0700 Subject: [PATCH 22/33] Add driver integration to integrator --- README.md | 2 +- .../BuildSettingsIntegrateAppender.swift | 19 ++++++- .../Prepare/Integrate/XCIntegrate.swift | 11 +++- .../Config/XCRemoteCacheConfig.swift | 10 ++-- ...jBuildSettingsIntegrateAppenderTests.swift | 51 ++++++++++++++++--- e2eTests/StandaloneSampleApp/.rcinfo | 2 +- 6 files changed, 77 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b37fa738..9d861e4c 100755 --- a/README.md +++ b/README.md @@ -359,7 +359,7 @@ Note: This step is not required if at least one of these is true: | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | | `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ | | `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ | -| `enable_swift_frontend_integration` | Enable experimental integration with swift-frontend added in Xcode 13 | `false` | ⬜️ | +| `enable_swift_driver_integration` | Enable experimental integration with swift driver, added in Xcode 13 | `false` | ⬜️ | ## Backend cache server diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift index 70c9c0ea..ec4f4b1c 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift @@ -21,6 +21,11 @@ import Foundation typealias BuildSettings = [String: Any] +struct BuildSettingsIntegrateAppenderOption: OptionSet { + let rawValue: Int + + static let disableSwiftDriverIntegration = BuildSettingsIntegrateAppenderOption(rawValue: 1 << 0) +} // Manages Xcode build settings protocol BuildSettingsIntegrateAppender { /// Appends XCRemoteCache-specific build settings @@ -35,18 +40,28 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender { private let repoRoot: URL private let fakeSrcRoot: URL private let sdksExclude: [String] + private let options: BuildSettingsIntegrateAppenderOption - init(mode: Mode, repoRoot: URL, fakeSrcRoot: URL, sdksExclude: [String]) { + init( + mode: Mode, + repoRoot: URL, + fakeSrcRoot: URL, + sdksExclude: [String], + options: BuildSettingsIntegrateAppenderOption + ) { self.mode = mode self.repoRoot = repoRoot self.fakeSrcRoot = fakeSrcRoot self.sdksExclude = sdksExclude + self.options = options } func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings { var result = buildSettings setBuildSetting(buildSettings: &result, key: "SWIFT_EXEC", value: wrappers.swiftc.path ) - setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" ) + if options.contains(.disableSwiftDriverIntegration) { + setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" ) + } // When generating artifacts, no need to shell-out all compilation commands to our wrappers if case .consumer = mode { setBuildSetting(buildSettings: &result, key: "CC", value: wrappers.cc.path ) diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift index 7e61e3ca..2c81af8a 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift @@ -89,7 +89,7 @@ public class XCIntegrate { binariesDir: binariesDir, fakeSrcRoot: fakeSrcRoot, outputPath: output, - useFontendIntegration: config.enableSwiftFrontendIntegration + useFontendIntegration: config.enableSwifDriverIntegration ) let configurationOracle = IncludeExcludeOracle( excludes: configurationsExclude.integrateArrayArguments, @@ -99,11 +99,18 @@ public class XCIntegrate { excludes: targetsExclude.integrateArrayArguments, includes: targetsInclude.integrateArrayArguments ) + let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption + if config.enableSwifDriverIntegration { + buildSettingsAppenderOptions = [] + } else { + buildSettingsAppenderOptions = [.disableSwiftDriverIntegration] + } let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender( mode: context.mode, repoRoot: context.repoRoot, fakeSrcRoot: context.fakeSrcRoot, - sdksExclude: sdksExclude.integrateArrayArguments + sdksExclude: sdksExclude.integrateArrayArguments, + options: buildSettingsAppenderOptions ) let lldbPatcher: LLDBInitPatcher switch lldbMode { diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index b8ff53ee..cad41e63 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -153,8 +153,8 @@ public struct XCRemoteCacheConfig: Encodable { /// If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch /// That might useful on CI, where a shallow clone is used var gracefullyHandleMissingCommonSha: Bool = false - /// Enable experimental integration with swift-frontend added in Xcode 13 - var enableSwiftFrontendIntegration: Bool = false + /// Enable experimental integration with swift driver, added in Xcode 13 + var enableSwifDriverIntegration: Bool = false } extension XCRemoteCacheConfig { @@ -215,7 +215,7 @@ extension XCRemoteCacheConfig { merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths merge.gracefullyHandleMissingCommonSha = scheme.gracefullyHandleMissingCommonSha ?? gracefullyHandleMissingCommonSha - merge.enableSwiftFrontendIntegration = scheme.enableSwiftFrontendIntegration ?? enableSwiftFrontendIntegration + merge.enableSwifDriverIntegration = scheme.enableSwifDriverIntegration ?? enableSwifDriverIntegration return merge } @@ -284,7 +284,7 @@ struct ConfigFileScheme: Decodable { let customRewriteEnvs: [String]? let irrelevantDependenciesPaths: [String]? let gracefullyHandleMissingCommonSha: Bool? - let enableSwiftFrontendIntegration: Bool? + let enableSwifDriverIntegration: Bool? // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84 enum CodingKeys: String, CodingKey { @@ -336,7 +336,7 @@ struct ConfigFileScheme: Decodable { case customRewriteEnvs = "custom_rewrite_envs" case irrelevantDependenciesPaths = "irrelevant_dependencies_paths" case gracefullyHandleMissingCommonSha = "gracefully_handle_missing_common_sha" - case enableSwiftFrontendIntegration = "enable_swift_frontend_integration" + case enableSwifDriverIntegration = "enable_swift_driver_integration" } } diff --git a/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift index 66da9aa7..819323c7 100644 --- a/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift @@ -49,7 +49,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let resultURL = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String) @@ -64,7 +65,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let resultURL: String = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String) @@ -79,7 +81,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlus: String = try XCTUnwrap(result["LDPLUSPLUS"] as? String) @@ -93,7 +96,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*"] + sdksExclude: ["watchOS*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlusWatchOS: String = try XCTUnwrap(result["LDPLUSPLUS[sdk=watchOS*]"] as? String) @@ -107,7 +111,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*"] + sdksExclude: ["watchOS*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let libtoolWatchOS: String = try XCTUnwrap(result["LIBTOOL[sdk=watchOS*]"] as? String) @@ -121,7 +126,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*", "watchsimulator*"] + sdksExclude: ["watchOS*", "watchsimulator*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlusWatchOS: String = try XCTUnwrap(result["LDPLUSPLUS[sdk=watchOS*]"] as? String) @@ -137,7 +143,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*", "watchsimulator*"] + sdksExclude: ["watchOS*", "watchsimulator*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let disabledWatchOS: String = try XCTUnwrap(result["XCRC_DISABLED[sdk=watchOS*]"] as? String) @@ -146,4 +153,34 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { XCTAssertEqual(disabledWatchOS, "YES") XCTAssertEqual(disabledWatchSimulator, "YES") } + + func testExcludesSwiftFrontendIntegrationForSpecificOption() throws { + let mode: Mode = .consumer + let appender = XcodeProjBuildSettingsIntegrateAppender( + mode: mode, + repoRoot: rootURL, + fakeSrcRoot: "/", + sdksExclude: [], + options: [.disableSwiftDriverIntegration] + ) + let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) + let useSwiftIntegrationDriver: String = try XCTUnwrap(result["SWIFT_USE_INTEGRATED_DRIVER"] as? String) + + XCTAssertEqual(useSwiftIntegrationDriver, "NO") + } + + func testDoesntExcludesSwiftFrontendIntegrationForEmptyOptions() throws { + let mode: Mode = .consumer + let appender = XcodeProjBuildSettingsIntegrateAppender( + mode: mode, + repoRoot: rootURL, + fakeSrcRoot: "/", + sdksExclude: [], + options: [] + ) + let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) + let useSwiftIntegrationDriver: String? = result["SWIFT_USE_INTEGRATED_DRIVER"] as? String + + XCTAssertNil(useSwiftIntegrationDriver) + } } diff --git a/e2eTests/StandaloneSampleApp/.rcinfo b/e2eTests/StandaloneSampleApp/.rcinfo index 75a50801..3cd0f26b 100644 --- a/e2eTests/StandaloneSampleApp/.rcinfo +++ b/e2eTests/StandaloneSampleApp/.rcinfo @@ -6,4 +6,4 @@ primary_branch: 'e2e-test-branch' mode: 'consumer' final_target': XCRemoteCacheSample' artifact_maximum_age: 0 # do not use local cache in ~/Library/Caches/XCRemoteCache -enable_swift_frontend_integration: true +enable_swift_driver_integration: true From 6ee9a108ad01054de103fb9f30f5580c4f9581c2 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 25 May 2023 21:52:51 -0700 Subject: [PATCH 23/33] Use swift-frontend --- Rakefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index 59b224f9..45a4e867 100644 --- a/Rakefile +++ b/Rakefile @@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze RELEASES_ROOT_DIR = File.join('releases').freeze EXECUTABLE_NAME = 'XCRemoteCache' -EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'xcld', 'xcldplusplus', 'xclipo'] +EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo'] PROJECT_NAME = 'XCRemoteCache' SWIFTLINT_ENABLED = true @@ -60,8 +60,9 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args| # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache` build_path_base = File.join(DERIVED_DATA_DIR, args.configuration) # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create - # a symbolic link between swiftc->xcswiftc + # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend system("cd #{build_path_base} && ln -s xcswiftc swiftc") + system("cd #{build_path_base} && ln -s xcswift-frontend swift-frontend") sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)} build_paths.push(sdk_build_paths) From 5c2d0d07585730831eae3cc6dbb50f16b1bf94b9 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 25 May 2023 22:03:40 -0700 Subject: [PATCH 24/33] Align fallback swiftc --- .../SwiftFrontend/XCSwiftFrontend.swift | 5 +- .../Commands/Swiftc/XCSwiftc.swift | 8 +-- .../XCSwiftcFrontendMain.swift | 2 +- Sources/xcswiftc/XCSwiftcMain.swift | 53 +++++++++++-------- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index fd167639..5f456b1b 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -240,7 +240,7 @@ public class XCSwiftFrontend: XCSwiftAbstract { return (config, context) } - override public func run() { + override public func run() throws { do { /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") @@ -259,7 +259,8 @@ public class XCSwiftFrontend: XCSwiftAbstract { try swiftFrontendOrchestrator.run(criticalSection: super.run) } catch { - printWarning("Cannot correctly orchestrate the \(command) with params \(inputArgs): error: \(error)") + defaultLog("Cannot correctly orchestrate the \(command) with params \(inputArgs): error: \(error)") + throw error } } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 9d906a4c..58a521ee 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -68,7 +68,7 @@ public class XCSwiftAbstract { } // swiftlint:disable:next function_body_length - public func run() { + public func run() throws { let fileManager = FileManager.default let (config, context) = buildContext() @@ -163,11 +163,7 @@ public class XCSwiftAbstract { invocationStorage: invocationStorage, shellOut: shellOut ) - do { - try orchestrator.run() - } catch { - exit(1, "Swiftc failed with error: \(error)") - } + try orchestrator.run() } } diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift index 03dda52f..528f86a3 100644 --- a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -104,7 +104,7 @@ public class XCSwiftcFrontendMain { env: env, dependenciesWriter: FileDependenciesWriter.init, touchFactory: FileTouch.init) - frontend.run() + try frontend.run() } catch { let swiftFrontendCommand = "swift-frontend" diff --git a/Sources/xcswiftc/XCSwiftcMain.swift b/Sources/xcswiftc/XCSwiftcMain.swift index acb04868..ceadab6b 100644 --- a/Sources/xcswiftc/XCSwiftcMain.swift +++ b/Sources/xcswiftc/XCSwiftcMain.swift @@ -60,30 +60,37 @@ public class XCSwiftcMain { let targetInputInput = target, let swiftFileListInput = swiftFileList else { - let swiftcCommand = "swiftc" - print("Fallbacking to compilation using \(swiftcCommand).") + executeFallback() + } + do { + let swiftcArgsInput = SwiftcArgInput( + objcHeaderOutput: objcHeaderOutputInput, + moduleName: moduleNameInput, + modulePathOutput: modulePathOutputInput, + filemap: filemapInput, + target: targetInputInput, + fileList: swiftFileListInput + ) + try XCSwiftc( + command: command, + inputArgs: swiftcArgsInput, + dependenciesWriter: FileDependenciesWriter.init, + touchFactory: FileTouch.init + ).run() + } catch { + executeFallback() + } + } + private func executeFallback() -> Never { + let swiftcCommand = "swiftc" + print("Fallbacking to compilation using \(swiftcCommand).") - let args = ProcessInfo().arguments - let paramList = [swiftcCommand] + args.dropFirst() - let cargs = paramList.map { strdup($0) } + [nil] - execvp(swiftcCommand, cargs) + let args = ProcessInfo().arguments + let paramList = [swiftcCommand] + args.dropFirst() + let cargs = paramList.map { strdup($0) } + [nil] + execvp(swiftcCommand, cargs) - /// C-function `execv` returns only when the command fails - exit(1) - } - let swiftcArgsInput = SwiftcArgInput( - objcHeaderOutput: objcHeaderOutputInput, - moduleName: moduleNameInput, - modulePathOutput: modulePathOutputInput, - filemap: filemapInput, - target: targetInputInput, - fileList: swiftFileListInput - ) - XCSwiftc( - command: command, - inputArgs: swiftcArgsInput, - dependenciesWriter: FileDependenciesWriter.init, - touchFactory: FileTouch.init - ).run() + /// C-function `execv` returns only when the command fails + exit(1) } } From bfd4f975f25a1ca83a24af4d4e53ef0ce7e917f2 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Thu, 25 May 2023 22:18:54 -0700 Subject: [PATCH 25/33] Use swift-frontend from Xcode --- Sources/xcswift-frontend/XCSwiftcFrontendMain.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift index 528f86a3..65580d8f 100644 --- a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -106,7 +106,9 @@ public class XCSwiftcFrontendMain { touchFactory: FileTouch.init) try frontend.run() } catch { - let swiftFrontendCommand = "swift-frontend" + let developerDir = env["DEVELOPER_DIR"]! + // limitation: always using the Xcode's toolchain + let swiftFrontendCommand = "\(developerDir)/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend" let args = ProcessInfo().arguments let paramList = [swiftFrontendCommand] + args.dropFirst() From eb4da759db959824d630528c491316eb817e6417 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Fri, 26 May 2023 18:16:36 -0700 Subject: [PATCH 26/33] Validate early --- .../Commands/SwiftFrontend/XCSwiftFrontend.swift | 6 ++++++ Sources/xcswift-frontend/XCSwiftcFrontendMain.swift | 1 + 2 files changed, 7 insertions(+) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index 5f456b1b..c580e66e 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -93,6 +93,12 @@ public struct SwiftFrontendArgInput { self.docPath = docPath } + public func validate() throws { + guard compile != emitModule else { + throw SwiftFrontendArgInputError.bothCompilationAndEmitAction + } + } + // swiftlint:disable:next cyclomatic_complexity function_body_length func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext { guard compile != emitModule else { diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift index 65580d8f..2fd4f842 100644 --- a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -98,6 +98,7 @@ public class XCSwiftcFrontendMain { docPath: docPath ) do { + try argInput.validate() let frontend = try XCSwiftFrontend( command: command, inputArgs: argInput, From 23a67dada8ca1d8196f017db2314350a9bfb33d0 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Fri, 26 May 2023 18:19:51 -0700 Subject: [PATCH 27/33] --amend --- .../Commands/SwiftFrontend/XCSwiftFrontend.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index c580e66e..3e1f7892 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -93,6 +93,9 @@ public struct SwiftFrontendArgInput { self.docPath = docPath } + /// An early validation of the swift-frontend args + /// Returns false an error, if the mode is not supported + /// and the fallback to the undelying command should be executed public func validate() throws { guard compile != emitModule else { throw SwiftFrontendArgInputError.bothCompilationAndEmitAction @@ -101,9 +104,6 @@ public struct SwiftFrontendArgInput { // swiftlint:disable:next cyclomatic_complexity function_body_length func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext { - guard compile != emitModule else { - throw SwiftFrontendArgInputError.bothCompilationAndEmitAction - } let inputPathsCount = inputPaths.count let primaryInputsCount = primaryInputPaths.count guard inputPathsCount > 0 else { From de4bb408106555afa5530c1fc1c4e278885ec385 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Fri, 26 May 2023 19:47:48 -0700 Subject: [PATCH 28/33] Cleanup --- .../SwiftFrontend/SwiftFrontendArgInput.swift | 202 +++++++++++++++++ .../SwiftFrontend/XCSwiftFrontend.swift | 204 +----------------- .../Swiftc/NoopSwiftcProductsGenerator.swift | 39 ++++ .../Swiftc/StaticSwiftcInputReader.swift | 46 ++++ .../Commands/Swiftc/SwiftcContext.swift | 3 +- .../Swiftc/SwiftcFilemapInputEditor.swift | 26 --- .../Swiftc/SwiftcProductsGenerator.swift | 18 -- .../Commands/Swiftc/XCSwiftc.swift | 22 +- .../Dependencies/CacheModeController.swift | 6 +- .../Dependencies/ListEditor.swift | 14 -- .../Dependencies/StaticFileListReader.swift | 34 +++ .../XCSwiftcFrontendMain.swift | 1 - 12 files changed, 341 insertions(+), 274 deletions(-) create mode 100644 Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift create mode 100644 Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift create mode 100644 Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift create mode 100644 Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift new file mode 100644 index 00000000..a255f5f2 --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift @@ -0,0 +1,202 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +enum SwiftFrontendArgInputError: Error { + // swift-frontend should either be compling or emiting a module + case bothCompilationAndEmitAction + // no .swift files have been passed as input files + case noCompilationInputs + // no -primary-file .swift files have been passed as input files + case noPrimaryFileCompilationInputs + // number of -emit-dependencies-path doesn't match compilation inputs + case dependenciesOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -serialize-diagnostics-path doesn't match compilation inputs + case diagnosticsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o doesn't match compilation inputs + case outputsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o for emit-module can be only 1 + case emitModulOuputCountIsNot1(parsed: Int) + // number of -emit-dependencies-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDependenciesOuputCountIsHigherThan1(parsed: Int) + // number of -serialize-diagnostics-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDiagnosticsOuputCountIsHigherThan1(parsed: Int) + // emit-module requires -emit-objc-header-path + case emitModuleMissingObjcHeaderPath + // -target is required + case emitMissingTarget + // -moduleName is required + case emiMissingModuleName +} + +public struct SwiftFrontendArgInput { + let compile: Bool + let emitModule: Bool + let objcHeaderOutput: String? + let moduleName: String? + let target: String? + let primaryInputPaths: [String] + let inputPaths: [String] + var outputPaths: [String] + var dependenciesPaths: [String] + // Extra params + // Diagnostics are not supported yet in the XCRemoteCache (cached artifacts assumes no warnings) + var diagnosticsPaths: [String] + // Unsed for now: + // .swiftsourceinfo and .swiftdoc will be placed next to the .swiftmodule + let sourceInfoPath: String? + let docPath: String? + + /// Manual initializer implementation required to be public + public init( + compile: Bool, + emitModule: Bool, + objcHeaderOutput: String?, + moduleName: String?, + target: String?, + primaryInputPaths: [String], + inputPaths: [String], + outputPaths: [String], + dependenciesPaths: [String], + diagnosticsPaths: [String], + sourceInfoPath: String?, + docPath: String? + ) { + self.compile = compile + self.emitModule = emitModule + self.objcHeaderOutput = objcHeaderOutput + self.moduleName = moduleName + self.target = target + self.primaryInputPaths = primaryInputPaths + self.inputPaths = inputPaths + self.outputPaths = outputPaths + self.dependenciesPaths = dependenciesPaths + self.diagnosticsPaths = diagnosticsPaths + self.sourceInfoPath = sourceInfoPath + self.docPath = docPath + } + + // swiftlint:disable:next cyclomatic_complexity function_body_length + func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext { + guard compile != emitModule else { + throw SwiftFrontendArgInputError.bothCompilationAndEmitAction + } + let inputPathsCount = inputPaths.count + let primaryInputsCount = primaryInputPaths.count + guard inputPathsCount > 0 else { + throw SwiftFrontendArgInputError.noCompilationInputs + } + guard let target = target else { + throw SwiftFrontendArgInputError.emitMissingTarget + } + guard let moduleName = moduleName else { + throw SwiftFrontendArgInputError.emiMissingModuleName + } + + if compile { + guard primaryInputsCount > 0 else { + throw SwiftFrontendArgInputError.noPrimaryFileCompilationInputs + } + guard [primaryInputsCount, 0].contains(dependenciesPaths.count) else { + throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: dependenciesPaths.count + ) + } + guard [primaryInputsCount, 0].contains(diagnosticsPaths.count) else { + throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: diagnosticsPaths.count + ) + } + guard outputPaths.count == primaryInputsCount else { + throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch( + expected: inputPathsCount, + parsed: outputPaths.count + ) + } + let primaryInputFilesURLs: [URL] = primaryInputPaths.map(URL.init(fileURLWithPath:)) + + let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps( + compileFilesScope: .subset(primaryInputFilesURLs), + emitModule: nil + ) + + let compilationFileMap = (0.. SwiftcContext { - let inputPathsCount = inputPaths.count - let primaryInputsCount = primaryInputPaths.count - guard inputPathsCount > 0 else { - throw SwiftFrontendArgInputError.noCompilationInputs - } - guard let target = target else { - throw SwiftFrontendArgInputError.emitMissingTarget - } - guard let moduleName = moduleName else { - throw SwiftFrontendArgInputError.emiMissingModuleName - } - - if compile { - guard primaryInputsCount > 0 else { - throw SwiftFrontendArgInputError.noPrimaryFileCompilationInputs - } - guard [primaryInputsCount, 0].contains(dependenciesPaths.count) else { - throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch( - expected: inputPathsCount, - parsed: dependenciesPaths.count - ) - } - guard [primaryInputsCount, 0].contains(diagnosticsPaths.count) else { - throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch( - expected: inputPathsCount, - parsed: diagnosticsPaths.count - ) - } - guard outputPaths.count == primaryInputsCount else { - throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch( - expected: inputPathsCount, - parsed: outputPaths.count - ) - } - let primaryInputFilesURLs: [URL] = primaryInputPaths.map(URL.init(fileURLWithPath:)) - - let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps( - compileFilesScope: .subset(primaryInputFilesURLs), - emitModule: nil - ) - - let compilationFileMap = (0.. { // don't lock individual compilation invocations for more than 10s private static let MaxLockingTimeout: TimeInterval = 10 @@ -228,19 +40,15 @@ public class XCSwiftFrontend: XCSwiftAbstract { ) } - override func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { + override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { let fileManager = FileManager.default let config: XCRemoteCacheConfig let context: SwiftcContext - do { - let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) - .readConfiguration() - context = try SwiftcContext(config: config, input: inputArgs) - } catch { - exit(1, "FATAL: XCSwiftFrontend initialization failed with error: \(error)") - } + let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) + .readConfiguration() + context = try SwiftcContext(config: config, input: inputArgs) // do not cache this context, as it is subject to change when // the emit-module finds that the cached artifact cannot be used return (config, context) @@ -250,7 +58,7 @@ public class XCSwiftFrontend: XCSwiftAbstract { do { /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") - let (_, context) = buildContext() + let (_, context) = try buildContext() let sharedLockFileURL = XCSwiftFrontend.generateLlbuildIdSharedLock(llbuildId: llbuildId, tmpDir: context.tempDir) let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift new file mode 100644 index 00000000..e6d4026f --- /dev/null +++ b/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift @@ -0,0 +1,39 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where +/// only individual .o files are created and not .swiftmodule of -Swift.h +/// (which is part of swift-frontend -emit-module invocation) +class NoopSwiftcProductsGenerator: SwiftcProductsGenerator { + func generateFrom( + artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], + artifactSwiftModuleObjCFile: URL + ) throws -> SwiftcProductsGeneratorOutput { + printWarning(""" + Invoking module generation from NoopSwiftcProductsGenerator which does nothing. \ + It might be an error of the swift-frontend mocking. + """) + // TODO: Refactor API: this url is never used: + // NoopSwiftcProductsGenerator is intended only for the swift-frontend + let trivialURL = URL(fileURLWithPath: "/non-existing") + return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL) + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift b/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift new file mode 100644 index 00000000..fd3d3fa4 --- /dev/null +++ b/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift @@ -0,0 +1,46 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +class StaticSwiftcInputReader: SwiftcInputReader { + private let moduleDependencies: URL? + private let swiftDependencies: URL? + private let compilationFiles: [SwiftFileCompilationInfo] + + init( + moduleDependencies: URL?, + swiftDependencies: URL?, + compilationFiles: [SwiftFileCompilationInfo] + ) { + self.moduleDependencies = moduleDependencies + self.swiftDependencies = swiftDependencies + self.compilationFiles = compilationFiles + } + + func read() throws -> SwiftCompilationInfo { + return .init( + info: .init( + dependencies: moduleDependencies, + swiftDependencies: swiftDependencies + ), + files: compilationFiles + ) + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index d64340b9..7e88a3b0 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -80,9 +80,8 @@ public struct SwiftcContext { outputs: CompilationFilesOutputs, target: String, inputs: CompilationFilesSource, - // TODO: make sure it is required /// any workspace file path - all other intermediate files for this compilation - /// are placed next to it. This path is used to infere the arch and TARGET_TEMP_DIR + /// are placed next to it. This path is used to infer the arch and TARGET_TEMP_DIR exampleWorkspaceFilePath: String ) throws { self.moduleName = moduleName diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift index 51f9e211..0d8776ed 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift @@ -59,32 +59,6 @@ public struct SwiftFileCompilationInfo: Encodable, Equatable { let swiftDependencies: URL? } -class StaticSwiftcInputReader: SwiftcInputReader { - private let moduleDependencies: URL? - private let swiftDependencies: URL? - private let compilationFiles: [SwiftFileCompilationInfo] - - init( - moduleDependencies: URL?, - swiftDependencies: URL?, - compilationFiles: [SwiftFileCompilationInfo] - ) { - self.moduleDependencies = moduleDependencies - self.swiftDependencies = swiftDependencies - self.compilationFiles = compilationFiles - } - - func read() throws -> SwiftCompilationInfo { - return .init( - info: .init( - dependencies: moduleDependencies, - swiftDependencies: swiftDependencies - ), - files: compilationFiles - ) - } -} - class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { private let file: URL diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift index 0d6496ce..74da9ca4 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift @@ -41,24 +41,6 @@ protocol SwiftcProductsGenerator { ) throws -> SwiftcProductsGeneratorOutput } -/// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where -/// only individual .o files are created and not .swiftmodule of -Swift.h -/// (which is part of swift-frontend -emit-module invocation) -class NoopSwiftcProductsGenerator: SwiftcProductsGenerator { - func generateFrom( - artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], - artifactSwiftModuleObjCFile: URL - ) throws -> SwiftcProductsGeneratorOutput { - printWarning(""" - Invoking module generation from NoopSwiftcProductsGenerator which does nothing. \ - It might be an error of the swift-frontend mocking. - """) - // TODO: Refactor API: this url is never used: - // NoopSwiftcProductsGenerator is intended only for the swift-frontend - let trivialURL = URL(fileURLWithPath: "/non-existing") - return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL) - } -} /// Generator that produces all products in the locations where Xcode expects it, using provided disk copier class DiskSwiftcProductsGenerator: SwiftcProductsGenerator { private let destinationSwiftmodulePaths: [SwiftmoduleFileExtension: URL] diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 58a521ee..eb31d36b 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -63,14 +63,14 @@ public class XCSwiftAbstract { self.touchFactory = touchFactory } - func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { + func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { fatalError("Need to override in \(Self.self)") } // swiftlint:disable:next function_body_length public func run() throws { let fileManager = FileManager.default - let (config, context) = buildContext() + let (config, context) = try buildContext() let swiftcCommand = config.swiftcCommand let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) @@ -82,8 +82,9 @@ public class XCSwiftAbstract { case .fileMap(let path): inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileManager: fileManager) case .map(let map): - // static + // static - passed via the arguments list // TODO: check if first 2 ars can always be `nil` + // with Xcode 13, inputs via cmd are only used for compilations inputReader = StaticSwiftcInputReader( moduleDependencies: nil, swiftDependencies: nil, @@ -168,18 +169,15 @@ public class XCSwiftAbstract { } public class XCSwiftc: XCSwiftAbstract { - override func buildContext() -> (XCRemoteCacheConfig, SwiftcContext) { + override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { let fileReader = FileManager.default let config: XCRemoteCacheConfig let context: SwiftcContext - do { - let srcRoot: URL = URL(fileURLWithPath: fileReader.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileReader) - .readConfiguration() - context = try SwiftcContext(config: config, input: inputArgs) - } catch { - exit(1, "FATAL: Swiftc initialization failed with error: \(error)") - } + let srcRoot: URL = URL(fileURLWithPath: fileReader.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileReader) + .readConfiguration() + context = try SwiftcContext(config: config, input: inputArgs) + return (config, context) } } diff --git a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift index 1600a3c0..175fc952 100644 --- a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift +++ b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift @@ -76,7 +76,7 @@ class PhaseCacheModeController: CacheModeController { } func enable(allowedInputFiles: [URL], dependencies: [URL]) throws { - try cleanupLlbuildLock() + try cleanupLlBuildLock() // marker file contains filepaths that contribute to the build products // and should invalidate all other target steps (swiftc,libtool etc.) let targetSensitiveFiles = dependencies + [modeMarker, Self.xcodeSelectLink] @@ -88,7 +88,7 @@ class PhaseCacheModeController: CacheModeController { } func disable() throws { - try cleanupLlbuildLock() + try cleanupLlBuildLock() guard !forceCached else { throw PhaseCacheModeControllerError.cannotUseRemoteCacheForForcedCacheMode } @@ -120,7 +120,7 @@ class PhaseCacheModeController: CacheModeController { return false } - private func cleanupLlbuildLock() throws { + private func cleanupLlBuildLock() throws { if fileManager.fileExists(atPath: llbuildLockFile.path) { do { try fileManager.removeItem(at: llbuildLockFile) diff --git a/Sources/XCRemoteCache/Dependencies/ListEditor.swift b/Sources/XCRemoteCache/Dependencies/ListEditor.swift index 8a7bc35e..fb2276e6 100644 --- a/Sources/XCRemoteCache/Dependencies/ListEditor.swift +++ b/Sources/XCRemoteCache/Dependencies/ListEditor.swift @@ -41,20 +41,6 @@ protocol ListWriter { func writerListFilesURLs(_ list: [URL]) throws } -class StaticFileListReader: ListReader { - private let list: [URL] - init(list: [URL]) { - self.list = list - } - func listFilesURLs() throws -> [URL] { - list - } - - func canRead() -> Bool { - true - } -} - /// Reads&Writes files that list files using one-file-per-line format class FileListEditor: ListReader, ListWriter { private let file: URL diff --git a/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift b/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift new file mode 100644 index 00000000..a4985241 --- /dev/null +++ b/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift @@ -0,0 +1,34 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +class StaticFileListReader: ListReader { + private let list: [URL] + init(list: [URL]) { + self.list = list + } + func listFilesURLs() throws -> [URL] { + list + } + + func canRead() -> Bool { + true + } +} diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift index 2fd4f842..65580d8f 100644 --- a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -98,7 +98,6 @@ public class XCSwiftcFrontendMain { docPath: docPath ) do { - try argInput.validate() let frontend = try XCSwiftFrontend( command: command, inputArgs: argInput, From df34ae7139300bdf6434ef6e80c9bded6d0cb99e Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 28 May 2023 05:57:32 -0700 Subject: [PATCH 29/33] Split log --- .../Commands/SwiftFrontend/XCSwiftFrontend.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift index fb96cf6e..1fd671ab 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -73,7 +73,9 @@ public class XCSwiftFrontend: XCSwiftAbstract { try swiftFrontendOrchestrator.run(criticalSection: super.run) } catch { - defaultLog("Cannot correctly orchestrate the \(command) with params \(inputArgs): error: \(error)") + // Splitting into 2 invocations as os_log truncates a massage + defaultLog("Cannot correctly orchestrate the \(command) with params \(inputArgs)") + defaultLog("Cannot correctly orchestrate error: \(error)") throw error } } From 6869838a282d9b597d96191b63c6d1d8f90d18d9 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 28 May 2023 06:27:45 -0700 Subject: [PATCH 30/33] Fix suffix --- .../XCSwiftcFrontendMain.swift | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift index 65580d8f..2e730fdb 100644 --- a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -73,9 +73,8 @@ public class XCSwiftcFrontendMain { case "-primary-file": // .swift primaryInputPaths.append(args[i + 1]) - inputPaths.append(args[i + 1]) default: - if arg.hasPrefix(".swift") { + if arg.hasSuffix(".swift") { inputPaths.append(arg) } } @@ -97,6 +96,12 @@ public class XCSwiftcFrontendMain { sourceInfoPath: sourceInfoPath, docPath: docPath ) + // swift-frontened is first invoked with some "probing" args like + // -print-target-info + guard emitModule != compile else { + runFallback(envs: env) + } + do { let frontend = try XCSwiftFrontend( command: command, @@ -106,17 +111,21 @@ public class XCSwiftcFrontendMain { touchFactory: FileTouch.init) try frontend.run() } catch { - let developerDir = env["DEVELOPER_DIR"]! - // limitation: always using the Xcode's toolchain - let swiftFrontendCommand = "\(developerDir)/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend" + runFallback(envs: env) + } + } - let args = ProcessInfo().arguments - let paramList = [swiftFrontendCommand] + args.dropFirst() - let cargs = paramList.map { strdup($0) } + [nil] - execvp(swiftFrontendCommand, cargs) + private func runFallback(envs env: [String: String]) -> Never { + let developerDir = env["DEVELOPER_DIR"]! + // limitation: always using the Xcode's toolchain + let swiftFrontendCommand = "\(developerDir)/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend" - /// C-function `execv` returns only when the command fails - exit(1) - } + let args = ProcessInfo().arguments + let paramList = [swiftFrontendCommand] + args.dropFirst() + let cargs = paramList.map { strdup($0) } + [nil] + execvp(swiftFrontendCommand, cargs) + + /// C-function `execv` returns only when the command fails + exit(1) } } From afa023663d226dd9b821f16f5e2ac6b60d8cea53 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 28 May 2023 07:04:18 -0700 Subject: [PATCH 31/33] Emit modules dependencies --- .../Commands/SwiftFrontend/SwiftFrontendArgInput.swift | 10 ++++++---- .../XCRemoteCache/Commands/Swiftc/SwiftcContext.swift | 4 +++- Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift | 5 ++++- Sources/XCRemoteCache/Utils/Array+Utils.swift | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift index a255f5f2..26cc2d8c 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift @@ -145,7 +145,8 @@ public struct SwiftFrontendArgInput { file: primaryInputFilesURLs[i], dependencies: dependenciesPaths.get(i).map(URL.init(fileURLWithPath:)), object: outputPaths.get(i).map(URL.init(fileURLWithPath:)), - swiftDependencies: dependenciesPaths.get(i).map(URL.init(fileURLWithPath:)) + // for now - swift-dependencies are not requested in the driver compilation mode + swiftDependencies: nil ) return new } @@ -156,7 +157,7 @@ public struct SwiftFrontendArgInput { steps: steps, outputs: .map(compilationFileMap), target: target, - inputs: .list(outputPaths), + inputs: .list(inputPaths), exampleWorkspaceFilePath: outputPaths[0] ) } else { @@ -185,7 +186,8 @@ public struct SwiftFrontendArgInput { compileFilesScope: .none, emitModule: SwiftcContext.SwiftcStepEmitModule( objcHeaderOutput: URL(fileURLWithPath: objcHeaderOutput), - modulePathOutput: URL(fileURLWithPath: outputPaths[0]) + modulePathOutput: URL(fileURLWithPath: outputPaths[0]), + dependencies: URL(fileURLWithPath: dependenciesPaths[0]) ) ) return try .init( @@ -194,7 +196,7 @@ public struct SwiftFrontendArgInput { steps: steps, outputs: .map([:]), target: target, - inputs: .list([]), + inputs: .list(inputPaths), exampleWorkspaceFilePath: objcHeaderOutput ) } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 7e88a3b0..03b9f20e 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -23,6 +23,7 @@ public struct SwiftcContext { public struct SwiftcStepEmitModule { let objcHeaderOutput: URL let modulePathOutput: URL + let dependencies: URL? } public enum SwiftcStepCompileFilesScope { case none @@ -126,7 +127,8 @@ public struct SwiftcContext { compileFilesScope: .all, emitModule: SwiftcStepEmitModule( objcHeaderOutput: URL(fileURLWithPath: (input.objcHeaderOutput)), - modulePathOutput: URL(fileURLWithPath: input.modulePathOutput) + modulePathOutput: URL(fileURLWithPath: input.modulePathOutput), + dependencies: nil ) ) let outputs = CompilationFilesOutputs.fileMap(input.filemap) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index eb31d36b..ec2a54b0 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -78,6 +78,9 @@ public class XCSwiftAbstract { let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager) let inputReader: SwiftcInputReader + // TODO: consider renaming + // outputs and outputs of the swift step are described in a map together + // and the file/list is named "output" switch context.outputs { case .fileMap(let path): inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileManager: fileManager) @@ -86,7 +89,7 @@ public class XCSwiftAbstract { // TODO: check if first 2 ars can always be `nil` // with Xcode 13, inputs via cmd are only used for compilations inputReader = StaticSwiftcInputReader( - moduleDependencies: nil, + moduleDependencies: context.steps.emitModule?.dependencies, swiftDependencies: nil, compilationFiles: Array(map.values) ) diff --git a/Sources/XCRemoteCache/Utils/Array+Utils.swift b/Sources/XCRemoteCache/Utils/Array+Utils.swift index fcf0c444..67e2e75a 100644 --- a/Sources/XCRemoteCache/Utils/Array+Utils.swift +++ b/Sources/XCRemoteCache/Utils/Array+Utils.swift @@ -21,7 +21,7 @@ import Foundation public extension Array { func get(_ i: Index) -> Element? { - guard count < i else { + guard count > i else { return nil } return self[i] From d95da61d11e1e45f03879844834557bb995319ea Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 28 May 2023 17:55:47 -0700 Subject: [PATCH 32/33] Support driver in cocoapods integration --- .../lib/cocoapods-xcremotecache/command/hooks.rb | 15 +++++++++------ .../lib/cocoapods-xcremotecache/gem_version.rb | 2 +- tasks/e2e.rb | 3 ++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb b/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb index 7a270f77..3c1c3110 100644 --- a/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb +++ b/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb @@ -123,7 +123,8 @@ def self.enable_xcremotecache( exclude_build_configurations, final_target, fake_src_root, - exclude_sdks_configurations + exclude_sdks_configurations, + enable_swift_driver_integration ) srcroot_relative_xc_location = parent_dir(xc_location, repo_distance) # location of the entrite CocoaPods project, relative to SRCROOT @@ -137,14 +138,15 @@ def self.enable_xcremotecache( elsif mode == 'producer' || mode == 'producer-fast' config.build_settings.delete('CC') if config.build_settings.key?('CC') end - reset_build_setting(config.build_settings, 'SWIFT_EXEC', "$SRCROOT/#{srcroot_relative_xc_location}/xcswiftc", exclude_sdks_configurations) + swiftc_name = enable_swift_driver_integration ? 'swiftc' : 'xcswiftc' + reset_build_setting(config.build_settings, 'SWIFT_EXEC', "$SRCROOT/#{srcroot_relative_xc_location}/#{swiftc_name}", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LIBTOOL', "$SRCROOT/#{srcroot_relative_xc_location}/xclibtool", exclude_sdks_configurations) # Setting LIBTOOL to '' breaks SwiftDriver intengration so resetting it to the original value 'libtool' for all excluded configurations add_build_setting_for_sdks(config.build_settings, 'LIBTOOL', 'libtool', exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LD', "$SRCROOT/#{srcroot_relative_xc_location}/xcld", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LDPLUSPLUS', "$SRCROOT/#{srcroot_relative_xc_location}/xcldplusplus", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LIPO', "$SRCROOT/#{srcroot_relative_xc_location}/xclipo", exclude_sdks_configurations) - reset_build_setting(config.build_settings, 'SWIFT_USE_INTEGRATED_DRIVER', 'NO', exclude_sdks_configurations) + reset_build_setting(config.build_settings, 'SWIFT_USE_INTEGRATED_DRIVER', 'NO', exclude_sdks_configurations) unless enable_swift_driver_integration reset_build_setting(config.build_settings, 'XCREMOTE_CACHE_FAKE_SRCROOT', fake_src_root, exclude_sdks_configurations) reset_build_setting(config.build_settings, 'XCRC_PLATFORM_PREFERRED_ARCH', "$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)", exclude_sdks_configurations) @@ -498,6 +500,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) check_platform = @@configuration['check_platform'] fake_src_root = @@configuration['fake_src_root'] exclude_sdks_configurations = @@configuration['exclude_sdks_configurations'] || [] + enable_swift_driver_integration = @@configuration['enable_swift_driver_integration'] || false xccc_location_absolute = "#{user_proj_directory}/#{xccc_location}" xcrc_location_absolute = "#{user_proj_directory}/#{xcrc_location}" @@ -521,7 +524,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) next if target.name.start_with?("Pods-") next if target.name.end_with?("Tests") next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end # Create .rcinfo into `Pods` directory as that .xcodeproj reads configuration from .xcodeproj location @@ -534,7 +537,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) next if target.source_build_phase.files_references.empty? next if target.name.end_with?("Tests") next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end generated_project.save() end @@ -575,7 +578,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) # Attach XCRC to the app targets user_project.targets.each do |target| next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 0, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 0, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end # Set Target sourcemap diff --git a/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb b/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb index ddf31972..b2acb8e4 100644 --- a/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb +++ b/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb @@ -13,5 +13,5 @@ # limitations under the License. module CocoapodsXcremotecache - VERSION = "0.0.16" + VERSION = "0.0.17" end diff --git a/tasks/e2e.rb b/tasks/e2e.rb index 9aa9dbd6..029299f2 100644 --- a/tasks/e2e.rb +++ b/tasks/e2e.rb @@ -25,7 +25,8 @@ 'primary_branch' => GIT_BRANCH, 'mode' => 'consumer', 'final_target' => 'XCRemoteCacheSample', - 'artifact_maximum_age' => 0 + 'artifact_maximum_age' => 0, + 'enable_swift_driver_integration' => true }.freeze DEFAULT_EXPECTATIONS = { 'misses' => 0, From 4a46d57f36d7acb25e123e37b5a2bb28dbdb9462 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 28 May 2023 18:40:51 -0700 Subject: [PATCH 33/33] Add -supplementary-output-file-map --- .../SwiftFrontend/SwiftFrontendArgInput.swift | 32 ++++++---- .../Commands/Swiftc/SwiftcContext.swift | 2 + .../Swiftc/SwiftcFilemapInputEditor.swift | 26 +++++++-- .../Commands/Swiftc/XCSwiftc.swift | 4 +- .../XCSwiftcFrontendMain.swift | 6 +- .../SwiftcFilemapInputEditorTests.swift | 58 ++++++++++++++++--- 6 files changed, 101 insertions(+), 27 deletions(-) diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift index 26cc2d8c..dda0b23d 100644 --- a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift @@ -63,6 +63,7 @@ public struct SwiftFrontendArgInput { // .swiftsourceinfo and .swiftdoc will be placed next to the .swiftmodule let sourceInfoPath: String? let docPath: String? + let supplementaryOutputFileMap: String? /// Manual initializer implementation required to be public public init( @@ -77,7 +78,8 @@ public struct SwiftFrontendArgInput { dependenciesPaths: [String], diagnosticsPaths: [String], sourceInfoPath: String?, - docPath: String? + docPath: String?, + supplementaryOutputFileMap: String? ) { self.compile = compile self.emitModule = emitModule @@ -91,6 +93,7 @@ public struct SwiftFrontendArgInput { self.diagnosticsPaths = diagnosticsPaths self.sourceInfoPath = sourceInfoPath self.docPath = docPath + self.supplementaryOutputFileMap = supplementaryOutputFileMap } // swiftlint:disable:next cyclomatic_complexity function_body_length @@ -139,23 +142,28 @@ public struct SwiftFrontendArgInput { emitModule: nil ) - let compilationFileMap = (0.. [String: Any]? { + switch fileFormat { + case .json: + return try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any] + case .yaml: + return try Yams.load(yaml: String(data: content, encoding: .utf8)!) as? [String: Any] + } + } } extension SwiftCompilationInfo { init(from object: [String: Any]) throws { - info = try SwiftModuleCompilationInfo(from: object[""]) + info = try SwiftModuleCompilationInfo(from: object["", default: [:]]) files = try object.reduce([]) { prev, new in let (key, value) = new if key.isEmpty { diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index ec2a54b0..a071f93c 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -83,7 +83,9 @@ public class XCSwiftAbstract { // and the file/list is named "output" switch context.outputs { case .fileMap(let path): - inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileManager: fileManager) + inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileFormat: .json, fileManager: fileManager) + case .supplementaryFileMap(let path): + inputReader = SwiftcFilemapInputEditor(URL(fileURLWithPath: path), fileFormat: .yaml, fileManager: fileManager) case .map(let map): // static - passed via the arguments list // TODO: check if first 2 ars can always be `nil` diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift index 2e730fdb..05141f49 100644 --- a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -42,6 +42,7 @@ public class XCSwiftcFrontendMain { var diagnosticsPaths: [String] = [] var sourceInfoPath: String? var docPath: String? + var supplementaryOutputFileMap: String? for i in 0..