diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift index 70c9c0ea..ec4f4b1c 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/BuildSettingsIntegrateAppender.swift @@ -21,6 +21,11 @@ import Foundation typealias BuildSettings = [String: Any] +struct BuildSettingsIntegrateAppenderOption: OptionSet { + let rawValue: Int + + static let disableSwiftDriverIntegration = BuildSettingsIntegrateAppenderOption(rawValue: 1 << 0) +} // Manages Xcode build settings protocol BuildSettingsIntegrateAppender { /// Appends XCRemoteCache-specific build settings @@ -35,18 +40,28 @@ class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender { private let repoRoot: URL private let fakeSrcRoot: URL private let sdksExclude: [String] + private let options: BuildSettingsIntegrateAppenderOption - init(mode: Mode, repoRoot: URL, fakeSrcRoot: URL, sdksExclude: [String]) { + init( + mode: Mode, + repoRoot: URL, + fakeSrcRoot: URL, + sdksExclude: [String], + options: BuildSettingsIntegrateAppenderOption + ) { self.mode = mode self.repoRoot = repoRoot self.fakeSrcRoot = fakeSrcRoot self.sdksExclude = sdksExclude + self.options = options } func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings { var result = buildSettings setBuildSetting(buildSettings: &result, key: "SWIFT_EXEC", value: wrappers.swiftc.path ) - setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" ) + if options.contains(.disableSwiftDriverIntegration) { + setBuildSetting(buildSettings: &result, key: "SWIFT_USE_INTEGRATED_DRIVER", value: "NO" ) + } // When generating artifacts, no need to shell-out all compilation commands to our wrappers if case .consumer = mode { setBuildSetting(buildSettings: &result, key: "CC", value: wrappers.cc.path ) diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift index acb52aae..4b435cc5 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift @@ -98,11 +98,15 @@ public class XCIntegrate { excludes: targetsExclude.integrateArrayArguments, includes: targetsInclude.integrateArrayArguments ) + let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption = [ + .disableSwiftDriverIntegration, + ] let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender( mode: context.mode, repoRoot: context.repoRoot, fakeSrcRoot: context.fakeSrcRoot, - sdksExclude: sdksExclude.integrateArrayArguments + sdksExclude: sdksExclude.integrateArrayArguments, + options: buildSettingsAppenderOptions ) let lldbPatcher: LLDBInitPatcher switch lldbMode { diff --git a/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift new file mode 100644 index 00000000..10ee7e0f --- /dev/null +++ b/Sources/XCRemoteCache/Commands/Swiftc/NoopSwiftcProductsGenerator.swift @@ -0,0 +1,38 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Products generator that doesn't create any swiftmodule. It is used in the compilation swift-frontend mocking, where +/// only individual .o files are created and not .swiftmodule of -Swift.h +/// (which is part of swift-frontend -emit-module invocation) +class NoopSwiftcProductsGenerator: SwiftcProductsGenerator { + func generateFrom( + artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], + artifactSwiftModuleObjCFile: URL + ) throws -> SwiftcProductsGeneratorOutput { + infoLog(""" + Invoking module generation from NoopSwiftcProductsGenerator does nothing. \ + It might be a side-effect of a plugin asking to generate a module. + """) + // NoopSwiftcProductsGenerator is intended only for the swift-frontend + let trivialURL = URL(fileURLWithPath: "/non-existing") + return SwiftcProductsGeneratorOutput(swiftmoduleDir: trivialURL, objcHeaderFile: trivialURL) + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift b/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift new file mode 100644 index 00000000..0e25e6be --- /dev/null +++ b/Sources/XCRemoteCache/Commands/Swiftc/StaticSwiftcInputReader.swift @@ -0,0 +1,46 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +class StaticSwiftcInputReader: SwiftcInputReader { + private let moduleDependencies: URL? + private let swiftDependencies: URL? + private let compilationFiles: [SwiftFileCompilationInfo] + + init( + moduleDependencies: URL?, + swiftDependencies: URL?, + compilationFiles: [SwiftFileCompilationInfo] + ) { + self.moduleDependencies = moduleDependencies + self.swiftDependencies = swiftDependencies + self.compilationFiles = compilationFiles + } + + func read() throws -> SwiftCompilationInfo { + return .init( + info: .init( + dependencies: moduleDependencies, + swiftDependencies: swiftDependencies + ), + files: compilationFiles + ) + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift index ef30547a..9bc70687 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/Swiftc.swift @@ -132,7 +132,7 @@ class Swiftc: SwiftcProtocol { // Read swiftmodule location from XCRemoteCache // arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo} - let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent + let moduleName = context.moduleName let allCompilations = try inputFilesReader.read() let artifactSwiftmoduleDir = artifactLocation .appendingPathComponent("swiftmodule") @@ -145,20 +145,24 @@ class Swiftc: SwiftcProtocol { } ) - // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h - let artifactSwiftModuleObjCDir = artifactLocation - .appendingPathComponent("include") - .appendingPathComponent(context.arch) - .appendingPathComponent(context.moduleName) - // Move cached xxxx-Swift.h to the location passed in arglist - // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir - let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir - .appendingPathComponent(context.objcHeaderOutput.lastPathComponent) + // emit module (if requested) + if let emitModule = context.steps.emitModule { + // Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h + let artifactSwiftModuleObjCDir = artifactLocation + .appendingPathComponent("include") + .appendingPathComponent(context.arch) + .appendingPathComponent(context.moduleName) + // Move cached xxxx-Swift.h to the location passed in arglist + // Alternatively, artifactSwiftModuleObjCFile could be built as a first .h + // file in artifactSwiftModuleObjCDir + let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir + .appendingPathComponent(emitModule.objcHeaderOutput.lastPathComponent) - _ = try productsGenerator.generateFrom( - artifactSwiftModuleFiles: artifactSwiftmoduleFiles, - artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile - ) + _ = try productsGenerator.generateFrom( + artifactSwiftModuleFiles: artifactSwiftmoduleFiles, + artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile + ) + } try plugins.forEach { try $0.generate(for: allCompilations) @@ -176,8 +180,10 @@ class Swiftc: SwiftcProtocol { try cachedDependenciesWriterFactory.generate(output: individualDeps) } } - // Save .d for the entire module - try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies) + // Save .d for the entire module (might not be required in the `swift-frontend -c` mode) + if let swiftDependencies = allCompilations.info.swiftDependencies { + try cachedDependenciesWriterFactory.generate(output: swiftDependencies) + } // Generate .d file with all deps in the "-master.d" (e.g. for WMO) if let wmoDeps = allCompilations.info.dependencies { try cachedDependenciesWriterFactory.generate(output: wmoDeps) diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 1e58ff4f..485c9d18 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -20,6 +20,53 @@ import Foundation public struct SwiftcContext { + /// Describes the action if the module emit should happen + /// that generates .swiftmodule and/or -Swift.h + public struct SwiftcStepEmitModule: Equatable { + // where the -Swift.h should be placed + let objcHeaderOutput: URL + // where should the .swiftmodule be placed + let modulePathOutput: URL + // might be passed as an explicit argument in the swiftc + // -emit-dependencies-path + let dependencies: URL? + } + + /// Which files (from the list of all files in the module) + /// should be compiled in this process + public enum SwiftcStepCompileFilesScope: Equatable { + /// used if only emit module should be done + case none + case all + case subset([URL]) + } + + /// Describes which steps should be done as a part of this process + public struct SwiftcSteps: Equatable { + /// which files should be compiled + let compileFilesScope: SwiftcStepCompileFilesScope + /// if a module should be generated + let emitModule: SwiftcStepEmitModule? + } + + /// Defines how a list of input files (*.swift) is passed to the invocation + public enum CompilationFilesSource: Equatable { + /// defined in a separate file (via @/.../*.SwiftFileList) + case fileList(String) + /// explicitly passed a list of files + case list([String]) + } + + /// Defines how a list of output files (*.d, *.o etc.) is passed to the invocation + public enum CompilationFilesInputs: Equatable { + /// defined in a separate file (via -output-file-map) + case fileMap(String) + /// defined in a separate file (via -supplementary-output-file-map) + case supplementaryFileMap(String) + /// explicitly passed in the invocation + case map([String: SwiftFileCompilationInfo]) + } + enum SwiftcMode: Equatable { case producer /// Commit sha of the commit to use during remote cache @@ -28,14 +75,13 @@ public struct SwiftcContext { case producerFast } - let objcHeaderOutput: URL + let steps: SwiftcSteps let moduleName: String - let modulePathOutput: URL - /// File that defines output files locations (.d, .swiftmodule etc.) - let filemap: URL + /// A source that defines output files locations (.d, .swiftmodule etc.) + let inputs: CompilationFilesInputs let target: String - /// File that contains input files for the swift module compilation - let fileList: URL + /// A source that contains all input files for the swift module compilation + let compilationFiles: CompilationFilesSource let tempDir: URL let arch: String let prebuildDependenciesPath: String @@ -43,29 +89,29 @@ public struct SwiftcContext { /// File that stores all compilation invocation arguments let invocationHistoryFile: URL - public init( config: XCRemoteCacheConfig, - objcHeaderOutput: String, moduleName: String, - modulePathOutput: String, - filemap: String, + steps: SwiftcSteps, + inputs: CompilationFilesInputs, target: String, - fileList: String + compilationFiles: CompilationFilesSource, + /// any workspace file path - all other intermediate files for this compilation + /// are placed next to it. This path is used to infer the arch and TARGET_TEMP_DIR + exampleWorkspaceFilePath: String ) throws { - self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput) self.moduleName = moduleName - self.modulePathOutput = URL(fileURLWithPath: modulePathOutput) - self.filemap = URL(fileURLWithPath: filemap) + self.steps = steps + self.inputs = inputs self.target = target - self.fileList = URL(fileURLWithPath: fileList) - // modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule + self.compilationFiles = compilationFiles + // exampleWorkspaceFilePath has a format $TARGET_TEMP_DIR/Objects-normal/$ARCH/some.file // That may be subject to change for other Xcode versions - tempDir = URL(fileURLWithPath: modulePathOutput) + tempDir = URL(fileURLWithPath: exampleWorkspaceFilePath) .deletingLastPathComponent() .deletingLastPathComponent() .deletingLastPathComponent() - arch = URL(fileURLWithPath: modulePathOutput).deletingLastPathComponent().lastPathComponent + arch = URL(fileURLWithPath: exampleWorkspaceFilePath).deletingLastPathComponent().lastPathComponent let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot) let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot) @@ -92,14 +138,25 @@ public struct SwiftcContext { config: XCRemoteCacheConfig, input: SwiftcArgInput ) throws { + let steps = SwiftcSteps( + compileFilesScope: .all, + emitModule: SwiftcStepEmitModule( + objcHeaderOutput: URL(fileURLWithPath: (input.objcHeaderOutput)), + modulePathOutput: URL(fileURLWithPath: input.modulePathOutput), + // in `swiftc`, .d dependencies are pass in the output filemap + dependencies: nil + ) + ) + let inputs = CompilationFilesInputs.fileMap(input.filemap) + let compilationFiles = CompilationFilesSource.fileList(input.fileList) try self.init( config: config, - objcHeaderOutput: input.objcHeaderOutput, moduleName: input.moduleName, - modulePathOutput: input.modulePathOutput, - filemap: input.filemap, + steps: steps, + inputs: inputs, target: input.target, - fileList: input.fileList + compilationFiles: compilationFiles, + exampleWorkspaceFilePath: input.modulePathOutput ) } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift index 7a803b78..6256d18e 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcFilemapInputEditor.swift @@ -18,11 +18,16 @@ // under the License. import Foundation +import Yams /// Errors with reading swiftc inputs enum SwiftcInputReaderError: Error { case readingFailed case invalidFormat + /// The file is not in the yaml format + case invalidYamlFormat + /// The yaml string contains illegal characters + case invalidYamlString case missingField(String) } @@ -45,10 +50,11 @@ struct SwiftCompilationInfo: Encodable, Equatable { struct SwiftModuleCompilationInfo: Encodable, Equatable { // not present for incremental builds let dependencies: URL? - let swiftDependencies: URL + // might be nil for the swift-frontend '-c' invocation + let swiftDependencies: URL? } -struct SwiftFileCompilationInfo: Encodable, Equatable { +public struct SwiftFileCompilationInfo: Encodable, Hashable { let file: URL // not present for WMO builds let dependencies: URL? @@ -60,11 +66,18 @@ struct SwiftFileCompilationInfo: Encodable, Equatable { class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { + enum Format { + case json + case yaml + } + private let file: URL + private let fileFormat: Format private let fileManager: FileManager - init(_ file: URL, fileManager: FileManager) { + init(_ file: URL, fileFormat: Format, fileManager: FileManager) { self.file = file + self.fileFormat = fileFormat self.fileManager = fileManager } @@ -72,7 +85,7 @@ class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { guard let content = fileManager.contents(atPath: file.path) else { throw SwiftcInputReaderError.readingFailed } - guard let representation = try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any] else { + guard let representation = try decodeFile(content: content) else { throw SwiftcInputReaderError.invalidFormat } return try SwiftCompilationInfo(from: representation) @@ -82,11 +95,23 @@ class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter { let data = try JSONSerialization.data(withJSONObject: info.dump(), options: [.prettyPrinted]) fileManager.createFile(atPath: file.path, contents: data, attributes: nil) } + + private func decodeFile(content: Data) throws -> [String: Any]? { + switch fileFormat { + case .json: + return try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any] + case .yaml: + guard let stringContent = String(data: content, encoding: .utf8) else { + throw SwiftcInputReaderError.invalidYamlString + } + return try Yams.load(yaml: stringContent) as? [String: Any] + } + } } extension SwiftCompilationInfo { init(from object: [String: Any]) throws { - info = try SwiftModuleCompilationInfo(from: object[""]) + info = try SwiftModuleCompilationInfo(from: object["", default: [:]]) files = try object.reduce([]) { prev, new in let (key, value) = new if key.isEmpty { @@ -111,14 +136,14 @@ extension SwiftModuleCompilationInfo { guard let dict = object as? [String: String] else { throw SwiftcInputReaderError.invalidFormat } - swiftDependencies = try dict.readURL(key: "swift-dependencies") + swiftDependencies = dict.readURL(key: "swift-dependencies") dependencies = dict.readURL(key: "dependencies") } func dump() -> [String: String] { return [ "dependencies": dependencies?.path, - "swift-dependencies": swiftDependencies.path, + "swift-dependencies": swiftDependencies?.path, ].compactMapValues { $0 } } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift index 86fbc5a3..a4c76290 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcOrchestrator.swift @@ -27,8 +27,10 @@ class SwiftcOrchestrator { private let mode: SwiftcContext.SwiftcMode // swiftc command that should be called to generate artifacts private let swiftcCommand: String - private let objcHeaderOutput: URL - private let moduleOutput: URL + // Might be nil if invoking from frontend compilation: `swift-frontend -c` + private let objcHeaderOutput: URL? + // Might be nil if invoking from frontend compilation: `swift-frontend -c` + private let moduleOutput: URL? private let arch: String private let artifactBuilder: ArtifactSwiftProductsBuilder private let shellOut: ShellOut @@ -39,8 +41,8 @@ class SwiftcOrchestrator { mode: SwiftcContext.SwiftcMode, swiftc: SwiftcProtocol, swiftcCommand: String, - objcHeaderOutput: URL, - moduleOutput: URL, + objcHeaderOutput: URL?, + moduleOutput: URL?, arch: String, artifactBuilder: ArtifactSwiftProductsBuilder, producerFallbackCommandProcessors: [ShellCommandsProcessor], @@ -128,10 +130,14 @@ class SwiftcOrchestrator { try processor.applyArgsRewrite(args) } try fallbackToDefaultAndWait(command: swiftcCommand, args: swiftcArgs) - // move generated .h to the location where artifact creator expects it - try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput) - // move generated .swiftmodule to the location where artifact creator expects it - try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput) + if let objcHeaderOutput = objcHeaderOutput { + // move generated .h to the location where artifact creator expects it + try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput) + } + if let moduleOutput = moduleOutput { + // move generated .swiftmodule to the location where artifact creator expects it + try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput) + } try producerFallbackCommandProcessors.forEach { try $0.postCommandProcessing() diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 7ea2aa20..ee40becc 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -45,15 +45,15 @@ public struct SwiftcArgInput { } } -public class XCSwiftc { - private let command: String - private let inputArgs: SwiftcArgInput +public class XCSwiftAbstract { + let command: String + let inputArgs: InputArgs private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter private let touchFactory: (URL, FileManager) -> Touch public init( command: String, - inputArgs: SwiftcArgInput, + inputArgs: InputArgs, dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, touchFactory: @escaping (URL, FileManager) -> Touch ) { @@ -63,26 +63,52 @@ public class XCSwiftc { self.touchFactory = touchFactory } + func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { + fatalError("Need to override in \(Self.self)") + } + // swiftlint:disable:next function_body_length - public func run() { + public func run() throws { let fileManager = FileManager.default - let config: XCRemoteCacheConfig - let context: SwiftcContext - do { - let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) - .readConfiguration() - context = try SwiftcContext(config: config, input: inputArgs) - } catch { - exit(1, "FATAL: Swiftc initialization failed with error: \(error)") - } + let (config, context) = try buildContext() + let swiftcCommand = config.swiftcCommand let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath) let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager) - let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) - let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) + let inputReader: SwiftcInputReader + switch context.inputs { + case .fileMap(let path): + inputReader = SwiftcFilemapInputEditor( + URL(fileURLWithPath: path), + fileFormat: .json, + fileManager: fileManager + ) + case .supplementaryFileMap(let path): + // Supplementary file map is endoded in the yaml file (contraty to + // the standard filemap, which is in json) + inputReader = SwiftcFilemapInputEditor( + URL(fileURLWithPath: path), + fileFormat: .yaml, + fileManager: fileManager + ) + case .map(let map): + // static - passed via the arguments list + inputReader = StaticSwiftcInputReader( + moduleDependencies: context.steps.emitModule?.dependencies, + // with Xcode 14, inputs via cmd are only used for compilations + swiftDependencies: nil, + compilationFiles: Array(map.values) + ) + } + let fileListReader: ListReader + switch context.compilationFiles { + case .fileList(let path): + fileListReader = FileListEditor(URL(fileURLWithPath: path), fileManager: fileManager) + case .list(let paths): + fileListReader = StaticFileListReader(list: paths.map(URL.init(fileURLWithPath:))) + } let artifactOrganizer = ZipArtifactOrganizer( targetTempDir: context.tempDir, // xcswiftc doesn't call artifact preprocessing @@ -101,11 +127,20 @@ public class XCSwiftc { moduleName: context.moduleName, fileManager: fileManager ) - let productsGenerator = DiskSwiftcProductsGenerator( - modulePathOutput: context.modulePathOutput, - objcHeaderOutput: context.objcHeaderOutput, - diskCopier: HardLinkDiskCopier(fileManager: fileManager) - ) + let productsGenerator: SwiftcProductsGenerator + if let emitModule = context.steps.emitModule { + productsGenerator = DiskSwiftcProductsGenerator( + modulePathOutput: emitModule.modulePathOutput, + objcHeaderOutput: emitModule.objcHeaderOutput, + diskCopier: HardLinkDiskCopier(fileManager: fileManager) + ) + } else { + // If the module was not requested for this proces (compiling files only) + // do nothing, when someone (e.g. a plugin) asks for the products generation + // This generation will happend in a separate process, where the module + // generation is requested + productsGenerator = NoopSwiftcProductsGenerator() + } let allInvocationsStorage = ExistingFileStorage( storageFile: context.invocationHistoryFile, command: swiftcCommand @@ -119,7 +154,7 @@ public class XCSwiftc { let shellOut = ProcessShellOut() let swiftc = Swiftc( - inputFileListReader: fileListEditor, + inputFileListReader: fileListReader, markerReader: markerReader, allowedFilesListScanner: allowedFilesListScanner, artifactOrganizer: artifactOrganizer, @@ -136,18 +171,28 @@ public class XCSwiftc { mode: context.mode, swiftc: swiftc, swiftcCommand: swiftcCommand, - objcHeaderOutput: context.objcHeaderOutput, - moduleOutput: context.modulePathOutput, + objcHeaderOutput: context.steps.emitModule?.objcHeaderOutput, + moduleOutput: context.steps.emitModule?.modulePathOutput, arch: context.arch, artifactBuilder: artifactBuilder, producerFallbackCommandProcessors: [], invocationStorage: invocationStorage, shellOut: shellOut ) - do { - try orchestrator.run() - } catch { - exit(1, "Swiftc failed with error: \(error)") - } + try orchestrator.run() + } +} + +public class XCSwiftc: XCSwiftAbstract { + override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { + let fileReader = FileManager.default + let config: XCRemoteCacheConfig + let context: SwiftcContext + let srcRoot: URL = URL(fileURLWithPath: fileReader.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileReader) + .readConfiguration() + context = try SwiftcContext(config: config, input: inputArgs) + + return (config, context) } } diff --git a/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift b/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift new file mode 100644 index 00000000..6abfcfe8 --- /dev/null +++ b/Sources/XCRemoteCache/Dependencies/StaticFileListReader.swift @@ -0,0 +1,36 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +class StaticFileListReader: ListReader { + private let list: [URL] + + init(list: [URL]) { + self.list = list + } + + func listFilesURLs() throws -> [URL] { + list + } + + func canRead() -> Bool { + true + } +} diff --git a/Sources/xcswiftc/XCSwiftcMain.swift b/Sources/xcswiftc/XCSwiftcMain.swift index acb04868..ceadab6b 100644 --- a/Sources/xcswiftc/XCSwiftcMain.swift +++ b/Sources/xcswiftc/XCSwiftcMain.swift @@ -60,30 +60,37 @@ public class XCSwiftcMain { let targetInputInput = target, let swiftFileListInput = swiftFileList else { - let swiftcCommand = "swiftc" - print("Fallbacking to compilation using \(swiftcCommand).") + executeFallback() + } + do { + let swiftcArgsInput = SwiftcArgInput( + objcHeaderOutput: objcHeaderOutputInput, + moduleName: moduleNameInput, + modulePathOutput: modulePathOutputInput, + filemap: filemapInput, + target: targetInputInput, + fileList: swiftFileListInput + ) + try XCSwiftc( + command: command, + inputArgs: swiftcArgsInput, + dependenciesWriter: FileDependenciesWriter.init, + touchFactory: FileTouch.init + ).run() + } catch { + executeFallback() + } + } + private func executeFallback() -> Never { + let swiftcCommand = "swiftc" + print("Fallbacking to compilation using \(swiftcCommand).") - let args = ProcessInfo().arguments - let paramList = [swiftcCommand] + args.dropFirst() - let cargs = paramList.map { strdup($0) } + [nil] - execvp(swiftcCommand, cargs) + let args = ProcessInfo().arguments + let paramList = [swiftcCommand] + args.dropFirst() + let cargs = paramList.map { strdup($0) } + [nil] + execvp(swiftcCommand, cargs) - /// C-function `execv` returns only when the command fails - exit(1) - } - let swiftcArgsInput = SwiftcArgInput( - objcHeaderOutput: objcHeaderOutputInput, - moduleName: moduleNameInput, - modulePathOutput: modulePathOutputInput, - filemap: filemapInput, - target: targetInputInput, - fileList: swiftFileListInput - ) - XCSwiftc( - command: command, - inputArgs: swiftcArgsInput, - dependenciesWriter: FileDependenciesWriter.init, - touchFactory: FileTouch.init - ).run() + /// C-function `execv` returns only when the command fails + exit(1) } } diff --git a/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift index 66da9aa7..819323c7 100644 --- a/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/XcodeProjBuildSettingsIntegrateAppenderTests.swift @@ -49,7 +49,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let resultURL = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String) @@ -64,7 +65,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let resultURL: String = try XCTUnwrap(result["XCRC_FAKE_SRCROOT"] as? String) @@ -79,7 +81,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: fakeRootURL, - sdksExclude: [] + sdksExclude: [], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlus: String = try XCTUnwrap(result["LDPLUSPLUS"] as? String) @@ -93,7 +96,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*"] + sdksExclude: ["watchOS*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlusWatchOS: String = try XCTUnwrap(result["LDPLUSPLUS[sdk=watchOS*]"] as? String) @@ -107,7 +111,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*"] + sdksExclude: ["watchOS*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let libtoolWatchOS: String = try XCTUnwrap(result["LIBTOOL[sdk=watchOS*]"] as? String) @@ -121,7 +126,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*", "watchsimulator*"] + sdksExclude: ["watchOS*", "watchsimulator*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let ldPlusPlusWatchOS: String = try XCTUnwrap(result["LDPLUSPLUS[sdk=watchOS*]"] as? String) @@ -137,7 +143,8 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { mode: mode, repoRoot: rootURL, fakeSrcRoot: "/", - sdksExclude: ["watchOS*", "watchsimulator*"] + sdksExclude: ["watchOS*", "watchsimulator*"], + options: [] ) let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) let disabledWatchOS: String = try XCTUnwrap(result["XCRC_DISABLED[sdk=watchOS*]"] as? String) @@ -146,4 +153,34 @@ class XcodeProjBuildSettingsIntegrateAppenderTests: XCTestCase { XCTAssertEqual(disabledWatchOS, "YES") XCTAssertEqual(disabledWatchSimulator, "YES") } + + func testExcludesSwiftFrontendIntegrationForSpecificOption() throws { + let mode: Mode = .consumer + let appender = XcodeProjBuildSettingsIntegrateAppender( + mode: mode, + repoRoot: rootURL, + fakeSrcRoot: "/", + sdksExclude: [], + options: [.disableSwiftDriverIntegration] + ) + let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) + let useSwiftIntegrationDriver: String = try XCTUnwrap(result["SWIFT_USE_INTEGRATED_DRIVER"] as? String) + + XCTAssertEqual(useSwiftIntegrationDriver, "NO") + } + + func testDoesntExcludesSwiftFrontendIntegrationForEmptyOptions() throws { + let mode: Mode = .consumer + let appender = XcodeProjBuildSettingsIntegrateAppender( + mode: mode, + repoRoot: rootURL, + fakeSrcRoot: "/", + sdksExclude: [], + options: [] + ) + let result = appender.appendToBuildSettings(buildSettings: buildSettings, wrappers: binaries) + let useSwiftIntegrationDriver: String? = result["SWIFT_USE_INTEGRATED_DRIVER"] as? String + + XCTAssertNil(useSwiftIntegrationDriver) + } } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift index ed9e08d5..8fc08c4d 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcContextTests.swift @@ -25,20 +25,25 @@ class SwiftcContextTests: FileXCTestCase { private var config: XCRemoteCacheConfig! private var input: SwiftcArgInput! private var remoteCommitFile: URL! + private var modulePathOutput: URL! + private var fileMapUrl: URL! + private var fileListUrl: URL! override func setUpWithError() throws { try super.setUpWithError() let workingDir = try prepareTempDir() remoteCommitFile = workingDir.appendingPathComponent("arc.rc") - let modulePathOutput = workingDir.appendingPathComponent("mpo") + modulePathOutput = workingDir.appendingPathComponent("mpo") + fileMapUrl = workingDir.appendingPathComponent("filemap") + fileListUrl = workingDir.appendingPathComponent("filelist") config = XCRemoteCacheConfig(remoteCommitFile: remoteCommitFile.path, sourceRoot: workingDir.path) input = SwiftcArgInput( objcHeaderOutput: "Target-Swift.h", - moduleName: "", + moduleName: "Module", modulePathOutput: modulePathOutput.path, - filemap: "", + filemap: fileMapUrl.path, target: "", - fileList: "" + fileList: fileListUrl.path ) try fileManager.write(toPath: remoteCommitFile.path, contents: "123".data(using: .utf8)) } @@ -77,4 +82,29 @@ class SwiftcContextTests: FileXCTestCase { XCTAssertEqual(context.mode, .producer) } + + func testStepsContainEmitingModuleAndAllCompilationScope() throws { + let context = try SwiftcContext(config: config, input: input) + + XCTAssertEqual(context.steps, .init( + compileFilesScope: .all, + emitModule: .init( + objcHeaderOutput: "Target-Swift.h", + modulePathOutput: modulePathOutput, + dependencies: nil) + ) + ) + } + + func testReadsInputsFromFileMap() throws { + let context = try SwiftcContext(config: config, input: input) + + XCTAssertEqual(context.inputs, .fileMap(fileMapUrl.path)) + } + + func testReadsCompilationFilesFromFileList() throws { + let context = try SwiftcContext(config: config, input: input) + + XCTAssertEqual(context.compilationFiles, .fileList(fileListUrl.path)) + } } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift index 80b2eb15..8bd688ba 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcFilemapInputEditorTests.swift @@ -34,19 +34,21 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { ) private let sampleInfoContentData = #"{"":{"swift-dependencies":"/"}}"#.data(using: .utf8)! private var inputFile: URL! - private var editor: SwiftcFilemapInputEditor! + private var editorJson: SwiftcFilemapInputEditor! + private var editorYaml: SwiftcFilemapInputEditor! override func setUpWithError() throws { try super.setUpWithError() try prepareTempDir() inputFile = workingDirectory!.appendingPathComponent("swift.json") - editor = SwiftcFilemapInputEditor(inputFile, fileManager: fileManager) + editorJson = SwiftcFilemapInputEditor(inputFile, fileFormat: .json, fileManager: fileManager) + editorYaml = SwiftcFilemapInputEditor(inputFile, fileFormat: .yaml, fileManager: fileManager) } func testReading() throws { try fileManager.spt_writeToFile(atPath: inputFile.path, contents: sampleInfoContentData) - let readInfo = try editor.read() + let readInfo = try editorJson.read() XCTAssertEqual(readInfo, sampleInfo) } @@ -80,13 +82,13 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { ]) try fileManager.spt_writeToFile(atPath: inputFile.path, contents: infoContentData) - let readInfo = try editor.read() + let readInfo = try editorJson.read() XCTAssertEqual(readInfo, expectedInfo) } func testWritingSavesContent() throws { - try editor.write(sampleInfo) + try editorJson.write(sampleInfo) let savedContent = try Data(contentsOf: inputFile) let content = try JSONSerialization.jsonObject(with: savedContent, options: []) as? [String: Any] @@ -108,7 +110,7 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { ), ]) - try editor.write(extendedInfo) + try editorJson.write(extendedInfo) let savedContent = try Data(contentsOf: inputFile) let content = try JSONSerialization.jsonObject(with: savedContent, options: []) as? [String: Any] @@ -119,12 +121,50 @@ class SwiftcFilemapInputEditorTests: FileXCTestCase { func testModifyingFileCompilationInfo() throws { try fileManager.spt_writeToFile(atPath: inputFile.path, contents: sampleInfoContentData) - let originalInfo = try editor.read() + let originalInfo = try editorJson.read() var modifiedInfo = originalInfo modifiedInfo.files = [file] - try editor.write(modifiedInfo) - let finalInfo = try editor.read() + try editorJson.write(modifiedInfo) + let finalInfo = try editorJson.read() XCTAssertEqual(finalInfo, modifiedInfo) } + + func testReadingSupplementaryInfoWithOptionalProperties() throws { + let infoContentData = #""" + "/file1.swift": + swift-dependencies: "/file1.swiftdeps" + dependencies: "/file1.d" + "/file2.swift": + dependencies: "/file2.d" + object: "/file2.o" + swift-dependencies: "/file2.swiftdeps" + """#.data(using: .utf8)! + let expectedInfo = SwiftCompilationInfo( + info: SwiftModuleCompilationInfo( + dependencies: nil, + swiftDependencies: nil + ), + files: [ + SwiftFileCompilationInfo( + file: "/file1.swift", + dependencies: "/file1.d", + object: nil, + swiftDependencies: "/file1.swiftdeps" + ), + SwiftFileCompilationInfo( + file: "/file2.swift", + dependencies: "/file2.d", + object: "/file2.o", + swiftDependencies: "/file2.swiftdeps" + ), + ]) + try fileManager.spt_writeToFile(atPath: inputFile.path, contents: infoContentData) + + let readInfo = try editorYaml.read() + + // `files` order doesn't match + XCTAssertEqual(readInfo.info, expectedInfo.info) + XCTAssertEqual(Set(readInfo.files), Set(expectedInfo.files)) + } } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift index 67bb1686..4d93cb4f 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcOrchestratorTests.swift @@ -231,4 +231,45 @@ class SwiftcOrchestratorTests: XCTestCase { XCTAssertEqual(artifactBuilder.addedObjCHeaders, [:]) } + + func testNotSetObjCHeaderIsNotCreated() throws { + let swiftc = SwiftcMock(mockingResult: .success) + let orchestrator = SwiftcOrchestrator( + mode: .producer, + swiftc: swiftc, + swiftcCommand: "", + objcHeaderOutput: nil, + moduleOutput: moduleOutputURL, + arch: "arch", + artifactBuilder: artifactBuilder, + producerFallbackCommandProcessors: [], + invocationStorage: invocationStorage, + shellOut: shellOutSpy + ) + + try orchestrator.run() + + XCTAssertEqual(artifactBuilder.addedObjCHeaders, [:]) + } + + func testNotSetModuleOutputIsNotCreated() throws { + let swiftc = SwiftcMock(mockingResult: .success) + let orchestrator = SwiftcOrchestrator( + mode: .producer, + swiftc: swiftc, + swiftcCommand: "", + objcHeaderOutput: objcHeaderURL, + moduleOutput: nil, + arch: "arch", + artifactBuilder: artifactBuilder, + producerFallbackCommandProcessors: [], + invocationStorage: invocationStorage, + shellOut: shellOutSpy + ) + + try orchestrator.run() + + XCTAssertEqual(artifactBuilder.addedModuleDefinitions, [:]) + } + } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift index 5df313ff..50bf67a2 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift @@ -62,7 +62,7 @@ class SwiftcTests: FileXCTestCase { input = SwiftcArgInput( objcHeaderOutput: "Target-Swift.h", - moduleName: "", + moduleName: "Target", modulePathOutput: modulePathOutput.path, filemap: "", target: "", @@ -276,7 +276,7 @@ class SwiftcTests: FileXCTestCase { func testCompilationUsesArchSpecificSwiftmoduleFiles() throws { let artifactRoot = URL(fileURLWithPath: "/cachedArtifact") - let artifactObjCHeader = URL(fileURLWithPath: "/cachedArtifact/include/archTest/Target-Swift.h") + let artifactObjCHeader = URL(fileURLWithPath: "/cachedArtifact/include/archTest/Target/Target-Swift.h") let artifactSwiftmodule = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftmodule") let artifactSwiftdoc = URL(fileURLWithPath: "/cachedArtifact/swiftmodule/archTest/Target.swiftdoc") let artifactSwiftSourceInfo = URL( diff --git a/Tests/XCRemoteCacheTests/Dependencies/StaticFileListReaderTests.swift b/Tests/XCRemoteCacheTests/Dependencies/StaticFileListReaderTests.swift new file mode 100644 index 00000000..6ce02fbd --- /dev/null +++ b/Tests/XCRemoteCacheTests/Dependencies/StaticFileListReaderTests.swift @@ -0,0 +1,36 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + +class StaticFileListReaderTests: XCTestCase { + func testCanAlwaysRead() throws { + let reader = StaticFileListReader(list: []) + + XCTAssertTrue(reader.canRead()) + } + + func testListsPassedUrls() throws { + let url: URL = "/file" + let reader = StaticFileListReader(list: [url]) + + XCTAssertEqual(try reader.listFilesURLs(), [url]) + } +}