diff --git a/README.md b/README.md index 55d42e80..cfe69faa 100755 --- a/README.md +++ b/README.md @@ -267,6 +267,41 @@ That command creates an empty file on a remote server which informs that for giv _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, `xclibtool` wrappers become no-op, so it is recommended to not add them for the `producer` mode._ +##### 7. Generalize `-Swift.h` (Optional only if using static library with a bridging header with public `NS_ENUM` exposed from ObjC) + +If a static library target contains a mixed target with a bridging header exposing an enum from ObjC in a public Swift API, your custom script that moves `*-Swift.h` to the shared location, it should also move `*-Swift.h.md5` next to it. + +Example: + +##### Existing script (Before): + +```shell +ditto "${SCRIPT_INPUT_FILE_0}" "${SCRIPT_OUTPUT_FILE_0}" +``` + +where +* `SCRIPT_INPUT_FILE_0="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"` +* `SCRIPT_OUTPUT_FILE_0="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"` + +##### Correct script (After): + +```shell +ditto "${SCRIPT_INPUT_FILE_0}" "${SCRIPT_OUTPUT_FILE_0}" +[ -f "${SCRIPT_INPUT_FILE_1}" ] && ditto "${SCRIPT_INPUT_FILE_1}" "${SCRIPT_OUTPUT_FILE_1}" || rm "${SCRIPT_OUTPUT_FILE_1}" +``` + +where +* `SCRIPT_INPUT_FILE_0="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"` +* `SCRIPT_INPUT_FILE_1="$(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME).md5"` +* `SCRIPT_OUTPUT_FILE_0="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME)"` +* `SCRIPT_OUTPUT_FILE_1="$(BUILT_PRODUCTS_DIR)/include/$(PRODUCT_MODULE_NAME)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME).md5"` + +Note: This step is not required if at least one of these is true: + +* you build a framework (not a static library) +* you don't expose `NS_ENUM` type from ObjC to Swift via a bridging header + + ## A full list of configuration parameters: | Property | Description | Default | Required | diff --git a/Sources/XCRemoteCache/Artifacts/ArtifactCreator.swift b/Sources/XCRemoteCache/Artifacts/ArtifactCreator.swift index 5d3a4ef4..277ddaed 100644 --- a/Sources/XCRemoteCache/Artifacts/ArtifactCreator.swift +++ b/Sources/XCRemoteCache/Artifacts/ArtifactCreator.swift @@ -42,6 +42,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator { private let modulesFolderPath: String private let dSYMPath: URL private let metaWriter: MetaWriter + private let artifactProcessor: ArtifactProcessor private let fileManager: FileManager init( @@ -52,6 +53,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator { modulesFolderPath: String, dSYMPath: URL, metaWriter: MetaWriter, + artifactProcessor: ArtifactProcessor, fileManager: FileManager ) { self.buildDir = buildDir @@ -62,6 +64,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator { self.fileManager = fileManager self.dSYMPath = dSYMPath self.metaWriter = metaWriter + self.artifactProcessor = artifactProcessor super.init(workingDir: tempDir, moduleName: moduleName, fileManager: fileManager) } @@ -87,6 +90,7 @@ class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator { /// - Parameter tempDir: Temp location to organize file hierarchy in the artifact /// - returns: URLs to include into the artifact package fileprivate func prepareSwiftArtifacts(tempDir: URL) throws -> [URL] { + try artifactProcessor.process(localArtifact: tempDir) var artifacts: [URL] = [] // Add optional directory with generated ObjC headers diff --git a/Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift b/Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift index 2bedc3cc..2139b4c7 100644 --- a/Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift +++ b/Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift @@ -31,7 +31,7 @@ enum ArtifactOrganizerLocationPreparationResult: Equatable { case preparedForArtifact(artifact: URL) } -/// Prepares .zip artifact for the local operations +/// Prepares existing .zip artifact for the local operations protocol ArtifactOrganizer { /// Prepares the location for the artifact unzipping /// - Parameter fileKey: artifact fileKey that corresponds to the zip filename on the remote cache server @@ -48,10 +48,13 @@ protocol ArtifactOrganizer { class ZipArtifactOrganizer: ArtifactOrganizer { private let cacheDir: URL + // all processors that should "prepare" the unzipped raw artifact + private let artifactProcessors: [ArtifactProcessor] private let fileManager: FileManager - init(targetTempDir: URL, fileManager: FileManager) { + init(targetTempDir: URL, artifactProcessors: [ArtifactProcessor], fileManager: FileManager) { cacheDir = targetTempDir.appendingPathComponent("xccache") + self.artifactProcessors = artifactProcessors self.fileManager = fileManager } @@ -93,6 +96,10 @@ class ZipArtifactOrganizer: ArtifactOrganizer { // when the command was interrupted (internal crash or `kill -9` signal) let tempDestination = destinationURL.appendingPathExtension("tmp") try Zip.unzipFile(artifact, destination: tempDestination, overwrite: true, password: nil) + + try artifactProcessors.forEach { processor in + try processor.process(rawArtifact: tempDestination) + } try fileManager.moveItem(at: tempDestination, to: destinationURL) return destinationURL } diff --git a/Sources/XCRemoteCache/Artifacts/ArtifactProcessor.swift b/Sources/XCRemoteCache/Artifacts/ArtifactProcessor.swift new file mode 100644 index 00000000..60b1f2ec --- /dev/null +++ b/Sources/XCRemoteCache/Artifacts/ArtifactProcessor.swift @@ -0,0 +1,80 @@ +// Copyright (c) 2022 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + + +/// Performs a pre/postprocessing on an artifact package +/// Could be a place for file reorganization (to support legacy package formats) and/or +/// remapp absolute paths in some package files +protocol ArtifactProcessor { + /// Processes a raw artifact in a directory. Raw artifact is a format of an artifact + /// that is stored in a remote cache server (generic) + /// - Parameter rawArtifact: directory that contains raw artifact content + func process(rawArtifact: URL) throws + + /// Processes a local artifact in a directory + /// - Parameter localArtifact: directory that contains local (machine-specific) artifact content + func process(localArtifact: URL) throws +} + +/// Processes downloaded artifact by replacing generic paths in generated ObjC headers placed in ./include +class UnzippedArtifactProcessor: ArtifactProcessor { + /// All directories in an artifact that should be processed by path remapping + private static let remappingDirs = ["include"] + private let fileRemapper: FileDependenciesRemapper + private let dirScanner: DirScanner + + init(fileRemapper: FileDependenciesRemapper, dirScanner: DirScanner) { + self.fileRemapper = fileRemapper + self.dirScanner = dirScanner + } + + private func findProcessingEligableFiles(path: String) throws -> [URL] { + let remappingURL = URL(fileURLWithPath: path) + let allFiles = try dirScanner.recursiveItems(at: remappingURL) + return allFiles.filter({ !$0.isHidden }) + } + + /// Replaces all generic paths in a raw artifact's `include` dir with + /// absolute paths, specific for a given machine and configuration + /// - Parameter rawArtifact: raw artifact location + func process(rawArtifact url: URL) throws { + for remappingDir in Self.remappingDirs { + let remappingPath = url.appendingPathComponent(remappingDir).path + let allFiles = try findProcessingEligableFiles(path: remappingPath) + try allFiles.forEach(fileRemapper.remap(fromGeneric:)) + } + } + + func process(localArtifact url: URL) throws { + for remappingDir in Self.remappingDirs { + let remappingPath = url.appendingPathComponent(remappingDir).path + let allFiles = try findProcessingEligableFiles(path: remappingPath) + try allFiles.forEach(fileRemapper.remap(fromLocal:)) + } + } +} + +fileprivate extension URL { + // Recognize hidden files starting with a dot + var isHidden: Bool { + lastPathComponent.hasPrefix(".") + } +} diff --git a/Sources/XCRemoteCache/Artifacts/ArtifactSwiftProductsBuilder.swift b/Sources/XCRemoteCache/Artifacts/ArtifactSwiftProductsBuilder.swift index 6e7f2f71..46631d99 100644 --- a/Sources/XCRemoteCache/Artifacts/ArtifactSwiftProductsBuilder.swift +++ b/Sources/XCRemoteCache/Artifacts/ArtifactSwiftProductsBuilder.swift @@ -88,7 +88,7 @@ class ArtifactSwiftProductsBuilderImpl: ArtifactSwiftProductsBuilder { throw ArtifactSwiftProductsBuilderError.populatingNonExistingObjCHeader } try fileManager.createDirectory(at: moduleObjCURL, withIntermediateDirectories: true, attributes: nil) - try fileManager.spt_forceLinkItem(at: headerURL, to: headerArtifactURL) + try fileManager.spt_forceCopyItem(at: headerURL, to: headerArtifactURL) } func includeModuleDefinitionsToTheArtifact(arch: String, moduleURL: URL) throws { diff --git a/Sources/XCRemoteCache/Artifacts/FileDependenciesRemapper.swift b/Sources/XCRemoteCache/Artifacts/FileDependenciesRemapper.swift new file mode 100644 index 00000000..9a3c6f44 --- /dev/null +++ b/Sources/XCRemoteCache/Artifacts/FileDependenciesRemapper.swift @@ -0,0 +1,81 @@ +// Copyright (c) 2022 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + + +enum FileDependenciesRemapperError: Error { + /// Thrown when the file to remap is invalid (e.g. doesn't exist or has unexpected format) + case invalidRemappingFile(URL) +} + +/// Replaces paths in a file content between generic (placeholder-based) +/// and local formats +protocol FileDependenciesRemapper { + /// Replaces all generic paths (with placeholders) to a local, machine + /// specific absolute paths + /// - Parameter url: location of a file that should be remapped in-place + func remap(fromGeneric url: URL) throws + /// Replaces all local, machine specific absolute paths to + /// generic ones + /// - Parameter url: location of a file that should be remapped in-place + func remap(fromLocal url: URL) throws +} + +/// Remaps absolute paths in a text files stored on a disk +/// Note: That class can be used only for text-based files, not binaries +class TextFileDependenciesRemapper: FileDependenciesRemapper { + private static let linesSeparator = "\n" + private let remapper: DependenciesRemapper + private let fileAccessor: FileAccessor + + init(remapper: DependenciesRemapper, fileAccessor: FileAccessor) { + self.remapper = remapper + self.fileAccessor = fileAccessor + } + + private func readFileLines(_ url: URL) throws -> [String] { + guard let content = try fileAccessor.contents(atPath: url.path) else { + // the file is empty + return [] + } + guard let contentString = String(data: content, encoding: .utf8) else { + throw FileDependenciesRemapperError.invalidRemappingFile(url) + } + return contentString.components(separatedBy: .newlines) + } + + private func storeFileLines(lines: [String], url: URL) throws { + let contentString = lines.joined(separator: "\n") + let contentData = contentString.data(using: String.Encoding.utf8) + try fileAccessor.write(toPath: url.path, contents: contentData) + } + + func remap(fromGeneric url: URL) throws { + let contentLines = try readFileLines(url) + let remappedContent = try remapper.replace(genericPaths: contentLines) + try storeFileLines(lines: remappedContent, url: url) + } + + func remap(fromLocal url: URL) throws { + let contentLines = try readFileLines(url) + let remappedContent = try remapper.replace(localPaths: contentLines) + try storeFileLines(lines: remappedContent, url: url) + } +} diff --git a/Sources/XCRemoteCache/Commands/Plugins/Thinning/Factories/ThinningConsumerArtifactsOrganizerFactory.swift b/Sources/XCRemoteCache/Commands/Plugins/Thinning/Factories/ThinningConsumerArtifactsOrganizerFactory.swift index 1ec4a343..42925505 100644 --- a/Sources/XCRemoteCache/Commands/Plugins/Thinning/Factories/ThinningConsumerArtifactsOrganizerFactory.swift +++ b/Sources/XCRemoteCache/Commands/Plugins/Thinning/Factories/ThinningConsumerArtifactsOrganizerFactory.swift @@ -27,13 +27,19 @@ protocol ThinningConsumerArtifactsOrganizerFactory { } class ThinningConsumerZipArtifactsOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory { + private let processors: [ArtifactProcessor] private let fileManager: FileManager - init(fileManager: FileManager) { + init(processors: [ArtifactProcessor], fileManager: FileManager) { + self.processors = processors self.fileManager = fileManager } func build(targetTempDir: URL) -> ArtifactOrganizer { - ZipArtifactOrganizer(targetTempDir: targetTempDir, fileManager: fileManager) + ZipArtifactOrganizer( + targetTempDir: targetTempDir, + artifactProcessors: processors, + fileManager: fileManager + ) } } diff --git a/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGenerator.swift index 74f023e0..2d1d23bd 100644 --- a/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGenerator.swift +++ b/Sources/XCRemoteCache/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGenerator.swift @@ -58,7 +58,7 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator { func generateFrom( artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], artifactSwiftModuleObjCFile: URL - ) throws -> URL { + ) throws -> SwiftcProductsGeneratorOutput { // Move cached -Swift.h file to the expected location try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput) for (ext, url) in sourceAtifactSwiftModuleFiles { @@ -79,6 +79,9 @@ class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator { } // Build parent dir of the .swiftmodule file that contains a module - return modulePathOutput.deletingLastPathComponent() + return SwiftcProductsGeneratorOutput( + swiftmoduleDir: modulePathOutput.deletingLastPathComponent(), + objcHeaderFile: objcHeaderOutput + ) } } diff --git a/Sources/XCRemoteCache/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizer.swift b/Sources/XCRemoteCache/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizer.swift index 7f86eee4..1d3b77fb 100644 --- a/Sources/XCRemoteCache/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizer.swift +++ b/Sources/XCRemoteCache/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizer.swift @@ -73,11 +73,12 @@ class UnzippedArtifactSwiftProductsOrganizer: SwiftProductsOrganizer { .appendingPathComponent(moduleName) .appendingPathComponent("\(moduleName)-Swift.h") - let generatedModuleDir = try productsGenerator.generateFrom( + let generatedModule = try productsGenerator.generateFrom( artifactSwiftModuleFiles: artifactSwiftmoduleFiles, artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile ) - try fingerprintSyncer.decorate(sourceDir: generatedModuleDir, fingerprint: fingerprint) + try fingerprintSyncer.decorate(sourceDir: generatedModule.swiftmoduleDir, fingerprint: fingerprint) + try fingerprintSyncer.decorate(file: generatedModule.objcHeaderFile, fingerprint: fingerprint) } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/Postbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/Postbuild.swift index b37cdaee..72ac8980 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/Postbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/Postbuild.swift @@ -238,13 +238,32 @@ class Postbuild { let moduleSwiftProductURL = context.productsDir .appendingPathComponent(context.modulesFolderPath) .appendingPathComponent("\(modulename).swiftmodule") + let objcHeaderSwiftProductURL = context.derivedSourcesDir + .appendingPathComponent("\(modulename)-Swift.h") + // This header is obly valid if building a frameworks + let objcHeaderSwiftPublicPathURL = context.publicHeadersFolderPath? + .appendingPathComponent("\(modulename)-Swift.h") if let fingerprint = contextSpecificFingerprint { try fingerprintSyncer.decorate( sourceDir: moduleSwiftProductURL, fingerprint: fingerprint ) + try fingerprintSyncer.decorate( + file: objcHeaderSwiftProductURL, + fingerprint: fingerprint + ) + if let objcPublic = objcHeaderSwiftPublicPathURL { + try fingerprintSyncer.decorate( + file: objcPublic, + fingerprint: fingerprint + ) + } } else { try fingerprintSyncer.delete(sourceDir: moduleSwiftProductURL) + try fingerprintSyncer.delete(sourceDir: objcHeaderSwiftProductURL) + if let objcPublic = objcHeaderSwiftPublicPathURL { + try fingerprintSyncer.delete(file: objcPublic) + } } } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index 3c88bfa5..90dc0ea7 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -74,7 +74,7 @@ public struct PostbuildContext { let builtProductsDir: URL /// Location to the product bundle. Can be nil for libraries let bundleDir: URL? - let derivedSourcesDir: URL + var derivedSourcesDir: URL /// List of all targets to downloaded from the thinning aggregation target var thinnedTargets: [String] /// Action type: build, indexbuild etc @@ -85,6 +85,8 @@ public struct PostbuildContext { let overlayHeadersPath: URL /// Regexes of files that should not be included in the dependency list let irrelevantDependenciesPaths: [String] + /// Location of public headers. Not always available (e.g. static libraries) + var publicHeadersFolderPath: URL? } extension PostbuildContext { @@ -138,5 +140,11 @@ extension PostbuildContext { /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") irrelevantDependenciesPaths = config.irrelevantDependenciesPaths + let publicHeadersPath: String = try env.readEnv(key: "PUBLIC_HEADERS_FOLDER_PATH") + if publicHeadersPath != "/usr/local/include" { + // '/usr/local/include' is a value of PUBLIC_HEADERS_FOLDER_PATH when no public headers are automatically + // generated and it is up to a project configuration to place it in a common location (e.g. static library) + publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath) + } } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index ea40dc57..c3795ab5 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -87,8 +87,14 @@ public class XCPostbuild { fingerprintFilesGenerator, algorithm: MD5Algorithm() ) - let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: context.targetTempDir, + // In postbuild we don't preprocess artifacts (no need to replace path placeholders) + artifactProcessors: [], + fileManager: fileManager + ) let metaWriter = JsonMetaWriter(fileWriter: fileManager, pretty: config.prettifyMetaFiles) + let fileRemapper = TextFileDependenciesRemapper(remapper: envsRemapper, fileAccessor: fileManager) let artifactCreator = BuildArtifactCreator( buildDir: context.productsDir, tempDir: context.targetTempDir, @@ -97,6 +103,7 @@ public class XCPostbuild { modulesFolderPath: context.modulesFolderPath, dSYMPath: context.dSYMPath, metaWriter: metaWriter, + artifactProcessor: UnzippedArtifactProcessor(fileRemapper: fileRemapper, dirScanner: fileManager), fileManager: fileManager ) let dirAccessor = DirAccessorComposer( @@ -200,7 +207,10 @@ public class XCPostbuild { if context.moduleName == config.thinningTargetModuleName { switch context.mode { case .consumer: + // no need to process artifacts in postbuild. Prebuild has already + // run a processor on a downloaded artifact let artifactOrganizerFactory = ThinningConsumerZipArtifactsOrganizerFactory( + processors: [], fileManager: fileManager ) let swiftProductsLocationProvider = diff --git a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift index 21f59042..dad6a2c1 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift @@ -151,13 +151,25 @@ public class XCPrebuild { filesFingerprintGenerator, algorithm: MD5Algorithm() ) - let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager) + let fileRemapper = TextFileDependenciesRemapper( + remapper: envsRemapper, + fileAccessor: fileManager + ) + let artifactProcessor = UnzippedArtifactProcessor(fileRemapper: fileRemapper, dirScanner: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: context.targetTempDir, + artifactProcessors: [artifactProcessor], + fileManager: fileManager + ) let metaReader = JsonMetaReader(fileAccessor: fileManager) var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = [] if config.thinningEnabled { if context.moduleName == config.thinningTargetModuleName, let thinnedTarget = context.thinnedTargets { - let organizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(fileManager: .default) + let organizerFactory = ThinningConsumerZipArtifactsOrganizerFactory( + processors: [artifactProcessor], + fileManager: fileManager + ) let aggregationPlugin = ThinningConsumerPrebuildPlugin( targetName: context.targetName, tempDir: context.targetTempDir, diff --git a/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift b/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift index 70ed2ccc..26c44707 100644 --- a/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift +++ b/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift @@ -78,7 +78,12 @@ public class XCCreateBinary { } let markerURL = tempDir.appendingPathComponent(config.modeMarkerPath) do { - let organizer = ZipArtifactOrganizer(targetTempDir: tempDir, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: tempDir, + // Creation binary doesn't call artifact preprocessing + artifactProcessors: [], + fileManager: fileManager + ) let dependenciesWriter = FileDatWriter(dependencyInfo, fileManager: fileManager) let markerReader = FileMarkerReader(markerURL, fileManager: fileManager) guard fileManager.fileExists(atPath: markerURL.path) else { diff --git a/Sources/XCRemoteCache/Commands/Swiftc/MirroredLinkingSwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/MirroredLinkingSwiftcProductsGenerator.swift index f4856aca..951add99 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/MirroredLinkingSwiftcProductsGenerator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/MirroredLinkingSwiftcProductsGenerator.swift @@ -55,7 +55,7 @@ class MirroredLinkingSwiftcProductsGenerator: SwiftcProductsGenerator { func generateFrom( artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], artifactSwiftModuleObjCFile: URL - ) throws -> URL { + ) throws -> SwiftcProductsGeneratorOutput { /// Predict moduleName from the `*.swiftmodule` artifact let foundSwiftmoduleFile = artifactSwiftModuleFiles[.swiftmodule] guard let mainSwiftmoduleFile = foundSwiftmoduleFile else { diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift index 879d4122..74da9ca4 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcProductsGenerator.swift @@ -26,14 +26,19 @@ enum DiskSwiftcProductsGeneratorError: Error { case unknownSwiftmoduleFile } +struct SwiftcProductsGeneratorOutput { + let swiftmoduleDir: URL + let objcHeaderFile: URL +} + /// Generates swiftc product to the expected location protocol SwiftcProductsGenerator { /// Generates products from given files - /// - Returns: location dir where .swiftmodule files have been placed + /// - Returns: location dir where .swiftmodule and ObjC header files have been placed func generateFrom( artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], artifactSwiftModuleObjCFile: URL - ) throws -> URL + ) throws -> SwiftcProductsGeneratorOutput } /// Generator that produces all products in the locations where Xcode expects it, using provided disk copier @@ -64,7 +69,7 @@ class DiskSwiftcProductsGenerator: SwiftcProductsGenerator { func generateFrom( artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], artifactSwiftModuleObjCFile: URL - ) throws -> URL { + ) throws -> SwiftcProductsGeneratorOutput { // Move cached -Swift.h file to the expected location try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput) for (ext, url) in sourceAtifactSwiftModuleFiles { @@ -85,6 +90,9 @@ class DiskSwiftcProductsGenerator: SwiftcProductsGenerator { } // Build parent dir of the .swiftmodule file that contains a module - return modulePathOutput.deletingLastPathComponent() + return SwiftcProductsGeneratorOutput( + swiftmoduleDir: modulePathOutput.deletingLastPathComponent(), + objcHeaderFile: objcHeaderOutput + ) } } diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 2d988eea..7ea2aa20 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -83,7 +83,12 @@ public class XCSwiftc { let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager) let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager) - let artifactOrganizer = ZipArtifactOrganizer(targetTempDir: context.tempDir, fileManager: fileManager) + let artifactOrganizer = ZipArtifactOrganizer( + targetTempDir: context.tempDir, + // xcswiftc doesn't call artifact preprocessing + artifactProcessors: [], + fileManager: fileManager + ) // TODO: check for allowedFile comparing a list of all inputfiles, not dependencies from a marker let makerReferencedFilesListScanner = FileListScannerImpl(markerReader, caseSensitive: false) let allowedFilesListScanner = ExceptionsFilteredFileListScanner( diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index 80ab7975..8459a0b0 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -107,7 +107,8 @@ public struct XCRemoteCacheConfig: Encodable { var turnOffRemoteCacheOnFirstTimeout: Bool = false /// List of all extensions that should carry over source fingerprints. Extensions of all product files that /// contain non-deterministic content (absolute paths, timestamp, etc) should be included - var productFilesExtensionsWithContentOverride = ["swiftmodule"] + /// .h files may contain absolute paths if NS_ENUM is used in a public API from Swift code + var productFilesExtensionsWithContentOverride = ["swiftmodule", "h"] /// If true, plugins for thinning support should be enabled var thinningEnabled: Bool = false /// Module name of a target that works as a helper for thinned targets diff --git a/Sources/XCRemoteCache/Dependencies/FingerprintSyncer.swift b/Sources/XCRemoteCache/Dependencies/FingerprintSyncer.swift index 083d5c5d..6a2201a2 100644 --- a/Sources/XCRemoteCache/Dependencies/FingerprintSyncer.swift +++ b/Sources/XCRemoteCache/Dependencies/FingerprintSyncer.swift @@ -30,6 +30,10 @@ protocol FingerprintSyncer { func decorate(sourceDir: URL, fingerprint: String) throws /// Deletes fingerprint overrides in the dir (if already created) func delete(sourceDir: URL) throws + /// Sets a fingerprint override for a singe file placed + func decorate(file: URL, fingerprint: String) throws + /// Deletes fingerprint override for a file (if already created) + func delete(file: URL) throws } class FileFingerprintSyncer: FingerprintSyncer { @@ -78,4 +82,25 @@ class FileFingerprintSyncer: FingerprintSyncer { try dirAccessor.removeItem(atPath: file.path) } } + + func decorate(file: URL, fingerprint: String) throws { + guard let fingerprintData = fingerprint.data(using: .utf8) else { + throw FingerprintSyncerError.invalidFingerprint + } + let fingerprintFile = file.appendingPathExtension(fingerprintExtension) + try dirAccessor.write(toPath: fingerprintFile.path, contents: fingerprintData) + } + + func delete(file: URL) throws { + guard case .file = try dirAccessor.itemType(atPath: file.path) else { + // no file to decorate (no module was generated) + return + } + let overrideURL = file.appendingPathExtension(fingerprintExtension) + guard case .file = try dirAccessor.itemType(atPath: overrideURL.path) else { + // no override + return + } + try dirAccessor.removeItem(atPath: overrideURL.path) + } } diff --git a/Sources/XCRemoteCache/FileAccess/DirAccessor.swift b/Sources/XCRemoteCache/FileAccess/DirAccessor.swift index 04cfd425..6ec3f95b 100644 --- a/Sources/XCRemoteCache/FileAccess/DirAccessor.swift +++ b/Sources/XCRemoteCache/FileAccess/DirAccessor.swift @@ -33,8 +33,13 @@ protocol DirScanner { /// Returns all items in a directory (shallow search) /// - Parameter at: url of an existing directory to search - /// - Throws: an error if dir doesn't exist or I/O error + /// - Throws: an error if a dir doesn't exist or on I/O error func items(at dir: URL) throws -> [URL] + + /// Returns all items in a directory (recursive search) + /// - Parameter at: url of an existing directory to search + /// - Throws: an error if a dir doesn't exist or on I/O error + func recursiveItems(at dir: URL) throws -> [URL] } typealias DirAccessor = FileAccessor & DirScanner @@ -54,4 +59,18 @@ extension FileManager: DirScanner { let resolvedDir = dir.resolvingSymlinksInPath() return try contentsOfDirectory(at: resolvedDir, includingPropertiesForKeys: nil, options: []) } + + func recursiveItems(at dir: URL) throws -> [URL] { + // Iterating DFS + var queue: [URL] = [dir] + var results: [URL] = [] + while let item = queue.popLast() { + if try itemType(atPath: item.path) == .dir { + try queue.append(contentsOf: items(at: item)) + } else { + results.append(item) + } + } + return results + } } diff --git a/Sources/XCRemoteCache/FileAccess/DirAccessorComposer.swift b/Sources/XCRemoteCache/FileAccess/DirAccessorComposer.swift index 4c42b8eb..1773a144 100644 --- a/Sources/XCRemoteCache/FileAccess/DirAccessorComposer.swift +++ b/Sources/XCRemoteCache/FileAccess/DirAccessorComposer.swift @@ -52,4 +52,8 @@ class DirAccessorComposer: DirAccessor { func items(at dir: URL) throws -> [URL] { try dirScanner.items(at: dir) } + + func recursiveItems(at dir: URL) throws -> [URL] { + try dirScanner.recursiveItems(at: dir) + } } diff --git a/Tests/XCRemoteCacheTests/Artifacts/BuildArtifactCreatorTests.swift b/Tests/XCRemoteCacheTests/Artifacts/BuildArtifactCreatorTests.swift index eb44e31f..4809da55 100644 --- a/Tests/XCRemoteCacheTests/Artifacts/BuildArtifactCreatorTests.swift +++ b/Tests/XCRemoteCacheTests/Artifacts/BuildArtifactCreatorTests.swift @@ -58,6 +58,7 @@ class BuildArtifactCreatorTests: FileXCTestCase { dSYM = executableURL.deletingPathExtension().appendingPathExtension(".dSYM") try fileManager.spt_createEmptyFile(executableURL) try fileManager.spt_createEmptyFile(headerURL) + let artifactProcessor = NoopArtifactProcessor() creator = BuildArtifactCreator( buildDir: buildDir, @@ -67,6 +68,7 @@ class BuildArtifactCreatorTests: FileXCTestCase { modulesFolderPath: "", dSYMPath: dSYM, metaWriter: JsonMetaWriter(fileWriter: fileManager, pretty: false), + artifactProcessor: artifactProcessor, fileManager: fileManager ) } diff --git a/Tests/XCRemoteCacheTests/Artifacts/TextFileDependenciesRemapperTests.swift b/Tests/XCRemoteCacheTests/Artifacts/TextFileDependenciesRemapperTests.swift new file mode 100644 index 00000000..fdf13f12 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Artifacts/TextFileDependenciesRemapperTests.swift @@ -0,0 +1,101 @@ +// Copyright (c) 2022 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache + +import XCTest + +class TextFileDependenciesRemapperTests: FileXCTestCase { + + let stringsRemapper = StringDependenciesRemapper( + mappings: [ + .init(generic: "$(SRCROOT)", local: "/example"), + ]) + let fileAccessor = FileAccessorFake(mode: .strict) + var remapper: TextFileDependenciesRemapper! + + override func setUp() { + super.setUp() + remapper = TextFileDependenciesRemapper( + remapper: stringsRemapper, + fileAccessor: fileAccessor + ) + } + + func testRemapsGenericPlaceholders() throws { + try fileAccessor.write(toPath: "/file", contents: "Some $(SRCROOT).") + + try remapper.remap(fromGeneric: "/file") + + try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "Some /example.") + } + + func testRemapsLocalPathsToPlaceholders() throws { + try fileAccessor.write(toPath: "/file", contents: "Some /example.") + + try remapper.remap(fromLocal: "/file") + + try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "Some $(SRCROOT).") + } + + func testPersistsEmptyLines() throws { + let multilineData = """ + Line1 + + Line 2 + """.data(using: .utf8) + try fileAccessor.write(toPath: "/file", contents: multilineData) + + try remapper.remap(fromGeneric: "/file") + + try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), multilineData) + } + + func testPersistsEmptyLineAtTheEnd() throws { + // swiftlint:disable trailing_whitespace + let multilineData = """ + Line1 + + Line 2 + + """.data(using: .utf8) + // swiftlint:enable trailing_whitespace + try fileAccessor.write(toPath: "/file", contents: multilineData) + + try remapper.remap(fromGeneric: "/file") + + try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), multilineData) + } + + func testReplacesMultipletimesInLine() throws { + try fileAccessor.write(toPath: "/file", contents: "$(SRCROOT) and $(SRCROOT)") + + try remapper.remap(fromGeneric: "/file") + + try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "/example and /example") + } + + func testReplacesInMultipleLine() throws { + try fileAccessor.write(toPath: "/file", contents: "$(SRCROOT)\n$(SRCROOT)") + + try remapper.remap(fromGeneric: "/file") + + try XCTAssertEqual(fileAccessor.contents(atPath: "/file"), "/example\n/example") + } +} diff --git a/Tests/XCRemoteCacheTests/Artifacts/UnzippedArtifactProcessorTests.swift b/Tests/XCRemoteCacheTests/Artifacts/UnzippedArtifactProcessorTests.swift new file mode 100644 index 00000000..74acd972 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Artifacts/UnzippedArtifactProcessorTests.swift @@ -0,0 +1,71 @@ +// Copyright (c) 2022 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache + +import XCTest + +class UnzippedArtifactProcessorTests: FileXCTestCase { + + private let fileAccessor = FileAccessorFake(mode: .strict) + private let remapper = StringDependenciesRemapper(mappings: [.init(generic: "$(SRCROOT)", local: "/local")]) + private var fileRemapper: FileDependenciesRemapper! + private var processor: UnzippedArtifactProcessor! + + override func setUp() { + super.setUp() + fileRemapper = TextFileDependenciesRemapper(remapper: remapper, fileAccessor: fileAccessor) + processor = UnzippedArtifactProcessor( + fileRemapper: fileRemapper, + dirScanner: fileAccessor + ) + } + + func testProcessingRawArtifactReplacesPlaceholders() throws { + try fileAccessor.write(toPath: "/artifact/include/file", contents: "Some $(SRCROOT)") + + try processor.process(rawArtifact: "/artifact") + + XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/include/file"), "Some /local") + } + + func testProcessingRawArtifactReplacesInNestedInclude() throws { + try fileAccessor.write(toPath: "/artifact/include/nested/file", contents: "Some $(SRCROOT)") + + try processor.process(rawArtifact: "/artifact") + + XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/include/nested/file"), "Some /local") + } + + func testProcessingRawArtifactDoesntReplacesInNonIncludeDir() throws { + try fileAccessor.write(toPath: "/artifact/some/file", contents: "Some $(SRCROOT)") + + try processor.process(rawArtifact: "/artifact") + + XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/some/file"), "Some $(SRCROOT)") + } + + func testDoesntProcessEmptyFiles() throws { + try fileAccessor.write(toPath: "/artifact/include/.hidden", contents: "Some $(SRCROOT)") + + try processor.process(rawArtifact: "/artifact") + + XCTAssertEqual(try fileAccessor.contents(atPath: "/artifact/include/.hidden"), "Some $(SRCROOT)") + } +} diff --git a/Tests/XCRemoteCacheTests/Artifacts/ZipArtifactOrganizerTests.swift b/Tests/XCRemoteCacheTests/Artifacts/ZipArtifactOrganizerTests.swift index 56293795..8be8478d 100644 --- a/Tests/XCRemoteCacheTests/Artifacts/ZipArtifactOrganizerTests.swift +++ b/Tests/XCRemoteCacheTests/Artifacts/ZipArtifactOrganizerTests.swift @@ -54,7 +54,11 @@ class ZipArtifactOrganizerTests: XCTestCase { func testPreparePlacesArtifactInTheActiveLocation() throws { let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt") - let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: workingDirectory, + artifactProcessors: [], + fileManager: fileManager + ) let preparedArtifact = try organizer.prepare(artifact: zipURL) @@ -64,7 +68,11 @@ class ZipArtifactOrganizerTests: XCTestCase { func testPreparingExistingArtifact() throws { let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt") - let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: workingDirectory, + artifactProcessors: [], + fileManager: fileManager + ) _ = try organizer.prepare(artifact: zipURL) let preparedArtifact = try organizer.prepare(artifact: zipURL) @@ -75,7 +83,11 @@ class ZipArtifactOrganizerTests: XCTestCase { func testPreparePlacesArtifactInTheFileKeyRelatedLocation() throws { let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt", zipFileName: "abb32_fileKey") - let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: workingDirectory, + artifactProcessors: [], + fileManager: fileManager + ) let expectedArtifactLocation = workingDirectory.appendingPathComponent("abb32_fileKey") let preparedArtifact = try organizer.prepare(artifact: zipURL) @@ -89,7 +101,11 @@ class ZipArtifactOrganizerTests: XCTestCase { let artifactLocation = workingDirectory.appendingPathComponent("xccache") .appendingPathComponent(sampleFileKey, isDirectory: true) try fileManager.createDirectory(at: artifactLocation, withIntermediateDirectories: true, attributes: nil) - let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: workingDirectory, + artifactProcessors: [], + fileManager: fileManager + ) let result = try organizer.prepareArtifactLocationFor(fileKey: sampleFileKey) if case .artifactExists(artifactDir: let u) = result { @@ -105,7 +121,11 @@ class ZipArtifactOrganizerTests: XCTestCase { .appendingPathComponent("xccache") .appendingPathComponent(sampleFileKey) .appendingPathExtension("zip") - let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: workingDirectory, + artifactProcessors: [], + fileManager: fileManager + ) let result = try organizer.prepareArtifactLocationFor(fileKey: sampleFileKey) @@ -126,7 +146,11 @@ class ZipArtifactOrganizerTests: XCTestCase { try fileManager.createDirectory(at: activeArtifact, withIntermediateDirectories: true, attributes: nil) try fileManager.spt_forceSymbolicLink(at: activeLink, withDestinationURL: activeArtifact) - let organizer = ZipArtifactOrganizer(targetTempDir: workingDirectory, fileManager: fileManager) + let organizer = ZipArtifactOrganizer( + targetTempDir: workingDirectory, + artifactProcessors: [], + fileManager: fileManager + ) let fileKey = try organizer.getActiveArtifactFilekey() diff --git a/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGeneratorTests.swift b/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGeneratorTests.swift index 79d20181..2c16901e 100644 --- a/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGeneratorTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/ThinningDiskSwiftcProductsGeneratorTests.swift @@ -67,7 +67,8 @@ class ThinningDiskSwiftcProductsGeneratorTests: FileXCTestCase { artifactSwiftModuleObjCFile: headerFile ) - XCTAssertEqual(generatedModulePath, destinationSwiftModuleDir) + XCTAssertEqual(generatedModulePath.swiftmoduleDir, destinationSwiftModuleDir) + XCTAssertEqual(generatedModulePath.objcHeaderFile, objCHeader) XCTAssertEqual(fileManager.contents(atPath: expectedSwiftSourceInfoFile.path), "sourceInfo".data(using: .utf8)) XCTAssertEqual(fileManager.contents(atPath: objCHeader.path), "header".data(using: .utf8)) } diff --git a/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizerTests.swift b/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizerTests.swift index 871bf44e..d6906e5f 100644 --- a/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizerTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/Plugins/Thinning/UnzippedArtifactSwiftProductsOrganizerTests.swift @@ -30,6 +30,7 @@ class UnzippedArtifactSwiftProductsOrganizerTests: XCTestCase { override func setUpWithError() throws { try super.setUpWithError() + let destination = SwiftcProductsGeneratorOutput(swiftmoduleDir: destination, objcHeaderFile: "") generator = SwiftcProductsGeneratorSpy(generatedDestination: destination) dirAccessor = DirAccessorFake() syncer = FileFingerprintSyncer( diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift index 86c6246e..38e3e779 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift @@ -44,6 +44,7 @@ class PostbuildContextTests: FileXCTestCase { "BUILT_PRODUCTS_DIR": "BUILT_PRODUCTS_DIR", "DERIVED_SOURCES_DIR": "DERIVED_SOURCES_DIR", "CURRENT_VARIANT": "normal", + "PUBLIC_HEADERS_FOLDER_PATH": "/usr/local/include", ] override func setUpWithError() throws { @@ -130,4 +131,23 @@ class PostbuildContextTests: FileXCTestCase { XCTAssertEqual(context.compilationTempDir, "/OBJECT_FILE_DIR_custom/x86_64") } + + func testGenericPublicHeaderDestinationIsSkipped() throws { + var envs = Self.SampleEnvs + envs["PUBLIC_HEADERS_FOLDER_PATH"] = "/usr/local/include" + + let context = try PostbuildContext(config, env: envs) + + XCTAssertNil(context.publicHeadersFolderPath) + } + + func testPublicHeaderFolderIsRelativeToProductsDir() throws { + var envs = Self.SampleEnvs + envs["BUILT_PRODUCTS_DIR"] = "/MyBuiltProductsDir" + envs["PUBLIC_HEADERS_FOLDER_PATH"] = "MyModule.grameworks/Headers" + + let context = try PostbuildContext(config, env: envs) + + XCTAssertEqual(context.publicHeadersFolderPath, "/MyBuiltProductsDir/MyModule.grameworks/Headers") + } } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index 33cb6313..bff7dab2 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -55,7 +55,8 @@ class PostbuildTests: FileXCTestCase { action: .build, modeMarkerPath: "", overlayHeadersPath: "", - irrelevantDependenciesPaths: [] + irrelevantDependenciesPaths: [], + publicHeadersFolderPath: nil ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), @@ -643,4 +644,79 @@ class PostbuildTests: FileXCTestCase { XCTAssertEqual(downloadedMeta, expectedMeta) } + + func testDecoratesDerivedSwiftHeaderWithEmptyModulesFolderPath() throws { + let dir = try prepareTempDir() + let derivedSourcesDir = dir + .appendingPathComponent("DerivedSources") + let swiftSwiftHeader = derivedSourcesDir + .appendingPathComponent("MyModule-Swift.h") + let swiftSwiftHeaderOverride = swiftSwiftHeader + .appendingPathExtension("md5") + + try fileManager.spt_createEmptyDir(derivedSourcesDir) + try fileManager.spt_createEmptyFile(swiftSwiftHeader) + postbuildContext.moduleName = "MyModule" + postbuildContext.derivedSourcesDir = derivedSourcesDir + let postbuild = Postbuild( + context: postbuildContext, + networkClient: network, + remapper: remapper, + fingerprintAccumulator: fingerprintGenerator, + artifactsOrganizer: organizer, + artifactCreator: artifactCreator, + fingerprintSyncer: syncer, + dependenciesReader: dependenciesReader, + dependencyProcessor: processor, + fingerprintOverrideManager: overrideManager, + dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil), + modeController: modeController, + metaReader: metaReader, + metaWriter: metaWriter, + creatorPlugins: [], + consumerPlugins: [] + ) + + try postbuild.performBuildCompletion() + + XCTAssertTrue(fileManager.fileExists(atPath: swiftSwiftHeaderOverride.path)) + } + + func testDecoratesPublicSwiftHeaderWithEmptyModulesFolderPath() throws { + let dir = try prepareTempDir() + let productsDir = dir + .appendingPathComponent("MyModule.framework") + .appendingPathComponent("Headers") + let swiftSwiftHeader = productsDir + .appendingPathComponent("MyModule-Swift.h") + let swiftSwiftHeaderOverride = swiftSwiftHeader + .appendingPathExtension("md5") + + try fileManager.spt_createEmptyDir(productsDir) + try fileManager.spt_createEmptyFile(swiftSwiftHeader) + postbuildContext.moduleName = "MyModule" + postbuildContext.publicHeadersFolderPath = productsDir + let postbuild = Postbuild( + context: postbuildContext, + networkClient: network, + remapper: remapper, + fingerprintAccumulator: fingerprintGenerator, + artifactsOrganizer: organizer, + artifactCreator: artifactCreator, + fingerprintSyncer: syncer, + dependenciesReader: dependenciesReader, + dependencyProcessor: processor, + fingerprintOverrideManager: overrideManager, + dSYMOrganizer: DSYMOrganizerFake(dSYMFile: nil), + modeController: modeController, + metaReader: metaReader, + metaWriter: metaWriter, + creatorPlugins: [], + consumerPlugins: [] + ) + + try postbuild.performBuildCompletion() + + XCTAssertTrue(fileManager.fileExists(atPath: swiftSwiftHeaderOverride.path)) + } } diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift index 41b9c69e..e5f23ccd 100644 --- a/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcTests.swift @@ -70,7 +70,9 @@ class SwiftcTests: FileXCTestCase { ) context = try SwiftcContext(config: config, input: input) markerWriter = MarkerWriterSpy() - productsGenerator = SwiftcProductsGeneratorSpy() + productsGenerator = SwiftcProductsGeneratorSpy( + generatedDestination: SwiftcProductsGeneratorOutput(swiftmoduleDir: "", objcHeaderFile: "") + ) let dependenciesWriterSpy = DependenciesWriterSpy() self.dependenciesWriterSpy = dependenciesWriterSpy dependenciesWriterFactory = { [dependenciesWriterSpy] _, _ in dependenciesWriterSpy } diff --git a/Tests/XCRemoteCacheTests/Dependencies/FileFingerprintSyncerTests.swift b/Tests/XCRemoteCacheTests/Dependencies/FileFingerprintSyncerTests.swift index 043d56be..cc8dc998 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/FileFingerprintSyncerTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/FileFingerprintSyncerTests.swift @@ -74,4 +74,51 @@ class FileFingerprintSyncerTests: FileXCTestCase { XCTAssertTrue(fileManager.fileExists(atPath: nonOverrideFile.path)) } + + func testDecoratesFile() throws { + let header = swiftmoduleDir.appendingPathComponent("Module-Swift.h") + let headerOverride = swiftmoduleDir.appendingPathComponent("Module-Swift.h.md5") + try fileManager.spt_createEmptyFile(header) + + + try syncer.decorate(file: header, fingerprint: "1") + + XCTAssertEqual(try String(contentsOf: headerOverride), "1") + } + + func testFileDecorateOverridesPreviousOverlay() throws { + let header = swiftmoduleDir.appendingPathComponent("Module-Swift.h") + let headerOverride = swiftmoduleDir.appendingPathComponent("Module-Swift.h.md5") + try fileManager.spt_createEmptyFile(header) + try "1".write(to: headerOverride, atomically: true, encoding: .utf8) + + try syncer.decorate(file: header, fingerprint: "2") + + XCTAssertEqual(try String(contentsOf: headerOverride), "2") + } + + func testDeletesFileOverride() throws { + let header = swiftmoduleDir.appendingPathComponent("Module-Swift.h") + let headerOverride = swiftmoduleDir.appendingPathComponent("Module-Swift.h.md5") + try fileManager.spt_createEmptyFile(header) + try fileManager.spt_createEmptyFile(headerOverride) + + + try syncer.delete(file: header) + + XCTAssertFalse(fileManager.fileExists(atPath: headerOverride.path)) + } + + func testDeletesDoesntDeleteWhenFileIsMissing() throws { + let nonExistingFile = swiftmoduleDir.appendingPathComponent("Module-Swift.h") + + XCTAssertNoThrow(try syncer.delete(file: nonExistingFile)) + } + + func testDeletesDoesntDeleteWhenOverrideIsMissing() throws { + let header = swiftmoduleDir.appendingPathComponent("Module-Swift.h") + try fileManager.spt_createEmptyFile(header) + + XCTAssertNoThrow(try syncer.delete(file: header)) + } } diff --git a/Tests/XCRemoteCacheTests/FileAccess/DirScannerTests.swift b/Tests/XCRemoteCacheTests/FileAccess/DirScannerTests.swift index 3844648e..cae32286 100644 --- a/Tests/XCRemoteCacheTests/FileAccess/DirScannerTests.swift +++ b/Tests/XCRemoteCacheTests/FileAccess/DirScannerTests.swift @@ -71,4 +71,17 @@ class FileManagerDirScannerTests: FileXCTestCase { try XCTAssertThrowsError(dirScanner.items(at: dir)) } + + func testFindsAllFilesRecursively() throws { + let dir = workingDirectory!.appendingPathComponent("dir") + let nestedDir = dir.appendingPathComponent("nested") + let nestedFile = nestedDir.appendingPathComponent("file") + try fileManager.spt_createEmptyFile(nestedFile) + + let allFiles = try dirScanner.recursiveItems(at: dir) + + // returned items may contain symbolic links in a path + let allFilesResolve = allFiles.map { $0.resolvingSymlinksInPath() } + XCTAssertEqual(allFilesResolve, [nestedFile]) + } } diff --git a/Tests/XCRemoteCacheTests/TestDoubles/DirAccessorFake.swift b/Tests/XCRemoteCacheTests/TestDoubles/DirAccessorFake.swift index 71a29146..7c922ab5 100644 --- a/Tests/XCRemoteCacheTests/TestDoubles/DirAccessorFake.swift +++ b/Tests/XCRemoteCacheTests/TestDoubles/DirAccessorFake.swift @@ -47,6 +47,16 @@ class DirAccessorFake: DirAccessor { } } + func recursiveItems(at dir: URL) throws -> [URL] { + memory.compactMap { url, _ in + // comparing paths to ignore dir or url's "isDir" property + if url.deletingLastPathComponent().path.starts(with: dir.path) { + return url + } + return nil + } + } + func contents(atPath path: String) throws -> Data? { memory[URL(fileURLWithPath: path)] } diff --git a/Tests/XCRemoteCacheTests/TestDoubles/FileAccessorFake.swift b/Tests/XCRemoteCacheTests/TestDoubles/FileAccessorFake.swift index 9f93377a..79880c91 100644 --- a/Tests/XCRemoteCacheTests/TestDoubles/FileAccessorFake.swift +++ b/Tests/XCRemoteCacheTests/TestDoubles/FileAccessorFake.swift @@ -60,3 +60,28 @@ class FileAccessorFake: FileAccessor { return storage[path]?.mdate } } + +extension FileAccessorFake: DirScanner { + func itemType(atPath path: String) throws -> ItemType { + if storage[path] != nil { + return .file + } + if try !recursiveItems(at: URL(fileURLWithPath: path)).isEmpty { + return .dir + } + return .nonExisting + } + + func items(at dir: URL) throws -> [URL] { + storage.keys.map(URL.init(fileURLWithPath:)).filter { + $0.deletingLastPathComponent() == dir + } + } + + func recursiveItems(at dir: URL) throws -> [URL] { + let paths = storage.keys.filter { + $0.hasPrefix(dir.path) + } + return paths.map(URL.init(fileURLWithPath:)) + } +} diff --git a/Tests/XCRemoteCacheTests/TestDoubles/NoopArtifactProcessor.swift b/Tests/XCRemoteCacheTests/TestDoubles/NoopArtifactProcessor.swift new file mode 100644 index 00000000..3a6a2ee6 --- /dev/null +++ b/Tests/XCRemoteCacheTests/TestDoubles/NoopArtifactProcessor.swift @@ -0,0 +1,28 @@ +// Copyright (c) 2022 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +import Foundation +@testable import XCRemoteCache + +/// No-operation processor +class NoopArtifactProcessor: ArtifactProcessor { + func process(rawArtifact url: URL) throws {} + func process(localArtifact url: URL) throws {} +} diff --git a/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorFake.swift b/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorFake.swift index e7e77d02..e75f35f8 100644 --- a/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorFake.swift +++ b/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorFake.swift @@ -39,7 +39,7 @@ class SwiftcProductsGeneratorFake: SwiftcProductsGenerator { func generateFrom( artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], artifactSwiftModuleObjCFile: URL - ) throws -> URL { + ) throws -> SwiftcProductsGeneratorOutput { let swiftmoduleDestBasename = swiftmoduleDest.deletingPathExtension() for (ext, url) in artifactSwiftModuleFiles { try dirAccessor.write( @@ -51,6 +51,9 @@ class SwiftcProductsGeneratorFake: SwiftcProductsGenerator { toPath: swiftmoduleObjCFile.path, contents: dirAccessor.contents(atPath: artifactSwiftModuleObjCFile.path) ) - return swiftmoduleDest.deletingLastPathComponent() + return SwiftcProductsGeneratorOutput( + swiftmoduleDir: swiftmoduleDest.deletingLastPathComponent(), + objcHeaderFile: swiftmoduleObjCFile + ) } } diff --git a/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorSpy.swift b/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorSpy.swift index 98a25c33..bd7946aa 100644 --- a/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorSpy.swift +++ b/Tests/XCRemoteCacheTests/TestDoubles/SwiftcProductsGeneratorSpy.swift @@ -22,16 +22,16 @@ import Foundation class SwiftcProductsGeneratorSpy: SwiftcProductsGenerator { private(set) var generated: [([SwiftmoduleFileExtension: URL], URL)] = [] - private let generationDestination: URL + private let generationDestination: SwiftcProductsGeneratorOutput - init(generatedDestination: URL = "") { + init(generatedDestination: SwiftcProductsGeneratorOutput) { generationDestination = generatedDestination } func generateFrom( artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL], artifactSwiftModuleObjCFile: URL - ) throws -> URL { + ) throws -> SwiftcProductsGeneratorOutput { generated.append(( artifactSwiftModuleFiles, artifactSwiftModuleObjCFile diff --git a/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj b/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj index e0dca9d0..b894e30d 100644 --- a/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj +++ b/e2eTests/StandaloneSampleApp/StandaloneApp.xcodeproj/project.pbxproj @@ -240,7 +240,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "ditto \"${SCRIPT_INPUT_FILE_0}\" \"${SCRIPT_OUTPUT_FILE_0}\"\nditto \"${SCRIPT_INPUT_FILE_1}\" \"${SCRIPT_OUTPUT_FILE_1}\" || true\n\n"; + shellScript = "ditto \"${SCRIPT_INPUT_FILE_0}\" \"${SCRIPT_OUTPUT_FILE_0}\"\n[ -f \"${SCRIPT_INPUT_FILE_1}\" ] && ditto \"${SCRIPT_INPUT_FILE_1}\" \"${SCRIPT_OUTPUT_FILE_1}\" || rm \"${SCRIPT_OUTPUT_FILE_1}\"\n\n"; }; /* End PBXShellScriptBuildPhase section */ diff --git a/tasks/e2e.rb b/tasks/e2e.rb index 825819c9..7700b441 100644 --- a/tasks/e2e.rb +++ b/tasks/e2e.rb @@ -26,8 +26,7 @@ Stats = Struct.new(:hits, :misses, :hit_rate) # run E2E tests - # TODO: add :run_standalone when support for bridging headers support is ready - task :run => [:run_cocoapods] + task :run => [:run_cocoapods, :run_standalone] # run E2E tests for CocoaPods-powered projects task :run_cocoapods do @@ -56,7 +55,6 @@ Dir.chdir(E2E_STANDALONE_SAMPLE_DIR) do clean_git # Run integrate the project - p "#{XCRC_BINARIES}/xcprepare integrate --input StandaloneApp.xcodeproj --mode producer --final-producer-target StandaloneApp" system("pwd") system("#{XCRC_BINARIES}/xcprepare integrate --input StandaloneApp.xcodeproj --mode producer --final-producer-target StandaloneApp") # Build the project to fill in the cache