From c573ced4f4e3b0a5359cec1436029778c0d16d5a Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sat, 7 May 2022 09:29:20 +0200 Subject: [PATCH 1/6] Add irrelevant_dependencies_paths property --- README.md | 1 + .../Commands/Postbuild/PostbuildContext.swift | 2 ++ .../Commands/Postbuild/XCPostbuild.swift | 5 ++++- .../Config/XCRemoteCacheConfig.swift | 6 ++++++ .../Dependencies/DependencyProcessor.swift | 8 +++++++- .../Commands/PostbuildTests.swift | 6 ++++-- .../DependencyProcessorImplTests.swift | 18 ++++++++++++------ 7 files changed, 36 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 254b2319..d9fdbb2d 100755 --- a/README.md +++ b/README.md @@ -315,6 +315,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ | | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ | | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | +| `irrelevant_dependencies_paths` | Paths that should not be included in a list of dependencies Note: Can contain ENV placeholders, e.g. `[$(COOL_LIBRARY)]`. | `[]` | ⬜️ | ## Backend cache server diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index abd9f0ea..1c398ea2 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -83,6 +83,7 @@ public struct PostbuildContext { /// location of the json file that define virtual files system overlay /// (mappings of the virtual location file -> local file path) let overlayHeadersPath: URL + let skippedDependenciesPaths: [String] } extension PostbuildContext { @@ -135,5 +136,6 @@ extension PostbuildContext { modeMarkerPath = config.modeMarkerPath /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") + skippedDependenciesPaths = config.irrelevantDependenciesPaths } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 0656af34..276faa4c 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -166,13 +166,16 @@ public class XCPostbuild { } remappers.append(envsRemapper) let pathRemapper = DependenciesRemapperComposite(remappers) + let skippedDependenciesPaths = try pathRemapper.replace(genericPaths: context.skippedDependenciesPaths) + let skippedDependenciesURLs = skippedDependenciesPaths.map(URL.init(fileURLWithPath:)) let dependencyProcessor = DependencyProcessorImpl( xcode: context.xcodeDir, product: context.productsDir, source: context.srcRoot, intermediate: context.targetTempDir, derivedFiles: context.derivedFilesDir, - bundle: context.bundleDir + bundle: context.bundleDir, + skipped: skippedDependenciesURLs ) // Override fingerprints for all produced '.swiftmodule' files let fingerprintOverrideManager = FingerprintOverrideManagerImpl( diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index be5bc27f..dec84b83 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -138,6 +138,9 @@ public struct XCRemoteCacheConfig: Encodable { /// A list of extra ENVs that should be used as placeholders in the dependency list /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process var customRewriteEnvs: [String] = [] + /// Paths that should not be included in a list of dependencies + /// Note: Can contain ENV placeholders, e.g. `[$(COOL_LIBRARY)]` + var irrelevantDependenciesPaths: [String] = [] } extension XCRemoteCacheConfig { @@ -193,6 +196,7 @@ extension XCRemoteCacheConfig { merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs + merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths return merge } @@ -257,6 +261,7 @@ struct ConfigFileScheme: Decodable { let disableCertificateVerification: Bool? let disableVFSOverlay: Bool? let customRewriteEnvs: [String]? + let irrelevantDependenciesPaths: [String]? // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84 enum CodingKeys: String, CodingKey { @@ -304,6 +309,7 @@ struct ConfigFileScheme: Decodable { case disableCertificateVerification = "disable_certificate_verification" case disableVFSOverlay = "disable_vfs_overlay" case customRewriteEnvs = "custom_rewrite_envs" + case irrelevantDependenciesPaths = "irrelevant_dependencies_paths" } } diff --git a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift index 4308f179..f2f711d1 100644 --- a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift +++ b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift @@ -30,6 +30,8 @@ public struct Dependency: Equatable { case derivedFile // Product of the target itself case ownProduct + // User-excluded path + case customSkipped case unknown } @@ -58,14 +60,16 @@ class DependencyProcessorImpl: DependencyProcessor { private let intermediatePath: String private let derivedFilesPath: String private let bundlePath: String? + private let skippedPaths: [String] - init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?) { + init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?, skipped: [URL]) { xcodePath = xcode.path.dirPath() productPath = product.path.dirPath() sourcePath = source.path.dirPath() intermediatePath = intermediate.path.dirPath() derivedFilesPath = derivedFiles.path.dirPath() bundlePath = bundle?.path.dirPath() + skippedPaths = skipped.map { $0.path.dirPath() } } func process(_ files: [URL]) -> [Dependency] { @@ -90,6 +94,8 @@ class DependencyProcessorImpl: DependencyProcessor { return Dependency(url: file, type: .product) } else if filePath.hasPrefix(sourcePath) { return Dependency(url: file, type: .source) + } else if skippedPaths.contains(where: { filePath.hasPrefix($0) }){ + return Dependency(url: file, type: .customSkipped) } else { return Dependency(url: file, type: .unknown) } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index cf95ba9e..be665edd 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -54,7 +54,8 @@ class PostbuildTests: FileXCTestCase { thinnedTargets: [], action: .build, modeMarkerPath: "", - overlayHeadersPath: "" + overlayHeadersPath: "", + skippedDependenciesPaths: [] ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), @@ -80,7 +81,8 @@ class PostbuildTests: FileXCTestCase { source: "/Source", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: nil + bundle: nil, + skipped: [] ) private var overrideManager = FingerprintOverrideManagerImpl( overridingFileExtensions: ["swiftmodule"], diff --git a/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift b/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift index a0ef4e2e..3e823605 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift @@ -28,7 +28,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/Source", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skipped: [] ) func testIntermediateFileIsSkippedForProductAndSourceSubdirectory() { @@ -39,7 +40,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: nil + bundle: nil, + skipped: [] ) XCTAssertEqual( @@ -56,7 +58,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skipped: [] ) XCTAssertEqual( @@ -150,7 +153,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/Source", intermediate: intermediateDirReal, derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skipped: [] ) let intermediateFileSymlink = createSymlink( @@ -179,7 +183,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: sourceDirReal, intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: "/Bundle" + bundle: "/Bundle", + skipped: [] ) let sourceFileSymlink = createSymlink( @@ -219,7 +224,8 @@ class DependencyProcessorImplTests: FileXCTestCase { source: "/", intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", - bundle: nil + bundle: nil, + skipped: [] ) XCTAssertEqual( From a55a4547ac59e75ef6ead1b7ddc63ac735c3ab7e Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 10 May 2022 12:39:40 +0200 Subject: [PATCH 2/6] Add skipped dependencies regex --- README.md | 2 +- .../Commands/Postbuild/PostbuildContext.swift | 5 +- .../Commands/Postbuild/XCPostbuild.swift | 4 +- .../Config/XCRemoteCacheConfig.swift | 6 +- .../Dependencies/DependencyProcessor.swift | 16 +++-- .../Commands/PostbuildTests.swift | 4 +- .../DependencyProcessorImplTests.swift | 70 ++++++++++++++++--- 7 files changed, 82 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d9fdbb2d..91347ca9 100755 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ | | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ | | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | -| `irrelevant_dependencies_paths` | Paths that should not be included in a list of dependencies Note: Can contain ENV placeholders, e.g. `[$(COOL_LIBRARY)]`. | `[]` | ⬜️ | +| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Be caution when adding entries here - excluding relevant dependencies might lead to targets overcaching. The regex can match either partially of fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files.. | `[]` | ⬜️ | ## Backend cache server diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index 1c398ea2..85ce00a3 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -83,7 +83,8 @@ public struct PostbuildContext { /// location of the json file that define virtual files system overlay /// (mappings of the virtual location file -> local file path) let overlayHeadersPath: URL - let skippedDependenciesPaths: [String] + /// Regexes of files that should not be included in the dependency list + let skippedDependenciesRegexes: [String] } extension PostbuildContext { @@ -136,6 +137,6 @@ extension PostbuildContext { modeMarkerPath = config.modeMarkerPath /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") - skippedDependenciesPaths = config.irrelevantDependenciesPaths + skippedDependenciesRegexes = config.irrelevantDependenciesPaths } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 276faa4c..59005f74 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -166,8 +166,6 @@ public class XCPostbuild { } remappers.append(envsRemapper) let pathRemapper = DependenciesRemapperComposite(remappers) - let skippedDependenciesPaths = try pathRemapper.replace(genericPaths: context.skippedDependenciesPaths) - let skippedDependenciesURLs = skippedDependenciesPaths.map(URL.init(fileURLWithPath:)) let dependencyProcessor = DependencyProcessorImpl( xcode: context.xcodeDir, product: context.productsDir, @@ -175,7 +173,7 @@ public class XCPostbuild { intermediate: context.targetTempDir, derivedFiles: context.derivedFilesDir, bundle: context.bundleDir, - skipped: skippedDependenciesURLs + skippedRegexes: context.skippedDependenciesRegexes ) // Override fingerprints for all produced '.swiftmodule' files let fingerprintOverrideManager = FingerprintOverrideManagerImpl( diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index dec84b83..aa189a65 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -138,8 +138,10 @@ public struct XCRemoteCacheConfig: Encodable { /// A list of extra ENVs that should be used as placeholders in the dependency list /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process var customRewriteEnvs: [String] = [] - /// Paths that should not be included in a list of dependencies - /// Note: Can contain ENV placeholders, e.g. `[$(COOL_LIBRARY)]` + /// Regexes of files that should not be included in a list of dependencies. Warning! Be caution when adding + /// entries here - excluding relevant dependencies might lead to targets overcaching. + /// The regex can match either partially of fully the filepath, e.g. `\\.modulemap$` will exclude + /// all `.modulemap` files. var irrelevantDependenciesPaths: [String] = [] } diff --git a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift index f2f711d1..f85bb922 100644 --- a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift +++ b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift @@ -60,16 +60,16 @@ class DependencyProcessorImpl: DependencyProcessor { private let intermediatePath: String private let derivedFilesPath: String private let bundlePath: String? - private let skippedPaths: [String] + private let skippedRegexes: [String] - init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?, skipped: [URL]) { + init(xcode: URL, product: URL, source: URL, intermediate: URL, derivedFiles: URL, bundle: URL?, skippedRegexes: [String]) { xcodePath = xcode.path.dirPath() productPath = product.path.dirPath() sourcePath = source.path.dirPath() intermediatePath = intermediate.path.dirPath() derivedFilesPath = derivedFiles.path.dirPath() bundlePath = bundle?.path.dirPath() - skippedPaths = skipped.map { $0.path.dirPath() } + self.skippedRegexes = skippedRegexes } func process(_ files: [URL]) -> [Dependency] { @@ -80,7 +80,9 @@ class DependencyProcessorImpl: DependencyProcessor { private func classify(_ files: [URL]) -> [Dependency] { return files.map { file -> Dependency in let filePath = file.resolvingSymlinksInPath().path - if filePath.hasPrefix(xcodePath) { + if skippedRegexes.contains(where: { filePath.range(of: $0, options: .regularExpression) != nil }) { + return Dependency(url: file, type: .customSkipped) + } else if filePath.hasPrefix(xcodePath) { return Dependency(url: file, type: .xcode) } else if filePath.hasPrefix(intermediatePath) { return Dependency(url: file, type: .intermediate) @@ -94,8 +96,6 @@ class DependencyProcessorImpl: DependencyProcessor { return Dependency(url: file, type: .product) } else if filePath.hasPrefix(sourcePath) { return Dependency(url: file, type: .source) - } else if skippedPaths.contains(where: { filePath.hasPrefix($0) }){ - return Dependency(url: file, type: .customSkipped) } else { return Dependency(url: file, type: .unknown) } @@ -120,7 +120,9 @@ class DependencyProcessorImpl: DependencyProcessor { // because in case of a hit, these will be taken from the artifact // - Customized DERIVED_FILE_DIR may change a directory of // derived files, which by default is under `*/Interemediates` - let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct, .derivedFile] + let irrelevantDependenciesType: [Dependency.Kind] = [ + .xcode, .intermediate, .ownProduct, .derivedFile, .customSkipped, + ] return !irrelevantDependenciesType.contains(dependency.type) } } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index be665edd..a8475b9f 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -55,7 +55,7 @@ class PostbuildTests: FileXCTestCase { action: .build, modeMarkerPath: "", overlayHeadersPath: "", - skippedDependenciesPaths: [] + skippedDependenciesRegexes: [] ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), @@ -82,7 +82,7 @@ class PostbuildTests: FileXCTestCase { intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", bundle: nil, - skipped: [] + skippedRegexes: [] ) private var overrideManager = FingerprintOverrideManagerImpl( overridingFileExtensions: ["swiftmodule"], diff --git a/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift b/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift index 3e823605..6ae2daaa 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/DependencyProcessorImplTests.swift @@ -29,10 +29,10 @@ class DependencyProcessorImplTests: FileXCTestCase { intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", bundle: "/Bundle", - skipped: [] + skippedRegexes: [] ) - func testIntermediateFileIsSkippedForProductAndSourceSubdirectory() { + func testIntermediateFileIsskippedRegexesForProductAndSourceSubdirectory() { let intermediateFile: URL = "/Intermediate/some" let processor = DependencyProcessorImpl( xcode: "/Xcode", @@ -41,7 +41,7 @@ class DependencyProcessorImplTests: FileXCTestCase { intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", bundle: nil, - skipped: [] + skippedRegexes: [] ) XCTAssertEqual( @@ -50,7 +50,7 @@ class DependencyProcessorImplTests: FileXCTestCase { ) } - func testBundleFileIsSkippedForProductAndSourceSubdirectory() { + func testBundleFileIsskippedRegexesForProductAndSourceSubdirectory() { let bundleFile: URL = "/Bundle/some" let processor = DependencyProcessorImpl( xcode: "/Xcode", @@ -59,7 +59,7 @@ class DependencyProcessorImplTests: FileXCTestCase { intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", bundle: "/Bundle", - skipped: [] + skippedRegexes: [] ) XCTAssertEqual( @@ -154,7 +154,7 @@ class DependencyProcessorImplTests: FileXCTestCase { intermediate: intermediateDirReal, derivedFiles: "/DerivedFiles", bundle: "/Bundle", - skipped: [] + skippedRegexes: [] ) let intermediateFileSymlink = createSymlink( @@ -184,7 +184,7 @@ class DependencyProcessorImplTests: FileXCTestCase { intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", bundle: "/Bundle", - skipped: [] + skippedRegexes: [] ) let sourceFileSymlink = createSymlink( @@ -225,7 +225,7 @@ class DependencyProcessorImplTests: FileXCTestCase { intermediate: "/Intermediate", derivedFiles: "/DerivedFiles", bundle: nil, - skipped: [] + skippedRegexes: [] ) XCTAssertEqual( @@ -233,4 +233,58 @@ class DependencyProcessorImplTests: FileXCTestCase { [] ) } + + func testSkippsFilesWithFullMatch() { + let source: URL = "/someFile.m" + let processor = DependencyProcessorImpl( + xcode: "/Xcode", + product: "/", + source: "/", + intermediate: "/Intermediate", + derivedFiles: "/DerivedFiles", + bundle: nil, + skippedRegexes: ["/someFile\\.m"] + ) + + XCTAssertEqual( + processor.process([source]), + [] + ) + } + + func testSkippsFilesWithPartialMatch() { + let derivedModulemap: URL = "/module.modulemap" + let processor = DependencyProcessorImpl( + xcode: "/Xcode", + product: "/product", + source: "/", + intermediate: "/Intermediate", + derivedFiles: "/DerivedFiles", + bundle: nil, + skippedRegexes: ["\\.modulemap$"] + ) + + XCTAssertEqual( + processor.process([derivedModulemap]), + [] + ) + } + + func testDoesntSkipFileIfInvalidRegex() { + let source: URL = "/someFile.m" + let processor = DependencyProcessorImpl( + xcode: "/Xcode", + product: "/product", + source: "/", + intermediate: "/Intermediate", + derivedFiles: "/DerivedFiles", + bundle: nil, + skippedRegexes: ["\\"] + ) + + XCTAssertEqual( + processor.process([source]), + [.init(url: source, type: .source)] + ) + } } From c39b286222aab2c9e69460e81b964ce191485e4c Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 10 May 2022 13:15:22 +0200 Subject: [PATCH 3/6] Fix swiftlint --- Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index aa189a65..a9028bda 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -139,9 +139,9 @@ public struct XCRemoteCacheConfig: Encodable { /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process var customRewriteEnvs: [String] = [] /// Regexes of files that should not be included in a list of dependencies. Warning! Be caution when adding - /// entries here - excluding relevant dependencies might lead to targets overcaching. + /// entries here - excluding relevant dependencies might lead to targets overcaching /// The regex can match either partially of fully the filepath, e.g. `\\.modulemap$` will exclude - /// all `.modulemap` files. + /// all `.modulemap` files var irrelevantDependenciesPaths: [String] = [] } From 3dfd6e296ff5b40676f312ff49b7cf981a4fa9b4 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Tue, 10 May 2022 21:31:34 +0200 Subject: [PATCH 4/6] Typo cleanup --- README.md | 2 +- Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift | 6 +++--- .../XCRemoteCache/Dependencies/DependencyProcessor.swift | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 91347ca9..3ab898dc 100755 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ | | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ | | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | -| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Be caution when adding entries here - excluding relevant dependencies might lead to targets overcaching. The regex can match either partially of fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files.. | `[]` | ⬜️ | +| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files.. | `[]` | ⬜️ | ## Backend cache server diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index a9028bda..80ab7975 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -138,9 +138,9 @@ public struct XCRemoteCacheConfig: Encodable { /// A list of extra ENVs that should be used as placeholders in the dependency list /// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process var customRewriteEnvs: [String] = [] - /// Regexes of files that should not be included in a list of dependencies. Warning! Be caution when adding - /// entries here - excluding relevant dependencies might lead to targets overcaching - /// The regex can match either partially of fully the filepath, e.g. `\\.modulemap$` will exclude + /// Regexes of files that should not be included in a list of dependencies. Warning! Add entries here + /// with caution - excluding dependencies that are relevant might lead to a target overcaching + /// Note: The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude /// all `.modulemap` files var irrelevantDependenciesPaths: [String] = [] } diff --git a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift index f85bb922..fc20375b 100644 --- a/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift +++ b/Sources/XCRemoteCache/Dependencies/DependencyProcessor.swift @@ -31,7 +31,7 @@ public struct Dependency: Equatable { // Product of the target itself case ownProduct // User-excluded path - case customSkipped + case userExcluded case unknown } @@ -81,7 +81,7 @@ class DependencyProcessorImpl: DependencyProcessor { return files.map { file -> Dependency in let filePath = file.resolvingSymlinksInPath().path if skippedRegexes.contains(where: { filePath.range(of: $0, options: .regularExpression) != nil }) { - return Dependency(url: file, type: .customSkipped) + return Dependency(url: file, type: .userExcluded) } else if filePath.hasPrefix(xcodePath) { return Dependency(url: file, type: .xcode) } else if filePath.hasPrefix(intermediatePath) { @@ -120,8 +120,9 @@ class DependencyProcessorImpl: DependencyProcessor { // because in case of a hit, these will be taken from the artifact // - Customized DERIVED_FILE_DIR may change a directory of // derived files, which by default is under `*/Interemediates` + // - User-specified (in .rcinfo) files to exclude let irrelevantDependenciesType: [Dependency.Kind] = [ - .xcode, .intermediate, .ownProduct, .derivedFile, .customSkipped, + .xcode, .intermediate, .ownProduct, .derivedFile, .userExcluded, ] return !irrelevantDependenciesType.contains(dependency.type) } From 1d6ac2c171b85f93c565b531c7885db11a248f37 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Wed, 11 May 2022 08:30:10 +0200 Subject: [PATCH 5/6] Apply reviewer suggestions --- README.md | 2 +- .../XCRemoteCache/Commands/Postbuild/PostbuildContext.swift | 4 ++-- Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3ab898dc..7a59fe13 100755 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, | `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ | | `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ | | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | -| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files.. | `[]` | ⬜️ | +| `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ | ## Backend cache server diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index 85ce00a3..3c88bfa5 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -84,7 +84,7 @@ public struct PostbuildContext { /// (mappings of the virtual location file -> local file path) let overlayHeadersPath: URL /// Regexes of files that should not be included in the dependency list - let skippedDependenciesRegexes: [String] + let irrelevantDependenciesPaths: [String] } extension PostbuildContext { @@ -137,6 +137,6 @@ extension PostbuildContext { modeMarkerPath = config.modeMarkerPath /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") - skippedDependenciesRegexes = config.irrelevantDependenciesPaths + irrelevantDependenciesPaths = config.irrelevantDependenciesPaths } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 59005f74..ea40dc57 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -173,7 +173,7 @@ public class XCPostbuild { intermediate: context.targetTempDir, derivedFiles: context.derivedFilesDir, bundle: context.bundleDir, - skippedRegexes: context.skippedDependenciesRegexes + skippedRegexes: context.irrelevantDependenciesPaths ) // Override fingerprints for all produced '.swiftmodule' files let fingerprintOverrideManager = FingerprintOverrideManagerImpl( From 181997e3d2287e607dcf16c0fc5e78a254c79661 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Wed, 11 May 2022 08:37:01 +0200 Subject: [PATCH 6/6] Test fix --- Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index a8475b9f..33cb6313 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -55,7 +55,7 @@ class PostbuildTests: FileXCTestCase { action: .build, modeMarkerPath: "", overlayHeadersPath: "", - skippedDependenciesRegexes: [] + irrelevantDependenciesPaths: [] ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default),