Merge pull request #71 from polac24/20220204-overlay-mapper

Enable virtual file system overlay replacements

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -79,6 +79,8 @@ public struct PostbuildContext {
     /// Action type: build, indexbuild etc.
     var action: BuildActionType
     let modeMarkerPath: String
+    /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
 }
 
 extension PostbuildContext {
@@ -127,5 +129,7 @@ extension PostbuildContext {
         thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init)
         action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown
         modeMarkerPath = config.modeMarkerPath
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -66,7 +66,7 @@ public class XCPostbuild {
             // Initialize dependencies
             let primaryGitBranch = GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch)
             let gitClient = GitClientImpl(repoRoot: config.repoRoot, primary: primaryGitBranch, shell: shellGetStdout)
-            let pathRemapper = try StringDependenciesRemapperFactory().build(
+            let envsRemapper = try StringDependenciesRemapperFactory().build(
                 orderKeys: DependenciesMapping.rewrittenEnvs,
                 envs: env,
                 customMappings: config.outOfBandMappings
@@ -145,6 +145,18 @@ public class XCPostbuild {
                 fileDependeciesReaderFactory: fileReaderFactory,
                 dirScanner: fileManager
             )
+            // As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
+            // do not fail in case of a missing headers overlay file. In the future, all overlay files could be
+            // captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode.
+            let overlayReader = JsonOverlayReader(
+                context.overlayHeadersPath,
+                mode: .bestEffort,
+                fileReader: fileManager
+            )
+            let overlayRemapper = try OverlayDependenciesRemapper(
+                overlayReader: overlayReader
+            )
+            let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper])
             let dependencyProcessor = DependencyProcessorImpl(
                 xcode: context.xcodeDir,
                 product: context.productsDir,

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -43,6 +43,8 @@ public struct PrebuildContext {
     let targetName: String
     /// List of all targets to downloaded from the thinning aggregation target
     var thinnedTargets: [String]?
+    /// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
+    let overlayHeadersPath: URL
 }
 
 extension PrebuildContext {
@@ -64,5 +66,7 @@ extension PrebuildContext {
         self.targetName = targetName
         let thinFocusedTargetsString: String? = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS")
         thinnedTargets = thinFocusedTargetsString?.split(separator: ",").map(String.init)
+        /// Note: The file has yaml extension, even it is in the json format
+        overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
     }
 }

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -115,11 +115,22 @@ public class XCPrebuild {
             )
             let client: NetworkClient = config.disableHttpCache ? networkClient : cacheNetworkClient
             let remoteNetworkClient = RemoteNetworkClientImpl(client, urlBuilder)
-            let pathRemapper = try StringDependenciesRemapperFactory().build(
+            let envsRemapper = try StringDependenciesRemapperFactory().build(
                 orderKeys: DependenciesMapping.rewrittenEnvs,
                 envs: env,
                 customMappings: config.outOfBandMappings
             )
+            // As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
+            // do not fail in case of a missing headers overlay file. 
+            let overlayReader = JsonOverlayReader(
+                context.overlayHeadersPath,
+                mode: .bestEffort,
+                fileReader: fileManager
+            )
+            let overlayRemapper = try OverlayDependenciesRemapper(
+                overlayReader: overlayReader
+            )
+            let pathRemapper = DependenciesRemapperComposite([overlayRemapper, envsRemapper])
             let filesFingerprintGenerator = FingerprintAccumulatorImpl(
                 algorithm: MD5Algorithm(),
                 fileManager: fileManager

Sources/XCRemoteCache/Dependencies/OverlayDependenciesRemapper.swift

@@ -0,0 +1,55 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+/// File paths remapper according the virtual file system mappings
+/// Warning: this class is not thread safe
+class OverlayDependenciesRemapper: DependenciesRemapper {
+    private var mappings: [OverlayMapping]
+
+    init(overlayReader: OverlayReader) throws {
+        mappings = try overlayReader.provideMappings()
+    }
+
+    private func mapPath(
+        _ path: String,
+        source: KeyPath<OverlayMapping,URL>,
+        destination: KeyPath<OverlayMapping,URL>
+    ) -> String {
+        guard let mapping = mappings.first(where: { $0[keyPath: source].path == path }) else {
+            // TODO: support partial mappings, where a directory path can be replaced with some other directory
+            // no direct mapping found
+            return path
+        }
+        return mapping[keyPath: destination].path
+    }
+
+    func replace(genericPaths: [String]) -> [String] {
+        Set(genericPaths.map {
+            mapPath($0, source: \.virtual, destination: \.local)
+        }).sorted()
+    }
+
+    func replace(localPaths: [String]) -> [String] {
+        Set(localPaths.map {
+            mapPath($0, source: \.local, destination: \.virtual)
+        }).sorted()
+    }
+}

Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift

@@ -51,7 +51,8 @@ class PostbuildTests: FileXCTestCase {
         derivedSourcesDir: "",
         thinnedTargets: [],
         action: .build,
-        modeMarkerPath: ""
+        modeMarkerPath: "",
+        overlayHeadersPath: ""
     )
     private var network = RemoteNetworkClientImpl(
         NetworkClientFake(fileManager: .default),

Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift

@@ -62,7 +62,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         contextCached = PrebuildContext(
             targetTempDir: sampleURL,
@@ -74,7 +75,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: true,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         organizer = ArtifactOrganizerFake(artifactRoot: artifactsRoot, unzippedExtension: "unzip")
         globalCacheSwitcher = InMemoryGlobalCacheSwitcher()
@@ -238,7 +240,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
 
         let prebuild = Prebuild(
@@ -268,7 +271,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: true,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         metaContent = try generateMeta(fingerprint: generator.generate(), filekey: "1")
         let downloadedArtifactPackage = artifactsRoot.appendingPathComponent("1")
@@ -330,7 +334,8 @@ class PrebuildTests: FileXCTestCase {
             forceCached: false,
             compilationHistoryFile: compilationHistory,
             turnOffRemoteCacheOnFirstTimeout: false,
-            targetName: ""
+            targetName: "",
+            overlayHeadersPath: ""
         )
         try globalCacheSwitcher.enable(sha: "1")
         let prebuild = Prebuild(

Tests/XCRemoteCacheTests/Dependencies/OverlayDependenciesRemapperTests.swift

@@ -0,0 +1,65 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class OverlayDependenciesRemapperTests: XCTestCase {
+    private let overlayReader = OverlayReaderFake(
+        mappings: [.init(virtual: "/file.h", local: "/Intermediate/Some/file.h")]
+    )
+
+    func testMappingFromLocalToGeneric() throws {
+        let reader = try OverlayDependenciesRemapper(
+            overlayReader: overlayReader
+        )
+
+        let dependencies = reader.replace(localPaths: ["/Intermediate/Some/file.h"])
+        XCTAssertEqual(dependencies, ["/file.h"])
+    }
+
+    func testMappingFromGenericToLocal() throws {
+        let reader = try OverlayDependenciesRemapper(
+            overlayReader: overlayReader
+        )
+
+        let dependencies = reader.replace(genericPaths: ["/file.h"])
+        XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h"])
+    }
+
+    func testGenericDependenciesAreMerged() throws {
+
+        let reader = try OverlayDependenciesRemapper(
+            overlayReader: overlayReader
+        )
+
+        let dependencies = reader.replace(localPaths: ["/Intermediate/Some/file.h", "/file.h"])
+        XCTAssertEqual(dependencies, ["/file.h"])
+    }
+
+    func testLocalDependenciesAreMerged() throws {
+        let reader = try OverlayDependenciesRemapper(
+            overlayReader: overlayReader
+        )
+
+        let dependencies = reader.replace(genericPaths: ["/Intermediate/Some/file.h", "/file.h"])
+        XCTAssertEqual(dependencies, ["/Intermediate/Some/file.h"])
+    }
+}
+

Tests/XCRemoteCacheTests/Dependencies/OverlayReaderTests.swift

@@ -51,6 +51,14 @@ class JsonOverlayReaderTests: XCTestCase {
         XCTAssertEqual(mappings, [])
     }
 
+    func testParsingEmptyOverlay() throws {
+        let file = try pathForTestData(name: "overlayReaderEmpty")
+        let reader = JsonOverlayReader(file, mode: .strict, fileReader: FileManager.default)
+        let mappings = try reader.provideMappings()
+
+        XCTAssertEqual(mappings, [])
+    }
+
     private func pathForTestData(name: String) throws -> URL {
         return try XCTUnwrap(Bundle.module.url(forResource: name, withExtension: "json", subdirectory: JsonOverlayReaderTests.resourcesSubdirectory))
     }

Tests/XCRemoteCacheTests/TestData/Dependencies/JsonOverlayReaderTests/overlayReaderEmpty.json

@@ -0,0 +1 @@
+{"case-sensitive":"false","roots":[],"version":0}

Tests/XCRemoteCacheTests/TestDoubles/OverlayReaderFake.swift

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+@testable import XCRemoteCache
+
+class OverlayReaderFake: OverlayReader {
+    private let mappings: [OverlayMapping]
+    init(mappings: [OverlayMapping]) {
+        self.mappings = mappings
+    }
+
+    func provideMappings() throws -> [OverlayMapping] {
+        return mappings
+    }
+}

tasks/e2e.rb

@@ -179,8 +179,7 @@ def self.run_cocoapods_scenario(template_path)
         dump_podfile(consumer_configuration, template_path)
         puts('Building consumer ...')
         Dir.chdir(E2E_COCOAPODS_SAMPLE_DIR) do
-            # TODO: Change DerivedData's path to emulate multi-machines setup. Blocked by #59
-            build_project({'derivedDataPath' => "#{DERIVED_DATA_PATH}"})
+            build_project({'derivedDataPath' => "#{DERIVED_DATA_PATH}_consumer"})
             valide_hit_rate
         end
     end