Merge remote-tracking branch 'origin/main' into graeme/partial-form-s…

…ave-refactor

Package.resolved

@@ -23,8 +23,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/duckduckgo-autofill.git",
       "state" : {
-        "branch" : "main",
-        "revision" : "88982a3802ac504e2f1a118a73bfdf2d8f4a7735"
+        "revision" : "88982a3802ac504e2f1a118a73bfdf2d8f4a7735",
+        "version" : "16.0.0"
       }
     },
     {

Package.swift

@@ -50,7 +50,7 @@ let package = Package(
         .library(name: "PrivacyStats", targets: ["PrivacyStats"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", branch: "main"),
+        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "16.0.0"),
         .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.4.2"),
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.3.0"),

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesManager.swift

@@ -242,26 +242,20 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         Logger.contentBlocking.debug("Lookup compiled rules")
         prepareSourceManagers()
         let initialCompilationTask = LookupRulesTask(sourceManagers: Array(sourceManagers.values))
-        let mutex = DispatchSemaphore(value: 0)
 
-        Task {
-            do {
-                try await initialCompilationTask.lookupCachedRulesLists()
-            } catch {
-                Logger.contentBlocking.debug("❌ Lookup failed: \(error.localizedDescription, privacy: .public)")
-            }
-            mutex.signal()
-        }
-        // We want to confine Compilation work to WorkQueue, so we wait to come back from async Task
-        mutex.wait()
+        let result: [LookupRulesTask.LookupResult]
+
+        do {
+            result = try initialCompilationTask.lookupCachedRulesLists()
 
-        if let result = initialCompilationTask.result {
             let rules = result.map(Rules.init(compilationResult:))
-            Logger.contentBlocking.debug("🟩 Found \(rules.count, privacy: .public) rules")
+            Logger.contentBlocking.debug("🟩 Lookup Found \(rules.count, privacy: .public) rules")
             applyRules(rules)
             return true
+        } catch {
+            Logger.contentBlocking.debug("❌ Lookup failed: \(error.localizedDescription, privacy: .public)")
+            return false
         }
-        return false
     }
 
     /*
@@ -273,18 +267,10 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
 
         let initialCompilationTask = LastCompiledRulesLookupTask(sourceRules: rulesSource.contentBlockerRulesLists,
                                                                  lastCompiledRules: lastCompiledRules)
-        let mutex = DispatchSemaphore(value: 0)
-        Task {
-            try? await initialCompilationTask.fetchCachedRulesLists()
-            mutex.signal()
-        }
-        // We want to confine Compilation work to WorkQueue, so we wait to come back from async Task
-        mutex.wait()
-
-        let rulesFound = initialCompilationTask.getFetchedRules()
+        let rules = initialCompilationTask.fetchCachedRulesLists()
 
-        if let rulesFound {
-            applyRules(rulesFound)
+        if let rules {
+            applyRules(rules)
         } else {
             lock.lock()
             state = .idle
@@ -294,7 +280,7 @@ public class ContentBlockerRulesManager: CompiledRuleListsSource {
         // No matter if rules were found or not, we need to schedule recompilation, after all
         scheduleCompilation()
 
-        return rulesFound != nil
+        return rules != nil
     }
 
     private func prepareSourceManagers() {

Sources/BrowserServicesKit/ContentBlocking/ContentBlockingRulesLastCompiledRulesLookupTask.swift

@@ -41,32 +41,49 @@ extension ContentBlockerRulesManager {
             self.lastCompiledRules = lastCompiledRules
         }
 
-        func fetchCachedRulesLists() async throws {
+        func fetchCachedRulesLists() -> [Rules]? {
             let sourceRulesNames = sourceRules.map { $0.name }
             let filteredBySourceLastCompiledRules = lastCompiledRules.filter { sourceRulesNames.contains($0.name) }
 
             guard filteredBySourceLastCompiledRules.count == sourceRules.count else {
                 // We should only load rule lists from cache, in case we can match every one of these
-                throw WKError(.contentRuleListStoreLookUpFailed)
+                return nil
             }
 
             var result: [CachedRulesList] = []
+            let group = DispatchGroup()
+
             for rules in filteredBySourceLastCompiledRules {
-                guard let ruleList = try await Task(operation: { @MainActor in
-                    try await WKContentRuleListStore.default().contentRuleList(forIdentifier: rules.identifier.stringValue)
-                }).value else { throw WKError(.contentRuleListStoreLookUpFailed) }
-
-                result.append(CachedRulesList(name: rules.name,
-                                              rulesList: ruleList,
-                                              tds: rules.trackerData,
-                                              rulesIdentifier: rules.identifier))
+                group.enter()
+
+                DispatchQueue.main.async {
+                    // This needs to be called from the main thread.
+                    WKContentRuleListStore.default().lookUpContentRuleList(forIdentifier: rules.identifier.stringValue) { ruleList, error in
+                        guard let ruleList, error == nil else {
+                            group.leave()
+                            return
+                        }
+
+                        result.append(CachedRulesList(name: rules.name,
+                                                      rulesList: ruleList,
+                                                      tds: rules.trackerData,
+                                                      rulesIdentifier: rules.identifier))
+                        group.leave()
+                    }
+                }
             }
-            self.result = result
+
+            let operationResult = group.wait(timeout: .now() + 6)
+
+            guard operationResult == .success, result.count == filteredBySourceLastCompiledRules.count else {
+                return nil
+            }
+
+            return getRules(from: result)
         }
 
-        public func getFetchedRules() -> [Rules]? {
-            guard let result else { return nil }
-            return result.map {
+        public func getRules(from cached: [CachedRulesList]) -> [Rules] {
+            return cached.map {
                 let surrogateTDS = ContentBlockerRulesManager.extractSurrogates(from: $0.tds)
                 let encodedData = try? JSONEncoder().encode(surrogateTDS)
                 let encodedTrackerData = String(data: encodedData!, encoding: .utf8)!

Sources/BrowserServicesKit/ContentBlocking/ContentBlockingRulesLookupTask.swift

@@ -28,31 +28,45 @@ extension ContentBlockerRulesManager {
 
         private let sourceManagers: [ContentBlockerRulesSourceManager]
 
-        public private(set) var result: [LookupResult]?
-
         init(sourceManagers: [ContentBlockerRulesSourceManager]) {
             self.sourceManagers = sourceManagers
         }
 
-        func lookupCachedRulesLists() async throws {
+        func lookupCachedRulesLists() throws -> [LookupResult]  {
+
+            let models = sourceManagers.compactMap { $0.makeModel() }
+            if models.count != sourceManagers.count {
+                // We should only load rule lists, in case we can match every one of the expected ones
+                throw WKError(.contentRuleListStoreLookUpFailed)
+            }
 
             var result = [LookupResult]()
-            for sourceManager in sourceManagers {
-                guard let model = sourceManager.makeModel() else {
-                    throw WKError(.contentRuleListStoreLookUpFailed)
-                }
+            let group = DispatchGroup()
+
+            for model in models {
+                group.enter()
 
-                guard let ruleList = try await Task(operation: { @MainActor in
-                    try await WKContentRuleListStore.default().contentRuleList(forIdentifier: model.rulesIdentifier.stringValue)
-                }).value else {
-                    // All lists must be found for this to be considered successful
-                    throw WKError(.contentRuleListStoreLookUpFailed)
+                DispatchQueue.main.async {
+                    // This needs to be called from the main thread.
+                    WKContentRuleListStore.default().lookUpContentRuleList(forIdentifier: model.rulesIdentifier.stringValue) { ruleList, error in
+                        guard let ruleList, error == nil else {
+                            group.leave()
+                            return
+                        }
+
+                        result.append((ruleList, model))
+                        group.leave()
+                    }
                 }
+            }
+
+            let operationResult = group.wait(timeout: .now() + 6)
 
-                result.append((ruleList, model))
+            guard operationResult == .success, result.count == models.count else {
+                throw WKError(.contentRuleListStoreLookUpFailed)
             }
-            self.result = result
-        }
 
+            return result
+        }
     }
 }

Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift

@@ -64,6 +64,7 @@ public enum PrivacyFeature: String {
     case contextualOnboarding
     case textZoom
     case adAttributionReporting
+    case experimentTest
 }
 
 /// An abstraction to be implemented by any "subfeature" of a given `PrivacyConfiguration` feature.
@@ -192,3 +193,8 @@ public enum SyncPromotionSubfeature: String, PrivacySubfeature {
     case bookmarks
     case passwords
 }
+
+public enum ExperimentTestSubfeatures: String, PrivacySubfeature {
+    public var parent: PrivacyFeature { .experimentTest }
+    case experimentTestAA
+}

Sources/NetworkProtection/Routing/VPNRoutingRange.swift

@@ -34,8 +34,13 @@ public enum VPNRoutingRange {
         "::1/128",    /* loopback */
     ]
 
+    public static let localNetworkRangeWithoutDNS: [NetworkProtection.IPAddressRange] = [
+        "172.16.0.0/12",  /* 255.240.0.0 */
+        "192.168.0.0/16", /* 255.255.0.0 */
+    ]
+
     public static let localNetworkRange: [NetworkProtection.IPAddressRange] = [
-        // "10.0.0.0/8",     /* 255.0.0.0   */
+        "10.0.0.0/8",     /* 255.0.0.0   */
         "172.16.0.0/12",  /* 255.240.0.0 */
         "192.168.0.0/16", /* 255.255.0.0 */
     ]

Sources/NetworkProtection/Routing/VPNRoutingTableResolver.swift

@@ -43,7 +43,7 @@ struct VPNRoutingTableResolver {
         var routes = VPNRoutingRange.alwaysExcludedIPv4Range
 
         if excludeLocalNetworks {
-            routes += VPNRoutingRange.localNetworkRange
+            routes += VPNRoutingRange.localNetworkRangeWithoutDNS
         }
 
         return routes

Sources/RemoteMessaging/Matchers/AppAttributeMatcher.swift

@@ -81,7 +81,7 @@ public struct CommonAppAttributeMatcher: AttributeMatching {
             assertionFailure("BundleIdentifier should not be empty")
         }
         self.init(bundleId: AppVersion.shared.identifier,
-                  appVersion: AppVersion.shared.versionAndBuildNumber,
+                  appVersion: AppVersion.shared.versionNumber,
                   isInternalUser: isInternalUser,
                   statisticsStore: statisticsStore,
                   variantManager: variantManager)

Sources/RemoteMessaging/Model/MatchingAttributes.swift

@@ -49,12 +49,27 @@ struct AppIdMatchingAttribute: SingleValueMatching {
 }
 
 struct AppVersionMatchingAttribute: StringRangeMatching {
-    static let defaultMaxValue: String = AppVersion.shared.versionAndBuildNumber
 
-    var min: String = MatchingAttributeDefaults.stringDefaultValue
-    var max: String = AppVersion.shared.versionAndBuildNumber
-    var value: String = MatchingAttributeDefaults.stringDefaultValue
+    static let defaultMaxValue: String = AppVersion.shared.versionNumber
+
+    var min: String
+    var max: String
+    var value: String
     var fallback: Bool?
+
+    // Legacy versions of the app require a build number in the version string in order to match correctly.
+    // To allow message authors to include a build number for backwards compatibility, while also allowing new clients to use the simpler version
+    // string, this initializer trims the build number before storing it.
+    init(min: String = MatchingAttributeDefaults.stringDefaultValue,
+         max: String  = AppVersion.shared.versionNumber,
+         value: String = MatchingAttributeDefaults.stringDefaultValue,
+         fallback: Bool?) {
+        self.min = min.trimmingBuildNumber
+        self.max = max.trimmingBuildNumber
+        self.value = value.trimmingBuildNumber
+        self.fallback = fallback
+    }
+
 }
 
 struct AtbMatchingAttribute: SingleValueMatching {
@@ -306,3 +321,17 @@ struct RangeStringNumericMatchingAttribute: Equatable {
         return version + String(repeating: ".0", count: matchComponents.count - versionComponents.count)
     }
 }
+
+private extension String {
+
+    var trimmingBuildNumber: String {
+        let components = self.split(separator: ".")
+
+        if components.count == 4 {
+            return components.dropLast().joined(separator: ".")
+        } else {
+            return self
+        }
+    }
+
+}