Merge branch 'main' into jjackson/crcid-send-receive-support

* main: (24 commits) change api (#1133) Ensure authToken is present before calling refreshAuthTokenIfNeeded Add 'locale' to report broken site params Add 'locale' to report broken site params Ensure authToken is present before calling refreshAuthTokenIfNeeded Privacy Pro Free Trials - Models and API (#1120) remove MaliciousSiteProtectionSubfeature (#1131) Malware protection 6: Malware integration (#1099) Add underlying error to PrivacyStatsError (#1130) Initial changes for compilation time tracking. (#1111) iOS System level credential provider (#1127) Increase ratio of complete form saves (#1124) Add PrivacyStatsError.failedToClearPrivacyStats (#1128) Update autofill to 16.0.0 (#1122) Update local network routing (#1117) Update RMF version matching to ignore build number (#1118) Fix threading issue between using a Semaphore and async/await (#1115) add experiment test fake feature (#1119) Bump Tests/BrowserServicesKitTests/Resources/privacy-reference-tests from `a603ff9` to `6133e7d` (#1109) experiment default metrics pixels (#1107) ...
duckduckgo · Dec 19, 2024 · 88449bb · 88449bb
2 parents c28ed68 + b71ed70
commit 88449bb
Show file tree

Hide file tree

Showing 150 changed files with 6,971 additions and 2,396 deletions.
diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit-Package.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit-Package.xcscheme
@@ -491,9 +491,9 @@
             buildForAnalyzing = "YES">
             <BuildableReference
                BuildableIdentifier = "primary"
-               BlueprintIdentifier = "PhishingDetection"
-               BuildableName = "PhishingDetection"
-               BlueprintName = "PhishingDetection"
+               BlueprintIdentifier = "MaliciousSiteProtection"
+               BuildableName = "MaliciousSiteProtection"
+               BlueprintName = "MaliciousSiteProtection"
                ReferencedContainer = "container:">
             </BuildableReference>
          </BuildActionEntry>
@@ -553,6 +553,20 @@
                ReferencedContainer = "container:">
             </BuildableReference>
          </BuildActionEntry>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "PixelExperimentKit"
+               BuildableName = "PixelExperimentKit"
+               BlueprintName = "PixelExperimentKit"
+               ReferencedContainer = "container:">
+            </BuildableReference>
+         </BuildActionEntry>
       </BuildActionEntries>
    </BuildAction>
    <TestAction
@@ -796,9 +810,9 @@
             skipped = "NO">
             <BuildableReference
                BuildableIdentifier = "primary"
-               BlueprintIdentifier = "PhishingDetectionTests"
-               BuildableName = "PhishingDetectionTests"
-               BlueprintName = "PhishingDetectionTests"
+               BlueprintIdentifier = "MaliciousSiteProtectionTests"
+               BuildableName = "MaliciousSiteProtectionTests"
+               BlueprintName = "MaliciousSiteProtectionTests"
                ReferencedContainer = "container:">
             </BuildableReference>
          </TestableReference>
@@ -842,6 +856,16 @@
                ReferencedContainer = "container:">
             </BuildableReference>
          </TestableReference>
+         <TestableReference
+            skipped = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "PixelExperimentKitTests"
+               BuildableName = "PixelExperimentKitTests"
+               BlueprintName = "PixelExperimentKitTests"
+               ReferencedContainer = "container:">
+            </BuildableReference>
+         </TestableReference>
       </Testables>
    </TestAction>
    <LaunchAction

diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit.xcscheme
@@ -165,15 +165,15 @@
             </BuildableReference>
          </TestableReference>
          <TestableReference
-             skipped = "NO">
-             <BuildableReference
-                BuildableIdentifier = "primary"
-                BlueprintIdentifier = "PhishingDetectionTests"
-                BuildableName = "PhishingDetectionTests"
-                BlueprintName = "PhishingDetectionTests"
-                ReferencedContainer = "container:">
-             </BuildableReference>
-          </TestableReference>
+            skipped = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "PhishingDetectionTests"
+               BuildableName = "PhishingDetectionTests"
+               BlueprintName = "PhishingDetectionTests"
+               ReferencedContainer = "container:">
+            </BuildableReference>
+         </TestableReference>
       </Testables>
    </TestAction>
    <LaunchAction

diff --git a/Package.resolved b/Package.resolved
@@ -14,17 +14,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "c4bb146afdf0c7a93fb9a7d95b1cb255708a470d",
-        "version" : "6.41.0"
+        "revision" : "93ea6c3e771bc0b743b38cefbff548c10add9898",
+        "version" : "6.42.0"
       }
     },
     {
       "identity" : "duckduckgo-autofill",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/duckduckgo-autofill.git",
       "state" : {
-        "revision" : "c992041d16ec10d790e6204dce9abf9966d1363c",
-        "version" : "15.1.0"
+        "revision" : "88982a3802ac504e2f1a118a73bfdf2d8f4a7735",
+        "version" : "16.0.0"
       }
     },
     {
@@ -50,8 +50,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/privacy-dashboard",
       "state" : {
-        "revision" : "49db79829dcb166b3524afdbc1c680890452ce1c",
-        "version" : "7.2.1"
+        "revision" : "022c845b06ace6a4aa712a4fa3e79da32193d5c6",
+        "version" : "7.4.0"
       }
     },
     {
@@ -63,6 +63,24 @@
         "version" : "3.0.0"
       }
     },
+    {
+      "identity" : "swift-clocks",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/swift-clocks.git",
+      "state" : {
+        "revision" : "b9b24b69e2adda099a1fa381cda1eeec272d5b53",
+        "version" : "1.0.5"
+      }
+    },
+    {
+      "identity" : "swift-concurrency-extras",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/swift-concurrency-extras",
+      "state" : {
+        "revision" : "163409ef7dae9d960b87f34b51587b6609a76c1f",
+        "version" : "1.3.0"
+      }
+    },
     {
       "identity" : "swifter",
       "kind" : "remoteSourceControl",
@@ -89,6 +107,15 @@
         "revision" : "5de0a610a7927b638a5fd463a53032c9934a2c3b",
         "version" : "3.0.0"
       }
+    },
+    {
+      "identity" : "xctest-dynamic-overlay",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/xctest-dynamic-overlay",
+      "state" : {
+        "revision" : "a3f634d1a409c7979cabc0a71b3f26ffa9fc8af1",
+        "version" : "1.4.3"
+      }
     }
   ],
   "version" : 2

diff --git a/Package.swift b/Package.swift
@@ -42,23 +42,25 @@ let package = Package(
         .library(name: "PixelKitTestingUtilities", targets: ["PixelKitTestingUtilities"]),
         .library(name: "SpecialErrorPages", targets: ["SpecialErrorPages"]),
         .library(name: "DuckPlayer", targets: ["DuckPlayer"]),
-        .library(name: "PhishingDetection", targets: ["PhishingDetection"]),
+        .library(name: "MaliciousSiteProtection", targets: ["MaliciousSiteProtection"]),
         .library(name: "Onboarding", targets: ["Onboarding"]),
+        .library(name: "PixelExperimentKit", targets: ["PixelExperimentKit"]),
         .library(name: "BrokenSitePrompt", targets: ["BrokenSitePrompt"]),
         .library(name: "PageRefreshMonitor", targets: ["PageRefreshMonitor"]),
         .library(name: "PrivacyStats", targets: ["PrivacyStats"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "15.1.0"),
+        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "16.0.0"),
         .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.4.2"),
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.3.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "3.0.0"),
-        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "6.41.0"),
-        .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "7.2.1"),
+        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "6.42.0"),
+        .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "7.4.0"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
-        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1")
+        .package(url: "https://github.com/1024jp/GzipSwift.git", exact: "6.0.1"),
+        .package(url: "https://github.com/pointfreeco/swift-clocks.git", exact: "1.0.5"),
     ],
     targets: [
         .target(
@@ -250,6 +252,7 @@ let package = Package(
                 "ContentBlocking",
                 "Persistence",
                 "BrowserServicesKit",
+                "MaliciousSiteProtection",
                 .product(name: "PrivacyDashboardResources", package: "privacy-dashboard")
             ],
             path: "Sources/PrivacyDashboard",
@@ -391,7 +394,8 @@ let package = Package(
             dependencies: [
                 "Common",
                 "UserScript",
-                "BrowserServicesKit"
+                "BrowserServicesKit",
+                "MaliciousSiteProtection",
             ],
             swiftSettings: [
                 .define("DEBUG", .when(configuration: .debug))
@@ -408,9 +412,11 @@ let package = Package(
             ]
         ),
         .target(
-            name: "PhishingDetection",
+            name: "MaliciousSiteProtection",
             dependencies: [
-                "Common"
+                "Common",
+                "Networking",
+                "PixelKit",
             ],
             swiftSettings: [
                 .define("DEBUG", .when(configuration: .debug))
@@ -428,6 +434,19 @@ let package = Package(
                 .define("DEBUG", .when(configuration: .debug))
             ]
         ),
+        .target(
+            name: "PixelExperimentKit",
+            dependencies: [
+                "PixelKit",
+                "BrowserServicesKit"
+            ],
+            resources: [
+                .process("Resources")
+            ],
+            swiftSettings: [
+                .define("DEBUG", .when(configuration: .debug))
+            ]
+        ),
         .target(
             name: "BrokenSitePrompt",
             dependencies: [
@@ -656,19 +675,21 @@ let package = Package(
         .testTarget(
             name: "DuckPlayerTests",
             dependencies: [
-                "DuckPlayer"
+                "DuckPlayer",
+                "BrowserServicesKitTestsUtils",
             ]
         ),
 
         .testTarget(
-            name: "PhishingDetectionTests",
+            name: "MaliciousSiteProtectionTests",
             dependencies: [
-                "PhishingDetection",
-                "PixelKit"
+                "TestUtils",
+                "MaliciousSiteProtection",
+                .product(name: "Clocks", package: "swift-clocks"),
             ],
             resources: [
-                .copy("Resources/hashPrefixes.json"),
-                .copy("Resources/filterSet.json")
+                .copy("Resources/phishingHashPrefixes.json"),
+                .copy("Resources/phishingFilterSet.json"),
             ]
         ),
         .testTarget(
@@ -677,6 +698,12 @@ let package = Package(
                 "Onboarding"
             ]
         ),
+        .testTarget(
+            name: "PixelExperimentKitTests",
+            dependencies: [
+                "PixelExperimentKit"
+            ]
+        ),
         .testTarget(
             name: "SpecialErrorPagesTests",
             dependencies: [

diff --git a/Sources/BrowserServicesKit/Autofill/AutofillUserScript+SecureVault.swift b/Sources/BrowserServicesKit/Autofill/AutofillUserScript+SecureVault.swift
@@ -491,6 +491,7 @@ extension AutofillUserScript {
         case userInitiated
         case autoprompt
         case formSubmission
+        case partialSave
         case passwordGeneration
         case emailProtection
     }

diff --git a/Sources/BrowserServicesKit/Autofill/AutofillUserScript.swift b/Sources/BrowserServicesKit/Autofill/AutofillUserScript.swift
@@ -17,9 +17,9 @@
 //
 
 import Common
-import WebKit
-import UserScript
 import os.log
+import UserScript
+@preconcurrency import WebKit
 
 var previousIncontextSignupPermanentlyDismissedAt: Double?
 var previousEmailSignedIn: Bool?

diff --git a/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerDebugEvents.swift b/Sources/BrowserServicesKit/ContentBlocking/ContentBlockerDebugEvents.swift
@@ -46,10 +46,9 @@ public enum ContentBlockerDebugEvents {
 
     case contentBlockingCompilationFailed(listName: String, component: Component)
 
-    case contentBlockingCompilationTime
-
     case contentBlockingLookupRulesSucceeded
     case contentBlockingFetchLRCSucceeded
     case contentBlockingLRCMissing
     case contentBlockingNoMatchInLRC
+    case contentBlockingCompilationTaskPerformance(iterationCount: Int, timeBucketAggregation: Double)
 }