Merge branch 'main' into sam/remove-atb-from-default-pixel-parameters

* main: (41 commits) Update BSK with autofill 10.1.0 (#2414) Bump submodules/privacy-reference-tests from `a3acc21` to `6b7ad1e` (#2408) Add Autoconsent onByDefault subfeature (#2423) Metadata reverted https://app.asana.com/0/0/1206476969305202/1206481077004998/f metadata updated Release 7.107.0-0 (#2421) More script fixes (#2419) More release script fixes (#2418) Release/7.106.0-4 (#2417) Reenable toggle on disallowing vpn (#2404) (#2416) Don't crash when AppRatingPromptEntity fetch errors (#2388) Add error codes to site breakage reports (#2413) Don't set dryRun for alpha builds (#2412) Add VPN redemption retry event (#2409) Changes to hotfix process (#2406) Release/7.106.0-3 changes (#2411) Fix tab loading (#2410) Improve waitlist invite code checks (#2398) Reenable toggle on disallowing vpn (#2404) Bump BrowserServicesKit to 103.0.2 (#2393) ...
duckduckgo · Feb 4, 2024 · b2f65f2 · b2f65f2
2 parents 64f72de + 1211ccd
commit b2f65f2
Show file tree

Hide file tree

Showing 52 changed files with 914 additions and 530 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -128,6 +128,9 @@ jobs:
 
     name: Make Release Build
 
+    # Dependabot doesn't have access to all secrets, so we skip this job
+    if: github.actor != 'dependabot[bot]'
+
     runs-on: macos-13-xlarge
     timeout-minutes: 30
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -20,15 +20,13 @@ on:
     branches:
     - release/**
     - hotfix/**
-    - coldfix/**
     - '!release/**-' # filter out PRs matching that pattern
     - '!hotfix/**-'
-    - '!coldfix/**-'
     types: [closed]
 
 jobs:
   make-release:
-    if: github.event.action == 0 || github.event.pull_request.merged == true # empty string returns 0; for case when workflow is triggered manually
+    if: github.event.action == 0 || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'Merge triggers release')) # empty string returns 0; for case when workflow is triggered manually
     runs-on: macos-13-xlarge
     name: Make App Store Connect Release
 
@@ -98,7 +96,7 @@ jobs:
 
     - name: Upload debug symbols to Asana
       if: ${{ always() && github.event.inputs.asana-task-url }}
-      env:
+      env:     
         ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
       run: |
         if [[ -f ${{ env.dsyms_path }} ]]; then

diff --git a/.gitignore b/.gitignore
@@ -74,3 +74,5 @@ fastlane/test_output
 
 Configuration/ExternalDeveloper.xcconfig
 scripts/assets
+
+DuckDuckGoTests/NetworkProtectionVPNLocationViewModelTests.swift*.plist
diff --git a/.maestro/release_tests/emailprotection.yaml b/.maestro/release_tests/emailprotection.yaml
@@ -18,7 +18,9 @@ tags:
 - scroll
 - assertVisible: Email Protection
 - tapOn: Email Protection
+- assertVisible: Email privacy, simplified.
 - assertVisible:
     id: searchEntry
+- tapOn: 
+    id: "searchEntry"
 - assertVisible: https://duckduckgo.com/email/
-- assertVisible: Email privacy, simplified.
diff --git a/Configuration/Version.xcconfig b/Configuration/Version.xcconfig
@@ -1 +1 @@
-MARKETING_VERSION = 7.106.0
+MARKETING_VERSION = 7.107.0
diff --git a/Core/AppPrivacyConfigurationDataProvider.swift b/Core/AppPrivacyConfigurationDataProvider.swift
@@ -23,8 +23,8 @@ import BrowserServicesKit
 final public class AppPrivacyConfigurationDataProvider: EmbeddedDataProvider {
 
     public struct Constants {
-        public static let embeddedDataETag = "\"4e984b6034f1e27fe85fdad5f4bf37c9\""
-        public static let embeddedDataSHA = "d599888e7b447bbaeb2d9a7fd7ccf06956fce8976c316be2f497561a6832613e"
+        public static let embeddedDataETag = "\"d0ae514c42e1e632584aba7a025b8b92\""
+        public static let embeddedDataSHA = "b304a2dbb2edc7443a4950bb2ba9f7604354cf32575dd5a9ca09acd5c4b78146"
     }
 
     public var embeddedDataEtag: String {

diff --git a/Core/CookieStorage.swift b/Core/CookieStorage.swift
@@ -22,16 +22,27 @@ import Foundation
 
 public class CookieStorage {
 
-    struct Constants {
-        static let key = "com.duckduckgo.allowedCookies"
+    struct Keys {
+        static let allowedCookies = "com.duckduckgo.allowedCookies"
+        static let consumed = "com.duckduckgo.consumedCookies"
     }
 
     private var userDefaults: UserDefaults
-
+
+    var isConsumed: Bool {
+        get {
+            userDefaults.bool(forKey: Keys.consumed, defaultValue: false)
+        }
+        set {
+            userDefaults.set(newValue, forKey: Keys.consumed)
+        }
+    }
+
+    /// Use the `updateCookies` function rather than the setter which is only visible for testing.
     var cookies: [HTTPCookie] {
         get {
             var storedCookies = [HTTPCookie]()
-            if let cookies = userDefaults.object(forKey: Constants.key) as? [[String: Any?]] {
+            if let cookies = userDefaults.object(forKey: Keys.allowedCookies) as? [[String: Any?]] {
                 for cookieData in cookies {
                     var properties = [HTTPCookiePropertyKey: Any]()
                     cookieData.forEach({
@@ -57,17 +68,76 @@ public class CookieStorage {
                 }
                 cookies.append(mappedCookie)
             }
-            userDefaults.setValue(cookies, forKey: Constants.key)
+            userDefaults.setValue(cookies, forKey: Keys.allowedCookies)
         }
+
     }
 
     public init(userDefaults: UserDefaults = UserDefaults.app) {
         self.userDefaults = userDefaults
     }
 
-    func clear() {
-        userDefaults.removeObject(forKey: Constants.key)
-        os_log("cleared cookies", log: .generalLog, type: .debug)
+    enum CookieDomainsOnUpdate {
+        case empty
+        case match
+        case missing
+        case different
     }
+
+    @discardableResult
+    func updateCookies(_ cookies: [HTTPCookie], keepingPreservedLogins preservedLogins: PreserveLogins) -> CookieDomainsOnUpdate {
+        isConsumed = false
+
+        let persisted = self.cookies
+
+        func cookiesByDomain(_ cookies: [HTTPCookie]) -> [String: [HTTPCookie]] {
+            var byDomain = [String: [HTTPCookie]]()
+            cookies.forEach { cookie in
+                var cookies = byDomain[cookie.domain, default: []]
+                cookies.append(cookie)
+                byDomain[cookie.domain] = cookies
+            }
+            return byDomain
+        }
+
+        let updatedCookiesByDomain = cookiesByDomain(cookies)
+        var persistedCookiesByDomain = cookiesByDomain(persisted)
 
+        // Do the diagnostics before the dicts get changed.
+        let diagnosticResult = evaluateDomains(
+            updatedDomains: updatedCookiesByDomain.keys.sorted(),
+            persistedDomains: persistedCookiesByDomain.keys.sorted()
+        )
+
+        updatedCookiesByDomain.keys.forEach {
+            persistedCookiesByDomain[$0] = updatedCookiesByDomain[$0]
+        }
+
+        persistedCookiesByDomain.keys.forEach {
+            guard $0 != "duckduckgo.com" else { return } // DDG cookies are for SERP settings only
+
+            if !preservedLogins.isAllowed(cookieDomain: $0) {
+                persistedCookiesByDomain.removeValue(forKey: $0)
+            }
+        }
+
+        let now = Date()
+        self.cookies = persistedCookiesByDomain.map { $0.value }.joined().compactMap { $0 }
+            .filter { $0.expiresDate == nil || $0.expiresDate! > now }
+
+        return diagnosticResult
+    }
+
+    private func evaluateDomains(updatedDomains: [String], persistedDomains: [String]) -> CookieDomainsOnUpdate {
+        if persistedDomains.isEmpty {
+            return .empty
+        } else if updatedDomains.count < persistedDomains.count {
+            return .missing
+        } else if updatedDomains == persistedDomains {
+            return .match
+        } else {
+            return .different
+        }
+    }
+
 }
diff --git a/Core/DailyPixel.swift b/Core/DailyPixel.swift
@@ -71,10 +71,16 @@ public final class DailyPixel {
     public static func fireDailyAndCount(pixel: Pixel.Event,
                                          error: Swift.Error? = nil,
                                          withAdditionalParameters params: [String: String] = [:],
+                                         includedParameters: [Pixel.QueryParameters] = [.atb, .appVersion],
                                          onDailyComplete: @escaping (Swift.Error?) -> Void = { _ in },
                                          onCountComplete: @escaping (Swift.Error?) -> Void = { _ in }) {
         if !pixel.hasBeenFiredToday(dailyPixelStorage: storage) {
-            Pixel.fire(pixelNamed: pixel.name + "_d", withAdditionalParameters: params, onComplete: onDailyComplete)
+            Pixel.fire(
+                pixelNamed: pixel.name + "_d",
+                withAdditionalParameters: params,
+                includedParameters: includedParameters,
+                onComplete: onDailyComplete
+            )
         } else {
             onDailyComplete(Error.alreadyFired)
         }
@@ -83,7 +89,12 @@ public final class DailyPixel {
         if let error {
             newParams.appendErrorPixelParams(error: error)
         }
-        Pixel.fire(pixelNamed: pixel.name + "_c", withAdditionalParameters: newParams, onComplete: onCountComplete)
+        Pixel.fire(
+            pixelNamed: pixel.name + "_c",
+            withAdditionalParameters: newParams,
+            includedParameters: includedParameters,
+            onComplete: onCountComplete
+        )
     }
 
     private static func updatePixelLastFireDate(pixel: Pixel.Event) {

diff --git a/Core/PixelEvent.swift b/Core/PixelEvent.swift
@@ -312,6 +312,7 @@ extension Pixel {
         case networkProtectionKeychainErrorFailedToCastKeychainValueToData
         case networkProtectionKeychainReadError
         case networkProtectionKeychainWriteError
+        case networkProtectionKeychainUpdateError
         case networkProtectionKeychainDeleteError
 
         case networkProtectionWireguardErrorCannotLocateTunnelFileDescriptor
@@ -341,6 +342,7 @@ extension Pixel {
         case networkProtectionWaitlistTermsAccepted
         case networkProtectionWaitlistNotificationShown
         case networkProtectionWaitlistNotificationLaunched
+        case networkProtectionWaitlistRetriedInviteCodeRedemption
 
         case networkProtectionGeoswitchingOpened
         case networkProtectionGeoswitchingSetNearest
@@ -521,6 +523,8 @@ extension Pixel {
         case emailIncontextModalExitEarlyContinue
 
         case compilationFailed
+
+        case appRatingPromptFetchError
     }
 
 }
@@ -810,6 +814,7 @@ extension Pixel.Event {
         case .networkProtectionKeychainErrorFailedToCastKeychainValueToData: return "m_netp_keychain_error_failed_to_cast_keychain_value_to_data"
         case .networkProtectionKeychainReadError: return "m_netp_keychain_error_read_failed"
         case .networkProtectionKeychainWriteError: return "m_netp_keychain_error_write_failed"
+        case .networkProtectionKeychainUpdateError: return "m_netp_keychain_error_update_failed"
         case .networkProtectionKeychainDeleteError: return "m_netp_keychain_error_delete_failed"
         case .networkProtectionWireguardErrorCannotLocateTunnelFileDescriptor: return "m_netp_wireguard_error_cannot_locate_tunnel_file_descriptor"
         case .networkProtectionWireguardErrorInvalidState: return "m_netp_wireguard_error_invalid_state"
@@ -833,6 +838,7 @@ extension Pixel.Event {
         case .networkProtectionWaitlistTermsAccepted: return "m_netp_waitlist_terms_accepted"
         case .networkProtectionWaitlistNotificationShown: return "m_netp_waitlist_notification_shown"
         case .networkProtectionWaitlistNotificationLaunched: return "m_netp_waitlist_notification_launched"
+        case .networkProtectionWaitlistRetriedInviteCodeRedemption: return "m_netp_waitlist_retried_invite_code_redemption"
 
         case .networkProtectionGeoswitchingOpened: return "m_netp_imp_geoswitching"
         case .networkProtectionGeoswitchingSetNearest: return "m_netp_ev_geoswitching_set_nearest"
@@ -1015,6 +1021,8 @@ extension Pixel.Event {
         // MARK: - Return user measurement
         case .debugReturnUserAddATB: return "m_debug_return_user_add_atb"
         case .debugReturnUserUpdateATB: return "m_debug_return_user_update_atb"
+
+        case .appRatingPromptFetchError: return "m_d_app_rating_prompt_fetch_error"
         }
 
     }

diff --git a/Core/WebCacheManager.swift b/Core/WebCacheManager.swift
@@ -74,7 +74,7 @@ public class WebCacheManager {
 
         let cookies = cookieStorage.cookies
 
-        guard !cookies.isEmpty else {
+        guard !cookies.isEmpty, !cookieStorage.isConsumed else {
             completion()
             return
         }
@@ -93,9 +93,9 @@ public class WebCacheManager {
 
         DispatchQueue.global(qos: .userInitiated).async {
             group.wait()
+            cookieStorage.isConsumed = true
 
             DispatchQueue.main.async {
-                cookieStorage.clear()
                 completion()
 
                 if cookieStorage.cookies.count > 0 {
@@ -162,7 +162,7 @@ public class WebCacheManager {
                                 logins: PreserveLogins,
                                 storeIdManager: DataStoreIdManager,
                                 completion: @escaping () -> Void) {
-
+        
         guard let containerId = storeIdManager.id else {
             completion()
             return
@@ -181,11 +181,8 @@ public class WebCacheManager {
             await checkForLeftBehindDataStores()
 
             storeIdManager.allocateNewContainerId()
-            // If cookies is empty it's likely that the webview was not used since the last fire button so
-            //  don't overwrite previously saved cookies
-            if let cookies, !cookies.isEmpty {
-                cookieStorage.cookies = cookies
-            }
+
+            cookieStorage.updateCookies(cookies ?? [], keepingPreservedLogins: logins)
 
             completion()
         }
@@ -208,7 +205,7 @@ public class WebCacheManager {
                     // From this point onwards... use containers
                     dataStoreIdManager.allocateNewContainerId()
                     Task { @MainActor in
-                        cookieStorage.cookies = cookies
+                        cookieStorage.updateCookies(cookies, keepingPreservedLogins: logins)
                         completion()
                     }
                 } else {
Original file line number	Diff line number	Diff line change
Expand Up		@@ -74,3 +74,5 @@ fastlane/test_output

		Configuration/ExternalDeveloper.xcconfig
		scripts/assets

		DuckDuckGoTests/NetworkProtectionVPNLocationViewModelTests.swift*.plist
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		MARKETING_VERSION = 7.106.0
		MARKETING_VERSION = 7.107.0