fix: batch event payloads into chunks of 100 to reduce errors on even…

…t processing (#193)

* fix: batch event payloads into chunks of 100 to reduce errors on event processing

* chore: add tests for batched event sending

* fix: conflicting sync vs timeout for event batch tests

* chore: update GH Action runners to use macos-130-large

* fix: refactor batching of events payload into its own private method

.github/workflows/ios.yaml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   build:
-    runs-on: macos-13
+    runs-on: macos-13-large
     steps:
       - name: Checkout Project
         uses: actions/checkout@v4
@@ -21,4 +21,3 @@ jobs:
           use-xcframeworks: true
       - name: Fastlane iOS Tests
         run: fastlane ios tests
-

.github/workflows/macos.yaml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   build:
-    runs-on: macos-13
+    runs-on: macos-13-large
     steps:
       - name: Checkout Project
         uses: actions/checkout@v4

.github/workflows/tvos.yaml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   build:
-    runs-on: macos-13
+    runs-on: macos-13-large
     steps:
       - name: Checkout Project
         uses: actions/checkout@v4
@@ -20,4 +20,4 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           use-xcframeworks: true
       - name: Fastlane tvOS Tests
-        run: fastlane tvos tests
+        run: fastlane tvos tests

.github/workflows/watchos.yaml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   build:
-    runs-on: macos-13
+    runs-on: macos-13-large
     steps:
       - name: Checkout Project
         uses: actions/checkout@v4

DevCycle/Networking/DevCycleService.swift

@@ -77,6 +77,7 @@ class DevCycleService: DevCycleServiceProtocol {
     var requestConsolidator: RequestConsolidator!
 
     private var newUser: DevCycleUser?
+    private var maxBatchSize = 100
 
     init(config: DVCConfig, cacheService: CacheServiceProtocol, options: DevCycleOptions? = nil) {
         let sessionConfig = URLSessionConfiguration.default
@@ -93,7 +94,6 @@ class DevCycleService: DevCycleServiceProtocol {
     }
 
     func publishEvents(events: [DevCycleEvent], user: DevCycleUser, completion: @escaping PublishEventsCompletionHandler) {
-        var eventsRequest = createEventsRequest()
         let userEncoder = JSONEncoder()
         userEncoder.dateEncodingStrategy = .iso8601
         guard let userId = user.userId, let userData = try? userEncoder.encode(user) else {
@@ -104,26 +104,8 @@ class DevCycleService: DevCycleServiceProtocol {
         guard let userBody = try? JSONSerialization.jsonObject(with: userData, options: .fragmentsAllowed) else {
             return completion((nil, nil, ClientError.InvalidUser))
         }
-
-        let requestBody: [String: Any] = [
-            "events": eventPayload,
-            "user": userBody
-        ]
-
-        eventsRequest.httpMethod = "POST"
-        eventsRequest.addValue("application/json", forHTTPHeaderField: "Content-Type")
-        eventsRequest.addValue("application/json", forHTTPHeaderField: "Accept")
-        eventsRequest.addValue(config.sdkKey, forHTTPHeaderField: "Authorization")
-        let jsonBody = try? JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted)
-        Log.debug("Post Events Payload: \(String(data: jsonBody!, encoding: .utf8) ?? "")")
-        eventsRequest.httpBody = jsonBody
-
-        self.makeRequest(request: eventsRequest) { data, response, error in
-            if error != nil || data == nil {
-                return completion((data, response, error))
-            }
-            return completion((data, response, nil))
-        }
+
+        self.batchEventsPayload(events: eventPayload, user: userBody, completion: completion)
     }
 
     func saveEntity(user: DevCycleUser, completion: @escaping SaveEntityCompletionHandler) {
@@ -310,4 +292,42 @@ class DevCycleService: DevCycleServiceProtocol {
 
         return eventsJSON
     }
+
+    private func batchEventsPayload(events: [[String:Any]], user: Any, completion: @escaping PublishEventsCompletionHandler) {
+        var eventsRequest = createEventsRequest()
+        eventsRequest.httpMethod = "POST"
+        eventsRequest.addValue("application/json", forHTTPHeaderField: "Content-Type")
+        eventsRequest.addValue("application/json", forHTTPHeaderField: "Accept")
+        eventsRequest.addValue(self.config.sdkKey, forHTTPHeaderField: "Authorization")
+
+        let totalEventsCount = events.count
+        var startIndex = 0
+        var endIndex = min(self.maxBatchSize, totalEventsCount)
+
+        while startIndex < totalEventsCount {
+            let batchEvents = Array(events[startIndex..<endIndex])
+
+            let requestBody: [String: Any] = [
+                "events": batchEvents,
+                "user": user
+            ]
+
+            let jsonBody = try? JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted)
+            Log.debug("Post Events Payload: \(String(data: jsonBody!, encoding: .utf8) ?? "")")
+            eventsRequest.httpBody = jsonBody
+
+            self.makeRequest(request: eventsRequest) { data, response, error in
+                if error != nil || data == nil {
+                    return completion((data, response, error))
+                }
+                // Continue with next batch
+                startIndex = endIndex
+                endIndex = min(endIndex + self.maxBatchSize, totalEventsCount)
+
+                if startIndex >= totalEventsCount {
+                    return completion((data, response, nil))
+                }
+            }
+        }
+    }
 }

DevCycleTests/Models/DevCycleClientTests.swift

@@ -242,6 +242,7 @@ class DevCycleClientTest: XCTestCase {
 
     func testVariableMethodReturnsDefaultedVariableWhenKeyIsNotInConfig() {
         let client = try! self.builder.user(self.user).sdkKey("my_sdk_key").build(onInitialized: nil)
+        client.setup(service: self.service)
         client.config?.userConfig = self.userConfig
         client.initialize(callback: nil)
 
@@ -252,7 +253,6 @@ class DevCycleClientTest: XCTestCase {
 
         let variableValue = client.variableValue(key: "some_non_existent_variable", defaultValue: false)
         XCTAssertFalse(variableValue)
-        client.close(callback: nil)
     }
 
     func testVariableStringDefaultValue() {
@@ -318,12 +318,13 @@ class DevCycleClientTest: XCTestCase {
         let nsDicDefault: NSDictionary = ["key":"val"]
         let variable2 = client.variable(key: "some_non_existent_variable", defaultValue: nsDicDefault)
         XCTAssertEqual(variable2.defaultValue, nsDicDefault)
-        XCTAssertEqual(variable2.type, DVCVariableTypes.JSON)        
+        XCTAssertEqual(variable2.type, DVCVariableTypes.JSON)
     }
 
     func testVariableMethodReturnsCorrectVariableForKey() {
         let client = try! self.builder.user(self.user).sdkKey("my_sdk_key").build(onInitialized: nil)
         client.initialize(callback: nil)
+        client.setup(service: self.service)
         client.config?.userConfig = self.userConfig
 
         let boolVar = client.variable(key: "bool-var", defaultValue: false)
@@ -354,6 +355,7 @@ class DevCycleClientTest: XCTestCase {
     func testVariableMethodReturnSameInstanceOfVariable() {
         let client = try! self.builder.user(self.user).sdkKey("my_sdk_key").build(onInitialized: nil)
         client.initialize(callback: nil)
+        client.setup(service: self.service)
         client.config?.userConfig = self.userConfig
 
         let boolVar = client.variable(key: "bool-var", defaultValue: false)
@@ -374,6 +376,7 @@ class DevCycleClientTest: XCTestCase {
     func testVariableMethodReturnsDifferentVariableForANewDefaultValue() {
         let client = try! self.builder.user(self.user).sdkKey("my_sdk_key").build(onInitialized: nil)
         client.initialize(callback: nil)
+        client.setup(service: self.service)
         client.config?.userConfig = self.userConfig
 
         var stringVar = client.variable(key: "string-var", defaultValue: "default value")

DevCycleTests/Models/EventQueueTests.swift

@@ -82,7 +82,6 @@ private class MockService: DevCycleServiceProtocol {
     func getConfig(user: DevCycleUser, enableEdgeDB: Bool, extraParams: RequestParams?, completion: @escaping ConfigCompletionHandler) {}
 
     func publishEvents(events: [DevCycleEvent], user: DevCycleUser, completion: @escaping PublishEventsCompletionHandler) {
-
         DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) {
             completion((nil, nil, nil))
         }

DevCycleTests/Networking/DevCycleServiceTests.swift

@@ -72,11 +72,52 @@ class DevCycleServiceTests: XCTestCase {
     }
 
     func testProcessConfigReturnsNilIfBrokenJson() throws {
-        let service = getService()
         let data = "{\"config\":\"key}".data(using: .utf8)
         let config = processConfig(data)
         XCTAssertNil(config)
     }
+
+    func testFlushingEvents() {
+        let service = MockDevCycleService()
+        let eventQueue = EventQueue()
+        let user = try! DevCycleUser.builder().userId("user1").build()
+        let expectation = XCTestExpectation(description: "Events are flushed in a single batch")
+
+        // Generate 205 custom events and add them to the queue
+        for i in 0..<10 {
+            let event = try! DevCycleEvent.builder().type("event_\(i)").build()
+            eventQueue.queue(event)
+        }
+        eventQueue.flush(service: service, user: user, callback: nil)
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
+            XCTAssertEqual(eventQueue.events.count, 0)
+            expectation.fulfill()
+        }
+        wait(for: [expectation], timeout: 1.0)
+        XCTAssertTrue(service.publishEventsCalled)
+        XCTAssertEqual(service.makeRequestCallCount, 1, "makeRequest should have been called 1 time")
+    }
+
+    func testFlushingEventsInBatches() {
+        let service = MockDevCycleService()
+        let eventQueue = EventQueue()
+        let user = try! DevCycleUser.builder().userId("user1").build()
+        let expectation = XCTestExpectation(description: "Events are serially queued and flushed in multiple batches")
+
+        // Generate 205 custom events and add them to the queue
+        for i in 0..<205 {
+            let event = try! DevCycleEvent.builder().type("event_\(i)").build()
+            eventQueue.queue(event)
+        }
+        eventQueue.flush(service: service, user: user, callback: nil)
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
+            XCTAssertEqual(eventQueue.events.count, 0)
+            expectation.fulfill()
+        }
+        wait(for: [expectation], timeout: 3.0)
+        XCTAssertTrue(service.publishEventsCalled)
+        XCTAssertEqual(service.makeRequestCallCount, 3, "makeRequest should have been called 3 times")
+    }
 }
 
 extension DevCycleServiceTests {
@@ -111,6 +152,117 @@ extension DevCycleServiceTests {
         }
     }
 
+    class MockDevCycleService: DevCycleServiceProtocol {
+        func getConfig(user: DevCycle.DevCycleUser, enableEdgeDB: Bool, extraParams: DevCycle.RequestParams?, completion: @escaping DevCycle.ConfigCompletionHandler) {
+            // Empty Stub
+        }
+
+        func saveEntity(user: DevCycle.DevCycleUser, completion: @escaping DevCycle.SaveEntityCompletionHandler) {
+            // Empty Stub
+        }
+
+        var publishEventsCalled = false
+        var makeRequestCallCount = 0
+        let testMaxBatchSize = 100
+        var sdkKey = "my_sdk_key"
+
+        func publishEvents(events: [DevCycleEvent], user: DevCycleUser, completion: @escaping PublishEventsCompletionHandler) {
+            publishEventsCalled = true
+
+            let userEncoder = JSONEncoder()
+            userEncoder.dateEncodingStrategy = .iso8601
+            guard let userId = user.userId, let userData = try? userEncoder.encode(user) else {
+                return completion((nil, nil, ClientError.MissingUser))
+            }
+
+            let eventPayload = self.generateEventPayload(events, userId, nil)
+            guard let userBody = try? JSONSerialization.jsonObject(with: userData, options: .fragmentsAllowed) else {
+                return completion((nil, nil, ClientError.InvalidUser))
+            }
+
+            self.batchEventsPayload(events: eventPayload, user: userBody, completion: completion)
+        }
+
+        func makeRequest(request: URLRequest, completion: @escaping CompletionHandler) {
+            self.makeRequestCallCount += 1
+
+            // Mock implementation for makeRequest
+            let mockData = "Successfully flushed \(self.testMaxBatchSize) events".data(using: .utf8)
+            let mockResponse = HTTPURLResponse(url: URL(string: "https://example.com")!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: nil)
+            completion((mockData, mockResponse, nil))
+        }
+
+        private func generateEventPayload(_ events: [DevCycleEvent], _ userId: String, _ featureVariables: [String:String]?) -> [[String:Any]] {
+            var eventsJSON: [[String:Any]] = []
+            let formatter = ISO8601DateFormatter()
+
+            for event in events {
+                if event.type == nil {
+                    continue
+                }
+                let eventDate: Date = event.clientDate ?? Date()
+                var eventToPost: [String: Any] = [
+                    "type": event.type!,
+                    "clientDate": formatter.string(from: eventDate),
+                    "user_id": userId,
+                    "featureVars": featureVariables ?? [:]
+                ]
+
+                if (event.target != nil) { eventToPost["target"] = event.target }
+                if (event.value != nil) { eventToPost["value"] = event.value }
+                if (event.metaData != nil) { eventToPost["metaData"] = event.metaData }
+                if (event.type != "variableDefaulted" && event.type != "variableEvaluated") {
+                    eventToPost["customType"] = event.type
+                    eventToPost["type"] = "customEvent"
+                }
+
+                eventsJSON.append(eventToPost)
+            }
+
+            return eventsJSON
+        }
+
+        private func batchEventsPayload(events: [[String:Any]], user: Any, completion: @escaping PublishEventsCompletionHandler) {
+                let url = URL(string: "http://test.com/v1/events")!
+                var eventsRequest = URLRequest(url: url)
+                eventsRequest.httpMethod = "POST"
+                eventsRequest.addValue("application/json", forHTTPHeaderField: "Content-Type")
+                eventsRequest.addValue("application/json", forHTTPHeaderField: "Accept")
+                eventsRequest.addValue(self.sdkKey, forHTTPHeaderField: "Authorization")
+
+                let totalEventsCount = events.count
+                var startIndex = 0
+                var endIndex = min(self.testMaxBatchSize, totalEventsCount)
+
+                while startIndex < totalEventsCount {
+                    let batchEvents = Array(events[startIndex..<endIndex])
+
+                    let requestBody: [String: Any] = [
+                        "events": batchEvents,
+                        "user": user
+                    ]
+
+                    let jsonBody = try? JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted)
+                    Log.debug("Post Events Payload: \(String(data: jsonBody!, encoding: .utf8) ?? "")")
+                    eventsRequest.httpBody = jsonBody
+
+                    self.makeRequest(request: eventsRequest) { data, response, error in
+                        if error != nil || data == nil {
+                            return completion((data, response, error))
+                        }
+                        // Continue with next batch
+                        startIndex = endIndex
+                        endIndex = min(endIndex + self.testMaxBatchSize, totalEventsCount)
+
+                        if startIndex >= totalEventsCount {
+                            return completion((data, response, nil))
+                        }
+                    }
+                }
+            }
+    }
+
+
     func getService(_ options: DevCycleOptions? = nil) -> DevCycleService {
         let user = getTestUser()
         let config = DVCConfig(sdkKey: "my_sdk_key", user: user)
@@ -122,7 +274,6 @@ extension DevCycleServiceTests {
             .userId("my_user")
             .build()
     }
-
 }