Merge pull request #111 from nprzy/days-requested

Allow batch mode to backfill more than 90 days of transaction history

src/main/kotlin/net/djvk/fireflyPlaidConnector2/sync/BatchSyncRunner.kt

@@ -18,6 +18,8 @@ import java.time.LocalDate
 import java.time.OffsetDateTime
 import java.util.*
 import kotlin.math.absoluteValue
+import kotlin.math.max
+import kotlin.math.min
 
 /**
  * Batch sync runner.
@@ -85,6 +87,14 @@ class BatchSyncRunner(
                             includeOriginalDescription = true,
                             includePersonalFinanceCategoryBeta = false,
                             includePersonalFinanceCategory = true,
+
+                            // The number of days of history to request from the financial institution the first time
+                            // Plaid fetches transactions for an item. This only matters the first time transactions
+                            // are requested for an item and after that it will be ignored. The max value is 730 and
+                            // the default for the API is 90. Since we only have one chance to get this right, we'll
+                            // enforce a minimum value of 180 days, regardless of how many days our config instructs
+                            // us to sync.
+                            daysRequested = max(min(syncDays + 1, 730), 180),
                         )
                     )
                     val plaidTxs: List<Transaction>

src/test/kotlin/net/djvk/fireflyPlaidConnector2/lib/MockUtil.kt

@@ -0,0 +1,89 @@
+package net.djvk.fireflyPlaidConnector2.lib
+
+import io.ktor.client.statement.*
+import io.ktor.http.*
+import io.ktor.util.reflect.*
+import net.djvk.fireflyPlaidConnector2.api.firefly.apis.AboutApi
+import net.djvk.fireflyPlaidConnector2.api.firefly.apis.AccountsApi
+import net.djvk.fireflyPlaidConnector2.api.firefly.apis.TransactionsApi
+import net.djvk.fireflyPlaidConnector2.api.firefly.models.SystemInfo
+import net.djvk.fireflyPlaidConnector2.api.firefly.models.SystemInfoData
+import net.djvk.fireflyPlaidConnector2.api.firefly.infrastructure.BodyProvider as FireflyBodyProvider
+import net.djvk.fireflyPlaidConnector2.api.firefly.infrastructure.HttpResponse as FireflyHttpResponse
+import net.djvk.fireflyPlaidConnector2.api.plaid.PlaidApiWrapper
+import net.djvk.fireflyPlaidConnector2.api.plaid.apis.PlaidApi
+import net.djvk.fireflyPlaidConnector2.sync.MINIMUM_FIREFLY_VERSION
+import net.djvk.fireflyPlaidConnector2.api.plaid.infrastructure.BodyProvider as PlaidBodyProvider
+import net.djvk.fireflyPlaidConnector2.api.plaid.infrastructure.HttpResponse as PlaidHttpResponse
+import org.mockito.kotlin.*
+import org.mockito.stubbing.Answer
+
+val OK_RESPONSE = mock<HttpResponse> {
+    on { status } doReturn HttpStatusCode.OK
+    on { headers } doReturn Headers.Empty
+}
+
+private class PlaidStubbedBodyProvider<T : Any>(val responseObj: T): PlaidBodyProvider<T> {
+    override suspend fun body(response: HttpResponse): T {
+        return responseObj
+    }
+
+    @Suppress("UNCHECKED_CAST")
+    override suspend fun <V : Any> typedBody(response: HttpResponse, type: TypeInfo): V {
+        return responseObj as V
+    }
+}
+
+fun <T : Any> createPlaidResponse(
+    response: T,
+    httpResponse: HttpResponse = OK_RESPONSE,
+): PlaidHttpResponse<T> {
+    return PlaidHttpResponse(httpResponse, PlaidStubbedBodyProvider(response))
+}
+
+class PlaidMock {
+    val api = mock<PlaidApi>()
+    val wrapper = mock<PlaidApiWrapper> {
+        onBlocking { executeRequest(any<suspend (PlaidApi) -> Any>(), any(), any()) } doSuspendableAnswer {
+            val requestExecutor = it.getArgument(0) as suspend (PlaidApi) -> Any
+            requestExecutor.invoke(api)
+        }
+    }
+}
+
+private class FireflyStubbedBodyProvider<T : Any>(val responseObj: T): FireflyBodyProvider<T> {
+    override suspend fun body(response: HttpResponse): T {
+        return responseObj
+    }
+
+    @Suppress("UNCHECKED_CAST")
+    override suspend fun <V : Any> typedBody(response: HttpResponse, type: TypeInfo): V {
+        return responseObj as V
+    }
+}
+
+fun <T : Any> createFireflyResponse(
+    response: T,
+    httpResponse: HttpResponse = OK_RESPONSE,
+): FireflyHttpResponse<T> {
+    return FireflyHttpResponse(httpResponse, FireflyStubbedBodyProvider(response))
+}
+
+class FireflyMock {
+    val aboutApi = mock<AboutApi>()
+    val transactionsApi = mock<TransactionsApi>()
+    val accountsApi = mock<AccountsApi>()
+
+    init {
+        val systemInfoData = SystemInfoData(
+            version = MINIMUM_FIREFLY_VERSION,
+            apiVersion = "1.0.0",
+            phpVersion = "1.0.0",
+            os = "testOs",
+            driver = "testDriver"
+        )
+        aboutApi.stub {
+            onBlocking { getAbout() } doAnswer { createFireflyResponse(SystemInfo(systemInfoData)) }
+        }
+    }
+}

src/test/kotlin/net/djvk/fireflyPlaidConnector2/lib/PlaidFixtures.kt

@@ -1,5 +1,6 @@
 package net.djvk.fireflyPlaidConnector2.lib
 
+import com.fasterxml.jackson.annotation.JsonProperty
 import net.djvk.fireflyPlaidConnector2.api.plaid.models.*
 import net.djvk.fireflyPlaidConnector2.transactions.FireflyAccountId
 import net.djvk.fireflyPlaidConnector2.transactions.PersonalFinanceCategoryEnum
@@ -231,6 +232,32 @@ object PlaidFixtures {
         )
     }
 
+    fun getItem(
+        itemId: kotlin.String = "testItemId1",
+        webhook: kotlin.String? = null,
+        error: PlaidError? = null,
+        availableProducts: List<Products> = listOf(Products.transactions),
+        billedProducts: List<Products> = listOf(Products.transactions),
+        consentExpirationTime: java.time.OffsetDateTime? = null,
+        updateType: Item.UpdateType = Item.UpdateType.background,
+        institutionId: String? = null,
+        products: List<Products>? = null,
+        consentedProducts: List<Products>? = null,
+    ): Item {
+        return Item(
+            itemId = itemId,
+            webhook = webhook,
+            error = error,
+            availableProducts = availableProducts,
+            billedProducts = billedProducts,
+            consentExpirationTime = consentExpirationTime,
+            updateType = updateType,
+            institutionId = institutionId,
+            products = products,
+            consentedProducts = consentedProducts,
+        )
+    }
+
     fun getStandardAccountMapping(): Map<PlaidAccountId, FireflyAccountId> {
         val out = mutableMapOf<PlaidAccountId, FireflyAccountId>()
         var index = 1

src/test/kotlin/net/djvk/fireflyPlaidConnector2/sync/BatchSyncRunnerTest.kt

@@ -0,0 +1,95 @@
+package net.djvk.fireflyPlaidConnector2.sync
+
+import kotlinx.coroutines.runBlocking
+import net.djvk.fireflyPlaidConnector2.api.plaid.models.TransactionsGetResponse
+import net.djvk.fireflyPlaidConnector2.config.AccountConfig
+import net.djvk.fireflyPlaidConnector2.config.properties.AccountConfigs
+import net.djvk.fireflyPlaidConnector2.lib.*
+import net.djvk.fireflyPlaidConnector2.transactions.TransactionConverter
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.Arguments
+import org.junit.jupiter.params.provider.MethodSource
+import org.mockito.kotlin.*
+
+internal class BatchSyncRunnerTest {
+    companion object {
+        fun createRunner(
+            plaid: PlaidMock,
+            firefly: FireflyMock,
+            syncDays: Int = 200,
+            setInitialBalance: Boolean = false,
+            plaidBatchSize: Int = 100,
+            syncHelper: SyncHelper? = null,
+            converter: TransactionConverter? = null,
+        ): BatchSyncRunner {
+            val defaultSyncHelper = SyncHelper(
+                plaidAccountsConfig = AccountConfigs(listOf(AccountConfig(1, "account1Token", "plaidAccount1"))),
+                fireflyAccessToken = "testToken",
+                fireflyAboutApi = firefly.aboutApi,
+                fireflyTxApi = firefly.transactionsApi,
+                fireflyAccountsApi = firefly.accountsApi,
+            )
+            val defaultTransactionConverter = TransactionConverter(
+                useNameForDestination = false,
+                timeZoneString = "America/New_York",
+                transferMatchWindowDays = 5,
+                enablePrimaryCategorization = true,
+                primaryCategoryPrefix = "primary-",
+                enableDetailedCategorization = true,
+                detailedCategoryPrefix = "detailed-",
+            )
+
+            return BatchSyncRunner(
+                syncDays,
+                setInitialBalance,
+                plaidBatchSize,
+                plaid.wrapper,
+                syncHelper ?: defaultSyncHelper,
+                firefly.accountsApi,
+                converter ?: defaultTransactionConverter,
+            )
+        }
+
+        @JvmStatic
+        fun provideDaysOfHistoryCases(): List<Arguments> {
+            return listOf(
+                Arguments.of("minimum of 180 (1 -> 180)", 1, 180),
+                Arguments.of("maximum of 730 (730 -> 730)", 730, 730),
+                Arguments.of("maximum of 730 (999 -> 730)", 9999, 730),
+                Arguments.of("syncDays plus one (180 -> 181)", 180, 181),
+                Arguments.of("syncDays plus one (500 -> 501)", 500, 501),
+            )
+        }
+    }
+
+    @ParameterizedTest(name = "{index} => {0}")
+    @MethodSource("provideDaysOfHistoryCases")
+    fun runRequestsExpectedDaysOfHistory(
+        testName: String,
+        configuredSyncDays: Int,
+        expectedDaysRequested: Int,
+    ) {
+        val firefly = FireflyMock()
+        val plaid = PlaidMock()
+
+        val runner = createRunner(plaid, firefly, syncDays = configuredSyncDays)
+        val response = TransactionsGetResponse(listOf(), listOf(), 0, PlaidFixtures.getItem(), "requestId1")
+
+        plaid.api.stub {
+            onBlocking { transactionsGet(any()) } doAnswer { createPlaidResponse(response) }
+        }
+
+        runBlocking {
+            runner.run()
+        }
+
+        verifyBlocking(plaid.api) {
+            transactionsGet(check { actual ->
+                assertThat(actual).extracting { it.options }
+                    .isNotNull()
+                assertThat(actual.options!!).extracting { it.daysRequested }.isEqualTo(expectedDaysRequested)
+            })
+        }
+    }
+}