From 6b6d9529e9f172e10dd95a5e649afceb2aee091d Mon Sep 17 00:00:00 2001 From: Adam Wushensky Date: Wed, 14 Jul 2021 16:52:34 -0700 Subject: [PATCH 1/5] Make onScanReady accurate even without warmUp --- .../java/com/getbouncer/scan/framework/Fetcher.kt | 11 +++++++++++ .../java/com/getbouncer/scan/payment/ModelManager.kt | 12 ++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt b/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt index dfaafa8a..a8ed1ace 100644 --- a/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt +++ b/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt @@ -21,6 +21,8 @@ import java.lang.Exception import java.net.URL import java.security.MessageDigest import java.security.NoSuchAlgorithmException +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine private const val CACHE_MODEL_MAX_COUNT = 3 @@ -115,6 +117,8 @@ interface Fetcher { * @param forImmediateUse: if there is a cached version of the model, return that immediately instead of downloading a new model */ suspend fun fetchData(forImmediateUse: Boolean, isOptional: Boolean): FetchedData + + suspend fun isCached(): Boolean } /** @@ -135,6 +139,8 @@ abstract class ResourceFetcher : Fetcher { modelHashAlgorithm = hashAlgorithm, assetFileName = assetFileName, ) + + override suspend fun isCached(): Boolean = true } /** @@ -226,6 +232,11 @@ sealed class WebFetcher(protected val context: Context) : Fetcher { } } + override suspend fun isCached(): Boolean = when (val meta = tryFetchLatestCachedData()) { + is FetchedModelFileMeta -> meta.modelFile != null + is FetchedModelResourceMeta -> true + } + /** * Get information about what version of the model to download. */ diff --git a/scan-payment/src/main/java/com/getbouncer/scan/payment/ModelManager.kt b/scan-payment/src/main/java/com/getbouncer/scan/payment/ModelManager.kt index e6179988..bba51d93 100644 --- a/scan-payment/src/main/java/com/getbouncer/scan/payment/ModelManager.kt +++ b/scan-payment/src/main/java/com/getbouncer/scan/payment/ModelManager.kt @@ -8,10 +8,10 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock abstract class ModelManager { - private lateinit var fetcher: Fetcher private val fetcherMutex = Mutex() - private var successfullyFetched = false + + private var onFetch: ((success: Boolean) -> Unit)? = null suspend fun fetchModel(context: Context, forImmediateUse: Boolean, isOptional: Boolean = false): FetchedData { fetcherMutex.withLock { @@ -19,10 +19,14 @@ abstract class ModelManager { fetcher = getModelFetcher(context.applicationContext) } } - return fetcher.fetchData(forImmediateUse, isOptional).also { successfullyFetched = it.successfullyFetched } + return fetcher.fetchData(forImmediateUse, isOptional).also { + onFetch?.invoke(it.successfullyFetched) + } } - fun isReady() = successfullyFetched + suspend fun isReady() = fetcherMutex.withLock { + if (this::fetcher.isInitialized) fetcher.isCached() else false + } @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) abstract fun getModelFetcher(context: Context): Fetcher From 175bd85bbd9d72d2b0cf1f1a475f0b8ede3871c3 Mon Sep 17 00:00:00 2001 From: Adam Wushensky Date: Mon, 19 Jul 2021 11:00:02 -0700 Subject: [PATCH 2/5] Clean up warmup flow --- .../cardscan/ui/CardScanActivity.kt | 34 +++------------ .../getbouncer/cardscan/ui/CardScanFlow.kt | 42 +++++++++---------- cardscan-ui/src/main/res/values/strings.xml | 4 -- 3 files changed, 25 insertions(+), 55 deletions(-) diff --git a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt index 3ad6fdc9..c3cf4946 100644 --- a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt +++ b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt @@ -31,6 +31,7 @@ import com.getbouncer.scan.ui.util.setTextSizeByRes import com.getbouncer.scan.ui.util.setVisible import com.getbouncer.scan.ui.util.show import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlinx.parcelize.Parcelize @@ -130,7 +131,7 @@ open class CardScanActivity : */ @JvmStatic fun warmUp(context: Context, apiKey: String, initializeNameAndExpiryExtraction: Boolean) { - CardScanFlow.warmUp(context, apiKey, initializeNameAndExpiryExtraction) + GlobalScope.launch { CardScanFlow.prepareScan(context, apiKey, initializeNameAndExpiryExtraction, false) } } /** @@ -157,7 +158,7 @@ open class CardScanActivity : enableEnterCardManually = enableEnterCardManually, enableExpiryExtraction = enableExpiryExtraction, enableNameExtraction = enableNameExtraction, - ) ?: return + ) activity.startActivityForResult(intent, REQUEST_CODE) } @@ -187,7 +188,7 @@ open class CardScanActivity : enableEnterCardManually = enableEnterCardManually, enableExpiryExtraction = enableExpiryExtraction, enableNameExtraction = enableNameExtraction, - ) ?: return + ) fragment.startActivityForResult(intent, REQUEST_CODE) } @@ -209,19 +210,9 @@ open class CardScanActivity : enableEnterCardManually: Boolean = false, enableExpiryExtraction: Boolean = false, enableNameExtraction: Boolean = false, - ): Intent? { + ): Intent { Config.apiKey = apiKey - if (!CardScanFlow.attemptedNameAndExpiryInitialization && (enableExpiryExtraction || enableNameExtraction)) { - Log.e( - Config.logTag, - "Attempting to run name and expiry without initializing text detector. " + - "Please invoke the warmup() function with initializeNameAndExpiryExtraction to true." - ) - showNameAndExpiryInitializationError(context) - return null - } - return Intent(context, CardScanActivity::class.java) .putExtra(PARAM_ENABLE_ENTER_MANUALLY, enableEnterCardManually) .putExtra(PARAM_ENABLE_EXPIRY_EXTRACTION, enableExpiryExtraction) @@ -268,21 +259,6 @@ open class CardScanActivity : */ @JvmStatic fun isScanReady() = CardScanFlow.isScanReady() - - /** - * Determine if the optional scan models are available (have been warmed up) - */ - @JvmStatic - fun isNameAndExpiryScanReady() = CardScanFlow.isNameAndExpiryScanReady() - - private fun showNameAndExpiryInitializationError(context: Context) { - AlertDialog.Builder(context) - .setTitle(R.string.bouncer_name_and_expiry_initialization_error) - .setMessage(R.string.bouncer_name_and_expiry_initialization_error_message) - .setPositiveButton(R.string.bouncer_name_and_expiry_initialization_error_ok) { dialog, _ -> dialog.dismiss() } - .setCancelable(false) - .show() - } } /** diff --git a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanFlow.kt b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanFlow.kt index 7c729a8a..bef950d9 100644 --- a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanFlow.kt +++ b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanFlow.kt @@ -18,6 +18,7 @@ import com.getbouncer.scan.framework.AggregateResultListener import com.getbouncer.scan.framework.AnalyzerLoopErrorListener import com.getbouncer.scan.framework.AnalyzerPool import com.getbouncer.scan.framework.Config +import com.getbouncer.scan.framework.FetchedData import com.getbouncer.scan.framework.FiniteAnalyzerLoop import com.getbouncer.scan.framework.ProcessBoundAnalyzerLoop import com.getbouncer.scan.framework.time.Duration @@ -36,12 +37,15 @@ import com.getbouncer.scan.payment.ml.SSDOcrModelManager import com.getbouncer.scan.payment.ml.TextDetect import com.getbouncer.scan.ui.ScanFlow import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.Job +import kotlinx.coroutines.async import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext data class SavedFrame( val pan: String?, @@ -67,12 +71,6 @@ open class CardScanFlow( private const val MAX_COMPLETION_LOOP_FRAMES_FAST_DEVICE = 8 private const val MAX_COMPLETION_LOOP_FRAMES_SLOW_DEVICE = 5 - /** - * This field represents whether the flow was initialized with name and expiry enabled. - */ - var attemptedNameAndExpiryInitialization = false - private set - /** * Warm up the analyzers for card scanner. This method is optional, but will increase the speed at which the * scan occurs. @@ -80,19 +78,25 @@ open class CardScanFlow( * @param context: A context to use for warming up the analyzers. */ @JvmStatic - fun warmUp(context: Context, apiKey: String, initializeNameAndExpiryExtraction: Boolean) { + suspend fun prepareScan( + context: Context, + apiKey: String, + initializeNameAndExpiryExtraction: Boolean, + forImmediateUse: Boolean, + ) = withContext(Dispatchers.IO) { Config.apiKey = apiKey + val deferredFetchers = mutableListOf>() - // pre-fetch all of the models used by this flow. - GlobalScope.launch(Dispatchers.IO) { SSDOcrModelManager.fetchModel(context, forImmediateUse = false) } - GlobalScope.launch(Dispatchers.IO) { CardDetectModelManager.fetchModel(context, forImmediateUse = false) } + deferredFetchers.add(async { SSDOcrModelManager.fetchModel(context, forImmediateUse) }) + deferredFetchers.add(async { CardDetectModelManager.fetchModel(context, forImmediateUse) }) if (initializeNameAndExpiryExtraction) { - attemptedNameAndExpiryInitialization = true - GlobalScope.launch(Dispatchers.IO) { TextDetectModelManager.fetchModel(context, forImmediateUse = false) } - GlobalScope.launch(Dispatchers.IO) { AlphabetDetectModelManager.fetchModel(context, forImmediateUse = false) } - GlobalScope.launch(Dispatchers.IO) { ExpiryDetectModelManager.fetchModel(context, forImmediateUse = false) } + deferredFetchers.add(async { TextDetectModelManager.fetchModel(context, forImmediateUse) }) + deferredFetchers.add(async { AlphabetDetectModelManager.fetchModel(context, forImmediateUse) }) + deferredFetchers.add(async { ExpiryDetectModelManager.fetchModel(context, forImmediateUse) }) } + + deferredFetchers.fold(true) { acc, deferred -> acc && deferred.await().successfullyFetched } } /** @@ -105,13 +109,7 @@ open class CardScanFlow( * Determine if the scan models are available (have been warmed up) */ @JvmStatic - fun isScanReady() = SSDOcrModelManager.isReady() && CardDetectModelManager.isReady() - - /** - * Determine if the optional scan models are available (have been warmed up) - */ - @JvmStatic - fun isNameAndExpiryScanReady() = TextDetectModelManager.isReady() && AlphabetDetectModelManager.isReady() && ExpiryDetectModelManager.isReady() + fun isScanReady() = runBlocking { SSDOcrModelManager.isReady() && CardDetectModelManager.isReady() } } /** diff --git a/cardscan-ui/src/main/res/values/strings.xml b/cardscan-ui/src/main/res/values/strings.xml index 7c7af59b..88add914 100644 --- a/cardscan-ui/src/main/res/values/strings.xml +++ b/cardscan-ui/src/main/res/values/strings.xml @@ -3,8 +3,4 @@ Processing, please wait Enter card manually - - Config Problem - Please initialize name/expiry models first in the warmup() function. - OK From 0b72cf704bd9bb6ded2fd9fb2c9c9841fa65364f Mon Sep 17 00:00:00 2001 From: Adam Wushensky Date: Mon, 19 Jul 2021 11:28:11 -0700 Subject: [PATCH 3/5] Add suspend method for awaiting warmup completion --- .../cardscan/ui/CardScanActivity.kt | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt index c3cf4946..b5a1f3ee 100644 --- a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt +++ b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt @@ -128,12 +128,27 @@ open class CardScanActivity : * speed at which the scan occurs. * * @param context: A context to use for warming up the analyzers. + * @param apiKey: the API key used to warm up the ML models + * @param initializeNameAndExpiryExtraction: if true, include name and expiry extraction + * @param forImmediateUse: if true, attempt to use cached models instead of downloading by default */ @JvmStatic - fun warmUp(context: Context, apiKey: String, initializeNameAndExpiryExtraction: Boolean) { - GlobalScope.launch { CardScanFlow.prepareScan(context, apiKey, initializeNameAndExpiryExtraction, false) } + fun warmUp(context: Context, apiKey: String, initializeNameAndExpiryExtraction: Boolean, forImmediateUse: Boolean = false) { + GlobalScope.launch { prepareScan(context, apiKey, initializeNameAndExpiryExtraction, forImmediateUse) } } + /** + * Warm up the analyzers and suspend the thread until it has completed. + * + * @param context: A context to use for warming up the analyzers. + * @param apiKey: the API key used to warm up the ML models + * @param initializeNameAndExpiryExtraction: if true, include name and expiry extraction + * @param forImmediateUse: if true, attempt to use cached models instead of downloading by default + */ + @JvmStatic + suspend fun prepareScan(context: Context, apiKey: String, initializeNameAndExpiryExtraction: Boolean, forImmediateUse: Boolean) = + CardScanFlow.prepareScan(context, apiKey, initializeNameAndExpiryExtraction, forImmediateUse) + /** * Start the card scanner activity. * From ac005a334f76c10468feb756d939823b0067cbf3 Mon Sep 17 00:00:00 2001 From: Adam Wushensky Date: Mon, 19 Jul 2021 11:36:52 -0700 Subject: [PATCH 4/5] Fix lint and compile --- .../main/java/com/getbouncer/cardscan/demo/LaunchActivity.java | 2 +- .../main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cardscan-demo/src/main/java/com/getbouncer/cardscan/demo/LaunchActivity.java b/cardscan-demo/src/main/java/com/getbouncer/cardscan/demo/LaunchActivity.java index 5548b67e..2f1e8974 100644 --- a/cardscan-demo/src/main/java/com/getbouncer/cardscan/demo/LaunchActivity.java +++ b/cardscan-demo/src/main/java/com/getbouncer/cardscan/demo/LaunchActivity.java @@ -71,7 +71,7 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { startActivity(new Intent(this, SingleActivityDemo.class)) ); - CardScanActivity.warmUp(this, API_KEY, true); + CardScanActivity.warmUp(this, API_KEY, true, false); } @Override diff --git a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt index b5a1f3ee..cdf9943f 100644 --- a/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt +++ b/cardscan-ui/src/main/java/com/getbouncer/cardscan/ui/CardScanActivity.kt @@ -4,13 +4,11 @@ import android.app.Activity import android.content.Context import android.content.Intent import android.os.Parcelable -import android.util.Log import android.view.Gravity import android.view.View import android.widget.ImageView import android.widget.ProgressBar import android.widget.TextView -import androidx.appcompat.app.AlertDialog import androidx.constraintlayout.widget.ConstraintLayout import androidx.constraintlayout.widget.ConstraintSet import androidx.fragment.app.Fragment From 5818777ce2eec4a5e04b480f2a5c62193547c3db Mon Sep 17 00:00:00 2001 From: Adam Wushensky Date: Mon, 19 Jul 2021 11:51:31 -0700 Subject: [PATCH 5/5] Fix lint more --- .../src/main/java/com/getbouncer/scan/framework/Fetcher.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt b/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt index a8ed1ace..d91a0582 100644 --- a/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt +++ b/scan-framework/src/main/java/com/getbouncer/scan/framework/Fetcher.kt @@ -21,8 +21,6 @@ import java.lang.Exception import java.net.URL import java.security.MessageDigest import java.security.NoSuchAlgorithmException -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine private const val CACHE_MODEL_MAX_COUNT = 3