diff --git a/.nais/config-loadtests.json b/.nais/config-loadtests.json index ce707ee7..47e85dc8 100644 --- a/.nais/config-loadtests.json +++ b/.nais/config-loadtests.json @@ -6,8 +6,7 @@ "ingress": "https://soknadsarkiverer-loadtests.intern.dev.nav.no", "outbound": [ "soknadsfillager-loadtests", - "arkiv-mock", - "innsending-api-loadtests" + "arkiv-mock" ], "outboundExternalHosts": [ "arkiv-mock-gcp.intern.dev.nav.no", @@ -18,6 +17,7 @@ "kafka-message-topic": "team-soknad.privat-soknadinnsending-messages-v1-loadtests", "kafka-metrics-topic": "team-soknad.privat-soknadinnsending-metrics-v1-loadtests", "filestorage-host": "https://soknadsfillager-gcp-loadtests.intern.dev.nav.no", + "innsending-api-host": "arkiv-mock-gcp.intern.dev.nav.no", "joark-host": "https://arkiv-mock-gcp.intern.dev.nav.no", "dokarkiv-scope": "api://dev-gcp.team-soknad.arkiv-mock/.default", "soknadsfillager-scope": "api://dev-gcp.team-soknad.soknadsfillager-loadtests/.default", diff --git a/arkiverer/src/main/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageService.kt b/arkiverer/src/main/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageService.kt index 250485c4..fd807601 100644 --- a/arkiverer/src/main/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageService.kt +++ b/arkiverer/src/main/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageService.kt @@ -52,36 +52,14 @@ class FilestorageService( } override fun deleteFilesFromFilestorage(key: String, data: Soknadarkivschema) { - val timer = metrics.filestorageDelLatencyStart() - val fileIds = getFileIds(data) - try { - - logger.info("$key: Calling file storage to delete $fileIds") - deleteFiles(key, fileIds) - logger.info("$key: Deleted these files: $fileIds") - - metrics.incDelFilestorageSuccesses() - } catch (e: Exception) { - metrics.incDelFilestorageErrors() - logger.warn( - "$key: Failed to delete files from file storage. Everything is saved to Joark correctly, " + - "so this error will be ignored. Affected file ids: '$fileIds'", e - ) - - } finally { - metrics.endTimer(timer) - } + logger.info("$key: The following files are archived and can be deleted (by sendsoknad or innsending-api) $fileIds") } private fun getFiles(key: String, fileIds: List) = mergeFetchResponsesAndSetOverallStatus(key, fileIds.map { getOneFile(key, it) } ) - private fun deleteFiles(key: String, fileIds: List) { - filesApi.deleteFiles(fileIds, key) - } - private fun returnFirstOrNull(files: List?): List? { if (files == null || files.isEmpty()) return null return mapToFileInfo(files.first()) diff --git a/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/ApplicationTests.kt b/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/ApplicationTests.kt index 80eeb671..18ddd466 100644 --- a/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/ApplicationTests.kt +++ b/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/ApplicationTests.kt @@ -130,7 +130,6 @@ class ApplicationTests : ContainerizedKafka() { )) verifyMockedPostRequests(1, safUrl) verifyMockedPostRequests(1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 1, @@ -159,7 +158,6 @@ class ApplicationTests : ContainerizedKafka() { )) verifyMockedPostRequests(1, safUrl) verifyMockedPostRequests(1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 1, @@ -185,7 +183,6 @@ class ApplicationTests : ContainerizedKafka() { RECEIVED hasCount 0, STARTED hasCount 0, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 0 )) verifyMockedPostRequests(0, journalPostUrl) - verifyDeleteRequestsToFilestorage(0) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 0, "send files to archive" hasCount 0, @@ -207,7 +204,6 @@ class ApplicationTests : ContainerizedKafka() { RECEIVED hasCount 1, STARTED hasCount maxNumberOfAttempts, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1 )) verifyMockedPostRequests(maxNumberOfAttempts, safUrl) - verifyDeleteRequestsToFilestorage(0) verifyMessageStartsWith(key, mapOf("**Archiving: FAILED" hasCount 1, "ok" hasCount 0, "Exception" hasCount maxNumberOfAttempts)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount maxNumberOfAttempts, @@ -259,7 +255,6 @@ class ApplicationTests : ContainerizedKafka() { )) verifyMockedPostRequests(1, safUrl) verifyMockedPostRequests(1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("ok" hasCount 1)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 1, @@ -290,7 +285,6 @@ class ApplicationTests : ContainerizedKafka() { )) verifyMockedPostRequests(numberOfFailures+1, safUrl) verifyMockedPostRequests(numberOfFailures+1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("ok" hasCount 1, "Exception" hasCount 1)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount numberOfFailures+1, @@ -299,7 +293,6 @@ class ApplicationTests : ContainerizedKafka() { )) verifyArchivingMetrics(getFilestorageSuccessesBefore + 2, { metrics.getGetFilestorageSuccesses() }) - verifyArchivingMetrics(delFilestorageSuccessesBefore + 1, { metrics.getDelFilestorageSuccesses() }) verifyArchivingMetrics(joarkErrorsBefore + 1, { metrics.getJoarkErrors() }) verifyArchivingMetrics(joarkSuccessesBefore + 1, { metrics.getJoarkSuccesses() }) verifyArchivingMetrics(tasksBefore + 0, { metrics.getTasks() }, "Should have created and finished task") @@ -323,7 +316,6 @@ class ApplicationTests : ContainerizedKafka() { )) verifyMockedPostRequests(attemptsToFail + 1, safUrl) verifyMockedPostRequests(attemptsToFail, journalPostUrl) - verifyDeleteRequestsToFilestorage(0) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount attemptsToFail, "send files to archive" hasCount 0, @@ -352,7 +344,6 @@ class ApplicationTests : ContainerizedKafka() { RECEIVED hasCount 1, STARTED hasCount 1, ARCHIVED hasCount 1, FINISHED hasCount 1, FAILURE hasCount 0 )) verifyMockedPostRequests(1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 1, @@ -362,7 +353,6 @@ class ApplicationTests : ContainerizedKafka() { verifyArchivingMetrics(getFilestorageSuccessesBefore + 1, { metrics.getGetFilestorageSuccesses() }) verifyArchivingMetrics(delFilestorageSuccessesBefore + 0, { metrics.getDelFilestorageSuccesses() }) - verifyArchivingMetrics(delFilestorageErrorsBefore + 1, { metrics.getDelFilestorageErrors() }) verifyArchivingMetrics(joarkErrorsBefore + 0, { metrics.getJoarkErrors() }) verifyArchivingMetrics(joarkSuccessesBefore + 1, { metrics.getJoarkSuccesses() }) } @@ -380,7 +370,6 @@ class ApplicationTests : ContainerizedKafka() { RECEIVED hasCount 1, STARTED hasCount maxNumberOfAttempts, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1 )) verifyMockedPostRequests(maxNumberOfAttempts, journalPostUrl) - verifyDeleteRequestsToFilestorage(0) verifyMessageStartsWith(key, mapOf("ok" hasCount 0, "Exception" hasCount maxNumberOfAttempts)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount maxNumberOfAttempts, @@ -406,7 +395,6 @@ class ApplicationTests : ContainerizedKafka() { )) verifyMockedPostRequests(attemptsToFail+1, safUrl) verifyMockedPostRequests(attemptsToFail + 1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("ok" hasCount 1, "Exception" hasCount attemptsToFail)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount attemptsToFail+1, @@ -437,7 +425,6 @@ class ApplicationTests : ContainerizedKafka() { verifyProcessingEvents(key, mapOf( RECEIVED hasCount 1, STARTED hasCount 1, ARCHIVED hasCount 1, FINISHED hasCount 1, FAILURE hasCount 0 )) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 1)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 1, @@ -447,7 +434,6 @@ class ApplicationTests : ContainerizedKafka() { verifyArchivingMetrics(getFilestorageErrorsBefore + 0, { metrics.getGetFilestorageErrors() }) verifyArchivingMetrics(getFilestorageSuccessesBefore + 1, { metrics.getGetFilestorageSuccesses() }) - verifyArchivingMetrics(delFilestorageSuccessesBefore + 1, { metrics.getDelFilestorageSuccesses() }) verifyArchivingMetrics(joarkErrorsBefore + 0, { metrics.getJoarkErrors() }) verifyArchivingMetrics(joarkSuccessesBefore + 0, { metrics.getJoarkSuccesses() }) verifyArchivingMetrics(tasksBefore, { metrics.getTasks() }) @@ -474,7 +460,6 @@ class ApplicationTests : ContainerizedKafka() { verifyProcessingEvents(key, mapOf( RECEIVED hasCount 1, STARTED hasCount 1, ARCHIVED hasCount 1, FINISHED hasCount 1, FAILURE hasCount 0 )) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 0, @@ -484,7 +469,6 @@ class ApplicationTests : ContainerizedKafka() { verifyArchivingMetrics(getFilestorageErrorsBefore + 0, { metrics.getGetFilestorageErrors() }) verifyArchivingMetrics(getFilestorageSuccessesBefore + 0, { metrics.getGetFilestorageSuccesses() }) - verifyArchivingMetrics(delFilestorageSuccessesBefore + 1, { metrics.getDelFilestorageSuccesses() }) verifyArchivingMetrics(joarkErrorsBefore + 0, { metrics.getJoarkErrors() }) verifyArchivingMetrics(joarkSuccessesBefore + 0, { metrics.getJoarkSuccesses() }) verifyArchivingMetrics(tasksBefore, { metrics.getTasks() }) @@ -511,7 +495,6 @@ class ApplicationTests : ContainerizedKafka() { verifyProcessingEvents(key, mapOf( RECEIVED hasCount 1, STARTED hasCount maxNumberOfAttempts, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1 )) - verifyDeleteRequestsToFilestorage(0) verifyMessageStartsWith(key, mapOf("ok" hasCount 0, "Exception" hasCount maxNumberOfAttempts)) verifyKafkaMetric(key, mapOf( "get files from filestorage" hasCount 0, @@ -549,7 +532,6 @@ class ApplicationTests : ContainerizedKafka() { verifyProcessingEvents(key, mapOf( RECEIVED hasCount 1, STARTED hasCount 2, ARCHIVED hasCount 0, FINISHED hasCount 1, FAILURE hasCount 0 )) - verifyDeleteRequestsToFilestorage(1) verifyMessageStartsWith(key, mapOf("ok" hasCount 1, "Exception" hasCount 1)) verifyKafkaMetric(key, mapOf( @@ -588,7 +570,6 @@ class ApplicationTests : ContainerizedKafka() { verifyProcessingEvents(key, mapOf( RECEIVED hasCount 1, STARTED hasCount tasksBefore.toInt() + 1, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1 )) - verifyDeleteRequestsToFilestorage(0) verifyMessageStartsWith(key, mapOf("ok" hasCount 0, "Exception" hasCount 6)) verifyKafkaMetric(key, mapOf( @@ -660,10 +641,6 @@ class ApplicationTests : ContainerizedKafka() { } - private fun verifyDeleteRequestsToFilestorage(expectedCount: Int) { - verifyMockedDeleteRequests(expectedCount, filestorageProperties.files + ".*") - } - private fun createSoknadarkivschema(behandlingsId: String) = createSoknadarkivschema(fileUuid, behandlingsId) diff --git a/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/IntegrationTests.kt b/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/IntegrationTests.kt index 66c19079..26922898 100644 --- a/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/IntegrationTests.kt +++ b/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/IntegrationTests.kt @@ -86,7 +86,6 @@ class IntegrationTests : ContainerizedKafka() { putDataOnKafkaTopic(createSoknadarkivschema()) verifyMockedPostRequests(initialRequests+2, journalPostUrl) - verifyDeleteRequestsToFilestorage(2) } @Test @@ -98,7 +97,6 @@ class IntegrationTests : ContainerizedKafka() { TimeUnit.SECONDS.sleep(1) verifyMockedPostRequests(0, journalPostUrl) - verifyDeleteRequestsToFilestorage(0) } @Test @@ -112,7 +110,6 @@ class IntegrationTests : ContainerizedKafka() { putDataOnKafkaTopic(soknadarkivschema) verifyMockedPostRequests(1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) } @Test @@ -125,7 +122,6 @@ class IntegrationTests : ContainerizedKafka() { putDataOnKafkaTopic(soknadarkivschema) verifyMockedPostRequests(0, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) } @Test @@ -138,12 +134,6 @@ class IntegrationTests : ContainerizedKafka() { putDataOnKafkaTopic(soknadarkivschema) verifyMockedPostRequests(1, journalPostUrl) - verifyDeleteRequestsToFilestorage(1) - } - - private fun verifyDeleteRequestsToFilestorage(expectedCount: Int) { - val url = filestorageProperties.files + ".*" - verifyMockedDeleteRequests(expectedCount, url) } private fun createSoknadarkivschema() = createSoknadarkivschema(fileId) diff --git a/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageServiceTests.kt b/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageServiceTests.kt index 34932b92..822854bf 100644 --- a/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageServiceTests.kt +++ b/arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageServiceTests.kt @@ -111,7 +111,7 @@ class FilestorageServiceTests { filestorageService.deleteFilesFromFilestorage(key, soknadarkivschema) - verify(exactly = 1) { filesApi.deleteFiles(any(), any()) } + verify(exactly = 0) { filesApi.deleteFiles(any(), any()) } } @Test @@ -122,7 +122,7 @@ class FilestorageServiceTests { filestorageService.deleteFilesFromFilestorage(key, soknadarkivschema) - verify(exactly = 1) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) } + verify(exactly = 0) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) } } @Test @@ -133,7 +133,7 @@ class FilestorageServiceTests { filestorageService.deleteFilesFromFilestorage(key, soknadarkivschema) - verify(exactly = 1) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) } + verify(exactly = 0) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) } }