Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/slett filer til arkivert soknad #148

Merged
merged 3 commits into from
Sep 28, 2023
Merged

Feature/slett filer til arkivert soknad #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6746709
Endret oppsett for kjøring av infrastruktur testing. Lagt inn mocking…
nils-arne Sep 25, 2023
c1044af
Fjernet kall til soknadsfillager for sletting av filer etter arkivering
nils-arne Sep 25, 2023
7816cbb
Endringer etter PR
nils-arne Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .nais/config-loadtests.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
"ingress": "https://soknadsarkiverer-loadtests.intern.dev.nav.no",
"outbound": [
"soknadsfillager-loadtests",
"arkiv-mock",
"innsending-api-loadtests"
"arkiv-mock"
],
"outboundExternalHosts": [
"arkiv-mock-gcp.intern.dev.nav.no",
Expand All @@ -18,6 +17,7 @@
"kafka-message-topic": "team-soknad.privat-soknadinnsending-messages-v1-loadtests",
"kafka-metrics-topic": "team-soknad.privat-soknadinnsending-metrics-v1-loadtests",
"filestorage-host": "https://soknadsfillager-gcp-loadtests.intern.dev.nav.no",
"innsending-api-host": "arkiv-mock-gcp.intern.dev.nav.no",
"joark-host": "https://arkiv-mock-gcp.intern.dev.nav.no",
"dokarkiv-scope": "api://dev-gcp.team-soknad.arkiv-mock/.default",
"soknadsfillager-scope": "api://dev-gcp.team-soknad.soknadsfillager-loadtests/.default",
Expand Down
24 changes: 1 addition & 23 deletions ...otlin/no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,36 +52,14 @@ class FilestorageService(
}

override fun deleteFilesFromFilestorage(key: String, data: Soknadarkivschema) {
val timer = metrics.filestorageDelLatencyStart()

val fileIds = getFileIds(data)
try {

logger.info("$key: Calling file storage to delete $fileIds")
deleteFiles(key, fileIds)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Vurdere å slette funksjonen også?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Slettet

logger.info("$key: Deleted these files: $fileIds")

metrics.incDelFilestorageSuccesses()
} catch (e: Exception) {
metrics.incDelFilestorageErrors()
logger.warn(
"$key: Failed to delete files from file storage. Everything is saved to Joark correctly, " +
"so this error will be ignored. Affected file ids: '$fileIds'", e
)

} finally {
metrics.endTimer(timer)
}
logger.info("$key: The following files are archived and can be deleted (by sendsoknad or innsending-api) $fileIds")
}

private fun getFiles(key: String, fileIds: List<String>) =
mergeFetchResponsesAndSetOverallStatus(key, fileIds.map { getOneFile(key, it) } )


private fun deleteFiles(key: String, fileIds: List<String>) {
filesApi.deleteFiles(fileIds, key)
}

private fun returnFirstOrNull(files: List<FileData>?): List<FileInfo>? {
if (files == null || files.isEmpty()) return null
return mapToFileInfo(files.first())
Expand Down
23 changes: 0 additions & 23 deletions arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/ApplicationTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,6 @@ class ApplicationTests : ContainerizedKafka() {
))
verifyMockedPostRequests(1, safUrl)
verifyMockedPostRequests(1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 1,
Expand Down Expand Up @@ -159,7 +158,6 @@ class ApplicationTests : ContainerizedKafka() {
))
verifyMockedPostRequests(1, safUrl)
verifyMockedPostRequests(1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 1,
Expand All @@ -185,7 +183,6 @@ class ApplicationTests : ContainerizedKafka() {
RECEIVED hasCount 0, STARTED hasCount 0, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 0
))
verifyMockedPostRequests(0, journalPostUrl)
verifyDeleteRequestsToFilestorage(0)
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 0,
"send files to archive" hasCount 0,
Expand All @@ -207,7 +204,6 @@ class ApplicationTests : ContainerizedKafka() {
RECEIVED hasCount 1, STARTED hasCount maxNumberOfAttempts, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1
))
verifyMockedPostRequests(maxNumberOfAttempts, safUrl)
verifyDeleteRequestsToFilestorage(0)
verifyMessageStartsWith(key, mapOf("**Archiving: FAILED" hasCount 1, "ok" hasCount 0, "Exception" hasCount maxNumberOfAttempts))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount maxNumberOfAttempts,
Expand Down Expand Up @@ -259,7 +255,6 @@ class ApplicationTests : ContainerizedKafka() {
))
verifyMockedPostRequests(1, safUrl)
verifyMockedPostRequests(1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("ok" hasCount 1))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 1,
Expand Down Expand Up @@ -290,7 +285,6 @@ class ApplicationTests : ContainerizedKafka() {
))
verifyMockedPostRequests(numberOfFailures+1, safUrl)
verifyMockedPostRequests(numberOfFailures+1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("ok" hasCount 1, "Exception" hasCount 1))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount numberOfFailures+1,
Expand All @@ -299,7 +293,6 @@ class ApplicationTests : ContainerizedKafka() {
))

verifyArchivingMetrics(getFilestorageSuccessesBefore + 2, { metrics.getGetFilestorageSuccesses() })
verifyArchivingMetrics(delFilestorageSuccessesBefore + 1, { metrics.getDelFilestorageSuccesses() })
verifyArchivingMetrics(joarkErrorsBefore + 1, { metrics.getJoarkErrors() })
verifyArchivingMetrics(joarkSuccessesBefore + 1, { metrics.getJoarkSuccesses() })
verifyArchivingMetrics(tasksBefore + 0, { metrics.getTasks() }, "Should have created and finished task")
Expand All @@ -323,7 +316,6 @@ class ApplicationTests : ContainerizedKafka() {
))
verifyMockedPostRequests(attemptsToFail + 1, safUrl)
verifyMockedPostRequests(attemptsToFail, journalPostUrl)
verifyDeleteRequestsToFilestorage(0)
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount attemptsToFail,
"send files to archive" hasCount 0,
Expand Down Expand Up @@ -352,7 +344,6 @@ class ApplicationTests : ContainerizedKafka() {
RECEIVED hasCount 1, STARTED hasCount 1, ARCHIVED hasCount 1, FINISHED hasCount 1, FAILURE hasCount 0
))
verifyMockedPostRequests(1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 1,
Expand All @@ -362,7 +353,6 @@ class ApplicationTests : ContainerizedKafka() {

verifyArchivingMetrics(getFilestorageSuccessesBefore + 1, { metrics.getGetFilestorageSuccesses() })
verifyArchivingMetrics(delFilestorageSuccessesBefore + 0, { metrics.getDelFilestorageSuccesses() })
verifyArchivingMetrics(delFilestorageErrorsBefore + 1, { metrics.getDelFilestorageErrors() })
verifyArchivingMetrics(joarkErrorsBefore + 0, { metrics.getJoarkErrors() })
verifyArchivingMetrics(joarkSuccessesBefore + 1, { metrics.getJoarkSuccesses() })
}
Expand All @@ -380,7 +370,6 @@ class ApplicationTests : ContainerizedKafka() {
RECEIVED hasCount 1, STARTED hasCount maxNumberOfAttempts, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1
))
verifyMockedPostRequests(maxNumberOfAttempts, journalPostUrl)
verifyDeleteRequestsToFilestorage(0)
verifyMessageStartsWith(key, mapOf("ok" hasCount 0, "Exception" hasCount maxNumberOfAttempts))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount maxNumberOfAttempts,
Expand All @@ -406,7 +395,6 @@ class ApplicationTests : ContainerizedKafka() {
))
verifyMockedPostRequests(attemptsToFail+1, safUrl)
verifyMockedPostRequests(attemptsToFail + 1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("ok" hasCount 1, "Exception" hasCount attemptsToFail))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount attemptsToFail+1,
Expand Down Expand Up @@ -437,7 +425,6 @@ class ApplicationTests : ContainerizedKafka() {
verifyProcessingEvents(key, mapOf(
RECEIVED hasCount 1, STARTED hasCount 1, ARCHIVED hasCount 1, FINISHED hasCount 1, FAILURE hasCount 0
))
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 1))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 1,
Expand All @@ -447,7 +434,6 @@ class ApplicationTests : ContainerizedKafka() {

verifyArchivingMetrics(getFilestorageErrorsBefore + 0, { metrics.getGetFilestorageErrors() })
verifyArchivingMetrics(getFilestorageSuccessesBefore + 1, { metrics.getGetFilestorageSuccesses() })
verifyArchivingMetrics(delFilestorageSuccessesBefore + 1, { metrics.getDelFilestorageSuccesses() })
verifyArchivingMetrics(joarkErrorsBefore + 0, { metrics.getJoarkErrors() })
verifyArchivingMetrics(joarkSuccessesBefore + 0, { metrics.getJoarkSuccesses() })
verifyArchivingMetrics(tasksBefore, { metrics.getTasks() })
Expand All @@ -474,7 +460,6 @@ class ApplicationTests : ContainerizedKafka() {
verifyProcessingEvents(key, mapOf(
RECEIVED hasCount 1, STARTED hasCount 1, ARCHIVED hasCount 1, FINISHED hasCount 1, FAILURE hasCount 0
))
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("**Archiving: OK" hasCount 1, "ok" hasCount 1, "Exception" hasCount 0))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 0,
Expand All @@ -484,7 +469,6 @@ class ApplicationTests : ContainerizedKafka() {

verifyArchivingMetrics(getFilestorageErrorsBefore + 0, { metrics.getGetFilestorageErrors() })
verifyArchivingMetrics(getFilestorageSuccessesBefore + 0, { metrics.getGetFilestorageSuccesses() })
verifyArchivingMetrics(delFilestorageSuccessesBefore + 1, { metrics.getDelFilestorageSuccesses() })
verifyArchivingMetrics(joarkErrorsBefore + 0, { metrics.getJoarkErrors() })
verifyArchivingMetrics(joarkSuccessesBefore + 0, { metrics.getJoarkSuccesses() })
verifyArchivingMetrics(tasksBefore, { metrics.getTasks() })
Expand All @@ -511,7 +495,6 @@ class ApplicationTests : ContainerizedKafka() {
verifyProcessingEvents(key, mapOf(
RECEIVED hasCount 1, STARTED hasCount maxNumberOfAttempts, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1
))
verifyDeleteRequestsToFilestorage(0)
verifyMessageStartsWith(key, mapOf("ok" hasCount 0, "Exception" hasCount maxNumberOfAttempts))
verifyKafkaMetric(key, mapOf(
"get files from filestorage" hasCount 0,
Expand Down Expand Up @@ -549,7 +532,6 @@ class ApplicationTests : ContainerizedKafka() {
verifyProcessingEvents(key, mapOf(
RECEIVED hasCount 1, STARTED hasCount 2, ARCHIVED hasCount 0, FINISHED hasCount 1, FAILURE hasCount 0
))
verifyDeleteRequestsToFilestorage(1)
verifyMessageStartsWith(key, mapOf("ok" hasCount 1, "Exception" hasCount 1))

verifyKafkaMetric(key, mapOf(
Expand Down Expand Up @@ -588,7 +570,6 @@ class ApplicationTests : ContainerizedKafka() {
verifyProcessingEvents(key, mapOf(
RECEIVED hasCount 1, STARTED hasCount tasksBefore.toInt() + 1, ARCHIVED hasCount 0, FINISHED hasCount 0, FAILURE hasCount 1
))
verifyDeleteRequestsToFilestorage(0)
verifyMessageStartsWith(key, mapOf("ok" hasCount 0, "Exception" hasCount 6))

verifyKafkaMetric(key, mapOf(
Expand Down Expand Up @@ -660,10 +641,6 @@ class ApplicationTests : ContainerizedKafka() {
}


private fun verifyDeleteRequestsToFilestorage(expectedCount: Int) {
verifyMockedDeleteRequests(expectedCount, filestorageProperties.files + ".*")
}

private fun createSoknadarkivschema(behandlingsId: String) = createSoknadarkivschema(fileUuid, behandlingsId)


Expand Down
10 changes: 0 additions & 10 deletions arkiverer/src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/IntegrationTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ class IntegrationTests : ContainerizedKafka() {
putDataOnKafkaTopic(createSoknadarkivschema())

verifyMockedPostRequests(initialRequests+2, journalPostUrl)
verifyDeleteRequestsToFilestorage(2)
}

@Test
Expand All @@ -98,7 +97,6 @@ class IntegrationTests : ContainerizedKafka() {

TimeUnit.SECONDS.sleep(1)
verifyMockedPostRequests(0, journalPostUrl)
verifyDeleteRequestsToFilestorage(0)
}

@Test
Expand All @@ -112,7 +110,6 @@ class IntegrationTests : ContainerizedKafka() {
putDataOnKafkaTopic(soknadarkivschema)

verifyMockedPostRequests(1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
}

@Test
Expand All @@ -125,7 +122,6 @@ class IntegrationTests : ContainerizedKafka() {
putDataOnKafkaTopic(soknadarkivschema)

verifyMockedPostRequests(0, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
}

@Test
Expand All @@ -138,12 +134,6 @@ class IntegrationTests : ContainerizedKafka() {
putDataOnKafkaTopic(soknadarkivschema)

verifyMockedPostRequests(1, journalPostUrl)
verifyDeleteRequestsToFilestorage(1)
}

private fun verifyDeleteRequestsToFilestorage(expectedCount: Int) {
val url = filestorageProperties.files + ".*"
verifyMockedDeleteRequests(expectedCount, url)
}

private fun createSoknadarkivschema() = createSoknadarkivschema(fileId)
Expand Down
6 changes: 3 additions & 3 deletions .../no/nav/soknad/arkivering/soknadsarkiverer/service/fileservice/FilestorageServiceTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ class FilestorageServiceTests {

filestorageService.deleteFilesFromFilestorage(key, soknadarkivschema)

verify(exactly = 1) { filesApi.deleteFiles(any(), any()) }
verify(exactly = 0) { filesApi.deleteFiles(any(), any()) }
}

@Test
Expand All @@ -122,7 +122,7 @@ class FilestorageServiceTests {

filestorageService.deleteFilesFromFilestorage(key, soknadarkivschema)

verify(exactly = 1) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) }
verify(exactly = 0) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) }
}

@Test
Expand All @@ -133,7 +133,7 @@ class FilestorageServiceTests {

filestorageService.deleteFilesFromFilestorage(key, soknadarkivschema)

verify(exactly = 1) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) }
verify(exactly = 0) { filesApi.deleteFiles(fileIdsAndResponses.take(numberOfFiles).map { it.first }, any()) }
}


Expand Down