Skip to content

Commit

Permalink
Merge pull request #171 from navikt/endringer-pa-metrics
Browse files Browse the repository at this point in the history
Endringer på metrics
  • Loading branch information
nils-arne authored Feb 11, 2024
2 parents 4f4fe56 + 2fd6b19 commit 3a96ec3
Show file tree
Hide file tree
Showing 7 changed files with 250 additions and 25 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,13 @@ package no.nav.soknad.arkivering.soknadsarkiverer.service
import kotlinx.coroutines.*
import no.nav.soknad.arkivering.avroschemas.InnsendingMetrics
import no.nav.soknad.arkivering.avroschemas.Soknadarkivschema
import no.nav.soknad.arkivering.soknadsarkiverer.Constants
import no.nav.soknad.arkivering.soknadsarkiverer.config.ArchivingException
import no.nav.soknad.arkivering.soknadsarkiverer.config.ShuttingDownException
import no.nav.soknad.arkivering.soknadsarkiverer.kafka.KafkaPublisher
import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.JournalpostClientInterface
import no.nav.soknad.arkivering.soknadsarkiverer.service.fileservice.*
import no.nav.soknad.arkivering.soknadsarkiverer.supervision.ArchivingMetrics
import no.nav.soknad.arkivering.soknadsfillager.model.FileData
import org.slf4j.LoggerFactory
import org.slf4j.MDC
import org.springframework.stereotype.Service
import java.io.PrintWriter
import java.io.StringWriter
Expand All @@ -29,7 +26,7 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
try {
val startTime = System.currentTimeMillis()
val journalpostId = journalpostClient.opprettJournalpost(key, data, files)
createMetric(key, "send files to archive", startTime)
createMetricAndPublishOnKafka(key, "send files to archive", startTime)
logger.info("$key: Opprettet journalpostId=$journalpostId for behandlingsid=${data.behandlingsid}")
createMessage(key, "**Archiving: OK. journalpostId=$journalpostId")

Expand All @@ -47,7 +44,13 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
return try {
val startTime = System.currentTimeMillis()
val files = makeParallelCallsToFetchFiles(key, data)
createMetric(key, "get files from filestorage", startTime)

createMetricAndPublishOnKafka(key, "get files from filestorage", startTime)
files.filter{it.content != null}.forEach {
metrics.setFileFetchSizeHistogram(it.content!!.size.toDouble(), data.arkivtema)
metrics.setFileFetchSize(it.content.size.toDouble())
}

files

} catch (e: ShuttingDownException) {
Expand Down Expand Up @@ -79,10 +82,11 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
val okResponse = responses.firstOrNull { it.status == ResponseStatus.Ok.value }
if (okResponse != null) {
metrics.incGetFilestorageSuccesses()
if (okResponse.files != null)
if (okResponse.files != null) {
return okResponse.files
else
} else {
return listOf()
}
}

val deletedResponse = responses.firstOrNull { it.status == "deleted" }
Expand All @@ -109,7 +113,7 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
try {
val startTime = System.currentTimeMillis()
filestorageService.deleteFilesFromFilestorage(key, data)
createMetric(key, "delete files from filestorage", startTime)
createMetricAndPublishOnKafka(key, "delete files from filestorage", startTime)
createMessage(key, "ok")

} catch (e: ShuttingDownException) {
Expand All @@ -127,7 +131,7 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
kafkaPublisher.putMessageOnTopic(key, message)
}

private fun createMetric(key: String, message: String, startTime: Long) {
private fun createMetricAndPublishOnKafka(key: String, message: String, startTime: Long) {
val duration = System.currentTimeMillis() - startTime
val metrics = InnsendingMetrics("soknadsarkiverer", message, startTime, duration)
kafkaPublisher.putMetricOnTopic(key, metrics)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ open class TaskListService(
MDC.clear()
metrics.endTimer(timer)
metrics.endHistogramTimer(histogram)
metrics.numberOfAttachmentHistogramSet(
metrics.setNumberOfAttachmentHistogram(
soknadarkivschema.mottatteDokumenter.size.toDouble(),
soknadarkivschema.arkivtema
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.api.Oppret
import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.converter.createOpprettJournalpostRequest
import no.nav.soknad.arkivering.soknadsarkiverer.service.fileservice.FileInfo
import no.nav.soknad.arkivering.soknadsarkiverer.supervision.ArchivingMetrics
import no.nav.soknad.arkivering.soknadsfillager.model.FileData
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.beans.factory.annotation.Value
Expand All @@ -32,7 +31,7 @@ class JournalpostClient(@Value("\${joark.host}") private val joarkHost: String,
val bidClient: WebClient = webClient.mutate().defaultHeader(NAV_CONSUMER_ID, "dialogstyring-bidrag").build()

override fun opprettJournalpost(key: String, soknadarkivschema: Soknadarkivschema, attachedFiles: List<FileInfo>): String {
val timer = metrics.joarkLatencyStart()
val timer = metrics.startJoarkLatency()
try {
logger.info("$key: About to create journalpost for behandlingsId: '${soknadarkivschema.behandlingsid}'")
val request = createOpprettJournalpostRequest(soknadarkivschema, attachedFiles)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
private val SUMMARY_FILESTORAGE_DEL_LATENCY = "latency_filestorage_del_operations"
private val SUMMARY_FILESTORAGE_DEL_LATENCY_DESC = "Latency for deleting from filestorage"

private val SUMMARY_FILE_FETCH_SIZE = "file_fetch_size"
private val SUMMARY_FILE_FETCH_SIZE_DESC = "Size of fetched file"
private val HISTOGRAM_FILE_FETCH_SIZE = "file_fetch_size_distribution"
private val HISTOGRAM_FILE_FETCH_SIZE_DESC = "Distribution of sizes of fetched files"

private val COUNTER_JOARK_SUCCESS = "counter_joark_success"
private val COUNTER_JOARK_SUCCESS_DESC = "Number of successes when sending to Joark"

Expand All @@ -67,10 +72,12 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
private val filestorageDelErrorCounter: Counter = registerCounter(COUNTER_FILESTORAGE_DEL_ERROR, COUNTER_FILESTORAGE_DEL_ERROR_DESC)
private val filestorageGetLatencySummary = registerSummary(SUMMARY_FILESTORAGE_GET_LATENCY, SUMMARY_FILESTORAGE_GET_LATENCY_DESC)
private val filestorageDelLatencySummary = registerSummary(SUMMARY_FILESTORAGE_DEL_LATENCY, SUMMARY_FILESTORAGE_DEL_LATENCY_DESC)
private val filefetchSizeSummary = registerSummary(SUMMARY_FILE_FETCH_SIZE, SUMMARY_FILE_FETCH_SIZE_DESC)
private val filefetchSizeHistogram = registerFileSizeHistogram(HISTOGRAM_FILE_FETCH_SIZE, HISTOGRAM_FILE_FETCH_SIZE_DESC)
private val joarkSuccessCounter: Counter = registerCounter(COUNTER_JOARK_SUCCESS, COUNTER_JOARK_SUCCESS_DESC)
private val joarkErrorCounter: Counter = registerCounter(COUNTER_JOARK_ERROR, COUNTER_JOARK_ERROR_DESC)
private val joarkLatencySummary = registerSummary(SUMMARY_JOARK_LATENCY, SUMMARY_JOARK_LATENCY_DESC)
private val archivingLatencyHistogram = registerHistogram(HISTOGRAM_ARCHIVING_LATENCY, HISTORGRAM_ARCHIVING_LATENCY_DESC)
private val archivingLatencyHistogram = registerLatencyHistogram(HISTOGRAM_ARCHIVING_LATENCY, HISTORGRAM_ARCHIVING_LATENCY_DESC)

private val HISTOGRAM_ATTACHMENT_NUMBER = "histogram_attachment_number"
private val HISTOGRAM_ATTACHMENT_NUMBER_DESC = "Histogram for number of attachment per application"
Expand Down Expand Up @@ -106,16 +113,41 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
.labelNames(APP_LABEL)
.register(registry)

private fun registerHistogram(name: String, help: String): Histogram =
private fun registerLatencyHistogram(name: String, help: String): Histogram =
Histogram
.build()
.namespace(SOKNAD_NAMESPACE)
.name(name)
.help(help)
.buckets(0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 100.2)
.buckets(0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 100.2, 240.0)
.labelNames(TEMA_LABEL)
.register(registry)

private fun registerFileSizeHistogram(name: String, help: String): Histogram {
val kB = 1024.0
val mB = kB * kB
return Histogram
.build()
.namespace(SOKNAD_NAMESPACE)
.name(name)
.help(help)
.buckets(
kB,
10 * kB,
50 * kB,
100 * kB,
500 * kB,
mB,
2 * mB,
5 * mB,
25 * mB,
50 * mB,
150 * mB
)
.labelNames(TEMA_LABEL)
.register(registry)
}

private fun registerAttachmentNumberHistogram(name: String, help: String): Histogram =
Histogram
.build()
Expand Down Expand Up @@ -156,9 +188,15 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
fun archivingLatencyStart(): Summary.Timer = archivingLatencySummary.labels(APP).startTimer()
fun filestorageGetLatencyStart(): Summary.Timer = filestorageGetLatencySummary.labels(APP).startTimer()
fun filestorageDelLatencyStart(): Summary.Timer = filestorageDelLatencySummary.labels(APP).startTimer()
fun joarkLatencyStart(): Summary.Timer = joarkLatencySummary.labels(APP).startTimer()
fun startJoarkLatency(): Summary.Timer = joarkLatencySummary.labels(APP).startTimer()
fun getJoarkLatency(): Summary.Child.Value = joarkLatencySummary.labels(APP).get()
fun archivingLatencyHistogramStart(tema: String): Histogram.Timer = archivingLatencyHistogram.labels(tema).startTimer()
fun numberOfAttachmentHistogramSet(number: Double, tema: String) = numberOfAttachmentHistogram.labels(tema).observe(number)
fun setNumberOfAttachmentHistogram(number: Double, tema: String) = numberOfAttachmentHistogram.labels(tema).observe(number)
fun getNumberOfAttachmentHistogram(tema: String) = numberOfAttachmentHistogram.labels(tema).get()
fun setFileFetchSize(size: Double) = filefetchSizeSummary.labels(APP).observe(size)
fun getFileFetchSize() = filefetchSizeSummary.labels(APP).get()
fun setFileFetchSizeHistogram(size: Double, tema: String) = filefetchSizeHistogram.labels(tema).observe(size)
fun getFileFetchSizeHistogram(tema: String) = filefetchSizeHistogram.labels(tema).get()

fun endTimer(timer: Summary.Timer) {
timer.observeDuration()
Expand All @@ -184,5 +222,6 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
registry.unregister(taskGauge)
registry.unregister(upOrDownGauge)
registry.unregister(numberOfAttachmentHistogram)
registry.unregister(filefetchSizeSummary)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ package no.nav.soknad.arkivering.soknadsarkiverer.service

import io.mockk.*
import io.prometheus.client.CollectorRegistry
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.*
import no.nav.soknad.arkivering.soknadsarkiverer.SoknadsarkivererApplication
import no.nav.soknad.arkivering.soknadsarkiverer.config.ArchivingException
import no.nav.soknad.arkivering.soknadsarkiverer.kafka.KafkaPublisher
import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.JournalpostClientInterface
Expand All @@ -18,11 +17,15 @@ import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import java.time.OffsetDateTime.now
import java.util.*

class ArchiverServiceTests {

private lateinit var metrics: ArchivingMetrics

private val filestorage = mockk<FilestorageService>().also {
every {
it.getFilesFromFilestorage(any(), any()) } returns FetchFileResponse(status = "ok",
Expand Down Expand Up @@ -68,8 +71,6 @@ class ArchiverServiceTests {
private val journalpostClient = mockk<JournalpostClientInterface>().also {
every { it.opprettJournalpost(any(), any(), any()) } returns UUID.randomUUID().toString()
}
@Autowired
private lateinit var metrics: ArchivingMetrics

private val kafkaPublisher = mockk<KafkaPublisher>().also {
every { it.putMetricOnTopic(any(), any(), any()) } just Runs
Expand Down Expand Up @@ -104,6 +105,38 @@ class ArchiverServiceTests {
private val journalpostClient2 = mockk<JournalpostClientInterface>().also {
every { it.opprettJournalpost(any(), any(), capture(filer)) } returns UUID.randomUUID().toString()
}

@Test
fun `Fetch file metrics test`() {
archiverService = ArchiverService(filestorageNotFound, innsendingApi, journalpostClient2, metrics, kafkaPublisher)
val key = UUID.randomUUID().toString()
val tema = "AAP"
val soknadschema =
createSoknadarkivschema(behandlingsId = key,
tema = tema,
fileIds = listOf(
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString()
))

runBlocking {
archiverService.fetchFiles(key, soknadschema)

val fetchObservation = metrics.getFileFetchSize()
assertTrue(fetchObservation != null)
assertEquals(7.0, fetchObservation.quantiles[0.99]!!)
val fetchFileHistogram = metrics.getFileFetchSizeHistogram(tema)
assertTrue(fetchFileHistogram != null)
assertEquals("content".length.toDouble(), fetchFileHistogram.sum)
}
}

@Test
fun `Archiving succeeds when all is up and running`() {
archiverService = ArchiverService(filestorageNotFound, innsendingApi, journalpostClient2, metrics, kafkaPublisher)
Expand All @@ -118,7 +151,6 @@ class ArchiverServiceTests {
verify(exactly = 1) { journalpostClient2.opprettJournalpost(eq(key), eq(soknadschema), any()) }
assertTrue(filer.isCaptured)
assertEquals(soknadschema.mottatteDokumenter.first().mottatteVarianter.size, filer.captured.size)

}

}
Expand Down
Loading

0 comments on commit 3a96ec3

Please sign in to comment.