Skip to content

Commit

Permalink
GH-2251 Fix Jetty Metrics Recording (#2258)
Browse files Browse the repository at this point in the history
  • Loading branch information
@solonovamax
solonovamax authored Nov 2, 2024
1 parent e73d4f4 commit 4818213
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 36 deletions.
8 changes: 4 additions & 4 deletions reposilite-plugins/prometheus-plugin/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ application {
dependencies {
compileOnly(project(":reposilite-backend"))

val prometheus = "1.3.1"
implementation("io.prometheus:prometheus-metrics-core:$prometheus")
implementation("io.prometheus:prometheus-metrics-instrumentation-jvm:$prometheus")
implementation("io.prometheus:prometheus-metrics-exporter-common:$prometheus")
val prometheusMetrics = "1.3.2"
implementation("io.prometheus:prometheus-metrics-core:$prometheusMetrics")
implementation("io.prometheus:prometheus-metrics-instrumentation-jvm:$prometheusMetrics")
implementation("io.prometheus:prometheus-metrics-exporter-common:$prometheusMetrics")

testImplementation(project(":reposilite-backend"))
}
Expand Down
52 changes: 28 additions & 24 deletions ...ns/prometheus-plugin/src/main/kotlin/com/reposilite/plugin/prometheus/PrometheusPlugin.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package com.reposilite.plugin.prometheus

import com.reposilite.maven.api.DeployEvent
import com.reposilite.maven.api.ResolvedFileEvent
import com.reposilite.plugin.api.Facade
import com.reposilite.plugin.api.Plugin
import com.reposilite.plugin.api.ReposilitePlugin
import com.reposilite.plugin.event
Expand All @@ -12,25 +11,27 @@ import com.reposilite.plugin.prometheus.metrics.QueuedThreadPoolMetrics
import com.reposilite.plugin.prometheus.metrics.ReposiliteMetrics
import com.reposilite.status.FailureFacade
import com.reposilite.status.StatusFacade
import com.reposilite.storage.api.toLocation
import com.reposilite.web.api.HttpServerConfigurationEvent
import com.reposilite.web.api.HttpServerStartedEvent
import com.reposilite.web.api.RoutingSetupEvent
import io.prometheus.metrics.instrumentation.jvm.JvmMetrics
import org.eclipse.jetty.server.Response
import org.eclipse.jetty.server.handler.StatisticsHandler
import org.eclipse.jetty.util.thread.QueuedThreadPool
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.DurationUnit


@Plugin(name = "prometheus", dependencies = ["failure", "statistics", "status"])
class PrometheusPlugin : ReposilitePlugin() {

override fun initialize(): PrometheusFacade {
logger.info("")
logger.info("--- Prometheus")

val prometheusPath = System.getProperty("reposilite.prometheus.path")
val prometheusPath = (System.getProperty("reposilite.prometheus.path")
?: System.getenv("REPOSILITE_PROMETHEUS_PATH")
?: "/metrics"
?: "/metrics").toLocation()

val prometheusUser = System.getProperty("reposilite.prometheus.user")
?: System.getenv("REPOSILITE_PROMETHEUS_USER")
Expand Down Expand Up @@ -66,57 +67,60 @@ class PrometheusPlugin : ReposilitePlugin() {


event { event: HttpServerConfigurationEvent ->
val server = event.config.pvt.jetty.server!!
val handler = StatisticsHandler()
server.handler = handler

JettyMetrics.register(handler)

event.config.router.mount {
it.before { ctx ->
ctx.attribute("timestamp", System.currentTimeMillis())
}

it.after { ctx ->
JettyMetrics.responseCounter.labelValues(ctx.statusCode().toString()).inc()
if (ctx.path().toLocation() == prometheusPath)
return@after

ReposiliteMetrics.responseCounter.labelValues(ctx.statusCode().toString()).inc()

val currentTime = System.currentTimeMillis()
val timestamp = ctx.attribute<Long>("timestamp")

if (timestamp != null)
JettyMetrics.responseTimeSummary.labelValues(ctx.statusCode().toString())
.observe((currentTime - timestamp).milliseconds.toDouble(DurationUnit.SECONDS))


val response = ctx.res()
if (response is Response) // should always succeed
JettyMetrics.responseSizeSummary.observe(response.contentCount.toDouble())
}
}

logger.info("Prometheus | Jetty StatisticsHandler metrics has been initialized.")
event.config.jetty.modifyServer { server ->
val handler = StatisticsHandler()
server.insertHandler(handler)
JettyMetrics.register(handler)

when (val threadPool = server.threadPool) {
is QueuedThreadPool -> {
QueuedThreadPoolMetrics.register(threadPool)
logger.info("Prometheus | Queued Thread Pool metrics has been initialized.")
}

when (val threadPool = server.threadPool) {
is QueuedThreadPool -> {
QueuedThreadPoolMetrics.register(threadPool)
logger.info("Prometheus | Queued Thread Pool metrics has been initialized.")
else -> logger.warn("Prometheus | Unsupported thread pool for metrics: ${threadPool.javaClass.name}, ignoring")
}

else -> logger.warn("Prometheus | Unsupported thread pool for metrics: ${threadPool.javaClass.name}, ignoring")
// yes, I need to nest events (I need access to the server and I can't get that from HttpServerStartedEvent)
event<HttpServerStartedEvent> { _ ->
for (connector in server.connectors) {
connector.addBean(JettyMetrics)
}

logger.info("Prometheus | Jetty metrics has been initialized.")
}
}
}

event { event: RoutingSetupEvent ->
event.registerRoutes(
PrometheusEndpoints(
prometheusFacade = prometheusFacade,
prometheusPath = prometheusPath
prometheusPath = prometheusPath.toString()
)
)
}

return prometheusFacade
}

}
25 changes: 17 additions & 8 deletions ...rometheus-plugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/JettyMetrics.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,33 @@ import io.prometheus.metrics.core.metrics.Counter
import io.prometheus.metrics.core.metrics.CounterWithCallback
import io.prometheus.metrics.core.metrics.GaugeWithCallback
import io.prometheus.metrics.core.metrics.Summary
import org.eclipse.jetty.io.Connection
import org.eclipse.jetty.server.handler.StatisticsHandler
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.DurationUnit
import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit


object JettyMetrics {
object JettyMetrics : Connection.Listener {
val responseTimeSummary: Summary = Summary.builder()
.name("jetty_response_time_seconds")
.help("Time spent for a response")
.labelNames("code")
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

val responseSizeSummary: Summary = Summary.builder()
.name("jetty_response_bytes")
.help("Size in bytes of responses")
val requestBytesSummary: Summary = Summary.builder()
.name("jetty_request_bytes")
.help("Size in bytes of incoming requests")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

val responseCounter: Counter = Counter.builder()
.name("reposilite_responses_total")
.help("Total response count")
.labelNames("code")
val responseBytesSummary: Summary = Summary.builder()
.name("jetty_response_bytes")
.help("Size in bytes of outgoing responses")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

fun register(statisticsHandler: StatisticsHandler) {
Expand Down Expand Up @@ -192,4 +194,11 @@ object JettyMetrics {
.callback { callback -> callback.call(statisticsHandler.responsesBytesTotal.toDouble()) }
.register()
}

override fun onOpened(connection: Connection) {}

override fun onClosed(connection: Connection) {
requestBytesSummary.observe(connection.bytesIn.toDouble())
responseBytesSummary.observe(connection.bytesOut.toDouble())
}
}
8 changes: 8 additions & 0 deletions ...heus-plugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/ReposiliteMetrics.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ import kotlin.time.DurationUnit
import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit

object ReposiliteMetrics {
// TODO: Remove this? See #2251
val responseFileSizeSummary: Summary = Summary.builder()
.name("reposilite_response_file_size_bytes")
.help("Size in bytes of response files")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

// TODO: Remove this? See #2251
val resolvedFileCounter: Counter = Counter.builder()
.name("reposilite_resolved_total")
.help("Total resolved files count")
Expand All @@ -28,6 +30,12 @@ object ReposiliteMetrics {
.help("Total successful deployments count")
.register()

val responseCounter: Counter = Counter.builder()
.name("reposilite_responses_total")
.help("Total response count, filtered to exclude /metrics")
.labelNames("code")
.register()

fun register(statusFacade: StatusFacade, failureFacade: FailureFacade) {
GaugeWithCallback.builder()
.name("reposilite_uptime_seconds")
Expand Down

0 comments on commit 4818213

Please sign in to comment.