Skip to content

Commit

Permalink
GH-2278 Fix failure when StatisticsHandler returns NaN (Fixes #2277)
Browse files Browse the repository at this point in the history
  • Loading branch information
@solonovamax
solonovamax authored Nov 27, 2024
1 parent e1cbbc3 commit fb062e5
Show file tree
Hide file tree
Showing 6 changed files with 463 additions and 87 deletions.
123 changes: 67 additions & 56 deletions ...rometheus-plugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/JettyMetrics.kt
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package com.reposilite.plugin.prometheus.metrics

import io.prometheus.metrics.core.metrics.Counter
import io.prometheus.metrics.core.metrics.CounterWithCallback
import io.prometheus.metrics.core.metrics.GaugeWithCallback
import io.prometheus.metrics.core.metrics.Summary
import io.prometheus.metrics.model.registry.PrometheusRegistry
import org.eclipse.jetty.io.Connection
import org.eclipse.jetty.server.handler.StatisticsHandler
import kotlin.time.Duration.Companion.milliseconds
Expand All @@ -12,155 +12,141 @@ import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit


object JettyMetrics : Connection.Listener {
val responseTimeSummary: Summary = Summary.builder()
.name("jetty_response_time_seconds")
.help("Time spent for a response")
.labelNames("code")
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

val requestBytesSummary: Summary = Summary.builder()
.name("jetty_request_bytes")
.help("Size in bytes of incoming requests")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

val responseBytesSummary: Summary = Summary.builder()
.name("jetty_response_bytes")
.help("Size in bytes of outgoing responses")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

fun register(statisticsHandler: StatisticsHandler) {
lateinit var responseTimeSummary: Summary
private set

private lateinit var requestBytesSummary: Summary

private lateinit var responseBytesSummary: Summary

fun register(statisticsHandler: StatisticsHandler, registry: PrometheusRegistry = PrometheusRegistry.defaultRegistry) {
CounterWithCallback.builder()
.name("jetty_requests_total")
.help("Number of requests")
.callback { callback -> callback.call(statisticsHandler.requests.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_requests_active")
.help("Number of requests currently active")
.callback { callback -> callback.call(statisticsHandler.requestsActive.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_requests_active_max")
.help("Maximum number of requests that have been active at once")
.callback { callback -> callback.call(statisticsHandler.requestsActiveMax.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_request_time_seconds_max")
.help("Maximum time spent handling requests")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeMax.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeMax.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_request_time_seconds_total")
.help("Total time spent in all request handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeTotal.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeTotal.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_request_time_seconds_mean")
.help("Mean time spent in all request handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeMean.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeMean.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_request_time_seconds_stddev")
.help("Standard deviation of time spent in all request handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeStdDev.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeStdDev.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_dispatched_total")
.help("Number of dispatches")
.callback { callback -> callback.call(statisticsHandler.dispatched.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_active")
.help("Number of dispatches currently active")
.callback { callback -> callback.call(statisticsHandler.dispatchedActive.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_active_max")
.help("Maximum number of active dispatches being handled")
.callback { callback -> callback.call(statisticsHandler.dispatchedActiveMax.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_time_seconds_max")
.help("Maximum time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMax.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMax.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_dispatched_time_seconds_total")
.help("Total time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeTotal.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeTotal.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_time_seconds_mean")
.help("Mean time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMean.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMean.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_time_seconds_stddev")
.help("Standard deviation of time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeStdDev.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeStdDev.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_async_requests_total")
.help("Total number of async requests")
.callback { callback -> callback.call(statisticsHandler.asyncRequests.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_async_requests_waiting")
.help("Currently waiting async requests")
.callback { callback -> callback.call(statisticsHandler.asyncRequestsWaiting.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_async_requests_waiting_max")
.help("Maximum number of waiting async requests")
.callback { callback -> callback.call(statisticsHandler.asyncRequestsWaitingMax.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_async_dispatches_total")
.help("Number of requested that have been asynchronously dispatched")
.callback { callback -> callback.call(statisticsHandler.asyncDispatches.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_expires_total")
.help("Number of async requests requests that have expired")
.callback { callback -> callback.call(statisticsHandler.expires.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_errors_total")
.help("Number of async errors that occurred")
.callback { callback -> callback.call(statisticsHandler.errors.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_responses_total")
Expand All @@ -173,26 +159,47 @@ object JettyMetrics : Connection.Listener {
callback.call(statisticsHandler.responses4xx.toDouble(), "4xx")
callback.call(statisticsHandler.responses5xx.toDouble(), "5xx")
}
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_responses_thrown_total")
.help("Number of requests that threw an exception")
.callback { callback -> callback.call(statisticsHandler.responsesThrown.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_stats_seconds")
.help("Time in seconds stats have been collected for")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.statsOnMs.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.statsOnMs.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_responses_bytes_total")
.help("Total number of bytes across all responses")
.callback { callback -> callback.call(statisticsHandler.responsesBytesTotal.toDouble()) }
.register()
.register(registry)

responseTimeSummary = Summary.builder()
.name("jetty_response_time_seconds")
.help("Time spent for a response")
.labelNames("code")
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)

requestBytesSummary = Summary.builder()
.name("jetty_request_bytes")
.help("Size in bytes of incoming requests")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)

responseBytesSummary = Summary.builder()
.name("jetty_response_bytes")
.help("Size in bytes of outgoing responses")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)
}

override fun onOpened(connection: Connection) {}
Expand All @@ -201,4 +208,8 @@ object JettyMetrics : Connection.Listener {
requestBytesSummary.observe(connection.bytesIn.toDouble())
responseBytesSummary.observe(connection.bytesOut.toDouble())
}

private fun Double.toMillisFinite() = if (this.isFinite()) this.milliseconds.toDouble(DurationUnit.SECONDS) else this

private fun Long.toMillisFinite() = this.milliseconds.toDouble(DurationUnit.SECONDS)
}
11 changes: 6 additions & 5 deletions ...lugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/QueuedThreadPoolMetrics.kt
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package com.reposilite.plugin.prometheus.metrics

import io.prometheus.metrics.core.metrics.GaugeWithCallback
import io.prometheus.metrics.model.registry.PrometheusRegistry
import org.eclipse.jetty.util.thread.QueuedThreadPool
import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit


object QueuedThreadPoolMetrics {
fun register(queuedThreadPool: QueuedThreadPool) {
fun register(queuedThreadPool: QueuedThreadPool, registry: PrometheusRegistry = PrometheusRegistry.defaultRegistry) {
GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_threads_state")
.help("Number of threads by state")
Expand All @@ -21,25 +22,25 @@ object QueuedThreadPoolMetrics {
callback.call(queuedThreadPool.utilizedThreads.toDouble(), "utilized")
callback.call(queuedThreadPool.maxAvailableThreads.toDouble(), "max_available")
}
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_utilization")
.help("Percentage of threads in use")
.unit(MetricsUnit.RATIO)
.callback { callback -> callback.call(queuedThreadPool.utilizationRate) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_jobs")
.help("Number of total jobs")
.callback { callback -> callback.call(queuedThreadPool.queueSize.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_low_on_threads")
.help("Number of total jobs")
.callback { callback -> callback.call(if (queuedThreadPool.isLowOnThreads) 1.0 else 0.0) }
.register()
.register(registry)
}
}
63 changes: 38 additions & 25 deletions ...heus-plugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/ReposiliteMetrics.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,48 +6,61 @@ import io.prometheus.metrics.core.metrics.Counter
import io.prometheus.metrics.core.metrics.CounterWithCallback
import io.prometheus.metrics.core.metrics.GaugeWithCallback
import io.prometheus.metrics.core.metrics.Summary
import io.prometheus.metrics.model.registry.PrometheusRegistry
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.DurationUnit
import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit

object ReposiliteMetrics {
// TODO: Remove this? See #2251
val responseFileSizeSummary: Summary = Summary.builder()
.name("reposilite_response_file_size_bytes")
.help("Size in bytes of response files")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()
lateinit var responseFileSizeSummary: Summary

// TODO: Remove this? See #2251
val resolvedFileCounter: Counter = Counter.builder()
.name("reposilite_resolved_total")
.help("Total resolved files count")
.register()

val mavenDeployCounter: Counter = Counter.builder()
.name("reposilite_deploy_total")
.help("Total successful deployments count")
.register()

val responseCounter: Counter = Counter.builder()
.name("reposilite_responses_total")
.help("Total response count, filtered to exclude /metrics")
.labelNames("code")
.register()

fun register(statusFacade: StatusFacade, failureFacade: FailureFacade) {
lateinit var resolvedFileCounter: Counter

lateinit var mavenDeployCounter: Counter

lateinit var responseCounter: Counter

fun register(
statusFacade: StatusFacade,
failureFacade: FailureFacade,
registry: PrometheusRegistry = PrometheusRegistry.defaultRegistry,
) {
GaugeWithCallback.builder()
.name("reposilite_uptime_seconds")
.help("Uptime of reposilite")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statusFacade.fetchInstanceStatus().uptime.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("reposilite_failures_total")
.help("Number of failures reposilite has encountered")
.callback { callback -> callback.call(failureFacade.getFailures().size.toDouble()) }
.register()
.register(registry)

responseFileSizeSummary = Summary.builder()
.name("reposilite_response_file_size_bytes")
.help("Size in bytes of response files")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)

resolvedFileCounter = Counter.builder()
.name("reposilite_resolved_total")
.help("Total resolved files count")
.register(registry)

mavenDeployCounter = Counter.builder()
.name("reposilite_deploy_total")
.help("Total successful deployments count")
.register(registry)

responseCounter = Counter.builder()
.name("reposilite_responses_total")
.help("Total response count, filtered to exclude /metrics")
.labelNames("code")
.register(registry)
}
}
Loading

0 comments on commit fb062e5

Please sign in to comment.