🩺 add application metrics based on business logic (#1910)

app/Providers/PrometheusServiceProvider.php

@@ -0,0 +1,104 @@
+<?php
+
+namespace App\Providers;
+
+use App\Models\HafasTrip;
+use App\Models\PolyLine;
+use App\Models\Status;
+use App\Models\User;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\ServiceProvider;
+use romanzipp\QueueMonitor\Enums\MonitorStatus;
+use Spatie\Prometheus\Facades\Prometheus;
+
+class PrometheusServiceProvider extends ServiceProvider
+{
+    public function register() {
+        /*
+         * Here you can register all the exporters that you
+         * want to export to prometheus
+         */
+        Prometheus::addGauge('Users count')
+                  ->helpText("How many users are registered on the website?")
+                  ->value(function() {
+                      return User::count();
+                  });
+
+        Prometheus::addGauge('Status count')
+                  ->helpText("How many statuses are posted on the website?")
+                  ->value(function() {
+                      return Status::count();
+                  });
+
+        Prometheus::addGauge('Hafas Trips count')
+                  ->helpText("How many hafas trips are posted grouped by operator and mode of transport?")
+                  ->labels(["operator", "category"])
+                  ->value(function() {
+                      return HafasTrip::groupBy("operator_id", "category")
+                                      ->selectRaw("count(*) AS total, operator_id, category")
+                                      ->with("operator")
+                                      ->get()
+                                      ->map(fn($item) => [$item->total, [$item->operator->name, $item->category]])
+                                      ->toArray();
+                  });
+
+        Prometheus::addGauge('Polylines count')
+                  ->helpText("How many polylines are saved grouped by source?")
+                  ->labels(["source"])
+                  ->value(function() {
+                      return PolyLine::groupBy("source")
+                                     ->selectRaw("count(*) AS total, source")
+                                     ->get()
+                                     ->map(fn($item) => [$item->total, [$item->source]])
+                                     ->toArray();
+                  });
+
+        Prometheus::addGauge("queue_size")
+                  ->helpText("How many items are currently in the job queue?")
+                  ->label("job_name")
+                  ->value(function() {
+                      if (config("queue.default") === "database") {
+                          return $this->getJobsByDisplayName("jobs");
+                      }
+
+                      return [Queue::size(), ["all"]];
+                  });
+
+        Prometheus::addGauge("failed_jobs_count")
+                  ->helpText("How many jobs have failed?")
+                  ->label("job_name")
+                  ->value(function() {
+                      return $this->getJobsByDisplayName("failed_jobs");
+                  });
+
+        Prometheus::addGauge("completed_jobs_count")
+                  ->helpText("How many jobs are done? Old items from queue monitor table are deleted after 7 days.")
+                  ->labels(["job_name", "status"])
+                  ->value(function() {
+                      return DB::table("queue_monitor")
+                               ->groupBy("name", "status")
+                               ->selectRaw("count(*) AS total, name, status")
+                               ->get()
+                               ->map(fn($item) => [$item->total, [$item->name, MonitorStatus::toNamedArray()[$item->status]]])
+                               ->toArray();
+                  });
+
+        Prometheus::addGauge("profile_image_count")
+                  ->helpText("How many profile images are stored?")
+                  ->value(function() {
+                      $iter = new \FilesystemIterator(public_path("uploads/avatars"));
+                      return iterator_count($iter);
+                  });
+    }
+
+
+    private function getJobsByDisplayName($table_name): array {
+        return DB::table($table_name)
+                 ->get("payload")
+                 ->map(fn($row) => json_decode($row->payload))
+                 ->countBy(fn($payload) => $payload->displayName)
+                 ->mapWithKeys(fn($total, $key) => [$total, [$key]])
+                 ->toArray();
+    }
+}

composer.json

@@ -34,6 +34,7 @@
         "revolution/laravel-mastodon-api": "^3.0",
         "romanzipp/laravel-queue-monitor": "^4.0",
         "spatie/icalendar-generator": "^2.0",
+        "spatie/laravel-prometheus": "^1.0",
         "spatie/laravel-sitemap": "^6.0",
         "spatie/laravel-validation-rules": "^3.2",
         "spatie/laravel-webhook-server": "^3.4",
@@ -55,7 +56,10 @@
     "config": {
         "optimize-autoloader": true,
         "preferred-install": "dist",
-        "sort-packages": true
+        "sort-packages": true,
+        "allow-plugins": {
+            "stephank/composer-plugin-nixify": true
+        }
     },
     "extra": {
         "laravel": {

config/app.php

@@ -191,6 +191,7 @@
         App\Providers\AuthServiceProvider::class,
         Laravel\Passport\PassportServiceProvider::class,
         // App\Providers\BroadcastServiceProvider::class,
+        App\Providers\PrometheusServiceProvider::class,
         App\Providers\EventServiceProvider::class,
         App\Providers\RouteServiceProvider::class,
 

config/prometheus.php

@@ -0,0 +1,43 @@
+<?php
+
+return [
+    'enabled' => true,
+
+    /*
+     * The urls that will return metrics.
+     */
+    'urls' => [
+        'default' => 'prometheus',
+    ],
+
+    /*
+     * Only these IP's will be allowed to visit the above urls.
+     * All IP's are allowed when empty.
+     */
+    'allowed_ips' => [
+        '127.0.0.1',
+        '::1'
+        // '1.2.3.4',
+    ],
+
+    /*
+     * This is the default namespace that will be
+     * used by all metrics
+     */
+    'default_namespace' => 'trwl',
+
+    /*
+     * The middleware that will be applied to the urls above
+     */
+    'middleware' => [
+        Spatie\Prometheus\Http\Middleware\AllowIps::class,
+    ],
+
+    /*
+     * You can override these classes to customize low-level behaviour of the package.
+     * In most cases, you can just use the defaults.
+     */
+    'actions' => [
+        'render_collectors' => Spatie\Prometheus\Actions\RenderCollectorsAction::class,
+    ],
+];