Merge branch 'develop' into dev-lb/preparations-for-multiple-datasources

# Conflicts:
#	app/DataProviders/Hafas.php

app/Console/Commands/CalculateMissingDuration.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Checkin;
+use Illuminate\Console\Command;
+use Symfony\Component\Console\Command\Command as CommandAlias;
+
+/**
+ * @deprecated Just created and marked as deprecated, because it is only needed for migrating old checkins.
+ *             Can be deleted after migration.
+ */
+class CalculateMissingDuration extends Command
+{
+    protected $signature   = 'trwl:calculate-missing-duration';
+    protected $description = 'Calculate missing duration for train checkins. Currently only needed for migrating old checkins.';
+
+    public function handle(): int {
+        while (true) {
+            Checkin::with(['HafasTrip.stopovers', 'originStation', 'destinationStation'])
+                   ->whereNull('duration')
+                   ->limit(250)
+                   ->each(function($checkin) {
+                       // foreach ($checkins as $checkin) {
+                       $duration = $checkin->duration;
+                       $this->info("Duration for checkin {$checkin->id} is {$duration}");
+                       //}
+                   });
+        }
+        return CommandAlias::SUCCESS;
+    }
+}

app/Console/Commands/PolylinesToFiles.php

@@ -16,6 +16,8 @@ public function handle(): int {
         $start = microtime(true);
         $rows  = DB::table('poly_lines')
                    ->where('polyline', '!=', '{}')
+                   ->orderBy('id', 'desc')
+                   ->limit(1000)
                    ->get();
         $this->info('Found ' . $rows->count() . ' polylines.');
         $affectedRows = 0;

app/DataProviders/Hafas.php

@@ -7,6 +7,8 @@
 use App\Enum\TravelType;
 use App\Enum\TripSource;
 use App\Exceptions\HafasException;
+use App\Helpers\CacheKey;
+use App\Helpers\HCK;
 use App\Http\Controllers\Controller;
 use App\Http\Controllers\TransportController;
 use App\Models\HafasOperator;
@@ -42,8 +44,12 @@ public function getStationByRilIdentifier(string $rilIdentifier): ?Station {
             if ($response->ok() && !empty($response->body()) && $response->body() !== '[]') {
                 $data    = json_decode($response->body(), false, 512, JSON_THROW_ON_ERROR);
                 $station = StationRepository::parseHafasStopObject($data);
+                CacheKey::increment(HCK::STATIONS_SUCCESS);
+            } else {
+                CacheKey::increment(HCK::STATIONS_NOT_OK);
             }
         } catch (Exception $exception) {
+            CacheKey::increment(HCK::STATIONS_FAILURE);
             report($exception);
         }
         return $station;
@@ -75,13 +81,21 @@ public function getStations(string $query, int $results = 10): Collection {
             );
 
             $data = json_decode($response->body(), false, 512, JSON_THROW_ON_ERROR);
+            if (!$response->ok()) {
+                CacheKey::increment(HCK::LOCATIONS_NOT_OK);
+            }
+
             if (empty($data) || !$response->ok()) {
                 return Collection::empty();
             }
 
+            CacheKey::increment(HCK::LOCATIONS_SUCCESS);
             return Repositories\StationRepository::parseHafasStops($data);
         } catch (JsonException $exception) {
             throw new HafasException($exception->getMessage());
+        } catch (Exception $exception) {
+            CacheKey::increment(HCK::LOCATIONS_FAILURE);
+            throw new HafasException($exception->getMessage());
         }
     }
 
@@ -100,6 +114,7 @@ public function getNearbyStations(float $latitude, float $longitude, int $result
             );
 
             if (!$response->ok()) {
+                CacheKey::increment(HCK::NEARBY_NOT_OK);
                 throw new HafasException(__('messages.exception.generalHafas'));
             }
 
@@ -111,8 +126,10 @@ public function getNearbyStations(float $latitude, float $longitude, int $result
                 $station->distance = $hafasStation->distance;
             }
 
+            CacheKey::increment(HCK::NEARBY_SUCCESS);
             return $stations;
         } catch (JsonException $exception) {
+            CacheKey::increment(HCK::NEARBY_FAILURE);
             throw new HafasException($exception->getMessage());
         }
     }
@@ -146,13 +163,16 @@ private function fetchDepartures(
         try {
             $response = $this->client()->get('/stops/' . $station->ibnr . '/departures', $query);
         } catch (Exception $exception) {
+            CacheKey::increment(HCK::DEPARTURES_FAILURE);
             throw new HafasException($exception->getMessage());
         }
 
         if (!$response->ok()) {
+            CacheKey::increment(HCK::DEPARTURES_NOT_OK);
             throw new HafasException(__('messages.exception.generalHafas'));
         }
 
+        CacheKey::increment(HCK::DEPARTURES_SUCCESS);
         return json_decode($response->body(), false, 512, JSON_THROW_ON_ERROR);
     }
 
@@ -244,20 +264,28 @@ public function getDepartures(
      * @throws HafasException|JsonException
      */
     public function fetchRawHafasTrip(string $tripId, string $lineName) {
-        $tripResponse = $this->client()->get("trips/" . rawurlencode($tripId), [
-            'lineName'  => $lineName,
-            'polyline'  => 'true',
-            'stopovers' => 'true'
-        ]);
+        try {
+            $tripResponse = $this->client()->get("trips/" . rawurlencode($tripId), [
+                'lineName'  => $lineName,
+                'polyline'  => 'true',
+                'stopovers' => 'true'
+            ]);
+        } catch (Exception $exception) {
+            CacheKey::increment(HCK::TRIPS_FAILURE);
+            throw new HafasException(__('messages.exception.generalHafas'));
+        }
 
         if ($tripResponse->ok()) {
+            CacheKey::increment(HCK::TRIPS_SUCCESS);
             return json_decode($tripResponse->body(), false, 512, JSON_THROW_ON_ERROR);
         }
         //sometimes HAFAS returnes 502 Bad Gateway
         if ($tripResponse->status() === 502) {
+            CacheKey::increment(HCK::TRIPS_502);
             Log::error('Cannot fetch trip with id: ' . $tripId);
             throw new HafasException(__('messages.exception.hafas.502'));
         }
+        CacheKey::increment(HCK::TRIPS_NOT_OK);
         Log::error('Unknown HAFAS Error (fetchRawHafasTrip)', [
             'status' => $tripResponse->status(),
             'body'   => $tripResponse->body()

app/Helpers/CacheKey.php

@@ -5,26 +5,27 @@
 
 use App\Models\User;
 use Carbon\Carbon;
+use Illuminate\Support\Facades\Cache;
 
 class CacheKey
 {
     // static keys
-    public const string STATUS_CREATED            = 'monitoring-counter-StatusCreated';
-    public const string STATUS_DELETED            = 'monitoring-counter-StatusDeleted';
-    public const string USER_CREATED              = 'monitoring-counter-UserCreated';
-    public const string USER_DELETED              = 'monitoring-counter-UserDeleted';
+    public const string STATUS_CREATED              = 'monitoring-counter-StatusCreated';
+    public const string STATUS_DELETED              = 'monitoring-counter-StatusDeleted';
+    public const string USER_CREATED                = 'monitoring-counter-UserCreated';
+    public const string USER_DELETED                = 'monitoring-counter-UserDeleted';
     public const string WEBHOOK_ABSENT              = 'monitoring-counter-WebhookAbsent';
     public const string LEADERBOARD_GLOBAL_POINTS   = 'LeaderboardGlobalPoints';
     public const string LEADERBOARD_GLOBAL_DISTANCE = 'LeaderboardGlobalDistance';
 
     // dynamic keys
     private const string LEADERBOARD_FRIENDS = 'LeaderboardFriends';
     private const string LEADERBOARD_MONTH   = 'LeaderboardMonth';
-    private const string STATISTICS_GLOBAL  = 'StatisticsGlobal';
+    private const string STATISTICS_GLOBAL   = 'StatisticsGlobal';
 
     // formatting keys
-    private const string FOR                = '%s-for-%s';
-    private const string FROM_TO            = '%s-from-%s-to-%s';
+    private const string FOR                 = '%s-for-%s';
+    private const string FROM_TO             = '%s-from-%s-to-%s';
 
     public static function getFriendsLeaderboardKey(int $userId): string {
         return sprintf(self::FOR, self::LEADERBOARD_FRIENDS, $userId);
@@ -54,4 +55,12 @@ public static function getYearInReviewKey(User $user, int $year): string {
     public static function getAccountDeletionNotificationTwoWeeksBeforeKey(User $user): string {
         return sprintf("account-deletion-notification-two-weeks-before-%s", $user->id);
     }
+
+    public static function increment(string $key): void {
+        if (Cache::has($key)) {
+            Cache::increment($key);
+        } else {
+            Cache::put($key, 1);
+        }
+    }
 }

app/Helpers/HCK.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Helpers;
+
+enum HCK
+{
+    public const string DEPARTURES_SUCCESS = 'monitoring-counter-HafasDeparturesSuccess';
+    public const string DEPARTURES_FAILURE = 'monitoring-counter-HafasDeparturesFailure';
+    public const string DEPARTURES_NOT_OK  = 'monitoring-counter-HafasDeparturesNotOk';
+    public const string TRIPS_SUCCESS      = 'monitoring-counter-HafasTripsSuccess';
+    public const string TRIPS_FAILURE      = 'monitoring-counter-HafasTripsFailure';
+    public const string TRIPS_NOT_OK       = 'monitoring-counter-HafasTripsNotOk';
+    public const string TRIPS_502          = 'monitoring-counter-HafasTrips502';
+    public const string STOPS_SUCCESS      = 'monitoring-counter-HafasStopsSuccess';
+    public const string STOPS_FAILURE      = 'monitoring-counter-HafasStopsFailure';
+    public const string STOPS_NOT_OK       = 'monitoring-counter-HafasStopsNotOk';
+    public const string STATIONS_SUCCESS   = 'monitoring-counter-HafasStationsSuccess';
+    public const string STATIONS_FAILURE   = 'monitoring-counter-HafasStationsFailure';
+    public const string STATIONS_NOT_OK    = 'monitoring-counter-HafasStationsNotOk';
+    public const string LOCATIONS_SUCCESS  = 'monitoring-counter-HafasLocationsSuccess';
+    public const string LOCATIONS_FAILURE  = 'monitoring-counter-HafasLocationsFailure';
+    public const string LOCATIONS_NOT_OK   = 'monitoring-counter-HafasLocationsNotOk';
+    public const string NEARBY_SUCCESS     = 'monitoring-counter-HafasNearbySuccess';
+    public const string NEARBY_FAILURE     = 'monitoring-counter-HafasNearbyFailure';
+    public const string NEARBY_NOT_OK      = 'monitoring-counter-HafasNearbyNotOk';
+
+    /**
+     * @return array {string: string}
+     */
+    public static function getFailures(): array {
+        return [
+            self::DEPARTURES_FAILURE => 'Departures',
+            self::TRIPS_FAILURE      => 'Trips',
+            self::STOPS_FAILURE      => 'Stops',
+            self::STATIONS_FAILURE   => 'Stations',
+            self::LOCATIONS_FAILURE  => 'Locations',
+            self::NEARBY_FAILURE     => 'Nearby',
+        ];
+    }
+
+    /**
+     * @return array {string: string}
+     */
+    public static function getNotOks(): array {
+        return [
+            self::DEPARTURES_NOT_OK => 'Departures',
+            self::TRIPS_NOT_OK      => 'Trips',
+            self::STOPS_NOT_OK      => 'Stops',
+            self::STATIONS_NOT_OK   => 'Stations',
+            self::LOCATIONS_NOT_OK  => 'Locations',
+            self::NEARBY_NOT_OK     => 'Nearby',
+        ];
+    }
+
+    /**
+     * @return array {string: string}
+     */
+    public static function getSuccesses(): array {
+        return [
+            self::DEPARTURES_SUCCESS => 'Departures',
+            self::TRIPS_SUCCESS      => 'Trips',
+            self::STOPS_SUCCESS      => 'Stops',
+            self::STATIONS_SUCCESS   => 'Stations',
+            self::LOCATIONS_SUCCESS  => 'Locations',
+            self::NEARBY_SUCCESS     => 'Nearby',
+        ];
+    }
+}

app/Listeners/RemoveAbsentWebhooksListener.php

@@ -4,7 +4,6 @@
 
 use App\Helpers\CacheKey;
 use App\Models\Webhook;
-use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Log;
 use Spatie\WebhookServer\Events\WebhookCallFailedEvent;
 
@@ -21,6 +20,6 @@ public function handle(WebhookCallFailedEvent $event) {
             "webhookId" => $webhookId,
             "userId"    => $event->headers["X-Trwl-User-Id"]
         ]);
-        Cache::increment(CacheKey::WEBHOOK_ABSENT);
+        CacheKey::increment(CacheKey::WEBHOOK_ABSENT);
     }
 }

app/Observers/StatusObserver.php

@@ -10,12 +10,11 @@
 use App\Notifications\StatusLiked;
 use App\Notifications\UserJoinedConnection;
 use Illuminate\Notifications\DatabaseNotification;
-use Illuminate\Support\Facades\Cache;
 
 class StatusObserver
 {
     public function created(Status $status): void {
-        Cache::increment(CacheKey::STATUS_CREATED);
+        CacheKey::increment(CacheKey::STATUS_CREATED);
         MentionHelper::createMentions($status);
     }
 
@@ -24,7 +23,7 @@ public function updated(Status $status): void {
     }
 
     public function deleted(Status $status): void {
-        Cache::increment(CacheKey::STATUS_DELETED);
+        CacheKey::increment(CacheKey::STATUS_DELETED);
 
         WebhookController::sendStatusWebhook(
             status: $status,

app/Observers/UserObserver.php

@@ -6,16 +6,15 @@
 use App\Models\User;
 use App\Notifications\StatusLiked;
 use Illuminate\Notifications\DatabaseNotification;
-use Illuminate\Support\Facades\Cache;
 
 class UserObserver
 {
     public function created(User $user): void {
-        Cache::increment(CacheKey::USER_CREATED);
+        CacheKey::increment(CacheKey::USER_CREATED);
     }
 
     public function deleted(User $user): void {
-        Cache::increment(CacheKey::USER_DELETED);
+        CacheKey::increment(CacheKey::USER_DELETED);
 
         //delete all notifications for this user (there is no cascade delete)
         DatabaseNotification::where('notifiable_id', $user->id)

app/Providers/PrometheusServiceProvider.php

@@ -3,6 +3,7 @@
 namespace App\Providers;
 
 use App\Helpers\CacheKey;
+use App\Helpers\HCK;
 use App\Models\PolyLine;
 use App\Models\Trip;
 use Illuminate\Support\Facades\Cache;
@@ -93,6 +94,27 @@ public function register() {
                       return $this->getJobsByDisplayName("failed_jobs");
                   });
 
+        Prometheus::addGauge("failed_hafas_requests_count")
+                  ->helpText("How many hafas requests have failed?")
+                  ->labels(["request_name"])
+                  ->value(function() {
+                      return $this->getHafasByType(HCK::getFailures());
+                  });
+
+        Prometheus::addGauge("not_ok_hafas_requests_count")
+                  ->helpText("How many hafas requests are not ok?")
+                  ->labels(["request_name"])
+                  ->value(function() {
+                      return $this->getHafasByType(HCK::getNotOks());
+                  });
+
+        Prometheus::addGauge("succeeded_hafas_requests_count")
+                  ->helpText("How many hafas requests have succeeded?")
+                  ->labels(["request_name"])
+                  ->value(function() {
+                      return $this->getHafasByType(HCK::getSuccesses());
+                  });
+
         Prometheus::addGauge("completed_jobs_count")
                   ->helpText("How many jobs are done? Old items from queue monitor table are deleted after 7 days.")
                   ->labels(["job_name", "status", "queue"])
@@ -192,4 +214,13 @@ public static function getJobsByDisplayName(string $tableName): array {
             array_values($counts)
         );
     }
+
+    private function getHafasByType(array $getFailures): array {
+        $values = [];
+        foreach ($getFailures as $key => $name) {
+            $values[$name] = Cache::get($key, 0);
+        }
+
+        return array_map(fn($value, $key) => [$value, [$key]], $values, array_keys($values));
+    }
 }