From 2e85d7358c1200d030ebb89c07ebbd2da904d796 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 14 Aug 2024 15:07:35 +0200 Subject: [PATCH] :sparkles: let user fetch wikidata entity for station (#2858) --- app/Exceptions/Wikidata/FetchException.php | 10 ++ .../API/v1/ExperimentalController.php | 59 +++++++++ .../Frontend/Admin/StationController.php | 59 ++------- .../Frontend/OpenData/WikidataController.php | 26 ++++ .../Wikidata/WikidataImportService.php | 59 +++++++++ .../views/open-data/wikidata/index.blade.php | 122 ++++++++++++++++++ routes/api.php | 7 + routes/web.php | 6 + 8 files changed, 297 insertions(+), 51 deletions(-) create mode 100644 app/Exceptions/Wikidata/FetchException.php create mode 100644 app/Http/Controllers/API/v1/ExperimentalController.php create mode 100644 app/Http/Controllers/Frontend/OpenData/WikidataController.php create mode 100644 resources/views/open-data/wikidata/index.blade.php diff --git a/app/Exceptions/Wikidata/FetchException.php b/app/Exceptions/Wikidata/FetchException.php new file mode 100644 index 000000000..3e11af025 --- /dev/null +++ b/app/Exceptions/Wikidata/FetchException.php @@ -0,0 +1,10 @@ +json(['error' => 'You are requesting too fast. Please try again later.'], 429); + } + + if (!self::checkStationRateLimit($stationId)) { + return response()->json(['error' => 'This station was already requested recently. Please try again later.'], 429); + } + + $station = Station::findOrFail($stationId); + if ($station->wikidata_id) { + return response()->json(['error' => 'This station already has a wikidata id.'], 400); + } + + try { + WikidataImportService::searchStation($station); + return response()->json(['message' => 'Wikidata information fetched successfully']); + } catch (FetchException $exception) { + return response()->json(['error' => $exception->getMessage()], 422); + } + } + + private static function checkGeneralRateLimit(): bool { + $key = "fetch-wikidata-user:" . auth()->id(); + if (RateLimiter::tooManyAttempts($key, 10)) { + return false; + } + RateLimiter::increment($key); + return true; + } + + private static function checkStationRateLimit(int $stationId): bool { + // request a station 1 time per 5 minutes + + $key = "fetch-wikidata-station:$stationId"; + if (RateLimiter::tooManyAttempts($key, 1)) { + return false; + } + RateLimiter::increment($key, 5 * 60); + return true; + } + +} diff --git a/app/Http/Controllers/Frontend/Admin/StationController.php b/app/Http/Controllers/Frontend/Admin/StationController.php index 0a57c6d20..1cf5e63ec 100644 --- a/app/Http/Controllers/Frontend/Admin/StationController.php +++ b/app/Http/Controllers/Frontend/Admin/StationController.php @@ -3,13 +3,13 @@ namespace App\Http\Controllers\Frontend\Admin; use App\Dto\Coordinate; +use App\Exceptions\Wikidata\FetchException; use App\Http\Controllers\Controller; use App\Models\Station; -use App\Models\StationName; use App\Objects\LineSegment; use App\Services\Wikidata\WikidataImportService; -use App\Services\Wikidata\WikidataQueryService; use Illuminate\Auth\Access\AuthorizationException; +use Illuminate\Http\JsonResponse; use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; use Illuminate\Support\Facades\Log; @@ -65,58 +65,15 @@ public function renderStation(int $id): View { * Needs to be cleaned up and refactored, if it should be used consistently. * Little testing if it works as expected. */ - public function fetchWikidata(int $id): void { + public function fetchWikidata(int $id): JsonResponse { $station = Station::findOrFail($id); $this->authorize('update', $station); - // P054 = IBNR - $sparqlQuery = <<ibnr}". } - SPARQL; - - $objects = (new WikidataQueryService())->setQuery($sparqlQuery)->execute()->getObjects(); - if (count($objects) > 1) { - Log::debug('More than one object found for station ' . $station->ibnr . ' (' . $station->id . ') - skipping'); - return; - } - - if (empty($objects)) { - Log::debug('No object found for station ' . $station->ibnr . ' (' . $station->id . ') - skipping'); - return; - } - - $object = $objects[0]; - $station->update(['wikidata_id' => $object->qId]); - Log::debug('Fetched object ' . $object->qId . ' for station ' . $station->name . ' (Trwl-ID: ' . $station->id . ')'); - - $ifopt = $object->getClaims('P12393')[0]['mainsnak']['datavalue']['value'] ?? null; - if ($station->ifopt_a === null && $ifopt !== null) { - $splitIfopt = explode(':', $ifopt); - $station->update([ - 'ifopt_a' => $splitIfopt[0] ?? null, - 'ifopt_b' => $splitIfopt[1] ?? null, - 'ifopt_c' => $splitIfopt[2] ?? null, - ]); - } - - $rl100 = $object->getClaims('P8671')[0]['mainsnak']['datavalue']['value'] ?? null; - if ($station->rilIdentifier === null && $rl100 !== null) { - $station->update(['rilIdentifier' => $rl100]); - } - - //get names - foreach ($object->getClaims('P2561') as $property) { - $text = $property['mainsnak']['datavalue']['value']['text'] ?? null; - $language = $property['mainsnak']['datavalue']['value']['language'] ?? null; - if ($language === null || $text === null) { - continue; - } - StationName::updateOrCreate([ - 'station_id' => $station->id, - 'language' => $language, - ], [ - 'name' => $text - ]); + try { + WikidataImportService::searchStation($station); + return response()->json(['success' => 'Wikidata information fetched successfully']); + } catch (FetchException $exception) { + return response()->json(['error' => $exception->getMessage()], 422); } } diff --git a/app/Http/Controllers/Frontend/OpenData/WikidataController.php b/app/Http/Controllers/Frontend/OpenData/WikidataController.php new file mode 100644 index 000000000..c5453506f --- /dev/null +++ b/app/Http/Controllers/Frontend/OpenData/WikidataController.php @@ -0,0 +1,26 @@ +join('train_checkins', 'train_checkins.destination_stopover_id', '=', 'train_stopovers.id') + ->where('train_checkins.user_id', auth()->id()) + ->whereNull('train_stations.wikidata_id') + ->select('train_stations.*') + ->limit(50) + ->get(); + + return view('open-data.wikidata.index', [ + 'destinationStationsWithoutWikidata' => $destinationStationsWithoutWikidata + ]); + } +} diff --git a/app/Services/Wikidata/WikidataImportService.php b/app/Services/Wikidata/WikidataImportService.php index 5cd599134..9cc0c14cc 100644 --- a/app/Services/Wikidata/WikidataImportService.php +++ b/app/Services/Wikidata/WikidataImportService.php @@ -3,7 +3,10 @@ namespace App\Services\Wikidata; use App\Dto\Wikidata\WikidataEntity; +use App\Exceptions\Wikidata\FetchException; use App\Models\Station; +use App\Models\StationName; +use Illuminate\Support\Facades\Log; class WikidataImportService { @@ -55,4 +58,60 @@ public static function importStation(string $qId): Station { ); } + /** + * @throws FetchException + */ + public static function searchStation(Station $station): void { + // P054 = IBNR + $sparqlQuery = <<ibnr}". } + SPARQL; + + $objects = (new WikidataQueryService())->setQuery($sparqlQuery)->execute()->getObjects(); + if (count($objects) > 1) { + Log::debug('More than one object found for station ' . $station->ibnr . ' (' . $station->id . ') - skipping'); + throw new FetchException('There are multiple Wikidata entitied with IBNR ' . $station->ibnr); + } + + if (empty($objects)) { + Log::debug('No object found for station ' . $station->ibnr . ' (' . $station->id . ') - skipping'); + throw new FetchException('No Wikidata entity found for IBNR ' . $station->ibnr); + } + + $object = $objects[0]; + $station->update(['wikidata_id' => $object->qId]); + activity()->performedOn($station)->log('Linked wikidata entity ' . $object->qId); + Log::debug('Fetched object ' . $object->qId . ' for station ' . $station->name . ' (Trwl-ID: ' . $station->id . ')'); + + $ifopt = $object->getClaims('P12393')[0]['mainsnak']['datavalue']['value'] ?? null; + if ($station->ifopt_a === null && $ifopt !== null) { + $splitIfopt = explode(':', $ifopt); + $station->update([ + 'ifopt_a' => $splitIfopt[0] ?? null, + 'ifopt_b' => $splitIfopt[1] ?? null, + 'ifopt_c' => $splitIfopt[2] ?? null, + ]); + } + + $rl100 = $object->getClaims('P8671')[0]['mainsnak']['datavalue']['value'] ?? null; + if ($station->rilIdentifier === null && $rl100 !== null) { + $station->update(['rilIdentifier' => $rl100]); + } + + //get names + foreach ($object->getClaims('P2561') as $property) { + $text = $property['mainsnak']['datavalue']['value']['text'] ?? null; + $language = $property['mainsnak']['datavalue']['value']['language'] ?? null; + if ($language === null || $text === null) { + continue; + } + StationName::updateOrCreate([ + 'station_id' => $station->id, + 'language' => $language, + ], [ + 'name' => $text + ]); + } + } + } diff --git a/resources/views/open-data/wikidata/index.blade.php b/resources/views/open-data/wikidata/index.blade.php new file mode 100644 index 000000000..4b524589f --- /dev/null +++ b/resources/views/open-data/wikidata/index.blade.php @@ -0,0 +1,122 @@ +@extends('layouts.app') + +@section('title', 'Open Data: Wikidata') + +@section('content') +
+
+
+

+ Open Data - Wikidata: Missing station information +

+ + @if(app()->getLocale() !== 'en') + + @endif + + + + + + + + + + + + + + + + @foreach($destinationStationsWithoutWikidata as $station) + + + + + + + + @endforeach + +
StationIBNRIFOPTRil100Wikidata
{{$station->name}}{{$station->ibnr}}{{$station->ifopt}}{{$station->rilIdentifier}} + +
+ + +
+
+
+@endsection diff --git a/routes/api.php b/routes/api.php index 4f755ca6f..c142297a7 100644 --- a/routes/api.php +++ b/routes/api.php @@ -13,6 +13,7 @@ use App\Http\Controllers\API\v1\AuthController as v1Auth; use App\Http\Controllers\API\v1\EventController; +use App\Http\Controllers\API\v1\ExperimentalController; use App\Http\Controllers\API\v1\ExportController; use App\Http\Controllers\API\v1\FollowController; use App\Http\Controllers\API\v1\IcsController; @@ -177,6 +178,12 @@ Route::get('/user/self/trusted-by', [TrustedUserController::class, 'indexTrustedBy']); Route::apiResource('report', ReportController::class); Route::apiResource('operators', OperatorController::class)->only(['index']); + + Route::prefix('experimental')->group(function() { + // undocumented, unstable, experimental endpoints. don't use in external applications! + + Route::post('/station/{id}/wikidata', [ExperimentalController::class, 'fetchWikidata']); + }); }); Route::group(['middleware' => ['privacy-policy']], static function() { diff --git a/routes/web.php b/routes/web.php index d1d3c602b..c19a5cbd9 100644 --- a/routes/web.php +++ b/routes/web.php @@ -17,6 +17,7 @@ use App\Http\Controllers\Frontend\EventController; use App\Http\Controllers\Frontend\IcsController; use App\Http\Controllers\Frontend\LeaderboardController; +use App\Http\Controllers\Frontend\OpenData\WikidataController; use App\Http\Controllers\Frontend\SettingsController; use App\Http\Controllers\Frontend\Social\MastodonController; use App\Http\Controllers\Frontend\Social\SocialController; @@ -128,6 +129,11 @@ Route::get('/daily/{dateString}', [DailyStatsController::class, 'renderDailyStats']) ->name('stats.daily'); }); + + Route::prefix('open-data')->group(function() { + Route::get('/wikidata', [WikidataController::class, 'indexHelpPage']) + ->name('open-data.wikidata'); + }); Route::prefix('settings')->group(function() {