Skip to content

Commit

Permalink
✨ search for wikidata objects; user import (#3057)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MrKrisKrisu
MrKrisKrisu authored Dec 26, 2024
1 parent a18f207 commit be399de
Show file tree
Hide file tree
Showing 7 changed files with 94 additions and 23 deletions.
2 changes: 1 addition & 1 deletion app/DataProviders/CachedHafas.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ function() use ($rilIdentifier) {
);
}

public function getStationsByFuzzyRilIdentifier(string $rilIdentifier): ?Collection {
public function getStationsByFuzzyRilIdentifier(string $rilIdentifier): Collection {
$key = CacheKey::getHafasStationsFuzzyKey($rilIdentifier);

return $this->remember(
Expand Down
15 changes: 10 additions & 5 deletions app/DataProviders/Hafas.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,17 @@ public function getStationByRilIdentifier(string $rilIdentifier): ?Station {
return $station;
}

public function getStationsByFuzzyRilIdentifier(string $rilIdentifier): ?Collection {
$stations = Station::where('rilIdentifier', 'LIKE', "$rilIdentifier%")->orderBy('rilIdentifier')->get();
if ($stations->count() > 0) {
return $stations;
public function getStationsByFuzzyRilIdentifier(string $rilIdentifier): Collection {
$stations = Station::where('rilIdentifier', 'LIKE', "$rilIdentifier%")
->orderBy('rilIdentifier')
->get();
if ($stations->count() === 0) {
$station = $this->getStationByRilIdentifier(rilIdentifier: $rilIdentifier);
if ($station !== null) {
$stations->push($station);
}
}
return collect([$this->getStationByRilIdentifier(rilIdentifier: $rilIdentifier)]);
return $stations;
}

/**
Expand Down
3 changes: 2 additions & 1 deletion app/Dto/Wikidata/WikidataEntity.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace App\Dto\Wikidata;

use Illuminate\Database\Eloquent\ModelNotFoundException;
use Illuminate\Support\Facades\Http;
use JsonException;

readonly class WikidataEntity
Expand All @@ -23,8 +24,8 @@ public static function fetch(string $qId): self {
}
$instance = new self();
$instance->qId = $qId;
$json = json_decode(file_get_contents('https://www.wikidata.org/wiki/Special:EntityData/' . $qId . '.json'), true, 512, JSON_THROW_ON_ERROR);

$json = Http::get('https://www.wikidata.org/wiki/Special:EntityData/' . $qId . '.json')->json();
if (!isset($json['entities'][$qId])) {
throw new ModelNotFoundException('Entity not found');
}
Expand Down
13 changes: 12 additions & 1 deletion app/Http/Controllers/API/v1/ExperimentalController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public function fetchWikidata(int $stationId): JsonResponse {
}
}

private static function checkGeneralRateLimit(): bool {
public static function checkGeneralRateLimit(): bool {
$key = "fetch-wikidata-user:" . auth()->id();
if (RateLimiter::tooManyAttempts($key, 20)) {
return false;
Expand All @@ -58,4 +58,15 @@ private static function checkStationRateLimit(int $stationId): bool {
return true;
}

public static function checkWikidataIdRateLimit(string $qId) {
// request a wikidata id 1 time per 5 minutes

$key = "fetch-wikidata-qid:$qId";
if (RateLimiter::tooManyAttempts($key, 1)) {
return false;
}
RateLimiter::increment($key, 5 * 60);
return true;
}

}
32 changes: 26 additions & 6 deletions app/Http/Controllers/TransportController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@

use App\DataProviders\DataProviderBuilder;
use App\DataProviders\DataProviderInterface;
use App\Exceptions\HafasException;
use App\Http\Controllers\API\v1\ExperimentalController;
use App\Http\Resources\StationResource;
use App\Models\Checkin;
use App\Models\PolyLine;
use App\Models\Station;
use App\Models\User;
use App\Services\Wikidata\WikidataImportService;
use Carbon\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;

/**
* @deprecated Content will be moved to the backend/frontend/API packages soon, please don't add new functions here!
Expand All @@ -32,23 +34,41 @@ public function __construct(string $dataProvider) {
* @param string $query
*
* @return Collection
* @throws HafasException
* @api v1
*/
public function getTrainStationAutocomplete(string $query): Collection {
if (!is_numeric($query) && strlen($query) <= 5 && ctype_upper($query)) {
$stations = $this->dataProvider->getStationsByFuzzyRilIdentifier(rilIdentifier: $query);
}

if (!isset($stations) || $stations[0] === null) {
} elseif (preg_match('/^Q\d+$/', $query)) {
$stations = self::getStationsByWikidataId($query);
} elseif (!isset($stations) || $stations[0] === null) {
$stations = $this->dataProvider->getStations($query);
}

return $stations->map(function(Station $station) {
return new StationResource($station);
});
}

private static function getStationsByWikidataId(string $wikidataId): Collection {
$stations = Station::where('wikidata_id', $wikidataId)->get();

if ($stations->isEmpty() && ExperimentalController::checkGeneralRateLimit() && ExperimentalController::checkWikidataIdRateLimit($wikidataId)) {
try {
Log::debug('Lookup Wikidata ID as User searched it', ['wikidataId' => $wikidataId]);
$station = WikidataImportService::importStation($wikidataId);
Log::info('Saved Station from Wikidata.', [$station->only(['id', 'name', 'wikidata_id'])]);
$stations->push($station);
} catch (\InvalidArgumentException $exception) {
// ignore in frontend, just log for debugging
Log::debug('Could not import Station from Wikidata: ' . $exception->getMessage(), ['wikidataId' => $wikidataId]);
} catch (\Exception $exception) {
report($exception);
}
}

return $stations;
}

/**
* Check if there are colliding CheckIns
*
Expand Down
51 changes: 43 additions & 8 deletions app/Services/Wikidata/WikidataImportService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace App\Services\Wikidata;

use App\Dto\Coordinate;
use App\Dto\Wikidata\WikidataEntity;
use App\Exceptions\Wikidata\FetchException;
use App\Models\Station;
Expand All @@ -11,9 +12,26 @@
class WikidataImportService
{

// supported types global definieren
private const SUPPORTED_TYPES = [
'Q55490', // Durchgangsbahnhof
'Q18543139', // Hauptbahnhof
'Q27996466', // Bahnhof (betrieblich)
'Q55488', // Bahnhof (Verkehrsanlage einer Bahn)
'Q124817561', // Betriebsstelle
'Q644371', // internationaler Flughafen
'Q21836433', // Flughafen
'Q953806', // Bushaltestelle
'Q2175765', // Straßenbahnhaltestelle
];

public static function importStation(string $qId): Station {
$wikidataEntity = WikidataEntity::fetch($qId);

if (!self::isTypeSupported($wikidataEntity)) {
throw new \InvalidArgumentException('Entity ' . $qId . ' is not a supported type');
}

$name = $wikidataEntity->getClaims('P1448')[0]['mainsnak']['datavalue']['value']['text'] //P1448 = official name
?? $wikidataEntity->getLabel('de') //german label
?? $wikidataEntity->getLabel(); //english label or null if also not available
Expand All @@ -22,16 +40,14 @@ public static function importStation(string $qId): Station {
throw new \InvalidArgumentException('No name found for entity ' . $qId);
}

$coordinates = $wikidataEntity->getClaims('P625')[0]['mainsnak']['datavalue']['value'] ?? null; //P625 = coordinate location
$coordinates = self::getCoordinates($wikidataEntity);
if ($coordinates === null) {
throw new \InvalidArgumentException('No coordinates found for entity ' . $qId);
}

$latitude = $coordinates['latitude'];
$longitude = $coordinates['longitude'];
$ibnr = $wikidataEntity->getClaims('P954')[0]['mainsnak']['datavalue']['value'] ?? null; //P954 = IBNR
$rl100 = $wikidataEntity->getClaims('P8671')[0]['mainsnak']['datavalue']['value'] ?? null; //P8671 = RL100
$ifopt = $wikidataEntity->getClaims('P12393')[0]['mainsnak']['datavalue']['value'] ?? null; //P12393 = IFOPT
$ibnr = $wikidataEntity->getClaims('P954')[0]['mainsnak']['datavalue']['value'] ?? null; //P954 = IBNR
$rl100 = $wikidataEntity->getClaims('P8671')[0]['mainsnak']['datavalue']['value'] ?? null; //P8671 = RL100
$ifopt = $wikidataEntity->getClaims('P12393')[0]['mainsnak']['datavalue']['value'] ?? null; //P12393 = IFOPT
if ($ifopt !== null) {
$splittedIfopt = explode(':', $ifopt);
}
Expand All @@ -44,8 +60,8 @@ public static function importStation(string $qId): Station {
return Station::create(
[
'name' => $name,
'latitude' => $latitude,
'longitude' => $longitude,
'latitude' => $coordinates->latitude,
'longitude' => $coordinates->longitude,
'wikidata_id' => $qId,
'rilIdentifier' => $rl100,
'ibnr' => $ibnr,
Expand Down Expand Up @@ -114,4 +130,23 @@ public static function searchStation(Station $station): void {
}
}

public static function isTypeSupported(WikidataEntity $entity): bool {
$instancesOf = $entity->getClaims('P31');
foreach ($instancesOf as $instanceOf) {
$instanceOfId = $instanceOf['mainsnak']['datavalue']['value']['id'];
if (in_array($instanceOfId, self::SUPPORTED_TYPES)) {
return true;
}
}
return false;
}

public static function getCoordinates(WikidataEntity $entity): ?Coordinate {
$coordinates = $entity->getClaims('P625')[0]['mainsnak']['datavalue']['value'] ?? null; //P625 = coordinate location
if ($coordinates === null) {
return null;
}
return new Coordinate($coordinates['latitude'], $coordinates['longitude']);
}

}
1 change: 0 additions & 1 deletion resources/vue/components/TripCreation/StationRow.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ export default {
let query = this.stationInput.replace(/%2F/, ' ').replace(/\//, ' ');
fetch(`/api/v1/stations/?query=${query}`).then((response) => {
response.json().then((result) => {
console.log(result.data);
this.autocompleteList = result.data;
this.loading = false;
});
Expand Down

0 comments on commit be399de

Please sign in to comment.