Properly handle latitude and longitude during Destination One import

They sometimes use a different separator.
The code is adjusted to always use same separator and precision.

That will prevent the same location from showing up multiple times due
to different latitude and longitude values.

Classes/Domain/Model/Location.php

@@ -75,8 +75,8 @@ public function __construct(
         $this->district = $district;
         $this->country = $country;
         $this->phone = $phone;
-        $this->latitude = $latitude;
-        $this->longitude = $longitude;
+        $this->latitude = $this->normalizeGeocoordinate($latitude);
+        $this->longitude = $this->normalizeGeocoordinate($longitude);
         $this->_languageUid = $languageUid;
 
         $this->globalId = $this->generateGlobalId();
@@ -158,8 +158,21 @@ private function generateGlobalId(): string
             $this->city,
             $this->district,
             $this->country,
-            $this->latitude,
-            $this->longitude,
         ]));
     }
+
+    private function normalizeGeocoordinate(string $coordinate): string
+    {
+        $numberOfCommas = substr_count($coordinate, ',');
+        $numberOfPoints = substr_count($coordinate, '.');
+
+        if (
+            $numberOfCommas === 1
+            && $numberOfPoints === 0
+        ) {
+            $coordinate = str_replace(',', '.', $coordinate);
+        }
+
+        return number_format((float) $coordinate, 6, '.', '');
+    }
 }

Classes/Domain/Repository/LocationRepository.php

@@ -3,7 +3,19 @@
 namespace Wrm\Events\Domain\Repository;
 
 use TYPO3\CMS\Extbase\Persistence\Repository;
+use Wrm\Events\Domain\Model\Location;
 
 class LocationRepository extends Repository
 {
+    public function findOneByGlobalId(string $globalId): ?Location
+    {
+        $query = $this->createQuery();
+
+        return $query
+            ->matching($query->equals('globalId', $globalId))
+            ->setLimit(1)
+            ->execute()
+            ->getFirst()
+        ;
+    }
 }

Classes/Updates/MigrateDuplicateLocations.php

@@ -0,0 +1,201 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Copyright (C) 2023 Daniel Siepmann <coding@daniel-siepmann.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+namespace Wrm\Events\Updates;
+
+use Generator;
+use TYPO3\CMS\Core\Database\Connection;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Install\Updates\UpgradeWizardInterface;
+use Wrm\Events\Domain\Model\Location;
+
+final class MigrateDuplicateLocations implements UpgradeWizardInterface
+{
+    /**
+     * @var ConnectionPool
+     */
+    private $connectionPool;
+
+    public function __construct(
+        ConnectionPool $connectionPool
+    ) {
+        $this->connectionPool = $connectionPool;
+    }
+
+    public function getIdentifier(): string
+    {
+        return self::class;
+    }
+
+    public function getTitle(): string
+    {
+        return 'Remove duplicate locations of EXT:event';
+    }
+
+    public function getDescription(): string
+    {
+        return 'Checks for duplicates and reduces them to one entry, fixing relations to events.';
+    }
+
+    public function updateNecessary(): bool
+    {
+        return true;
+    }
+
+    public function executeUpdate(): bool
+    {
+        $duplicates = [];
+
+        foreach ($this->getLocations() as $location) {
+            $locationObject = $this->buildObject($location);
+            if ($locationObject->getGlobalId() === $location['global_id']) {
+                continue;
+            }
+
+            $uid = (int)$location['uid'];
+            $matchingLocations = $this->getMatchingLocations(
+                $locationObject->getGlobalId(),
+                $uid
+            );
+
+            // Already have entries for the new id, this one is duplicate
+            if ($matchingLocations !== []) {
+                $duplicates[$uid] = $matchingLocations[0];
+                continue;
+            }
+
+            // No duplicates, update this one
+            $this->updateLocation($locationObject, $uid);
+        }
+
+        $this->removeDuplicates(array_keys($duplicates));
+        $this->updateRelations($duplicates);
+        return true;
+    }
+
+    public function getPrerequisites(): array
+    {
+        return [];
+    }
+
+    public static function register(): void
+    {
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/install']['update'][self::class] = self::class;
+    }
+
+    /**
+     * @return Generator<array>
+     */
+    private function getLocations(): Generator
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('tx_events_domain_model_location');
+        $queryBuilder->select(
+            'name',
+            'street',
+            'zip',
+            'city',
+            'district',
+            'country',
+            'phone',
+            'latitude',
+            'longitude',
+            'global_id',
+            'uid',
+            'sys_language_uid'
+        );
+        $queryBuilder->from('tx_events_domain_model_location');
+        $queryBuilder->orderBy('uid', 'asc');
+        $result = $queryBuilder->execute();
+
+        foreach ($result->fetchAllAssociative() as $location) {
+            yield $location;
+        }
+    }
+
+    private function getMatchingLocations(
+        string $globalId,
+        int $uid
+    ): array {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('tx_events_domain_model_location');
+        $queryBuilder->select('uid');
+        $queryBuilder->from('tx_events_domain_model_location');
+        $queryBuilder->where($queryBuilder->expr()->eq('global_id', $queryBuilder->createNamedParameter($globalId)));
+        $queryBuilder->andWhere($queryBuilder->expr()->neq('uid', $queryBuilder->createNamedParameter($uid)));
+
+        return $queryBuilder->execute()->fetchFirstColumn();
+    }
+
+    private function buildObject(array $location): Location
+    {
+        return new Location(
+            $location['name'],
+            $location['street'],
+            $location['zip'],
+            $location['city'],
+            $location['district'],
+            $location['country'],
+            $location['phone'],
+            $location['latitude'],
+            $location['longitude'],
+            $location['sys_language_uid']
+        );
+    }
+
+    private function updateLocation(Location $location, int $uid): void
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('tx_events_domain_model_location');
+        $queryBuilder->update('tx_events_domain_model_location');
+        $queryBuilder->where($queryBuilder->expr()->eq('uid', $queryBuilder->createNamedParameter($uid)));
+        $queryBuilder->set('global_id', $location->getGlobalId());
+        $queryBuilder->set('latitude', $location->getLatitude());
+        $queryBuilder->set('longitude', $location->getLongitude());
+        $queryBuilder->execute();
+    }
+
+    /**
+     * @param int[] $uids
+     */
+    private function removeDuplicates(array $uids): void
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('tx_events_domain_model_location');
+        $queryBuilder->delete('tx_events_domain_model_location');
+        $queryBuilder->where($queryBuilder->expr()->in('uid', $queryBuilder->createNamedParameter($uids, Connection::PARAM_INT_ARRAY)));
+        $queryBuilder->execute();
+    }
+
+    /**
+     * @param int[] $uids
+     */
+    private function updateRelations(array $migration): void
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('tx_events_domain_model_event');
+        $queryBuilder->update('tx_events_domain_model_event');
+
+        foreach ($migration as $legacyLocationUid => $newLocationUid) {
+            $finalBuilder = clone $queryBuilder;
+            $finalBuilder->where($finalBuilder->expr()->eq('location', $finalBuilder->createNamedParameter($legacyLocationUid)));
+            $finalBuilder->set('location', $newLocationUid);
+            $finalBuilder->execute();
+        }
+    }
+}

Configuration/Services.yaml

@@ -37,6 +37,9 @@ services:
         identifier: 'WrmEventsAddSpecialPropertiesToDate'
         event: TYPO3\CMS\Extbase\Event\Persistence\AfterObjectThawedEvent
 
+  Wrm\Events\Updates\MigrateDuplicateLocations:
+    public: true
+
   Wrm\Events\Updates\MigrateOldLocations:
     public: true
 

Documentation/Changelog/3.4.0.rst

@@ -4,7 +4,8 @@
 Breaking
 --------
 
-Nothing
+* Determining global_id of locations has changed.
+  An Update Wizard is provided in order to migrate existing data.
 
 Features
 --------
@@ -74,6 +75,10 @@ Fixes
   Those might not exist in newer systems where migration is not necessary.
   The wizard now properly checks for existence before querying the data.
 
+* Prevent duplicate location entries from Destination One import.
+  They seem to differ in writing of latitude and longitude.
+  A update wizard is provided to clean up existing duplicates.
+
 Tasks
 -----
 

Tests/Functional/Import/DestinationDataTest/Assertions/ImportsWithLocations.php

@@ -0,0 +1,52 @@
+<?php
+
+return [
+    'tx_events_domain_model_location' => [
+        [
+            'uid' => 1,
+            'pid' => 2,
+            'cruser_id' => 0,
+            'hidden' => 0,
+            'starttime' => 0,
+            'endtime' => 0,
+            'sys_language_uid' => -1,
+            'l10n_parent' => 0,
+            't3ver_oid' => 0,
+            't3ver_wsid' => 0,
+            't3ver_state' => 0,
+            'name' => 'Schillerhaus Rudolstadt',
+            'street' => 'Schillerstraße 25',
+            'district' => '',
+            'city' => 'Rudolstadt',
+            'zip' => '07407',
+            'country' => 'Deutschland',
+            'latitude' => '50.720971023258805',
+            'longitude' => '11.335229873657227',
+            'phone' => '+ 49 3672 / 486470',
+            'global_id' => '2e49097eda43aa83a6d98e25416078f7552525088ee403a6de272cb10301baf6',
+        ],
+        [
+            'uid' => 2,
+            'pid' => 2,
+            'cruser_id' => 0,
+            'hidden' => 0,
+            'starttime' => 0,
+            'endtime' => 0,
+            'sys_language_uid' => -1,
+            'l10n_parent' => 0,
+            't3ver_oid' => 0,
+            't3ver_wsid' => 0,
+            't3ver_state' => 0,
+            'name' => 'Schillerhaus Rudolstadt',
+            'street' => 'Other address',
+            'district' => '',
+            'city' => 'Rudolstadt',
+            'zip' => '07407',
+            'country' => 'Deutschland',
+            'latitude' => '50.720971023258805',
+            'longitude' => '11.335229873657227',
+            'phone' => '+ 49 3672 / 486470',
+            'global_id' => '8b3bfd7f6cd0a9899dbfa6c2e80614b1fa6100710bb28c281f3e3dd31d9d1ef1',
+        ],
+    ],
+];

Tests/Functional/Import/DestinationDataTest/Assertions/ImportsWithSingleLocationOnDuplicates.php

@@ -0,0 +1,20 @@
+<?php
+
+return [
+    'tx_events_domain_model_location' => [
+        [
+            'uid' => 1,
+            'pid' => 2,
+            'name' => 'Schillerhaus Rudolstadt',
+            'street' => 'Schillerstraße 25',
+            'district' => '',
+            'city' => 'Rudolstadt',
+            'zip' => '07407',
+            'country' => 'Deutschland',
+            'latitude' => '50.720971023258805',
+            'longitude' => '11.335229873657227',
+            'phone' => '+ 49 3672 / 486470',
+            'global_id' => '2e49097eda43aa83a6d98e25416078f7552525088ee403a6de272cb10301baf6',
+        ],
+    ],
+];