Merge pull request #355 from City-of-Turku/feature/import-parking-mac…

…hines-from-wfs

Feature/import parking machines from wfs

mobility_data/README.md

@@ -170,7 +170,6 @@ To import data type:
 ./manage.py import_wfs BarbecuePlace
 ```
 
-
 ### Playgrounds
 ```
 ./manage.py import_wfs PlayGround
@@ -190,7 +189,7 @@ Imports the outdoor gym devices from the services.unit model. i.e., sets referen
 
 ### Parking machines
 ```
-./manage.py import_parking_machines
+./manage.py import_wfs ParkingMachine
 ```
 
 ### School and kindergarten accessibility areas

mobility_data/importers/data/wfs_importer_config.yml

@@ -1,4 +1,26 @@
 features:
+  - content_type_name: ParkingMachine
+    wfs_layer: GIS:Pysakointiautomaatit
+    translate_fi_address_field: Osoite
+    translate_fi_address_municipality_id: turku
+    extra_fields:
+      maksutapa_fi:
+        wfs_field: Maksutapa
+      maksutapa_sv:
+        wfs_field: Maksutapa_sv
+      maksutapa_en:
+        wfs_field: Maksutapa_en
+      maksuvyohyke:
+        wfs_field: Maksuvyohyke  
+      taksa:
+        wfs_field: Taksa
+      muu_tieto_fi:
+        wfs_field: Muu_tieto
+      muu_tieto_sv:
+        wfs_field: Muu_tieto_sv
+      muu_tieto_en:
+        wfs_field: Muu_tieto_en
+
   - content_type_name: StreetAreaInformation
     wfs_layer: GIS:Katualueet
     max_features: 100000

mobility_data/importers/data/wfs_importer_config_example.yml

@@ -16,6 +16,13 @@ features:
     municipality: muni_field
     # Optional, if set, include only if geometry is inside the boundarys of Turku, default=False
     locates_in_turku: True
+    # Optional, Add the field from which the Finnish address is fetched
+    # and the importer will assign and lookup the Swedish and the English translations to it.  
+    # Suitable if only Finnish address is available in the source data.
+    translate_fi_address_field: field_with_finnish_address
+    # Required if "translate_fi_address_field" is used. 
+    # municipality id of the municipality from which to lookup the address translations.
+    translate_fi_address_municipality_id: turku
     # Optional, include only if 'field_name' contains the given string.
     include:
       field_name: this_must_be_in_field_name

mobility_data/importers/wfs.py

@@ -11,10 +11,13 @@
 from mobility_data.importers.utils import (
     delete_mobile_units,
     get_or_create_content_type_from_config,
+    get_street_name_translations,
+    LANGUAGES,
     locates_in_turku,
     log_imported_message,
     MobileUnitDataBase,
     save_to_database,
+    split_string_at_first_digit,
 )
 
 DEFAULT_SOURCE_DATA_SRID = 3877
@@ -40,6 +43,7 @@ def __init__(self):
         super().__init__()
 
     def add_feature(self, feature, config):
+        municipality = None
         create_multipolygon = False
         if "create_multipolygon" in config:
             create_multipolygon = config["create_multipolygon"]
@@ -97,14 +101,32 @@ def add_feature(self, feature, config):
                 self.municipality = Municipality.objects.filter(
                     id=municipality_id
                 ).first()
-
         if "fields" in config:
             for attr, field in config["fields"].items():
                 for lang, field_name in field.items():
                     # attr can have fallback definitons if None
                     if getattr(self, attr)[lang] is None:
                         getattr(self, attr)[lang] = feature[field_name].as_string()
 
+        if "translate_fi_address_municipality_id" in config:
+            municipality = Municipality.objects.filter(
+                id=config["translate_fi_address_municipality_id"].lower()
+            ).first()
+
+        if "translate_fi_address_field" in config:
+            address = feature[config["translate_fi_address_field"]].as_string()
+            if not address[0].isdigit():
+                street_name, street_number = split_string_at_first_digit(address)
+            else:
+                street_name = address
+                street_number = ""
+            muni = municipality if municipality else self.municipality
+            translated_street_names = get_street_name_translations(
+                street_name.strip(), muni
+            )
+            for lang in LANGUAGES:
+                self.address[lang] = f"{translated_street_names[lang]} {street_number}"
+
         if "extra_fields" in config:
             for field, attr in config["extra_fields"].items():
                 val = None
@@ -168,9 +190,12 @@ def import_wfs_feature(config, data_file=None):
     assert len(ds) == 1
     layer = ds[0]
     for feature in layer:
-        object = MobilityData()
-        if object.add_feature(feature, config):
-            objects.append(object)
+        try:
+            object = MobilityData()
+            if object.add_feature(feature, config):
+                objects.append(object)
+        except Exception as e:
+            logger.warning(f"Discarding feature {feature}, cause: {e}")
     content_type = get_or_create_content_type_from_config(config["content_type_name"])
     num_created, num_deleted = save_to_database(objects, content_type)
     log_imported_message(logger, content_type, num_created, num_deleted)

mobility_data/management/commands/import_parking_machines.py

@@ -1,3 +1,9 @@
+"""
+Deprecated, the parking machines will in future be imported with the WFS importer.
+All code related to this importer can be removed after the importing
+from WFS feature is in the production environment.
+"""
+
 import logging
 
 from django.core.management import BaseCommand

mobility_data/tasks.py

@@ -155,7 +155,7 @@ def import_wfs(args=None, name="import_wfs"):
 
 @shared_task_email
 def import_parking_machines(name="import_parking_machines"):
-    management.call_command("import_parking_machines")
+    management.call_command("import_wfs", "ParkingMachine")
 
 
 @shared_task_email

mobility_data/tests/test_api.py

@@ -1,6 +1,4 @@
 import pytest
-from django.conf import settings
-from django.contrib.gis.geos import Point
 from rest_framework.reverse import reverse
 
 
@@ -47,9 +45,7 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit):
     assert result["extra"]["test_string"] == "4242"
     assert result["extra"]["test_int"] == 4242
     assert result["extra"]["test_float"] == 42.42
-    assert result["geometry"] == Point(
-        235404.6706163187, 6694437.919005549, srid=settings.DEFAULT_SRID
-    )
+    assert "POINT" in result["geometry"]
     url = reverse(
         "mobility_data:mobile_units-detail",
         args=["ba6c2903-d36f-4c61-b828-19084fc7a64b"],