Merge pull request #355 from City-of-Turku/feature/import-parking-mac…

…hines-from-wfs

Feature/import parking machines from wfs

mobility_data/README.md

@@ -170,7 +170,6 @@ To import data type:
 ./manage.py import_wfs BarbecuePlace
 ```
 
-
 ### Playgrounds
 ```
 ./manage.py import_wfs PlayGround
@@ -190,7 +189,7 @@ Imports the outdoor gym devices from the services.unit model. i.e., sets referen
 
 ### Parking machines
 ```
-./manage.py import_parking_machines
+./manage.py import_wfs ParkingMachine
 ```
 
 ### School and kindergarten accessibility areas

mobility_data/importers/data/wfs_importer_config.yml

@@ -1,4 +1,26 @@
 features:
+ - content_type_name: ParkingMachine
+ wfs_layer: GIS:Pysakointiautomaatit
+ translate_fi_address_field: Osoite
+ translate_fi_address_municipality_id: turku
+ extra_fields:
+ maksutapa_fi:
+ wfs_field: Maksutapa
+ maksutapa_sv:
+ wfs_field: Maksutapa_sv
+ maksutapa_en:
+ wfs_field: Maksutapa_en
+ maksuvyohyke:
+ wfs_field: Maksuvyohyke 
+ taksa:
+ wfs_field: Taksa
+ muu_tieto_fi:
+ wfs_field: Muu_tieto
+ muu_tieto_sv:
+ wfs_field: Muu_tieto_sv
+ muu_tieto_en:
+ wfs_field: Muu_tieto_en
+
  - content_type_name: StreetAreaInformation
  wfs_layer: GIS:Katualueet
  max_features: 100000

mobility_data/importers/data/wfs_importer_config_example.yml

@@ -16,6 +16,13 @@ features:
  municipality: muni_field
  # Optional, if set, include only if geometry is inside the boundarys of Turku, default=False
  locates_in_turku: True
+ # Optional, Add the field from which the Finnish address is fetched
+ # and the importer will assign and lookup the Swedish and the English translations to it. 
+ # Suitable if only Finnish address is available in the source data.
+ translate_fi_address_field: field_with_finnish_address
+ # Required if "translate_fi_address_field" is used. 
+ # municipality id of the municipality from which to lookup the address translations.
+ translate_fi_address_municipality_id: turku
  # Optional, include only if 'field_name' contains the given string.
  include:
  field_name: this_must_be_in_field_name

mobility_data/importers/wfs.py

@@ -11,10 +11,13 @@
 from mobility_data.importers.utils import (
  delete_mobile_units,
  get_or_create_content_type_from_config,
+ get_street_name_translations,
+ LANGUAGES,
  locates_in_turku,
  log_imported_message,
  MobileUnitDataBase,
  save_to_database,
+ split_string_at_first_digit,
 )
 
 DEFAULT_SOURCE_DATA_SRID = 3877
@@ -40,6 +43,7 @@ def __init__(self):
  super().__init__()
 
  def add_feature(self, feature, config):
+ municipality = None
  create_multipolygon = False
  if "create_multipolygon" in config:
  create_multipolygon = config["create_multipolygon"]
@@ -97,14 +101,32 @@ def add_feature(self, feature, config):
  self.municipality = Municipality.objects.filter(
  id=municipality_id
  ).first()
-
  if "fields" in config:
  for attr, field in config["fields"].items():
  for lang, field_name in field.items():
  # attr can have fallback definitons if None
  if getattr(self, attr)[lang] is None:
  getattr(self, attr)[lang] = feature[field_name].as_string()
 
+ if "translate_fi_address_municipality_id" in config:
+ municipality = Municipality.objects.filter(
+ id=config["translate_fi_address_municipality_id"].lower()
+ ).first()
+
+ if "translate_fi_address_field" in config:
+ address = feature[config["translate_fi_address_field"]].as_string()
+ if not address[0].isdigit():
+ street_name, street_number = split_string_at_first_digit(address)
+ else:
+ street_name = address
+ street_number = ""
+ muni = municipality if municipality else self.municipality
+ translated_street_names = get_street_name_translations(
+ street_name.strip(), muni
+ )
+ for lang in LANGUAGES:
+ self.address[lang] = f"{translated_street_names[lang]} {street_number}"
+
  if "extra_fields" in config:
  for field, attr in config["extra_fields"].items():
  val = None
@@ -168,9 +190,12 @@ def import_wfs_feature(config, data_file=None):
  assert len(ds) == 1
  layer = ds[0]
  for feature in layer:
- object = MobilityData()
- if object.add_feature(feature, config):
- objects.append(object)
+ try:
+ object = MobilityData()
+ if object.add_feature(feature, config):
+ objects.append(object)
+ except Exception as e:
+ logger.warning(f"Discarding feature {feature}, cause: {e}")
  content_type = get_or_create_content_type_from_config(config["content_type_name"])
  num_created, num_deleted = save_to_database(objects, content_type)
  log_imported_message(logger, content_type, num_created, num_deleted)

mobility_data/management/commands/import_parking_machines.py

@@ -1,3 +1,9 @@
+"""
+Deprecated, the parking machines will in future be imported with the WFS importer.
+All code related to this importer can be removed after the importing
+from WFS feature is in the production environment.
+"""
+
 import logging
 
 from django.core.management import BaseCommand

mobility_data/tasks.py

@@ -155,7 +155,7 @@ def import_wfs(args=None, name="import_wfs"):
 
 @shared_task_email
 def import_parking_machines(name="import_parking_machines"):
- management.call_command("import_parking_machines")
+ management.call_command("import_wfs", "ParkingMachine")
 
 
 @shared_task_email

mobility_data/tests/test_api.py

@@ -1,6 +1,4 @@
 import pytest
-from django.conf import settings
-from django.contrib.gis.geos import Point
 from rest_framework.reverse import reverse
 
 
@@ -47,9 +45,7 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit):
  assert result["extra"]["test_string"] == "4242"
  assert result["extra"]["test_int"] == 4242
  assert result["extra"]["test_float"] == 42.42
- assert result["geometry"] == Point(
- 235404.6706163187, 6694437.919005549, srid=settings.DEFAULT_SRID
- )
+ assert "POINT" in result["geometry"]
  url = reverse(
  "mobility_data:mobile_units-detail",
  args=["ba6c2903-d36f-4c61-b828-19084fc7a64b"],