diff --git a/mobility_data/README.md b/mobility_data/README.md index b0b6c1db9..0db5ac0aa 100644 --- a/mobility_data/README.md +++ b/mobility_data/README.md @@ -170,7 +170,6 @@ To import data type: ./manage.py import_wfs BarbecuePlace ``` - ### Playgrounds ``` ./manage.py import_wfs PlayGround @@ -190,7 +189,7 @@ Imports the outdoor gym devices from the services.unit model. i.e., sets referen ### Parking machines ``` -./manage.py import_parking_machines +./manage.py import_wfs ParkingMachine ``` ### School and kindergarten accessibility areas diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index 9a0bf5f76..46b99e769 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -1,4 +1,26 @@ features: + - content_type_name: ParkingMachine + wfs_layer: GIS:Pysakointiautomaatit + translate_fi_address_field: Osoite + translate_fi_address_municipality_id: turku + extra_fields: + maksutapa_fi: + wfs_field: Maksutapa + maksutapa_sv: + wfs_field: Maksutapa_sv + maksutapa_en: + wfs_field: Maksutapa_en + maksuvyohyke: + wfs_field: Maksuvyohyke + taksa: + wfs_field: Taksa + muu_tieto_fi: + wfs_field: Muu_tieto + muu_tieto_sv: + wfs_field: Muu_tieto_sv + muu_tieto_en: + wfs_field: Muu_tieto_en + - content_type_name: StreetAreaInformation wfs_layer: GIS:Katualueet max_features: 100000 diff --git a/mobility_data/importers/data/wfs_importer_config_example.yml b/mobility_data/importers/data/wfs_importer_config_example.yml index ea98129bf..640d717fe 100644 --- a/mobility_data/importers/data/wfs_importer_config_example.yml +++ b/mobility_data/importers/data/wfs_importer_config_example.yml @@ -16,6 +16,13 @@ features: municipality: muni_field # Optional, if set, include only if geometry is inside the boundarys of Turku, default=False locates_in_turku: True + # Optional, Add the field from which the Finnish address is fetched + # and the importer will assign and lookup the Swedish and the English translations to it. + # Suitable if only Finnish address is available in the source data. + translate_fi_address_field: field_with_finnish_address + # Required if "translate_fi_address_field" is used. + # municipality id of the municipality from which to lookup the address translations. + translate_fi_address_municipality_id: turku # Optional, include only if 'field_name' contains the given string. include: field_name: this_must_be_in_field_name diff --git a/mobility_data/importers/wfs.py b/mobility_data/importers/wfs.py index 666a46f4a..506b4de66 100644 --- a/mobility_data/importers/wfs.py +++ b/mobility_data/importers/wfs.py @@ -11,10 +11,13 @@ from mobility_data.importers.utils import ( delete_mobile_units, get_or_create_content_type_from_config, + get_street_name_translations, + LANGUAGES, locates_in_turku, log_imported_message, MobileUnitDataBase, save_to_database, + split_string_at_first_digit, ) DEFAULT_SOURCE_DATA_SRID = 3877 @@ -40,6 +43,7 @@ def __init__(self): super().__init__() def add_feature(self, feature, config): + municipality = None create_multipolygon = False if "create_multipolygon" in config: create_multipolygon = config["create_multipolygon"] @@ -97,7 +101,6 @@ def add_feature(self, feature, config): self.municipality = Municipality.objects.filter( id=municipality_id ).first() - if "fields" in config: for attr, field in config["fields"].items(): for lang, field_name in field.items(): @@ -105,6 +108,25 @@ def add_feature(self, feature, config): if getattr(self, attr)[lang] is None: getattr(self, attr)[lang] = feature[field_name].as_string() + if "translate_fi_address_municipality_id" in config: + municipality = Municipality.objects.filter( + id=config["translate_fi_address_municipality_id"].lower() + ).first() + + if "translate_fi_address_field" in config: + address = feature[config["translate_fi_address_field"]].as_string() + if not address[0].isdigit(): + street_name, street_number = split_string_at_first_digit(address) + else: + street_name = address + street_number = "" + muni = municipality if municipality else self.municipality + translated_street_names = get_street_name_translations( + street_name.strip(), muni + ) + for lang in LANGUAGES: + self.address[lang] = f"{translated_street_names[lang]} {street_number}" + if "extra_fields" in config: for field, attr in config["extra_fields"].items(): val = None @@ -168,9 +190,12 @@ def import_wfs_feature(config, data_file=None): assert len(ds) == 1 layer = ds[0] for feature in layer: - object = MobilityData() - if object.add_feature(feature, config): - objects.append(object) + try: + object = MobilityData() + if object.add_feature(feature, config): + objects.append(object) + except Exception as e: + logger.warning(f"Discarding feature {feature}, cause: {e}") content_type = get_or_create_content_type_from_config(config["content_type_name"]) num_created, num_deleted = save_to_database(objects, content_type) log_imported_message(logger, content_type, num_created, num_deleted) diff --git a/mobility_data/management/commands/import_parking_machines.py b/mobility_data/management/commands/import_parking_machines.py index 97a032f87..39a764b6d 100644 --- a/mobility_data/management/commands/import_parking_machines.py +++ b/mobility_data/management/commands/import_parking_machines.py @@ -1,3 +1,9 @@ +""" +Deprecated, the parking machines will in future be imported with the WFS importer. +All code related to this importer can be removed after the importing +from WFS feature is in the production environment. +""" + import logging from django.core.management import BaseCommand diff --git a/mobility_data/tasks.py b/mobility_data/tasks.py index c59023c98..70e5e9918 100644 --- a/mobility_data/tasks.py +++ b/mobility_data/tasks.py @@ -155,7 +155,7 @@ def import_wfs(args=None, name="import_wfs"): @shared_task_email def import_parking_machines(name="import_parking_machines"): - management.call_command("import_parking_machines") + management.call_command("import_wfs", "ParkingMachine") @shared_task_email diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index 34b95695d..2946a09ea 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -1,6 +1,4 @@ import pytest -from django.conf import settings -from django.contrib.gis.geos import Point from rest_framework.reverse import reverse @@ -47,9 +45,7 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): assert result["extra"]["test_string"] == "4242" assert result["extra"]["test_int"] == 4242 assert result["extra"]["test_float"] == 42.42 - assert result["geometry"] == Point( - 235404.6706163187, 6694437.919005549, srid=settings.DEFAULT_SRID - ) + assert "POINT" in result["geometry"] url = reverse( "mobility_data:mobile_units-detail", args=["ba6c2903-d36f-4c61-b828-19084fc7a64b"],