From ac680008bacab6964d2730e6a26963b320bd38af Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 09:42:27 +0300 Subject: [PATCH 1/9] Update info about importing parking machines --- mobility_data/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mobility_data/README.md b/mobility_data/README.md index b0b6c1db9..0db5ac0aa 100644 --- a/mobility_data/README.md +++ b/mobility_data/README.md @@ -170,7 +170,6 @@ To import data type: ./manage.py import_wfs BarbecuePlace ``` - ### Playgrounds ``` ./manage.py import_wfs PlayGround @@ -190,7 +189,7 @@ Imports the outdoor gym devices from the services.unit model. i.e., sets referen ### Parking machines ``` -./manage.py import_parking_machines +./manage.py import_wfs ParkingMachine ``` ### School and kindergarten accessibility areas From 56bc7da9dcb090adadcd12168727cc700025741a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 09:42:58 +0300 Subject: [PATCH 2/9] Import parking machines with the WFS importer --- mobility_data/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/tasks.py b/mobility_data/tasks.py index c59023c98..70e5e9918 100644 --- a/mobility_data/tasks.py +++ b/mobility_data/tasks.py @@ -155,7 +155,7 @@ def import_wfs(args=None, name="import_wfs"): @shared_task_email def import_parking_machines(name="import_parking_machines"): - management.call_command("import_parking_machines") + management.call_command("import_wfs", "ParkingMachine") @shared_task_email From bb47ff6c9096315e67d9bf58076cb8ba454c8eae Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 09:43:34 +0300 Subject: [PATCH 3/9] Fix faulty assertion --- mobility_data/tests/test_api.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index 34b95695d..084c82d22 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -1,6 +1,4 @@ import pytest -from django.conf import settings -from django.contrib.gis.geos import Point from rest_framework.reverse import reverse @@ -47,8 +45,8 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): assert result["extra"]["test_string"] == "4242" assert result["extra"]["test_int"] == 4242 assert result["extra"]["test_float"] == 42.42 - assert result["geometry"] == Point( - 235404.6706163187, 6694437.919005549, srid=settings.DEFAULT_SRID + assert ( + result["geometry"] == "SRID=3067;POINT (235404.67061631865 6694437.919005549)" ) url = reverse( "mobility_data:mobile_units-detail", From c6e4429336801574251e42cb04b502a49bfbfff5 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 09:44:11 +0300 Subject: [PATCH 4/9] Add configuration for parking machines --- .../importers/data/wfs_importer_config.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index 9a0bf5f76..46b99e769 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -1,4 +1,26 @@ features: + - content_type_name: ParkingMachine + wfs_layer: GIS:Pysakointiautomaatit + translate_fi_address_field: Osoite + translate_fi_address_municipality_id: turku + extra_fields: + maksutapa_fi: + wfs_field: Maksutapa + maksutapa_sv: + wfs_field: Maksutapa_sv + maksutapa_en: + wfs_field: Maksutapa_en + maksuvyohyke: + wfs_field: Maksuvyohyke + taksa: + wfs_field: Taksa + muu_tieto_fi: + wfs_field: Muu_tieto + muu_tieto_sv: + wfs_field: Muu_tieto_sv + muu_tieto_en: + wfs_field: Muu_tieto_en + - content_type_name: StreetAreaInformation wfs_layer: GIS:Katualueet max_features: 100000 From 35443cfba648a1108e839c95001080bba1352592 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 10:12:21 +0300 Subject: [PATCH 5/9] Add feature to store address with translations --- mobility_data/importers/wfs.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/mobility_data/importers/wfs.py b/mobility_data/importers/wfs.py index 666a46f4a..506b4de66 100644 --- a/mobility_data/importers/wfs.py +++ b/mobility_data/importers/wfs.py @@ -11,10 +11,13 @@ from mobility_data.importers.utils import ( delete_mobile_units, get_or_create_content_type_from_config, + get_street_name_translations, + LANGUAGES, locates_in_turku, log_imported_message, MobileUnitDataBase, save_to_database, + split_string_at_first_digit, ) DEFAULT_SOURCE_DATA_SRID = 3877 @@ -40,6 +43,7 @@ def __init__(self): super().__init__() def add_feature(self, feature, config): + municipality = None create_multipolygon = False if "create_multipolygon" in config: create_multipolygon = config["create_multipolygon"] @@ -97,7 +101,6 @@ def add_feature(self, feature, config): self.municipality = Municipality.objects.filter( id=municipality_id ).first() - if "fields" in config: for attr, field in config["fields"].items(): for lang, field_name in field.items(): @@ -105,6 +108,25 @@ def add_feature(self, feature, config): if getattr(self, attr)[lang] is None: getattr(self, attr)[lang] = feature[field_name].as_string() + if "translate_fi_address_municipality_id" in config: + municipality = Municipality.objects.filter( + id=config["translate_fi_address_municipality_id"].lower() + ).first() + + if "translate_fi_address_field" in config: + address = feature[config["translate_fi_address_field"]].as_string() + if not address[0].isdigit(): + street_name, street_number = split_string_at_first_digit(address) + else: + street_name = address + street_number = "" + muni = municipality if municipality else self.municipality + translated_street_names = get_street_name_translations( + street_name.strip(), muni + ) + for lang in LANGUAGES: + self.address[lang] = f"{translated_street_names[lang]} {street_number}" + if "extra_fields" in config: for field, attr in config["extra_fields"].items(): val = None @@ -168,9 +190,12 @@ def import_wfs_feature(config, data_file=None): assert len(ds) == 1 layer = ds[0] for feature in layer: - object = MobilityData() - if object.add_feature(feature, config): - objects.append(object) + try: + object = MobilityData() + if object.add_feature(feature, config): + objects.append(object) + except Exception as e: + logger.warning(f"Discarding feature {feature}, cause: {e}") content_type = get_or_create_content_type_from_config(config["content_type_name"]) num_created, num_deleted = save_to_database(objects, content_type) log_imported_message(logger, content_type, num_created, num_deleted) From 1fa7c6672ba9b8266b9a6101822578d2c6a288e5 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 10:14:47 +0300 Subject: [PATCH 6/9] Add translate_fi_address_field and translate_fi_address_municipality_id --- .../importers/data/wfs_importer_config_example.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mobility_data/importers/data/wfs_importer_config_example.yml b/mobility_data/importers/data/wfs_importer_config_example.yml index ea98129bf..4ab3ea159 100644 --- a/mobility_data/importers/data/wfs_importer_config_example.yml +++ b/mobility_data/importers/data/wfs_importer_config_example.yml @@ -16,6 +16,13 @@ features: municipality: muni_field # Optional, if set, include only if geometry is inside the boundarys of Turku, default=False locates_in_turku: True + # Optional, Add the field from which the Finnish address is fetched + # and get the Swedish and the English translations to it. + # Suitable if only Finnish address is available in the source data. + translate_fi_address_field: field_with_finnish_address + # Required if "translate_fi_address_field" is used. + # municipality id of the municipality from which to lookup the address translations. + translate_fi_address_municipality_id: turku # Optional, include only if 'field_name' contains the given string. include: field_name: this_must_be_in_field_name From 100336e38d434d9737bbd44a7b1419c8a6a94ec2 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 10:16:27 +0300 Subject: [PATCH 7/9] Add comment that the importer is deprecated --- .../management/commands/import_parking_machines.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mobility_data/management/commands/import_parking_machines.py b/mobility_data/management/commands/import_parking_machines.py index 97a032f87..39a764b6d 100644 --- a/mobility_data/management/commands/import_parking_machines.py +++ b/mobility_data/management/commands/import_parking_machines.py @@ -1,3 +1,9 @@ +""" +Deprecated, the parking machines will in future be imported with the WFS importer. +All code related to this importer can be removed after the importing +from WFS feature is in the production environment. +""" + import logging from django.core.management import BaseCommand From 9ecf9bb198dedd9a14a5af24472e666e2e779532 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 10:25:22 +0300 Subject: [PATCH 8/9] Fix comment for translate_fi_address_field --- mobility_data/importers/data/wfs_importer_config_example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/importers/data/wfs_importer_config_example.yml b/mobility_data/importers/data/wfs_importer_config_example.yml index 4ab3ea159..640d717fe 100644 --- a/mobility_data/importers/data/wfs_importer_config_example.yml +++ b/mobility_data/importers/data/wfs_importer_config_example.yml @@ -17,7 +17,7 @@ features: # Optional, if set, include only if geometry is inside the boundarys of Turku, default=False locates_in_turku: True # Optional, Add the field from which the Finnish address is fetched - # and get the Swedish and the English translations to it. + # and the importer will assign and lookup the Swedish and the English translations to it. # Suitable if only Finnish address is available in the source data. translate_fi_address_field: field_with_finnish_address # Required if "translate_fi_address_field" is used. From 9d1ef0954e937939baa9335b84ab3becb2e2ec4b Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 22 May 2024 10:57:04 +0300 Subject: [PATCH 9/9] Fix geometry assertion --- mobility_data/tests/test_api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index 084c82d22..2946a09ea 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -45,9 +45,7 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): assert result["extra"]["test_string"] == "4242" assert result["extra"]["test_int"] == 4242 assert result["extra"]["test_float"] == 42.42 - assert ( - result["geometry"] == "SRID=3067;POINT (235404.67061631865 6694437.919005549)" - ) + assert "POINT" in result["geometry"] url = reverse( "mobility_data:mobile_units-detail", args=["ba6c2903-d36f-4c61-b828-19084fc7a64b"],