From a9d3bd52397eb38c09bc23c5538178244f5a65b1 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 12:11:50 +0200 Subject: [PATCH 01/97] Rename type_name to name, add name --- mobility_data/models/content_type.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mobility_data/models/content_type.py b/mobility_data/models/content_type.py index 9dbec462e..11fa5d4e4 100644 --- a/mobility_data/models/content_type.py +++ b/mobility_data/models/content_type.py @@ -5,17 +5,18 @@ class BaseType(models.Model): id = models.UUIDField(default=uuid.uuid4, primary_key=True, editable=False) - name = models.CharField(max_length=64, null=True, unique=True) + type_name = models.CharField(max_length=64, null=True, unique=True) + name = models.CharField(max_length=128, null=True) description = models.TextField( null=True, verbose_name="Optional description of the content type." ) class Meta: abstract = True - ordering = ["name"] + ordering = ["type_name"] def __str__(self): - return self.name + return self.type_name class ContentType(BaseType): From e7da3b4273fdc2d268a098d871468f11cbf67b36 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 12:12:30 +0200 Subject: [PATCH 02/97] Add initial version of content_types configurations --- .../importers/data/content_types.yml | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 mobility_data/importers/data/content_types.yml diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml new file mode 100644 index 000000000..add901ade --- /dev/null +++ b/mobility_data/importers/data/content_types.yml @@ -0,0 +1,113 @@ +content_types: + - content_type_name: BikeServiceStation + name: + fi: Pyöränkorjauspiste + sv: Cykelservicestation + en: Bike service stations + - content_type_name: BicycleStand + name: + fi: Pyöräpysäköinti + sv: Cykelparkering + en: Bicycle parking + - content_type_name: ChargingStation + name: + fi: Sähkölatauspiste + sv: Elladningsstation för bilar + en: Car e-charging point + - content_type_name: NoStaffParking + name: + fi: Yleiset pysäköintialueet + sv: Allmänna parkeringsplatser + en: Public parking spaces + - content_type_name: DisabledParking + name: + fi: Liikkumisesteisen pysäköinti + sv: Parkering för rörelsehindrade + en: Parking for disabled + - content_type_name: FoliParkAndRideCarsStop + name: + fi: Föli auto liityntäpysäköinti + sv: Föli bil infartsparkering + en: Föli car park-and-ride stop + - content_type_name: FoliParkAndRideBikesStop + name: + fi: Föli pyörä liityntäpysäköinti + sv: Föli cykel infartsparkering + en: Föli bicycle park-and-ride stop + - content_type_name: FoliStop + name: + fi: Föli pysäkki + sv: Föli busshållplats + en: Föli bus stop + - content_type_name: GasFillingStation + name: + fi: Kaasutankkausasema + sv: Tankstation med gas + en: Gas filling station + - content_type_name: LoadingUnloadingPlace + name: + fi: Kourmauspaikka + sv: Lastningsplats + en: Loading place + # Content types from lounaistieto shapefiles importer + - content_type_name: BusStopSouthwestFinland + name: + fi: Bussipysäkki + sv: Busshållplats + en: Bus stop + description: + fi: Bussipysäkkejä Varsinais-Suomessa. + sv: Busshållplatser i Egentliga Finland. + en: Bus stops in Southwest Finland. + - content_type_name: FerryDock + name: + fi: Lossi laituri + sv: Färjeläge + en: Ferry dock + description: + fi: Lossi laitureita Varsinais-Suomessa. + sv: Färjelägen i Egentliga Finland. + en: Ferry docks in Southwest Finland. + - content_type_name: CommonFerryRoute + name: + fi: Yhteysalusreitti. + sv: Förbindelsefartygsrutt. + en: Common ferry route. + description: + fi: Yhteysalusreittejä Varsinais-Suomessa. + sv: Förbindelsefartygsrutter i Egentliga Finland. + en: Common ferry routes in Southwest Finland. + - content_type_name: FishingSpot + name: + fi: Kalastuspaikka + sv: Fiskeplats + en: Fishing spot + description: + name: + fi: Kalastuspaikkoja Varsinais-Suomessa. + sv: Fiskeplatser i Egentliga Finland. + en: Fishing spots in Southwest Finland. + - content_type_name: FerryRoute + name: + fi: Lossi reittejä Varsinais-Suomessa. + sv: Färjerutter i Egentliga Finland. + en: Ferry routes(yhteysalusreitti) in Southwest Finland. + - content_type_name: SlipwaySouthwestFinland + name: + fi: Veneenlaskupaikka + sv: Sjösättningsplat för båt + en: Slipway for boat + description: + fi: Veneenlaskupaikkoja Varsinais-Suomessa. + sv: Sjösättningsplatser för båtar i Egentliga Finland. + en: Slipways for boats in Southwest Finland. + - content_type_name: RecreationalRoute + name: + fi: Virkistysreitti + sv: Rekreationsrutt + en: Recreational route + description: + fi: Virkistysreittejä Varsinais-Suomessa. + sv: Rekreationsrutter i Egentliga Finland + en: Recreational routes in Southwest Finland + # End of lounaistieto shapefile importer content types \ No newline at end of file From cc129305d1de424bc7585d59d0526f885d77075a Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 12:15:25 +0200 Subject: [PATCH 03/97] Add functions to create content_type from YAML config --- mobility_data/importers/utils.py | 64 ++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/mobility_data/importers/utils.py b/mobility_data/importers/utils.py index c49ba1dd4..28c1286c6 100644 --- a/mobility_data/importers/utils.py +++ b/mobility_data/importers/utils.py @@ -6,6 +6,7 @@ from enum import Enum import requests +import yaml from django.conf import settings from django.contrib.gis.db.models.functions import Distance from django.contrib.gis.gdal import DataSource as GDALDataSource @@ -27,6 +28,23 @@ "https://tie.digitraffic.fi/api/traffic-message/v1/area-geometries/" + f"{GEOMETRY_ID}?includeGeometry=true" ) + + +def get_root_dir(): + """ + Returns the root directory of the project. + """ + if hasattr(settings, "PROJECT_ROOT"): + return settings.PROJECT_ROOT + else: + return settings.BASE_DIR + + +CONTENT_TYPES_CONFIG_FILE = ( + f"{get_root_dir()}/mobility_data/importers/data/content_types.yml" +) + + LANGUAGES = ["fi", "sv", "en"] @@ -67,8 +85,8 @@ def fetch_json(url): return response.json() -def delete_mobile_units(name): - MobileUnit.objects.filter(content_types__name=name).delete() +def delete_mobile_units(type_name): + MobileUnit.objects.filter(content_types__type_name=type_name).delete() def create_mobile_unit_as_unit_reference(unit_id, content_type): @@ -230,16 +248,6 @@ def locates_in_turku(feature, source_data_srid): return turku_boundary.contains(geometry) -def get_root_dir(): - """ - Returns the root directory of the project. - """ - if hasattr(settings, "PROJECT_ROOT"): - return settings.PROJECT_ROOT - else: - return settings.BASE_DIR - - def get_file_name_from_data_source(content_type): """ Returns the stored file name in the DataSource table for @@ -252,3 +260,35 @@ def get_file_name_from_data_source(content_type): file_name = str(data_source_qs.first().data_file.file) return file_name return None + + +def get_yaml_config(file): + return yaml.safe_load(open(file, "r", encoding="utf-8")) + + +def get_content_type_config(type_name): + configs = get_yaml_config(CONTENT_TYPES_CONFIG_FILE) + for config in configs.get("content_types", None): + if type_name == config.get("content_type_name", None): + return config + return None + + +def get_or_create_content_type_from_config(type_name): + config = get_content_type_config(type_name) + if config is None: + raise Exception( + f"Configuration not found for {type_name} in {CONTENT_TYPES_CONFIG_FILE}" + ) + + content_type, _ = ContentType.objects.get_or_create(type_name=type_name) + for lang in ["fi", "sv", "en"]: + setattr(content_type, f"name_{lang}", config["name"].get(lang, None)) + if "description" in config: + setattr( + content_type, + f"description_{lang}", + config["description"].get(lang, None), + ) + content_type.save() + return content_type From f879a6fa2fb1d980c284b7454dda41e696f7615a Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 13:17:44 +0200 Subject: [PATCH 04/97] Make ContentType read only --- mobility_data/admin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobility_data/admin.py b/mobility_data/admin.py index 726049193..6a63ed522 100644 --- a/mobility_data/admin.py +++ b/mobility_data/admin.py @@ -93,12 +93,12 @@ def type_name(self, obj): class ContentTypeAdmin(admin.ModelAdmin): - # readonly_fields = ("id", "type_name", "name", "description") - readonly_fields = ("id", "name", "description") + def has_change_permission(self, request, obj=None): + return False class GroupTypeAdmin(admin.ModelAdmin): - readonly_fields = ("id", "name", "description") + readonly_fields = ("id", "type_name", "description") class DataSourceAdmin(admin.ModelAdmin): From 94ac4980b4e90651fddf0f9e8977be3dbdc7ec40 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 13:18:21 +0200 Subject: [PATCH 05/97] Add name and description to translations --- mobility_data/translation.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mobility_data/translation.py b/mobility_data/translation.py index 294cce2bb..97497fe36 100644 --- a/mobility_data/translation.py +++ b/mobility_data/translation.py @@ -1,6 +1,6 @@ from modeltranslation.translator import TranslationOptions, translator -from mobility_data.models import MobileUnit, MobileUnitGroup +from mobility_data.models import ContentType, MobileUnit, MobileUnitGroup class MobileUnitGroupTranslationOptions(TranslationOptions): @@ -15,3 +15,10 @@ class MobileUnitTranslationOptions(TranslationOptions): translator.register(MobileUnit, MobileUnitTranslationOptions) + + +class ContentTypeTranslationOptions(TranslationOptions): + fields = ("name", "description") + + +translator.register(ContentType, ContentTypeTranslationOptions) From ba26fe9ef5132b45d965f5d27b5d44bedc3ec954 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 13:19:23 +0200 Subject: [PATCH 06/97] Filter type_name field of ContentType --- mobility_data/api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py index 580710ed7..652ed3a65 100644 --- a/mobility_data/api/views.py +++ b/mobility_data/api/views.py @@ -194,11 +194,11 @@ def list(self, request): # TODO, remove when front end is updated. if type_name in type_name_mappings: type_name = type_name_mappings[type_name] - if not ContentType.objects.filter(name=type_name).exists(): + if not ContentType.objects.filter(type_name=type_name).exists(): return Response( "type_name does not exist.", status=status.HTTP_400_BAD_REQUEST ) - queryset = MobileUnit.objects.filter(content_types__name=type_name) + queryset = MobileUnit.objects.filter(content_types__type_name=type_name) # If the data locates in the services_unit table (i.e., MobileUnit has a unit_id) # get the unit_ids to retrieve the Units for filtering(bbox and extra) unit_ids = list( From c0676d1a6c01e066b96a2fa63781fdd20ae9dcc2 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 13:20:04 +0200 Subject: [PATCH 07/97] Add multilingual name and description fields --- mobility_data/api/serializers/content_type.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mobility_data/api/serializers/content_type.py b/mobility_data/api/serializers/content_type.py index c9fd91c85..61adbe208 100644 --- a/mobility_data/api/serializers/content_type.py +++ b/mobility_data/api/serializers/content_type.py @@ -6,4 +6,13 @@ class ContentTypeSerializer(serializers.ModelSerializer): class Meta: model = ContentType - fields = ["id", "name", "description"] + fields = [ + "id", + "type_name", + "name", + "name_sv", + "name_en", + "description", + "description_sv", + "description_en", + ] From 8f80c0a324a5f0fc2f33305b5319f774bfe4e010 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 13:23:03 +0200 Subject: [PATCH 08/97] Create ContentType from config, remove custom ContentType creation function --- mobility_data/importers/bicycle_stands.py | 11 ++------ .../importers/bike_service_stations.py | 11 ++------ mobility_data/importers/charging_stations.py | 11 ++------ .../disabled_and_no_staff_parking.py | 28 +++++-------------- .../importers/foli_parkandride_stop.py | 25 ++--------------- mobility_data/importers/foli_stops.py | 15 ++++------ .../importers/gas_filling_station.py | 11 ++------ .../importers/loading_unloading_places.py | 11 ++------ 8 files changed, 25 insertions(+), 98 deletions(-) diff --git a/mobility_data/importers/bicycle_stands.py b/mobility_data/importers/bicycle_stands.py index 967721b49..e60ab6b3d 100644 --- a/mobility_data/importers/bicycle_stands.py +++ b/mobility_data/importers/bicycle_stands.py @@ -22,7 +22,7 @@ delete_mobile_units, get_closest_address_full_name, get_municipality_name, - get_or_create_content_type, + get_or_create_content_type_from_config, get_root_dir, get_street_name_translations, locates_in_turku, @@ -260,19 +260,12 @@ def delete_bicycle_stands(): delete_mobile_units(CONTENT_TYPE_NAME) -@db.transaction.atomic -def create_bicycle_stand_content_type(): - description = "Bicycle stands in The Turku Region." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_bicycle_stands() - content_type = create_bicycle_stand_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: mobile_unit = MobileUnit.objects.create( extra=object.extra, diff --git a/mobility_data/importers/bike_service_stations.py b/mobility_data/importers/bike_service_stations.py index f7d00ad9a..13fbee6ba 100644 --- a/mobility_data/importers/bike_service_stations.py +++ b/mobility_data/importers/bike_service_stations.py @@ -8,7 +8,7 @@ from mobility_data.importers.utils import ( delete_mobile_units, get_file_name_from_data_source, - get_or_create_content_type, + get_or_create_content_type_from_config, get_root_dir, get_street_name_translations, set_translated_field, @@ -95,19 +95,12 @@ def delete_bike_service_stations(): delete_mobile_units(CONTENT_TYPE_NAME) -@db.transaction.atomic -def create_bike_service_station_content_type(): - description = "Bike service stations in the Turku region." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_bike_service_stations() - content_type = create_bike_service_station_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: mobile_unit = MobileUnit.objects.create( extra=object.extra, diff --git a/mobility_data/importers/charging_stations.py b/mobility_data/importers/charging_stations.py index be6a5f36b..51adab8d7 100644 --- a/mobility_data/importers/charging_stations.py +++ b/mobility_data/importers/charging_stations.py @@ -11,7 +11,7 @@ delete_mobile_units, get_file_name_from_data_source, get_municipality_name, - get_or_create_content_type, + get_or_create_content_type_from_config, get_postal_code, get_root_dir, get_street_name_translations, @@ -168,18 +168,11 @@ def delete_charging_stations(): delete_mobile_units(CONTENT_TYPE_NAME) -@db.transaction.atomic -def create_charging_station_content_type(): - description = "Charging stations in province of Southwest Finland." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_charging_stations() - content_type = create_charging_station_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: is_active = object.is_active diff --git a/mobility_data/importers/disabled_and_no_staff_parking.py b/mobility_data/importers/disabled_and_no_staff_parking.py index ff9cfdb0e..2c25b0a8b 100644 --- a/mobility_data/importers/disabled_and_no_staff_parking.py +++ b/mobility_data/importers/disabled_and_no_staff_parking.py @@ -10,7 +10,7 @@ delete_mobile_units, FieldTypes, get_file_name_from_data_source, - get_or_create_content_type, + get_or_create_content_type_from_config, get_root_dir, set_translated_field, ) @@ -168,32 +168,18 @@ def delete_disabled_parkings(): delete_mobile_units(DISABLED_PARKING_CONTENT_TYPE_NAME) -@db.transaction.atomic -def get_and_create_no_staff_parking_content_type(): - description = "No staff parkings in the Turku region." - content_type, _ = get_or_create_content_type( - NO_STAFF_PARKING_CONTENT_TYPE_NAME, description - ) - return content_type - - -@db.transaction.atomic -def get_and_create_disabled_parking_content_type(): - description = "Parkings for disabled in the Turku region." - content_type, _ = get_or_create_content_type( - DISABLED_PARKING_CONTENT_TYPE_NAME, description - ) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_no_staff_parkings() delete_disabled_parkings() - no_staff_parking_content_type = get_and_create_no_staff_parking_content_type() - disabled_parking_content_type = get_and_create_disabled_parking_content_type() + no_staff_parking_content_type = get_or_create_content_type_from_config( + NO_STAFF_PARKING_CONTENT_TYPE_NAME + ) + disabled_parking_content_type = get_or_create_content_type_from_config( + DISABLED_PARKING_CONTENT_TYPE_NAME + ) for object in objects: mobile_unit = MobileUnit.objects.create( diff --git a/mobility_data/importers/foli_parkandride_stop.py b/mobility_data/importers/foli_parkandride_stop.py index b1ff60df4..ac912f6fc 100644 --- a/mobility_data/importers/foli_parkandride_stop.py +++ b/mobility_data/importers/foli_parkandride_stop.py @@ -8,7 +8,7 @@ from .utils import ( delete_mobile_units, fetch_json, - get_or_create_content_type, + get_or_create_content_type_from_config, set_translated_field, ) @@ -61,24 +61,6 @@ def get_parkandride_stop_objects(): return car_stops, bike_stops -@db.transaction.atomic -def get_and_create_foli_parkandride_bike_stop_content_type(): - description = "Föli park and ride bike stop." - content_type, _ = get_or_create_content_type( - FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME, description - ) - return content_type - - -@db.transaction.atomic -def get_and_create_foli_parkandride_car_stop_content_type(): - description = "Föli park and ride car stop." - content_type, _ = get_or_create_content_type( - FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME, description - ) - return content_type - - @db.transaction.atomic def save_to_database(objects, content_type_name, delete_tables=True): assert ( @@ -88,10 +70,7 @@ def save_to_database(objects, content_type_name, delete_tables=True): if delete_tables: delete_mobile_units(content_type_name) - if content_type_name == FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME: - content_type = get_and_create_foli_parkandride_bike_stop_content_type() - elif content_type_name == FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME: - content_type = get_and_create_foli_parkandride_car_stop_content_type() + content_type = get_or_create_content_type_from_config(content_type_name) for object in objects: mobile_unit = MobileUnit.objects.create( diff --git a/mobility_data/importers/foli_stops.py b/mobility_data/importers/foli_stops.py index a86a25cef..8b5f02ca0 100644 --- a/mobility_data/importers/foli_stops.py +++ b/mobility_data/importers/foli_stops.py @@ -6,7 +6,11 @@ from mobility_data.models import MobileUnit -from .utils import delete_mobile_units, fetch_json, get_or_create_content_type +from .utils import ( + delete_mobile_units, + fetch_json, + get_or_create_content_type_from_config, +) URL = "http://data.foli.fi/gtfs/stops" CONTENT_TYPE_NAME = "FoliStop" @@ -32,19 +36,12 @@ def get_foli_stops(): return [FoliStop(json_data[stop_code]) for stop_code in json_data] -@db.transaction.atomic -def get_and_create_foli_stop_content_type(): - description = "Föli stops." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_mobile_units(CONTENT_TYPE_NAME) - content_type = get_and_create_foli_stop_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: mobile_unit = MobileUnit.objects.create( name=object.name, diff --git a/mobility_data/importers/gas_filling_station.py b/mobility_data/importers/gas_filling_station.py index 40430042b..15bbfcdd5 100644 --- a/mobility_data/importers/gas_filling_station.py +++ b/mobility_data/importers/gas_filling_station.py @@ -11,7 +11,7 @@ from .utils import ( delete_mobile_units, fetch_json, - get_or_create_content_type, + get_or_create_content_type_from_config, get_street_name_and_number, get_street_name_translations, LANGUAGES, @@ -96,19 +96,12 @@ def delete_gas_filling_stations(): delete_mobile_units(CONTENT_TYPE_NAME) -@db.transaction.atomic -def create_gas_filling_station_content_type(): - description = "Gas filling stations in province of Southwest Finland." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_gas_filling_stations() - content_type = create_gas_filling_station_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: is_active = object.is_active mobile_unit = MobileUnit.objects.create( diff --git a/mobility_data/importers/loading_unloading_places.py b/mobility_data/importers/loading_unloading_places.py index b45988ba0..9e58a8155 100644 --- a/mobility_data/importers/loading_unloading_places.py +++ b/mobility_data/importers/loading_unloading_places.py @@ -11,7 +11,7 @@ delete_mobile_units, FieldTypes, get_file_name_from_data_source, - get_or_create_content_type, + get_or_create_content_type_from_config, get_root_dir, set_translated_field, ) @@ -136,19 +136,12 @@ def delete_loading_and_unloading_places(): delete_mobile_units(CONTENT_TYPE_NAME) -@db.transaction.atomic -def get_and_create_loading_and_unloading_place_content_type(): - description = "Loading and uloading places in the Turku region." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_loading_and_unloading_places() - content_type = get_and_create_loading_and_unloading_place_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: mobile_unit = MobileUnit.objects.create( extra=object.extra, From 9da81862f72e005fb213178d193625e3f3eca61b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 13:25:23 +0200 Subject: [PATCH 09/97] Test created ContentType field. change name to type_name --- .../test_import_bike_service_stations.py | 22 +++++++++++++++++-- .../tests/test_import_charging_stations.py | 20 +++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/mobility_data/tests/test_import_bike_service_stations.py b/mobility_data/tests/test_import_bike_service_stations.py index 307e61af2..70fe149bd 100644 --- a/mobility_data/tests/test_import_bike_service_stations.py +++ b/mobility_data/tests/test_import_bike_service_stations.py @@ -1,6 +1,7 @@ import pytest from mobility_data.importers.bike_service_stations import CONTENT_TYPE_NAME +from mobility_data.importers.utils import get_content_type_config from mobility_data.models import ContentType, MobileUnit from .utils import import_command @@ -11,8 +12,16 @@ def test_import_bike_service_stations(): import_command( "import_bike_service_stations", test_mode="bike_service_stations.geojson" ) - assert ContentType.objects.filter(name=CONTENT_TYPE_NAME).count() == 1 - assert MobileUnit.objects.filter(content_types__name=CONTENT_TYPE_NAME).count() == 3 + assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 + assert ( + MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() + == 3 + ) + config = get_content_type_config(CONTENT_TYPE_NAME) + content_type = ContentType.objects.get(type_name=CONTENT_TYPE_NAME) + content_type.name_fi = config["name"]["fi"] + content_type.name_sv = config["name"]["sv"] + content_type.name_en = config["name"]["en"] kupittaankentta = MobileUnit.objects.get(name="Kupittaankenttä") assert kupittaankentta.name_sv == "Kuppisplan" assert kupittaankentta.name_en == "Kupittaa court" @@ -31,3 +40,12 @@ def test_import_bike_service_stations(): assert roola.name_sv == "Röölä" assert roola.name_en == "Röölä" assert roola.extra["in_terrain"] == "Kyllä" + # Test that dublicates are not created + import_command( + "import_bike_service_stations", test_mode="bike_service_stations.geojson" + ) + assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 + assert ( + MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() + == 3 + ) diff --git a/mobility_data/tests/test_import_charging_stations.py b/mobility_data/tests/test_import_charging_stations.py index a6098fefc..6efb617cc 100644 --- a/mobility_data/tests/test_import_charging_stations.py +++ b/mobility_data/tests/test_import_charging_stations.py @@ -5,6 +5,7 @@ CHARGING_STATION_SERVICE_NAMES, CONTENT_TYPE_NAME, ) +from mobility_data.importers.utils import get_content_type_config from mobility_data.models import ContentType, MobileUnit from .utils import import_command @@ -20,8 +21,16 @@ def test_import_charging_stations( address, ): import_command("import_charging_stations", test_mode="charging_stations.csv") - assert ContentType.objects.filter(name=CONTENT_TYPE_NAME).count() == 1 - assert MobileUnit.objects.filter(content_types__name=CONTENT_TYPE_NAME).count() == 3 + assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 + assert ( + MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() + == 3 + ) + config = get_content_type_config(CONTENT_TYPE_NAME) + content_type = ContentType.objects.get(type_name=CONTENT_TYPE_NAME) + content_type.name_fi = config["name"]["fi"] + content_type.name_sv = config["name"]["sv"] + content_type.name_en = config["name"]["en"] aimopark = MobileUnit.objects.get(name="Aimopark, Yliopistonkatu 29") assert aimopark.address == "Yliopistonkatu 29" assert aimopark.address_sv == "Universitetsgatan 29" @@ -54,3 +63,10 @@ def test_import_charging_stations( ratapihankatu.name_en == f"{CHARGING_STATION_SERVICE_NAMES['en']}, Ratapihankatu 53" ) + # Test that dublicates are not created + import_command("import_charging_stations", test_mode="charging_stations.csv") + assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 + assert ( + MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() + == 3 + ) From 391130115a7bf710144c036d93221cdbab07fe00 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 13:25:57 +0200 Subject: [PATCH 10/97] Change ContentType name field to type_name --- .../test_import_disabled_and_no_staff_parkings.py | 6 +++--- .../tests/test_import_foli_parkandride_stops.py | 10 ++++++++-- mobility_data/tests/test_import_foli_stops.py | 2 +- .../tests/test_import_gas_filling_stations.py | 7 +++++-- .../tests/test_import_loading_and_unloading_places.py | 2 +- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/mobility_data/tests/test_import_disabled_and_no_staff_parkings.py b/mobility_data/tests/test_import_disabled_and_no_staff_parkings.py index e5120173a..c56527a78 100644 --- a/mobility_data/tests/test_import_disabled_and_no_staff_parkings.py +++ b/mobility_data/tests/test_import_disabled_and_no_staff_parkings.py @@ -25,7 +25,7 @@ def test_geojson_import(municipalities): kupittaan_maauimala = MobileUnit.objects.get(name="Kupittaan maauimala") assert kupittaan_maauimala.content_types.all().count() == 1 assert ( - kupittaan_maauimala.content_types.first().name + kupittaan_maauimala.content_types.first().type_name == DISABLED_PARKING_CONTENT_TYPE_NAME ) assert kupittaan_maauimala @@ -42,7 +42,7 @@ def test_geojson_import(municipalities): assert kupittaan_maauimala.extra["rajoitustyyppi"]["en"] == "Special area" kupittaan_seikkailupuisto = MobileUnit.objects.get(name="Kupittaan seikkailupuisto") assert ( - kupittaan_seikkailupuisto.content_types.first().name + kupittaan_seikkailupuisto.content_types.first().type_name == NO_STAFF_PARKING_CONTENT_TYPE_NAME ) assert kupittaan_seikkailupuisto @@ -55,7 +55,7 @@ def test_geojson_import(municipalities): kupittaan_urheiluhalli = MobileUnit.objects.get(name="Kupittaan urheiluhalli") assert kupittaan_urheiluhalli assert ( - kupittaan_urheiluhalli.content_types.first().name + kupittaan_urheiluhalli.content_types.first().type_name == NO_STAFF_PARKING_CONTENT_TYPE_NAME ) assert kupittaan_urheiluhalli.name_en == "Kupittaa sports hall" diff --git a/mobility_data/tests/test_import_foli_parkandride_stops.py b/mobility_data/tests/test_import_foli_parkandride_stops.py index fde5640ff..09d70079f 100644 --- a/mobility_data/tests/test_import_foli_parkandride_stops.py +++ b/mobility_data/tests/test_import_foli_parkandride_stops.py @@ -2,6 +2,7 @@ import pytest +from mobility_data.importers.utils import get_content_type_config from mobility_data.models import ContentType, MobileUnit from .utils import get_test_fixture_json_data @@ -25,10 +26,15 @@ def test_import_foli_stops(fetch_json_mock, municipalities): save_to_database(bike_stops, FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME) cars_stops_content_type = ContentType.objects.get( - name=FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME + type_name=FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME ) + config = get_content_type_config(FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME) + cars_stops_content_type.name_fi = config["name"]["fi"] + cars_stops_content_type.name_sv = config["name"]["sv"] + cars_stops_content_type.name_en = config["name"]["en"] + bikes_stops_content_type = ContentType.objects.get( - name=FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME + type_name=FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME ) # Fixture data contains two park and ride stops for cars and bikes. assert MobileUnit.objects.filter(content_types=cars_stops_content_type).count() == 2 diff --git a/mobility_data/tests/test_import_foli_stops.py b/mobility_data/tests/test_import_foli_stops.py index 4ebb8f450..b0ba3a4be 100644 --- a/mobility_data/tests/test_import_foli_stops.py +++ b/mobility_data/tests/test_import_foli_stops.py @@ -17,7 +17,7 @@ def test_import_foli_stops(fetch_json_mock): objects = foli_stops.get_foli_stops() foli_stops.save_to_database(objects) assert ContentType.objects.count() == 1 - assert ContentType.objects.first().name == foli_stops.CONTENT_TYPE_NAME + assert ContentType.objects.first().type_name == foli_stops.CONTENT_TYPE_NAME assert MobileUnit.objects.count() == 3 turun_satama = MobileUnit.objects.get(name="Turun satama (Silja)") assert turun_satama.content_types.all().count() == 1 diff --git a/mobility_data/tests/test_import_gas_filling_stations.py b/mobility_data/tests/test_import_gas_filling_stations.py index 29f226206..fa7a31650 100644 --- a/mobility_data/tests/test_import_gas_filling_stations.py +++ b/mobility_data/tests/test_import_gas_filling_stations.py @@ -10,8 +10,11 @@ def test_importer(municipalities): import_command("import_gas_filling_stations", test_mode="gas_filling_stations.json") - assert ContentType.objects.filter(name=CONTENT_TYPE_NAME).count() == 1 - assert MobileUnit.objects.filter(content_types__name=CONTENT_TYPE_NAME).count() == 2 + assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 + assert ( + MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() + == 2 + ) assert MobileUnit.objects.get(name="Raisio Kuninkoja") unit = MobileUnit.objects.get(name="Turku Satama") assert unit.address == "Tuontiväylä 42 abc 1-2" diff --git a/mobility_data/tests/test_import_loading_and_unloading_places.py b/mobility_data/tests/test_import_loading_and_unloading_places.py index 5aeb10674..519f8f5dc 100644 --- a/mobility_data/tests/test_import_loading_and_unloading_places.py +++ b/mobility_data/tests/test_import_loading_and_unloading_places.py @@ -23,7 +23,7 @@ def test_import(municipalities): assert turku_muni lantinen_rantakatu = MobileUnit.objects.get(name="Läntinen Rantakatu") assert lantinen_rantakatu.content_types.all().count() == 1 - assert lantinen_rantakatu.content_types.first().name == CONTENT_TYPE_NAME + assert lantinen_rantakatu.content_types.first().type_name == CONTENT_TYPE_NAME assert lantinen_rantakatu.name_sv == "Östra Strandgatan" assert lantinen_rantakatu.name_en == "Läntinen Rantakatu" assert lantinen_rantakatu.address_fi == "Läntinen Rantakatu 13" From ef2d2f8cf85593f15bd9cb019edf5f54345a18ae Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 14:22:50 +0200 Subject: [PATCH 11/97] Add boating, outdoor gym and parking --- .../importers/data/content_types.yml | 64 +++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index add901ade..b01f66cb2 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -4,51 +4,61 @@ content_types: fi: Pyöränkorjauspiste sv: Cykelservicestation en: Bike service stations + - content_type_name: BicycleStand name: fi: Pyöräpysäköinti sv: Cykelparkering en: Bicycle parking + - content_type_name: ChargingStation name: fi: Sähkölatauspiste sv: Elladningsstation för bilar en: Car e-charging point + - content_type_name: NoStaffParking name: fi: Yleiset pysäköintialueet sv: Allmänna parkeringsplatser en: Public parking spaces + - content_type_name: DisabledParking name: fi: Liikkumisesteisen pysäköinti sv: Parkering för rörelsehindrade en: Parking for disabled + - content_type_name: FoliParkAndRideCarsStop name: fi: Föli auto liityntäpysäköinti sv: Föli bil infartsparkering en: Föli car park-and-ride stop + - content_type_name: FoliParkAndRideBikesStop name: fi: Föli pyörä liityntäpysäköinti sv: Föli cykel infartsparkering en: Föli bicycle park-and-ride stop + - content_type_name: FoliStop name: fi: Föli pysäkki sv: Föli busshållplats en: Föli bus stop + - content_type_name: GasFillingStation name: fi: Kaasutankkausasema sv: Tankstation med gas en: Gas filling station + - content_type_name: LoadingUnloadingPlace name: fi: Kourmauspaikka sv: Lastningsplats en: Loading place + # Content types from lounaistieto shapefiles importer - content_type_name: BusStopSouthwestFinland name: @@ -59,6 +69,7 @@ content_types: fi: Bussipysäkkejä Varsinais-Suomessa. sv: Busshållplatser i Egentliga Finland. en: Bus stops in Southwest Finland. + - content_type_name: FerryDock name: fi: Lossi laituri @@ -68,15 +79,17 @@ content_types: fi: Lossi laitureita Varsinais-Suomessa. sv: Färjelägen i Egentliga Finland. en: Ferry docks in Southwest Finland. + - content_type_name: CommonFerryRoute name: - fi: Yhteysalusreitti. - sv: Förbindelsefartygsrutt. - en: Common ferry route. + fi: Yhteysalusreitti + sv: Förbindelsefartygsrutt + en: Common ferry route description: fi: Yhteysalusreittejä Varsinais-Suomessa. sv: Förbindelsefartygsrutter i Egentliga Finland. en: Common ferry routes in Southwest Finland. + - content_type_name: FishingSpot name: fi: Kalastuspaikka @@ -87,11 +100,17 @@ content_types: fi: Kalastuspaikkoja Varsinais-Suomessa. sv: Fiskeplatser i Egentliga Finland. en: Fishing spots in Southwest Finland. + - content_type_name: FerryRoute name: + fi: Lossi reitti + sv: Färjerutte + en: Ferry route + description: fi: Lossi reittejä Varsinais-Suomessa. sv: Färjerutter i Egentliga Finland. en: Ferry routes(yhteysalusreitti) in Southwest Finland. + - content_type_name: SlipwaySouthwestFinland name: fi: Veneenlaskupaikka @@ -101,6 +120,7 @@ content_types: fi: Veneenlaskupaikkoja Varsinais-Suomessa. sv: Sjösättningsplatser för båtar i Egentliga Finland. en: Slipways for boats in Southwest Finland. + - content_type_name: RecreationalRoute name: fi: Virkistysreitti @@ -110,4 +130,40 @@ content_types: fi: Virkistysreittejä Varsinais-Suomessa. sv: Rekreationsrutter i Egentliga Finland en: Recreational routes in Southwest Finland - # End of lounaistieto shapefile importer content types \ No newline at end of file + # End of lounaistieto shapefile importer content types + + - content_type_name: GuestMarina + name: + fi: Vierasvenesatama + sv: Gästhamn + en: Guest harbour + + - content_type_name: BoatParking + name: + fi: Lyhytaikainen veneparkki + sv: Båtparkering + en: Boat parking + + - content_type_name: Marina + name: + fi: Veneparkki + sv: Båtplatser + en: Marina berths + + - content_type_name: OutdoorGymDevice + name: + fi: Ulkoliikuntalaite + sv: Utomhus tränigsredskap + en: Outdoor gym device + + - content_type_name: ParkingMachine + name: + fi: Pysäköintiautomaatti + sv: Parkeringsautomat + en: Parking machine + + - content_type_name: ShareCarParkingPlace + name: + fi: Yhteiskäyttöautojen pysäköintipaikka + sv: Bilpoolbilars parkeringsplats + en: Parking place for car sharing cars \ No newline at end of file From 782ea2c389f3a81a5ee9a858d00a202869955be4 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 14:23:42 +0200 Subject: [PATCH 12/97] Remove obsolete content_type_description --- .../importers/data/lounaistieto_shapefiles_config.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mobility_data/importers/data/lounaistieto_shapefiles_config.yml b/mobility_data/importers/data/lounaistieto_shapefiles_config.yml index 022bbf198..b1e3b0b0d 100644 --- a/mobility_data/importers/data/lounaistieto_shapefiles_config.yml +++ b/mobility_data/importers/data/lounaistieto_shapefiles_config.yml @@ -1,6 +1,5 @@ data_sources: - content_type_name: BusStopSouthwestFinland - content_type_description: "Bus stops in Southwest Finland." data_url: "https://data.lounaistieto.fi/data/dataset/ee440090-7303-4453-8639-b5f711669acb/resource/14060749-5215-41af-9312-5393c966f987/download/tieverkkodata.zip/bussipysakit.shp" fields: name: @@ -19,7 +18,6 @@ data_sources: pystunn196: pystunn196 - content_type_name: FerryDock - content_type_description: "Ferry docks in Southwest Finland." data_url: "https://data.lounaistieto.fi/data/dataset/ea4dcae2-5832-403c-bf7e-b19783ee9a70/resource/faa02c52-2da8-4603-b8bf-4afca42d39a5/download/yhteysalusreitit_laiturit.zip/Laiturit.shp" encoding: latin_1 filter_by_southwest_finland: True @@ -34,7 +32,6 @@ data_sources: reittialue: REITTIALUE - content_type_name: CommonFerryRoute - content_type_description: "Common ferry routes(yhteysalusreitti) in Southwest Finland." data_url: "https://data.lounaistieto.fi/data/dataset/ea4dcae2-5832-403c-bf7e-b19783ee9a70/resource/faa02c52-2da8-4603-b8bf-4afca42d39a5/download/yhteysalusreitit_laiturit.zip/Yhteysalusreitit.shp" encoding: latin_1 filter_by_southwest_finland: True @@ -47,7 +44,6 @@ data_sources: yhteysalus: Yhteysalus - content_type_name: FishingSpot - content_type_description: "Fishing spots in Soutwest Finland." data_url: 'https://data.lounaistieto.fi/data/dataset/50451034-b760-4b39-8f88-701d9c968d88/resource/cd41b91f-f684-4cdc-bbf8-e2c4eaca3a70/download/kalapaikat.zip' srid: 3067 encoding: latin_1 @@ -73,7 +69,6 @@ data_sources: tietod: Tiedot - content_type_name: FerryRoute - content_type_description: "Ferry routes in Southwest Finland" data_url: 'https://data.lounaistieto.fi/data/dataset/4809cdd7-7d6d-46f7-86b1-5d6f3fbd97aa/resource/a3905406-cb79-465d-927c-1d6b53c0385e/download/lossireitit_euref.zip' srid: 3067 encoding: latin_1 @@ -87,7 +82,6 @@ data_sources: pituus: PITUUS - content_type_name: MarinaSouthwestFinland - content_type_description: "Marinas in Southwest Finland, the data contains marinas, moorings, connection piers" data_url: 'https://data.lounaistieto.fi/data/dataset/6069c4d5-cbed-4e32-9f21-502688f26192/resource/214e5337-8232-4e02-8324-81285b1a1024/download/venesatamat_kaikki.zip' srid: 3067 encoding: utf-8 @@ -109,7 +103,6 @@ data_sources: www: www - content_type_name: "SlipwaySouthwestFinland" - content_type_description: "Slipways for boats." data_url: 'https://data.lounaistieto.fi/data/dataset/84116aeb-4ced-48ca-9c93-58edb946ab96/resource/2bf2497e-4ed0-4b38-b1e9-ecd1bbb28856/download/venerampit.zip' srid: 3067 encoding: latin_1 @@ -123,7 +116,6 @@ data_sources: päiv_aika: PÄIV_AIKA - content_type_name: RecreationalRoute - content_type_description: 'Recreational routes in Southwest Finland' data_url: 'https://data.lounaistieto.fi/data/dataset/3e92c2b6-7b62-49d1-b2fd-ff4917208f2a/resource/f20f006f-7592-45a8-9698-7c6fa09af068/download/virkistysaineisto.zip' srid: 3067 encoding: latin_1 From 7bafabfa942bd31c1d2cacf441559d5025c88a1b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 14:24:20 +0200 Subject: [PATCH 13/97] Filter with field type_name --- mobility_data/tests/test_import_parking_machines.py | 2 +- mobility_data/tests/test_import_share_car_parking_places.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mobility_data/tests/test_import_parking_machines.py b/mobility_data/tests/test_import_parking_machines.py index 3c69e1c1a..7c835669d 100644 --- a/mobility_data/tests/test_import_parking_machines.py +++ b/mobility_data/tests/test_import_parking_machines.py @@ -17,7 +17,7 @@ def test_import_parking_machines(get_json_data_mock): ) objects = parking_machines.get_parking_machine_objects() parking_machines.save_to_database(objects) - assert ContentType.objects.first().name == parking_machines.CONTENT_TYPE_NAME + assert ContentType.objects.first().type_name == parking_machines.CONTENT_TYPE_NAME assert MobileUnit.objects.count() == 3 satamakatu = MobileUnit.objects.first() assert satamakatu.content_types.all().count() == 1 diff --git a/mobility_data/tests/test_import_share_car_parking_places.py b/mobility_data/tests/test_import_share_car_parking_places.py index e3bac978d..3d74acbc9 100644 --- a/mobility_data/tests/test_import_share_car_parking_places.py +++ b/mobility_data/tests/test_import_share_car_parking_places.py @@ -11,8 +11,8 @@ def test_import_car_share_parking_places(): import_command( "import_share_car_parking_places", test_mode="share_car_parking_places.geojson" ) - assert ContentType.objects.filter(name=CONTENT_TYPE_NAME).count() == 1 - assert MobileUnit.objects.filter(content_types__name=CONTENT_TYPE_NAME).count() == 3 + assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 + assert MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() == 3 linnankatu = MobileUnit.objects.get( name="Yhteiskäyttöautojen pysäköintipaikka, Linnankatu 29" ) From 8812e33c7350d0f038375385e551b6e543f347da Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 7 Mar 2023 14:25:01 +0200 Subject: [PATCH 14/97] Create ContentType from config, remove custom ContentType creation function --- .../importers/lounaistieto_shapefiles.py | 15 +------- mobility_data/importers/marinas.py | 38 ++----------------- .../importers/outdoor_gym_devices.py | 12 +----- mobility_data/importers/parking_machines.py | 11 +----- .../importers/share_car_parking_places.py | 11 +----- 5 files changed, 12 insertions(+), 75 deletions(-) diff --git a/mobility_data/importers/lounaistieto_shapefiles.py b/mobility_data/importers/lounaistieto_shapefiles.py index 010d7f349..3987a2994 100644 --- a/mobility_data/importers/lounaistieto_shapefiles.py +++ b/mobility_data/importers/lounaistieto_shapefiles.py @@ -8,7 +8,7 @@ from mobility_data.importers.utils import ( delete_mobile_units, - get_or_create_content_type, + get_or_create_content_type_from_config, set_translated_field, ) from mobility_data.models import MobileUnit @@ -99,17 +99,6 @@ def add_feature(self, feature, config, srid): return True -@db.transaction.atomic -def get_and_create_datasource_content_type(config): - if "content_type_description" in config: - description = config["content_type_description"] - else: - description = "" - name = config["content_type_name"] - ct, _ = get_or_create_content_type(name, description) - return ct - - @db.transaction.atomic def delete_content_type(config): delete_mobile_units(config["content_type_name"]) @@ -117,7 +106,7 @@ def delete_content_type(config): @db.transaction.atomic def save_to_database(objects, config): - content_type = get_and_create_datasource_content_type(config) + content_type = get_or_create_content_type_from_config(config["content_type_name"]) if not content_type: return for object in objects: diff --git a/mobility_data/importers/marinas.py b/mobility_data/importers/marinas.py index 678e51567..88b3a302f 100644 --- a/mobility_data/importers/marinas.py +++ b/mobility_data/importers/marinas.py @@ -13,7 +13,7 @@ from mobility_data.models import MobileUnit from .berths import get_berths -from .utils import delete_mobile_units, get_or_create_content_type +from .utils import delete_mobile_units, get_or_create_content_type_from_config MARINA_URL = "{}{}".format( settings.TURKU_WFS_URL, @@ -47,47 +47,17 @@ def delete_guest_marina(): delete_mobile_units(GUEST_MARINA_CONTENT_TYPE_NAME) -db.transaction.atomic - - -def create_guest_marina_content_type(): - description = "Guest marina in Turku." - content_type, _ = get_or_create_content_type( - GUEST_MARINA_CONTENT_TYPE_NAME, description - ) - return content_type - @db.transaction.atomic def delete_boat_parking(): delete_mobile_units(BOAT_PARKING_CONTENT_TYPE_NAME) -db.transaction.atomic - - -def create_boat_parking_content_type(): - description = "Boat parking in Turku." - content_type, _ = get_or_create_content_type( - BOAT_PARKING_CONTENT_TYPE_NAME, description - ) - return content_type - - @db.transaction.atomic def delete_marinas(): delete_mobile_units(MARINA_CONTENT_TYPE_NAME) -db.transaction.atomic - - -def create_marina_content_type(): - description = "Marinas in the Turku region." - content_type, _ = get_or_create_content_type(MARINA_CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def import_marinas(delete=True): marinas = [] @@ -97,7 +67,7 @@ def import_marinas(delete=True): ds = DataSource(MARINA_URL) for feature in ds[0]: marinas.append(Marina(feature)) - content_type = create_marina_content_type() + content_type = get_or_create_content_type_from_config(MARINA_CONTENT_TYPE_NAME) for marina in marinas: mobile_unit = MobileUnit.objects.create( geometry=marina.geometry, @@ -124,9 +94,9 @@ def import_guest_marina_and_boat_parking(delete=True): type_name = feature["Muu_venesatama"].as_string() content_type = None if type_name == GUEST_MARINA: - content_type = create_guest_marina_content_type() + content_type = get_or_create_content_type_from_config(GUEST_MARINA_CONTENT_TYPE_NAME) elif type_name == BOAT_PARKING: - content_type = create_boat_parking_content_type() + content_type = get_or_create_content_type_from_config(BOAT_PARKING_CONTENT_TYPE_NAME) mobile_unit = MobileUnit.objects.create(geometry=geometry) mobile_unit.content_types.add(content_type) diff --git a/mobility_data/importers/outdoor_gym_devices.py b/mobility_data/importers/outdoor_gym_devices.py index 13bc292aa..ce7f9f84d 100644 --- a/mobility_data/importers/outdoor_gym_devices.py +++ b/mobility_data/importers/outdoor_gym_devices.py @@ -5,20 +5,12 @@ from mobility_data.models import MobileUnit from services.models import Service, Unit -from .utils import delete_mobile_units, get_or_create_content_type +from .utils import delete_mobile_units, get_or_create_content_type_from_config logger = logging.getLogger("mobility_data") SERVICE_NAME = "Outdoor Gym Devices" CONTENT_TYPE_NAME = "OutdoorGymDevice" -db.transaction.atomic - - -def create_content_type(): - description = "Outdoor gym devices in Turku." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - db.transaction.atomic @@ -34,7 +26,7 @@ def save_outdoor_gym_devices(): except Service.DoesNotExist: return 0 - content_type = create_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) units_qs = Unit.objects.filter(services=service) for unit in units_qs: mobile_unit = MobileUnit.objects.create(unit_id=unit.id) diff --git a/mobility_data/importers/parking_machines.py b/mobility_data/importers/parking_machines.py index eaca1c259..917230dd0 100644 --- a/mobility_data/importers/parking_machines.py +++ b/mobility_data/importers/parking_machines.py @@ -8,7 +8,7 @@ delete_mobile_units, FieldTypes, get_file_name_from_data_source, - get_or_create_content_type, + get_or_create_content_type_from_config, get_root_dir, set_translated_field, ) @@ -129,18 +129,11 @@ def get_parking_machine_objects(): return [ParkingMachine(feature) for feature in json_data] -@db.transaction.atomic -def get_and_create_parking_machine_content_type(): - description = "Parking machines in the City of Turku." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_mobile_units(CONTENT_TYPE_NAME) - content_type = get_and_create_parking_machine_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: mobile_unit = MobileUnit.objects.create( geometry=object.geometry, diff --git a/mobility_data/importers/share_car_parking_places.py b/mobility_data/importers/share_car_parking_places.py index 87e25f9d0..31c7b26ab 100644 --- a/mobility_data/importers/share_car_parking_places.py +++ b/mobility_data/importers/share_car_parking_places.py @@ -8,7 +8,7 @@ from mobility_data.importers.utils import ( delete_mobile_units, get_file_name_from_data_source, - get_or_create_content_type, + get_or_create_content_type_from_config, get_root_dir, set_translated_field, ) @@ -82,18 +82,11 @@ def delete_car_share_parking_places(): delete_mobile_units(CONTENT_TYPE_NAME) -@db.transaction.atomic -def create_car_share_parking_place_content_type(): - description = "Car share parking places in the Turku region." - content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description) - return content_type - - @db.transaction.atomic def save_to_database(objects, delete_tables=True): if delete_tables: delete_car_share_parking_places() - content_type = create_car_share_parking_place_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) for object in objects: mobile_unit = MobileUnit.objects.create(extra=object.extra) mobile_unit.content_types.add(content_type) From ede64e73b5132f7500672c5fb6af223ad38c91c3 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 09:31:16 +0200 Subject: [PATCH 15/97] Add traffic signs --- .../importers/data/content_types.yml | 139 +++++++++++++++++- 1 file changed, 138 insertions(+), 1 deletion(-) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index b01f66cb2..bb3fc3cd6 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -166,4 +166,141 @@ content_types: name: fi: Yhteiskäyttöautojen pysäköintipaikka sv: Bilpoolbilars parkeringsplats - en: Parking place for car sharing cars \ No newline at end of file + en: Parking place for car sharing cars + + - content_type_name: TrafficSign + # Umbrella content type for traffic signs + name: + fi: Liikennemerkki + sv: Trafikmärke + en: Traffic sign + # WFS importer content types + - content_type_name: PlayGround + name: + fi: Leikkipaikka + sv: Lekplats + en: Playground + + - content_type_name: BarbecuePlace + name: + fi: Grillipaikka + sv: Grillplats + en: Barbecue place + + - content_type_name: TicketMachineSign + # Liikennemerkki: 990 Lippuautomaatti + name: + fi: Lippuautomaatti + sv: Biljettautomat + en: Ticket machine + + - content_type_name: TaxiStandSign + # Liikennemerkki: 534 Taksiasema + name: + fi: Taksiasema liikennemerkki + sv: Taxistation trafikmärke + en: Taxi stand sign + + - content_type_name: SingleTrackRailwayLevelCrossingSign + # Liikennemerkki: 176 Yksiraiteisen rautatien tasoristeys + name: + fi: Yksiraiteisen rautatien tasoristeys liikennemerkki + sv: Enspårig plankorsning trafikmärke + en: Single track railway level crossing sign + description: + fi: + sv: + en: Single track railway level crossing with booms signs in the city of Turku. + + - content_type_name: RailwayLevelCrossingWithBoomsSign + # Liikennemerkki: 172 Rautatien tasoristeys, jossa on puomit + name: + fi: Rautatieen tasoristeys jossa puomit liikennemerkki + sv: Plankorsning med bummar trafikmärke + en: Railway level crossing with booms sign + + - content_type_name: RailwayLevelCrossingWithoutBoomsSign + # Liikennemerkki: 171 Rautatien tasoristeys ilman puomeja + name: + fi: Rautatien tasoristeys ilman puomeja liikennemerkki + sv: Plankorsning utan bummar trafikmärke + en: Railway level crossing without boom sign + + - content_type_name: ParkingLotSign + # Liikennemerkki: 521 Pysäköintipaikka + name: + fi: Pysäköintipaika liikennemerkki + sv: Parkeringsplats trafikmärke + en: Parking lot sign + + - content_type_name: PaidParkingSign + # Liikennemerkki: 855b Maksullinen pysäköinti + name: + fi: Maksullinen pysäköinti liikennemerkki + sv: Avgiftsbelagd parkerings trafikmärke + en: Paid parking sign + + - content_type_name: ObligationToUseParkingDiscSign + # Liikennemerkki: 856b Pysäköintikiekon käyttövelvollisuus + name: + fi: Pysäköintikiekon käyttövelvollisuus liikennemerkki + sv: Skyldighet att använda parkeringsskiva trafikmärke + en: Obligation to use parking disc sign + + - content_type_name: ParkingForbiddenAreaSign + # Liikennemerkki: 373 Pysäköintikieltoalue + name: + fi: Pysäköintikieltoalue liikennemerkki + sv: Område för parkeringsförbjud trafikmärke + en: Parking forbidden area sign + + - content_type_name: ParkingForbiddenSign + # Liikennemerkki: 372 Pysäköinti kielletty + name: + fi: Pysäköinti kielletty liikennemerkki + sv: Parkering förbjudet trafikmärke + en: Parking forbidden sign + + - content_type_name: LongDistanceBusStopSign + # Liikennemerkki: 532 Kaukoliikenteen linja-auton pysäkki + name: + fi: Kaukoliikenteen linja-auton pysäkki liikennemerkki + sv: Fjärrtrafik busshållplats trafikmärke + en: Long distance bus stop sign + + - content_type_name: LocalTrafficBusStopSign + # Liikennemerkki: 531 Paikallisliikenteen linja-auton pysäkki + name: + fi: Paikallisliikenteen linja-auton pysäkki + sv: Busshållsplats för lokaltrafik trafikmärke + en: Local traffic bus stop sign + + - content_type_name: ParkingTerminalSign + # Liikennemerkki: 991 Pysäköintiautomaatti + name: + fi: Pysäköintiautomaatti liikennemerkki + sv: Parkeringsautomat trafikmärke + en: Parking terminal sign + + - content_type_name: CrossWalkSign + # Liikennemerkki: 511 Suojatie + name: + fi: Suojatie liikennemerkki + sv: Skyddsväg trafikmärke + en: Crosswalk sign + + - content_type_name: RouteForDisabledSign + # Liikennemerkki: 683 Vammaisille tarkoitettu reitti + name: + fi: Liikkkumisesteiselle tarkoitettu reitti liikennemerkki + sv: Rutt för röreslehindrade trafikmärke + en: Route for disabled sign + + - content_type_name: DisabledParkingSign + # Liikennemerkki: 836 Invalidin ajoneuvo + name: + fi: Liikkumisesteisen pysäköinti liikennemerkki + sv: Parkering för rörelsehindrade trafikmärke + en: Disabled parking sign + + # End of WFS importer content types \ No newline at end of file From 5514fa8fa72a22fd8bff54a93d48e6ef4be7faa1 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 12:41:52 +0200 Subject: [PATCH 16/97] Add WFS importer content types --- .../importers/data/content_types.yml | 97 ++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index bb3fc3cd6..32b090d66 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -302,5 +302,100 @@ content_types: fi: Liikkumisesteisen pysäköinti liikennemerkki sv: Parkering för rörelsehindrade trafikmärke en: Disabled parking sign - + + - content_type_name: PaddlingTrail + name: + fi: Melontareitti + sv: Paddlingrutt + en: Paddling trail + + - content_type_name: HikingTrail + name: + fi: Retkeilyreitti + sv: Vandringsrutt + en: Hiking trail + + - content_type_name: NatureTrail + name: + fi: Luontopolku + sv: Naturstig + en: Nature trail + + - content_type_name: FitnessTrail + name: + fi: Kuntoilureitti + sv: Konditionsrutt + en: Fitness trail + + - content_type_name: PaavonPolku + name: + fi: Paavon polku + sv: Paavos stig + en: Paavo trail + + - content_type_name: PaymentZone + name: + fi: Pysäköinnin maksuvyöhyke + sv: Parkeringsavgiftszon + en: Parking charging zone + + - content_type_name: ScooterParkingArea + name: + fi: Sähköpotkulaudan pysäköintipaikka + sv: Elsparkcykelns parkeringsplats + en: Electric scooter parking place + + - content_type_name: ScooterSpeedLimitArea + name: + fi: Sähköpotkulaudan nopeusrajoitusalue + sv: Elsparkscykelns hastighetsbegränsningszon + en: Electric scooter speed limit zone + + - content_type_name: ScooterNoParkingArea + name: + fi: Sähköpotkulaudan pysäköintikieltoalue + sv: Elsparkscykelns parkeringsförbudszon + en: Electric scooter no parking zone + + - content_type_name: PublicToilet + name: + fi: Yleinen vessa + sv: Allmän toalet + en: Public restroom + + - content_type_name: PublicTable + name: + fi: Yleinen pöytä + sv: Allmän bord + eb: Public table + + - content_type_name: PublicBench + name: + fi: Yleinen penkki + sv: Allmän bänk + en: Public bench + + - content_type_name: PublicFurnitureGroup + name: + fi: Yleinen kalusteryhmä + sv: Allmän möbelgrupp + en: Public furniture group + + - content_type_name: BrushSaltedBicycleNetwork + name: + fi: Harjasuolattu pyörätie + sv: Sopsaltad cykelväg + en: Brush salted bicycle road + + - content_type_name: BrushSandedBicycleNetwork + name: + fi: Harjahiekoitettava pyörätie + sv: Sopsandad cykelväg + en: Brush sanded bicycle road + + - content_type_name: SpeedLimitZone + name: + fi: Nopeusrajoitusalue + sv: Hastighetsbegränsningszon + en: Speed limit zone # End of WFS importer content types \ No newline at end of file From a69e0f1c3d2d4038d4046f35a35b3921a0dd4caf Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 12:42:54 +0200 Subject: [PATCH 17/97] Test multilingual ContentType name, change name to type_name --- mobility_data/tests/test_api.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index a8f20b136..7861f862b 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -11,7 +11,10 @@ def test_content_type(api_client, content_types): assert response.status_code == 200 assert len(response.json()["results"]) == len(content_types) result = response.json()["results"][0] - assert result["name"] == "Test" + assert result["type_name"] == "Test" + assert result["name"] == "fi" + assert result["name_sv"] == "sv" + assert result["name_en"] == "en" assert result["description"] == "test content type" @@ -21,7 +24,7 @@ def test_group_type(api_client, group_type): response = api_client.get(url) assert response.status_code == 200 result = response.json()["results"][0] - assert result["name"] == "TestGroup" + assert result["type_name"] == "TestGroup" assert result["description"] == "test group type" @@ -56,8 +59,8 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): # Test multiple content types result = response.json() assert len(result["content_types"]) == 2 - assert result["content_types"][0]["name"] == "Test" - assert result["content_types"][1]["name"] == "Test2" + assert result["content_types"][0]["type_name"] == "Test" + assert result["content_types"][1]["type_name"] == "Test2" # Test string in extra field url = ( reverse("mobility_data:mobile_units-list") @@ -115,7 +118,7 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): result = response.json() assert result["name"] == "Test unit" assert result["description"] == "desc" - assert result["content_types"][0]["name"] == "Test unit" + assert result["content_types"][0]["type_name"] == "Test unit" assert result["geometry"] == "POINT (24.24 62.22)" assert result["geometry_coords"]["lon"] == 24.24 assert result["geometry_coords"]["lat"] == 62.22 From 8f80eea00c4519d37bfc526b01f737b9ddf42b6f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 13:40:17 +0200 Subject: [PATCH 18/97] Create ContentType from config --- smbackend_turku/importers/bicycle_stands.py | 5 +++-- smbackend_turku/importers/bike_service_stations.py | 5 +++-- smbackend_turku/importers/stations.py | 13 +++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/smbackend_turku/importers/bicycle_stands.py b/smbackend_turku/importers/bicycle_stands.py index 0136d6704..f53ce8bf8 100644 --- a/smbackend_turku/importers/bicycle_stands.py +++ b/smbackend_turku/importers/bicycle_stands.py @@ -1,7 +1,8 @@ from mobility_data.importers.bicycle_stands import ( - create_bicycle_stand_content_type, + CONTENT_TYPE_NAME, get_bicycle_stand_objects, ) +from mobility_data.importers.utils import get_or_create_content_type_from_config from smbackend_turku.importers.utils import BaseExternalSource @@ -13,7 +14,7 @@ def __init__(self, logger=None, config=None, test_data=None): def import_bicycle_stands(self): self.logger.info("Importing Bicycle Stands...") - content_type = create_bicycle_stand_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) filtered_objects = get_bicycle_stand_objects() super().save_objects_as_units(filtered_objects, content_type) diff --git a/smbackend_turku/importers/bike_service_stations.py b/smbackend_turku/importers/bike_service_stations.py index ad468fc41..4c2bec58b 100644 --- a/smbackend_turku/importers/bike_service_stations.py +++ b/smbackend_turku/importers/bike_service_stations.py @@ -1,7 +1,8 @@ from mobility_data.importers.bike_service_stations import ( - create_bike_service_station_content_type, + CONTENT_TYPE_NAME, get_bike_service_station_objects, ) +from mobility_data.importers.utils import get_or_create_content_type_from_config from smbackend_turku.importers.utils import BaseExternalSource @@ -13,7 +14,7 @@ def __init__(self, config=None, logger=None, test_data=None): def import_bike_service_stations(self): self.logger.info("Importing Bike service stations...") - content_type = create_bike_service_station_content_type() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) filtered_objects = get_bike_service_station_objects(geojson_file=self.test_data) super().save_objects_as_units(filtered_objects, content_type) diff --git a/smbackend_turku/importers/stations.py b/smbackend_turku/importers/stations.py index 22f16b141..c4dbd0fb0 100644 --- a/smbackend_turku/importers/stations.py +++ b/smbackend_turku/importers/stations.py @@ -1,11 +1,12 @@ from mobility_data.importers.charging_stations import ( - create_charging_station_content_type, + CONTENT_TYPE_NAME as CHARGING_STATION_CONTENT_TYPE_NAME, get_charging_station_objects, ) from mobility_data.importers.gas_filling_station import ( - create_gas_filling_station_content_type, + CONTENT_TYPE_NAME as GAS_FILLING_STATION_CONTENT_TYPE_NAME, get_filtered_gas_filling_station_objects, ) +from mobility_data.importers.utils import get_or_create_content_type_from_config from smbackend_turku.importers.utils import BaseExternalSource @@ -17,7 +18,9 @@ def __init__(self, config=None, logger=None, test_data=None): def import_gas_filling_stations(self): self.logger.info("Importing gas filling stations...") - content_type = create_gas_filling_station_content_type() + content_type = get_or_create_content_type_from_config( + GAS_FILLING_STATION_CONTENT_TYPE_NAME + ) filtered_objects = get_filtered_gas_filling_station_objects( json_data=self.test_data ) @@ -33,7 +36,9 @@ def __init__(self, logger=None, config=None, importer=None, test_data=None): def import_charging_stations(self): self.logger.info("Importing charging stations...") filtered_objects = get_charging_station_objects(csv_file=self.test_data) - content_type = create_charging_station_content_type() + content_type = get_or_create_content_type_from_config( + CHARGING_STATION_CONTENT_TYPE_NAME + ) super().save_objects_as_units(filtered_objects, content_type) From 4de02a0e1a8de586982017224402658c3e7eb281 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 13:43:23 +0200 Subject: [PATCH 19/97] Add type_names and multilingual name to content type --- mobility_data/tests/conftest.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mobility_data/tests/conftest.py b/mobility_data/tests/conftest.py index a344ab0de..91eff2084 100644 --- a/mobility_data/tests/conftest.py +++ b/mobility_data/tests/conftest.py @@ -53,21 +53,24 @@ def content_types(): content_types = [ ContentType.objects.create( id="aa6c2903-d36f-4c61-b828-19084fc7a64b", - name="Test", + type_name="Test", + name_fi="fi", + name_sv="sv", + name_en="en", description="test content type", ) ] content_types.append( ContentType.objects.create( id="ba6c2903-d36f-4c61-b828-19084fc7a64b", - name="Test2", + type_name="Test2", description="test content type2", ) ) content_types.append( ContentType.objects.create( id="ca6c2903-d36f-4c61-b828-19084fc7a64b", - name="Test unit", + type_name="Test unit", description="test content type3", ) ) @@ -78,7 +81,7 @@ def content_types(): @pytest.fixture def group_type(): group_type = GroupType.objects.create( - name="TestGroup", description="test group type" + type_name="TestGroup", description="test group type" ) return group_type From 0b77c5be1725194d3060e1d61bbb2a13ffbc484a Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 13:47:16 +0200 Subject: [PATCH 20/97] Create ContentType from config --- mobility_data/importers/wfs.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/mobility_data/importers/wfs.py b/mobility_data/importers/wfs.py index bb15af06e..1c5f402e7 100644 --- a/mobility_data/importers/wfs.py +++ b/mobility_data/importers/wfs.py @@ -9,7 +9,7 @@ from mobility_data.importers.utils import ( delete_mobile_units, - get_or_create_content_type, + get_or_create_content_type_from_config, locates_in_turku, set_translated_field, ) @@ -23,17 +23,6 @@ WFS_URL = "{wfs_url}?service=WFS&request=GetFeature&typeName={wfs_layer}&outputFormat=GML3&maxFeatures={max_features}" -@db.transaction.atomic -def get_or_create_content_type_using_yaml_config(config): - if "content_type_description" in config: - description = config["content_type_description"] - else: - description = "" - name = config["content_type_name"] - ct, _ = get_or_create_content_type(name, description) - return ct - - @db.transaction.atomic def delete_content_type_using_yaml_config(config): content_type_name = config["content_type_name"] @@ -42,7 +31,7 @@ def delete_content_type_using_yaml_config(config): @db.transaction.atomic def save_to_database_using_yaml_config(objects, config): - content_type = get_or_create_content_type_using_yaml_config(config) + content_type = get_or_create_content_type_from_config(config["content_type_name"]) if not content_type: return for object in objects: From d1e66ea379aeca0b29ad3525bea656613486a61c Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 13:47:53 +0200 Subject: [PATCH 21/97] Serialize all fields --- mobility_data/api/serializers/group_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/api/serializers/group_type.py b/mobility_data/api/serializers/group_type.py index 88eb06e1a..dbd6b8e76 100644 --- a/mobility_data/api/serializers/group_type.py +++ b/mobility_data/api/serializers/group_type.py @@ -6,4 +6,4 @@ class GroupTypeSerializer(serializers.ModelSerializer): class Meta: model = GroupType - fields = ["id", "name", "description"] + fields = "__all__" From 31314e5e78659fa2d63253371f4458fbca38110b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 13:49:09 +0200 Subject: [PATCH 22/97] Create ContentType from config --- mobility_data/importers/marinas.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mobility_data/importers/marinas.py b/mobility_data/importers/marinas.py index 88b3a302f..9cab1ee97 100644 --- a/mobility_data/importers/marinas.py +++ b/mobility_data/importers/marinas.py @@ -47,7 +47,6 @@ def delete_guest_marina(): delete_mobile_units(GUEST_MARINA_CONTENT_TYPE_NAME) - @db.transaction.atomic def delete_boat_parking(): delete_mobile_units(BOAT_PARKING_CONTENT_TYPE_NAME) @@ -94,9 +93,13 @@ def import_guest_marina_and_boat_parking(delete=True): type_name = feature["Muu_venesatama"].as_string() content_type = None if type_name == GUEST_MARINA: - content_type = get_or_create_content_type_from_config(GUEST_MARINA_CONTENT_TYPE_NAME) + content_type = get_or_create_content_type_from_config( + GUEST_MARINA_CONTENT_TYPE_NAME + ) elif type_name == BOAT_PARKING: - content_type = get_or_create_content_type_from_config(BOAT_PARKING_CONTENT_TYPE_NAME) + content_type = get_or_create_content_type_from_config( + BOAT_PARKING_CONTENT_TYPE_NAME + ) mobile_unit = MobileUnit.objects.create(geometry=geometry) mobile_unit.content_types.add(content_type) From b9a935277761f3e6a527eebb6d99501b74e2d97c Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 13:49:46 +0200 Subject: [PATCH 23/97] Filter ContentType with type_name field --- mobility_data/tests/test_import_accessories.py | 10 ++++++---- mobility_data/tests/test_import_payment_zones.py | 2 +- .../tests/test_import_scooter_restrictions.py | 8 +++++--- .../tests/test_import_share_car_parking_places.py | 5 ++++- mobility_data/tests/test_import_speed_limits.py | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mobility_data/tests/test_import_accessories.py b/mobility_data/tests/test_import_accessories.py index c81b005da..0391b95f3 100644 --- a/mobility_data/tests/test_import_accessories.py +++ b/mobility_data/tests/test_import_accessories.py @@ -29,7 +29,7 @@ def test_import_accessories( data_file=f"{settings.BASE_DIR}/mobility_data/tests/data/accessories.gml", ) - public_toilet_content_type = ContentType.objects.get(name="PublicToilet") + public_toilet_content_type = ContentType.objects.get(type_name="PublicToilet") public_toilet_units_qs = MobileUnit.objects.filter( content_types=public_toilet_content_type ) @@ -51,7 +51,7 @@ def test_import_accessories( assert extra["Valmistaja_koodi"] == 0 assert extra["Varustelaji_koodi"] == 4022 - bench_content_type = ContentType.objects.get(name="PublicBench") + bench_content_type = ContentType.objects.get(type_name="PublicBench") bench_units_qs = MobileUnit.objects.filter(content_types=bench_content_type) # Bench id 107620803 locates in Kaarina and therefore is not included. assert bench_units_qs.count() == 1 @@ -62,13 +62,15 @@ def test_import_accessories( point.transform(settings.DEFAULT_SRID) bench_unit.geometry.equals_exact(point, tolerance=0.0001) - table_content_type = ContentType.objects.get(name="PublicTable") + table_content_type = ContentType.objects.get(type_name="PublicTable") table_units_qs = MobileUnit.objects.filter(content_types=table_content_type) assert table_units_qs.count() == 2 assert table_units_qs[0].content_types.all().count() == 1 assert table_units_qs[0].content_types.first() == table_content_type - furniture_group_content_type = ContentType.objects.get(name="PublicFurnitureGroup") + furniture_group_content_type = ContentType.objects.get( + type_name="PublicFurnitureGroup" + ) furniture_group_units_qs = MobileUnit.objects.filter( content_types=furniture_group_content_type ) diff --git a/mobility_data/tests/test_import_payment_zones.py b/mobility_data/tests/test_import_payment_zones.py index 4f2535443..c8a91adf9 100644 --- a/mobility_data/tests/test_import_payment_zones.py +++ b/mobility_data/tests/test_import_payment_zones.py @@ -25,7 +25,7 @@ def test_import_payment_zones(): ) assert ContentType.objects.all().count() == 1 content_type = ContentType.objects.first() - assert content_type.name == "PaymentZone" + assert content_type.type_name == "PaymentZone" assert MobileUnit.objects.all().count() == 2 payment_zone0 = MobileUnit.objects.first() payment_zone1 = MobileUnit.objects.all()[1] diff --git a/mobility_data/tests/test_import_scooter_restrictions.py b/mobility_data/tests/test_import_scooter_restrictions.py index d0393c29a..b0ce709a7 100644 --- a/mobility_data/tests/test_import_scooter_restrictions.py +++ b/mobility_data/tests/test_import_scooter_restrictions.py @@ -38,7 +38,7 @@ def test_import_scooter_restrictions(): data_file=f"{settings.BASE_DIR}/mobility_data/tests/data/scooter_parkings.gml", ) # Test scooter parking - parking_content_type = ContentType.objects.get(name="ScooterParkingArea") + parking_content_type = ContentType.objects.get(type_name="ScooterParkingArea") assert parking_content_type parking_units_qs = MobileUnit.objects.filter(content_types=parking_content_type) assert parking_units_qs.count() == 3 @@ -53,7 +53,9 @@ def test_import_scooter_restrictions(): data_file=f"{settings.BASE_DIR}/mobility_data/tests/data/scooter_speed_limits.gml", ) # Test scooter speed limits - speed_limit_content_type = ContentType.objects.get(name="ScooterSpeedLimitArea") + speed_limit_content_type = ContentType.objects.get( + type_name="ScooterSpeedLimitArea" + ) assert speed_limit_content_type speed_limits_qs = MobileUnit.objects.filter(content_types=speed_limit_content_type) assert speed_limits_qs.count() == 3 @@ -71,7 +73,7 @@ def test_import_scooter_restrictions(): data_file=f"{settings.BASE_DIR}/mobility_data/tests/data/scooter_no_parking_zones.gml", ) # Test scooter no parking zones - no_parking_content_type = ContentType.objects.get(name="ScooterNoParkingArea") + no_parking_content_type = ContentType.objects.get(type_name="ScooterNoParkingArea") assert no_parking_content_type no_parking_qs = MobileUnit.objects.filter(content_types=no_parking_content_type) assert no_parking_qs.count() == 3 diff --git a/mobility_data/tests/test_import_share_car_parking_places.py b/mobility_data/tests/test_import_share_car_parking_places.py index 3d74acbc9..0292d3425 100644 --- a/mobility_data/tests/test_import_share_car_parking_places.py +++ b/mobility_data/tests/test_import_share_car_parking_places.py @@ -12,7 +12,10 @@ def test_import_car_share_parking_places(): "import_share_car_parking_places", test_mode="share_car_parking_places.geojson" ) assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 - assert MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() == 3 + assert ( + MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() + == 3 + ) linnankatu = MobileUnit.objects.get( name="Yhteiskäyttöautojen pysäköintipaikka, Linnankatu 29" ) diff --git a/mobility_data/tests/test_import_speed_limits.py b/mobility_data/tests/test_import_speed_limits.py index 724da5708..28f3d8771 100644 --- a/mobility_data/tests/test_import_speed_limits.py +++ b/mobility_data/tests/test_import_speed_limits.py @@ -26,7 +26,7 @@ def test_import_speed_limits(): assert ContentType.objects.all().count() == 1 content_type = ContentType.objects.first() - assert content_type.name == "SpeedLimitZone" + assert content_type.type_name == "SpeedLimitZone" assert MobileUnit.objects.all().count() == 3 zone_80 = MobileUnit.objects.first() From e99cb1eabe9b0aed9a1f96a55b4bfde86c52c206 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 15:03:32 +0200 Subject: [PATCH 24/97] Remove content_type_description fields --- .../importers/data/wfs_importer_config.yml | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index f86db8084..723f60fb0 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -1,6 +1,5 @@ features: - content_type_name: PlayGround - content_type_description: Playgrounds in the city of Turku. wfs_layer: GIS:Viheralueet max_features: 50000 include: @@ -36,7 +35,6 @@ features: wfs_field: Peruskorjausvuosi - content_type_name: BarbecuePlace - content_type_description: Barbecue places in the city of Turku. wfs_layer: GIS:Varusteet max_features: 100000 include: @@ -73,119 +71,102 @@ features: wfs_field: Asennus - content_type_name: TicketMachineSign - content_type_description: Ticket machine signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 990 Lippuautomaatti max_features: 50000 - content_type_name: TaxiStandSign - content_type_description: Taxi stand signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 534 Taksiasema max_features: 50000 - content_type_name: SingleTrackRailwayLevelCrossingSign - content_type_description: Single track railway level crossing with booms signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 176 Yksiraiteisen rautatien tasoristeys max_features: 50000 - content_type_name: RailwayLevelCrossingWithBoomsSign - content_type_description: Railway level crossing with booms signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 172 Rautatien tasoristeys, jossa on puomit max_features: 50000 - content_type_name: RailwayLevelCrossingWithoutBoomsSign - content_type_description: Railway level crossing without booms signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 171 Rautatien tasoristeys ilman puomeja max_features: 50000 - content_type_name: ParkingLotSign - content_type_description: Parking lot signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 521 Pysäköintipaikka max_features: 50000 - content_type_name: PaidParkingSign - content_type_description: Paid parking signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 855b Maksullinen pysäköinti max_features: 50000 - content_type_name: ObligationToUseParkingDiscSign - content_type_description: Obligation to use parking disc signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 856b Pysäköintikiekon käyttövelvollisuus max_features: 50000 - content_type_name: ParkingForbiddenAreaSign - content_type_description: Parking forbidden area signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 373 Pysäköintikieltoalue max_features: 50000 - content_type_name: ParkingForbiddenSign - content_type_description: Parking forbidden signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 372 Pysäköinti kielletty max_features: 50000 - content_type_name: LongDistanceBusStopSign - content_type_description: Long-distance bus stop signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 532 Kaukoliikenteen linja-auton pysäkki max_features: 50000 - content_type_name: LocalTrafficBusStopSign - content_type_description: Local traffic bus stop signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 531 Paikallisliikenteen linja-auton pysäkki max_features: 50000 - content_type_name: ParkingTerminalSign - content_type_description: Parking terminal signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 991 Pysäköintiautomaatti max_features: 50000 - content_type_name: CrossWalkSign - content_type_description: Crosswalk signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 511 Suojatie max_features: 50000 - content_type_name: RouteForDisabledSign - content_type_description: Route for disabled signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 683 Vammaisille tarkoitettu reitti max_features: 50000 - content_type_name: DisabledParkingSign - content_type_description: Disabled parking signs in the city of Turku. wfs_layer: GIS:Liikennemerkit include: Varustelaji: 836 Invalidin ajoneuvo max_features: 50000 - content_type_name: PaddlingTrail - content_type_description: Paddling trails in Southwest Finland # if not defined use the Turku WFS url set in environment wfs_url: https://geoserver.lounaistieto.fi/geoserver/virma/ows wfs_layer: virma_reitit_melontareitti @@ -215,7 +196,6 @@ features: wfs_type: double - content_type_name: HikingTrail - content_type_description: Hiking trails in Southwest Finland wfs_url: https://geoserver.lounaistieto.fi/geoserver/virma/ows wfs_layer: virma_reitit_retkeilyreitti srid: 3067 @@ -247,7 +227,6 @@ features: wfs_type: double - content_type_name: NatureTrail - content_type_description: Nature trails in Southwest Finland wfs_url: https://geoserver.lounaistieto.fi/geoserver/virma/ows wfs_layer: virma_reitit_luontopolku srid: 3067 @@ -275,7 +254,6 @@ features: wfs_type: double - content_type_name: FitnessTrail - content_type_description: Fitness trails in Southwest Finland wfs_url: https://geoserver.lounaistieto.fi/geoserver/virma/ows wfs_layer: virma_reitit_kuntoreitti srid: 3067 @@ -307,7 +285,6 @@ features: wfs_type: double - content_type_name: PaavonPolku - content_type_description: Paavo trails are marked trails in the terrain that introduce walkers to the nearby nature and urban environment. Blue signs painted on trees and stones. wfs_url: https://geoserver.lounaistieto.fi/geoserver/virma/ows wfs_layer: virma_reitit_kuntoreitti srid: 3067 @@ -357,19 +334,15 @@ features: wfs_field: paatospykala - content_type_name: ScooterParkingArea - content_type_description: Scooter parking zones in the Turku region. wfs_layer: GIS:Sahkopotkulautaparkki - content_type_name: ScooterSpeedLimitArea - content_type_description: Scooter speed limit zones in the Turku region. wfs_layer: GIS:Sahkopotkulauta_nopeusrajoitus - content_type_name: ScooterNoParkingArea - content_type_description: Scooter no parking zones in the Turku region. wfs_layer: GIS:Sahkopotkulauta_pysakointikielto - content_type_name: PublicToilet - content_type_description: Public toilets in the Turku region. wfs_layer: GIS:Varusteet max_features: 10000 # Default is False, if True include only if geometry locates in Turku. @@ -411,7 +384,6 @@ features: wfs_type: int - content_type_name: PublicTable - content_type_description: Public tables in the Turku region. wfs_layer: GIS:Varusteet max_features: 10000 locates_in_turku: True @@ -451,7 +423,6 @@ features: wfs_type: int - content_type_name: PublicBench - content_type_description: Benches in the Turku region. wfs_layer: GIS:Varusteet max_features: 10000 locates_in_turku: True @@ -491,7 +462,6 @@ features: wfs_type: int - content_type_name: PublicFurnitureGroup - content_type_description: Furniture groups in the Turku region. wfs_layer: GIS:Varusteet max_features: 10000 locates_in_turku: True @@ -531,15 +501,12 @@ features: wfs_type: int - content_type_name: BrushSaltedBicycleNetwork - content_type_description: Brush salted bicycle network in the region of Turku. wfs_layer: GIS:Harjasuolatut_pyoratiet - content_type_name: BrushSandedBicycleNetwork - content_type_description: Brush sanded bicycle network In the region of Turku. wfs_layer: GIS:Harjahiekoitetut_pyoratiet - content_type_name: SpeedLimitZone - content_type_description: Speed limit zones in the Turku region. wfs_layer: GIS:Nopeusrajoitusalueet # If geometry contains multiple polygons create one multipolygon from the polygons create_multipolygon: True From ef5e8287143575be267bd9cbe504a1071a09cb0f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 15:04:55 +0200 Subject: [PATCH 25/97] __str__ returns type_name if not None --- mobility_data/models/content_type.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobility_data/models/content_type.py b/mobility_data/models/content_type.py index 11fa5d4e4..2acf722f9 100644 --- a/mobility_data/models/content_type.py +++ b/mobility_data/models/content_type.py @@ -16,7 +16,9 @@ class Meta: ordering = ["type_name"] def __str__(self): - return self.type_name + if self.type_name: + return self.type_name + return str(self.id) class ContentType(BaseType): From e51b62bb8b915ab3f76b910e81243523bf3ae36f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 15:06:28 +0200 Subject: [PATCH 26/97] Migration that renames ContentType and GroupType name to type_name --- ..._and_grouptype_rename_name_to_type_name.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py diff --git a/mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py b/mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py new file mode 100644 index 000000000..aa0b6a828 --- /dev/null +++ b/mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py @@ -0,0 +1,34 @@ +# Generated by Django 4.1.2 on 2023-03-07 09:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "mobility_data", + "0038_alter_contenttype_and_grouptype_ordering_to_field_name", + ), + ] + + operations = [ + migrations.AlterModelOptions( + name="contenttype", + options={"ordering": ["type_name"]}, + ), + migrations.AlterModelOptions( + name="grouptype", + options={"ordering": ["type_name"]}, + ), + migrations.RenameField( + model_name="contenttype", + old_name="name", + new_name="type_name", + ), + migrations.RenameField( + model_name="grouptype", + old_name="name", + new_name="type_name", + ), + ] From 845a8e83cff4d93832c67362a55cb85a327317bf Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 15:07:57 +0200 Subject: [PATCH 27/97] Add migration that adds name to ContentType and GroupType --- .../0040_contenttype_name_grouptype_name.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 mobility_data/migrations/0040_contenttype_name_grouptype_name.py diff --git a/mobility_data/migrations/0040_contenttype_name_grouptype_name.py b/mobility_data/migrations/0040_contenttype_name_grouptype_name.py new file mode 100644 index 000000000..6303c45d3 --- /dev/null +++ b/mobility_data/migrations/0040_contenttype_name_grouptype_name.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.2 on 2023-03-07 09:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("mobility_data", "0039_contentype_and_grouptype_rename_name_to_type_name"), + ] + + operations = [ + migrations.AddField( + model_name="contenttype", + name="name", + field=models.CharField(max_length=128, null=True), + ), + migrations.AddField( + model_name="grouptype", + name="name", + field=models.CharField(max_length=128, null=True), + ), + ] From 04e373c734f70e62f9a068439865ac507f7ce0e6 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 8 Mar 2023 15:11:20 +0200 Subject: [PATCH 28/97] Make ContentType and Grouptype name and description multilingual --- ..._make_name_and_description_multilingual.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py diff --git a/mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py b/mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py new file mode 100644 index 000000000..c6d5e252d --- /dev/null +++ b/mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py @@ -0,0 +1,49 @@ +# Generated by Django 4.1.2 on 2023-03-07 09:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("mobility_data", "0040_contenttype_name_grouptype_name"), + ] + + operations = [ + migrations.AddField( + model_name="contenttype", + name="description_en", + field=models.TextField( + null=True, verbose_name="Optional description of the content type." + ), + ), + migrations.AddField( + model_name="contenttype", + name="description_fi", + field=models.TextField( + null=True, verbose_name="Optional description of the content type." + ), + ), + migrations.AddField( + model_name="contenttype", + name="description_sv", + field=models.TextField( + null=True, verbose_name="Optional description of the content type." + ), + ), + migrations.AddField( + model_name="contenttype", + name="name_en", + field=models.CharField(max_length=128, null=True), + ), + migrations.AddField( + model_name="contenttype", + name="name_fi", + field=models.CharField(max_length=128, null=True), + ), + migrations.AddField( + model_name="contenttype", + name="name_sv", + field=models.CharField(max_length=128, null=True), + ), + ] From 89fa61b8fbc5fd1753c1c9872b8fca29eeea791e Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 10 Mar 2023 11:26:13 +0200 Subject: [PATCH 29/97] Add comment --- street_maintenance/api/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/street_maintenance/api/views.py b/street_maintenance/api/views.py index b8440ef1b..75e228330 100644 --- a/street_maintenance/api/views.py +++ b/street_maintenance/api/views.py @@ -58,6 +58,8 @@ class LargeResultsSetPagination(PageNumberPagination): """ page_size_query_param = "page_size" + # Works are fetched to the remote data storage on a single page, to prevent + # duplicates. max_page_size = 200_000 From 5ba7c171d2aa53268f14cf5cada3d3f07fe69398 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 10 Mar 2023 11:26:32 +0200 Subject: [PATCH 30/97] Change related_name --- street_maintenance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/street_maintenance/models.py b/street_maintenance/models.py index 9ce318535..3591c3117 100644 --- a/street_maintenance/models.py +++ b/street_maintenance/models.py @@ -24,7 +24,7 @@ class MaintenanceWork(models.Model): maintenance_unit = models.ForeignKey( "MaintenanceUnit", on_delete=models.CASCADE, - related_name="maintenance_unit", + related_name="maintenance_work", null=True, ) From afaef50d7bf2d43644aad3e2a371f2da8f423df8 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 10 Mar 2023 11:27:03 +0200 Subject: [PATCH 31/97] Remove obsolete period --- .../management/commands/import_street_maintenance_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/street_maintenance/management/commands/import_street_maintenance_history.py b/street_maintenance/management/commands/import_street_maintenance_history.py index 6dcf795d1..2512840f1 100644 --- a/street_maintenance/management/commands/import_street_maintenance_history.py +++ b/street_maintenance/management/commands/import_street_maintenance_history.py @@ -110,13 +110,13 @@ def handle(self, *args, **options): f"Deleted {num_del_units} obsolete Units for provider {provider}" ) logger.info( - f"Created {num_created_units} units of total {tot_num_units} units for provider {provider}." + f"Created {num_created_units} units of total {tot_num_units} units for provider {provider}" ) logger.info( f"Deleted {num_del_works} obsolete Works for provider {provider}" ) logger.info( - f"Created {num_created_works} Works of total {tot_num_works} Works for provider {provider}." + f"Created {num_created_works} Works of total {tot_num_works} Works for provider {provider}" ) if num_created_works > 0: From 3a44a66531f409305375ac54a038cc9f0afdcf76 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 10 Mar 2023 11:32:56 +0200 Subject: [PATCH 32/97] Add tests for duplicate works and events --- street_maintenance/tests/test_importers.py | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/street_maintenance/tests/test_importers.py b/street_maintenance/tests/test_importers.py index ecfdbc9a3..888c3f026 100644 --- a/street_maintenance/tests/test_importers.py +++ b/street_maintenance/tests/test_importers.py @@ -43,6 +43,13 @@ def test_yit_units( assert num_created_units == 0 assert num_del_units == 1 assert MaintenanceUnit.objects.count() == 1 + # Test duplicate unit + unit_dup = MaintenanceUnit.objects.first() + unit_dup.pk = 42 + unit_dup.save() + num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") + assert num_created_units == 0 + assert num_del_units == 1 @pytest.mark.django_db @@ -90,6 +97,15 @@ def test_yit_works( assert num_del_works == 1 assert work_id == MaintenanceWork.objects.first().id assert MaintenanceWork.objects.count() == 1 + # Create duplicate work + work_dup = MaintenanceWork.objects.first() + work_dup.pk = 42 + work_dup.save() + num_created_works, num_del_works = create_yit_maintenance_works( + "test_access_token", 3 + ) + assert num_created_works == 0 + assert num_del_works == 1 @pytest.mark.django_db @@ -134,6 +150,22 @@ def test_kuntec( assert num_del_works == 1 assert work_id == MaintenanceWork.objects.first().id assert MaintenanceWork.objects.count() == 1 + # Test duplicate unit + unit_dup = MaintenanceUnit.objects.first() + unit_dup.pk = 42 + unit_dup.save() + get_json_data_mock.return_value = get_kuntec_units_mock_data(1) + num_created_units, num_del_units = create_kuntec_maintenance_units() + assert num_created_units == 0 + assert num_del_units == 1 + # Create duplicate work + work_dup = MaintenanceWork.objects.first() + work_dup.pk = 42 + work_dup.save() + get_json_data_mock.return_value = get_kuntec_works_mock_data(1) + num_created_works, num_del_works = create_kuntec_maintenance_works(3) + assert num_created_works == 0 + assert num_del_works == 1 @pytest.mark.django_db @@ -177,6 +209,22 @@ def test_infraroad( assert num_del_works == 2 assert work_id == MaintenanceWork.objects.first().id assert MaintenanceWork.objects.count() == 1 + # Test duplicate Unit + unit_dup = MaintenanceUnit.objects.first() + unit_dup.pk = 42 + unit_dup.save() + get_json_data_mock.return_value = get_fluentprogress_units_mock_data(1) + num_created_units, num_del_units = create_maintenance_units(INFRAROAD) + assert num_created_units == 0 + assert num_del_units == 1 + # Test duplicate work + work_dup = MaintenanceWork.objects.first() + work_dup.pk = 42 + work_dup.save() + get_json_data_mock.return_value = get_fluentprogress_works_mock_data(1) + num_created_works, num_del_works = create_maintenance_works(INFRAROAD, 1, 10) + assert num_created_works == 0 + assert num_del_works == 1 @pytest.mark.django_db From ad35e8653db7ba043ccc0f926297a9e2bda088a9 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 10 Mar 2023 11:33:41 +0200 Subject: [PATCH 33/97] Add migration the changes related_name --- ...013_maintenancework_change_related_name.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 street_maintenance/migrations/0013_maintenancework_change_related_name.py diff --git a/street_maintenance/migrations/0013_maintenancework_change_related_name.py b/street_maintenance/migrations/0013_maintenancework_change_related_name.py new file mode 100644 index 000000000..7af974ae0 --- /dev/null +++ b/street_maintenance/migrations/0013_maintenancework_change_related_name.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.2 on 2023-03-10 07:03 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("street_maintenance", "0012_maintenancework_add_field_original_event_names"), + ] + + operations = [ + migrations.AlterField( + model_name="maintenancework", + name="maintenance_unit", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="maintenance_work", + to="street_maintenance.maintenanceunit", + ), + ), + ] From ceef1408dd87de4c57f70ae819e8566b1e58fc8b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 10 Mar 2023 11:34:48 +0200 Subject: [PATCH 34/97] Handle possible duplicate Works or Units and refactor code --- .../management/commands/utils.py | 135 ++++++++++-------- 1 file changed, 76 insertions(+), 59 deletions(-) diff --git a/street_maintenance/management/commands/utils.py b/street_maintenance/management/commands/utils.py index a3597e11a..f920e14b5 100644 --- a/street_maintenance/management/commands/utils.py +++ b/street_maintenance/management/commands/utils.py @@ -215,8 +215,8 @@ def precalculate_geometry_history(provider): discarded_linestrings += results[0] discarded_points += results[1] GeometryHistory.objects.bulk_create(objects) - logger.info(f"Discarded {discarded_points} Points") - logger.info(f"Discarded {discarded_linestrings} LineStrings") + logger.info(f"Discarded {discarded_points} points in linestring generation") + logger.info(f"Discarded {discarded_linestrings} invalid LineStrings") logger.info(f"Created {len(objects)} HistoryGeometry rows for provider: {provider}") @@ -237,6 +237,37 @@ def get_linestring_in_boundary(linestring, boundary): return False +def handle_unit(filter, objs_to_delete): + num_created = 0 + queryset = MaintenanceUnit.objects.filter(**filter) + queryset_count = queryset.count() + if queryset_count == 0: + MaintenanceUnit.objects.create(**filter) + num_created += 1 + else: + # Keep the first element and if duplicates leave them for deletion. + id = queryset.first().id + if id in objs_to_delete: + objs_to_delete.remove(id) + return num_created + + +def handle_work(filter, objs_to_delete): + num_created = 0 + queryset = MaintenanceWork.objects.filter(**filter) + queryset_count = queryset.count() + + if queryset_count == 0: + MaintenanceWork.objects.create(**filter) + num_created += 1 + else: + # Keep the first element and if duplicates leave them for deletion. + id = queryset.first().id + if id in objs_to_delete: + objs_to_delete.remove(queryset.first().id) + return num_created + + @db.transaction.atomic def create_yit_maintenance_works(access_token, history_size): contract = get_yit_contract(access_token) @@ -296,18 +327,15 @@ def create_yit_maintenance_works(access_token, history_size): except MaintenanceUnit.DoesNotExist: logger.warning(f"Maintenance unit: {unit_id}, not found.") continue + filter = { + "timestamp": route["startTime"], + "maintenance_unit": unit, + "geometry": geometry, + "events": events, + "original_event_names": original_event_names, + } + num_created += handle_work(filter, objs_to_delete) - obj, created = MaintenanceWork.objects.get_or_create( - timestamp=route["startTime"], - maintenance_unit=unit, - events=events, - original_event_names=original_event_names, - geometry=geometry, - ) - if obj.id in objs_to_delete: - objs_to_delete.remove(obj.id) - if created: - num_created += 1 MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() return num_created, len(objs_to_delete) @@ -360,18 +388,14 @@ def create_kuntec_maintenance_works(history_size): if not geometry: continue timestamp = route["start"]["time"] - - obj, created = MaintenanceWork.objects.get_or_create( - timestamp=timestamp, - maintenance_unit=unit, - events=events, - original_event_names=original_event_names, - geometry=geometry, - ) - if obj.id in objs_to_delete: - objs_to_delete.remove(obj.id) - if created: - num_created += 1 + filter = { + "timestamp": timestamp, + "maintenance_unit": unit, + "geometry": geometry, + "events": events, + "original_event_names": original_event_names, + } + num_created += handle_work(filter, objs_to_delete) MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() return num_created, len(objs_to_delete) @@ -431,17 +455,14 @@ def create_maintenance_works(provider, history_size, fetch_size): # If no events found discard the work if len(events) == 0: continue - obj, created = MaintenanceWork.objects.get_or_create( - timestamp=timestamp, - maintenance_unit=unit, - geometry=point, - events=events, - original_event_names=original_event_names, - ) - if obj.id in objs_to_delete: - objs_to_delete.remove(obj.id) - if created: - num_created += 1 + filter = { + "timestamp": timestamp, + "maintenance_unit": unit, + "geometry": point, + "events": events, + "original_event_names": original_event_names, + } + num_created += handle_work(filter, objs_to_delete) MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() return num_created, len(objs_to_delete) @@ -456,13 +477,12 @@ def create_maintenance_units(provider): for unit in get_json_data(URLS[provider][UNITS]): # The names of the unit is derived from the events. names = [n for n in unit["last_location"]["events"]] - obj, created = MaintenanceUnit.objects.get_or_create( - unit_id=unit["id"], names=names, provider=provider - ) - if obj.id in objs_to_delete: - objs_to_delete.remove(obj.id) - if created: - num_created += 1 + filter = { + "unit_id": unit["id"], + "names": names, + "provider": provider, + } + num_created += handle_unit(filter, objs_to_delete) MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() return num_created, len(objs_to_delete) @@ -518,15 +538,12 @@ def create_kuntec_maintenance_units(): names.append(io["label"]) # If names, we have a unit with at least one io_din with State On. if len(names) > 0: - unit_id = unit["unit_id"] - obj, created = MaintenanceUnit.objects.get_or_create( - unit_id=unit_id, names=names, provider=KUNTEC - ) - if obj.id in objs_to_delete: - objs_to_delete.remove(obj.id) - if created: - num_created += 1 - + filter = { + "unit_id": unit["unit_id"], + "names": names, + "provider": KUNTEC, + } + num_created += handle_unit(filter, objs_to_delete) else: no_io_din += 1 MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() @@ -557,13 +574,13 @@ def create_yit_maintenance_units(access_token): ) for unit in vehicles: names = [unit["vehicleTypeName"]] - obj, created = MaintenanceUnit.objects.get_or_create( - unit_id=unit["id"], names=names, provider=YIT - ) - if obj.id in objs_to_delete: - objs_to_delete.remove(obj.id) - if created: - num_created += 1 + filter = { + "unit_id": unit["id"], + "names": names, + "provider": YIT, + } + num_created += handle_unit(filter, objs_to_delete) + MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() return num_created, len(objs_to_delete) From 3ef967797694194c7b913a0865e7d6d96c689cfc Mon Sep 17 00:00:00 2001 From: juuso-j Date: Mon, 13 Mar 2023 13:02:00 +0200 Subject: [PATCH 35/97] Use MobileUnitDataBase class and generic save_to_database, remove obsolete code --- mobility_data/importers/bicycle_stands.py | 39 ++----------- .../importers/bike_service_stations.py | 51 ++++------------ mobility_data/importers/charging_stations.py | 50 ++++------------ .../disabled_and_no_staff_parking.py | 58 ++++--------------- .../importers/foli_parkandride_stop.py | 54 +++++------------ mobility_data/importers/foli_stops.py | 31 ++-------- .../importers/loading_unloading_places.py | 48 +++------------ .../importers/lounaistieto_shapefiles.py | 43 +++----------- mobility_data/importers/parking_machines.py | 26 +-------- .../importers/share_car_parking_places.py | 30 +--------- mobility_data/importers/wfs.py | 37 +++--------- 11 files changed, 81 insertions(+), 386 deletions(-) diff --git a/mobility_data/importers/bicycle_stands.py b/mobility_data/importers/bicycle_stands.py index e60ab6b3d..fa79658e6 100644 --- a/mobility_data/importers/bicycle_stands.py +++ b/mobility_data/importers/bicycle_stands.py @@ -5,7 +5,6 @@ import logging import os -from django import db from django.conf import settings from django.contrib.gis.gdal import DataSource from django.contrib.gis.geos import GEOSGeometry @@ -15,18 +14,15 @@ Municipality, ) -from mobility_data.models import MobileUnit from services.models import Unit from .utils import ( - delete_mobile_units, get_closest_address_full_name, get_municipality_name, - get_or_create_content_type_from_config, get_root_dir, get_street_name_translations, locates_in_turku, - set_translated_field, + MobileUnitDataBase, ) CONTENT_TYPE_NAME = "BicycleStand" @@ -53,7 +49,7 @@ ).boundary -class BicyleStand: +class BicyleStand(MobileUnitDataBase): WFS_HULL_LOCKABLE_STR = "runkolukitusmahdollisuus" GEOJSON_HULL_LOCKABLE_STR = "runkolukittava" @@ -68,11 +64,8 @@ class BicyleStand: ] def __init__(self): - self.geometry = None - self.municipality = None - self.name = {} + super().__init__() self.prefix_name = {} - self.address = {} self.related_unit = None self.extra = {f: None for f in self.EXTRA_FIELDS} @@ -140,7 +133,7 @@ def set_geojson_feature(self, feature): # The last part is always the number address_number = address[-1] translated_street_names = get_street_name_translations( - street_name, municipality_name + street_name, self.municipality ) self.address["fi"] = f"{translated_street_names['fi']} {address_number}" self.address["sv"] = f"{translated_street_names['sv']} {address_number}" @@ -253,27 +246,3 @@ def get_bicycle_stand_objects(data_source=None): logger.info(f"Retrieved {len(bicycle_stands)} bicycle stands.") return bicycle_stands - - -@db.transaction.atomic -def delete_bicycle_stands(): - delete_mobile_units(CONTENT_TYPE_NAME) - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_bicycle_stands() - - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - for object in objects: - mobile_unit = MobileUnit.objects.create( - extra=object.extra, - municipality=object.municipality, - ) - mobile_unit.content_types.add(content_type) - mobile_unit.geometry = object.geometry - set_translated_field(mobile_unit, "name", object.name) - if object.address: - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() diff --git a/mobility_data/importers/bike_service_stations.py b/mobility_data/importers/bike_service_stations.py index 13fbee6ba..1a5870c00 100644 --- a/mobility_data/importers/bike_service_stations.py +++ b/mobility_data/importers/bike_service_stations.py @@ -1,19 +1,16 @@ import logging -from django import db from django.conf import settings from django.contrib.gis.gdal import DataSource as GDALDataSource from django.contrib.gis.geos import GEOSGeometry +from munigeo.models import Municipality from mobility_data.importers.utils import ( - delete_mobile_units, get_file_name_from_data_source, - get_or_create_content_type_from_config, get_root_dir, get_street_name_translations, - set_translated_field, + MobileUnitDataBase, ) -from mobility_data.models import MobileUnit logger = logging.getLogger("mobility_data") @@ -23,14 +20,9 @@ CONTENT_TYPE_NAME = "BikeServiceStation" -class BikeServiceStation: +class BikeServiceStation(MobileUnitDataBase): def __init__(self, feature): - self.name = {} - self.address = {} - self.description = {} - self.extra = {} - self.address_zip = None - self.municipality = None + super().__init__() self.geometry = GEOSGeometry(feature.geom.wkt, srid=SOURCE_DATA_SRID) self.geometry.transform(settings.DEFAULT_SRID) targets = feature["Kohde"].as_string().split("/") @@ -40,9 +32,7 @@ def __init__(self, feature): # Addresses are in format: # Uudenmaankatu 18, 20700 Turku / Nylandsgatan 18, 20700 Turku addresses = feature["Osoite"].as_string().split("/") - self.address_zip, self.municipality = ( - addresses[0].split(",")[1].strip().split(" ") - ) + self.address_zip, municipality = addresses[0].split(",")[1].strip().split(" ") # remove zip code and municipality addresses = [address.split(",")[0].strip() for address in addresses] for i, language in enumerate(LANGUAGES): @@ -62,11 +52,14 @@ def __init__(self, feature): street_name, number = addresses[0].split(" ") self.address[ language - ] = f"{get_street_name_translations(street_name, self.municipality)[language]} number" + ] = f"{get_street_name_translations(street_name, municipality)[language]} number" # Source data does not contain English addresses, assign the Finnsh else: self.address[language] = addresses[0] - + try: + self.municipality = Municipality.objects.get(name=municipality) + except Municipality.DoesNotExist: + self.municipality = None self.extra["additional_details"] = feature["Lisätieto"].as_string() self.extra["in_terrain"] = feature["Maastossa"].as_string() @@ -88,27 +81,3 @@ def get_bike_service_station_objects(geojson_file=None): for feature in data_layer: bicycle_repair_points.append(BikeServiceStation(feature)) return bicycle_repair_points - - -@db.transaction.atomic -def delete_bike_service_stations(): - delete_mobile_units(CONTENT_TYPE_NAME) - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_bike_service_stations() - - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - for object in objects: - mobile_unit = MobileUnit.objects.create( - extra=object.extra, - geometry=object.geometry, - address_zip=object.address_zip, - ) - mobile_unit.content_types.add(content_type) - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "description", object.description) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() diff --git a/mobility_data/importers/charging_stations.py b/mobility_data/importers/charging_stations.py index 51adab8d7..fbebc6441 100644 --- a/mobility_data/importers/charging_stations.py +++ b/mobility_data/importers/charging_stations.py @@ -1,22 +1,18 @@ import csv import logging -from django import db from django.conf import settings from django.contrib.gis.geos import Point - -from mobility_data.models import MobileUnit +from munigeo.models import Municipality from .utils import ( - delete_mobile_units, get_file_name_from_data_source, get_municipality_name, - get_or_create_content_type_from_config, get_postal_code, get_root_dir, get_street_name_translations, LANGUAGES, - set_translated_field, + MobileUnitDataBase, ) logger = logging.getLogger("mobility_data") @@ -60,16 +56,12 @@ } -class ChargingStation: +class ChargingStation(MobileUnitDataBase): def __init__(self, values): - self.is_active = True - self.extra = {} + super().__init__() self.extra["chargers"] = [] self.extra["administrator"] = {} - self.address = {} - self.name = {} # Contains Only steet_name and number - self.street_address = {} x = float(values["x"].replace(",", ".")) y = float(values["y"].replace(",", ".")) self.geometry = Point(x, y, srid=SOURCE_DATA_SRID) @@ -78,7 +70,12 @@ def __init__(self, values): self.extra["method_of_use"] = values["method_of_use"] self.extra["other"] = values["other"] self.extra["payment"] = values["payment"] - self.municipality = get_municipality_name(self.geometry) + try: + self.municipality = Municipality.objects.get( + name=get_municipality_name(self.geometry) + ) + except Municipality.DoesNotExist: + self.municipality = None self.address_zip = get_postal_code(self.geometry) tmp = values["address"].split(" ") address_number = None @@ -161,30 +158,3 @@ def get_charging_station_objects(csv_file=None): # create list from dict values. objects = [obj for obj in charging_stations.values()] return objects - - -@db.transaction.atomic -def delete_charging_stations(): - delete_mobile_units(CONTENT_TYPE_NAME) - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_charging_stations() - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - - for object in objects: - is_active = object.is_active - mobile_unit = MobileUnit.objects.create( - is_active=is_active, - geometry=object.geometry, - extra=object.extra, - address_zip=object.address_zip, - ) - mobile_unit.content_types.add(content_type) - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() - - logger.info(f"Saved {len(objects)} charging stations to database.") diff --git a/mobility_data/importers/disabled_and_no_staff_parking.py b/mobility_data/importers/disabled_and_no_staff_parking.py index 2c25b0a8b..1fd6982d7 100644 --- a/mobility_data/importers/disabled_and_no_staff_parking.py +++ b/mobility_data/importers/disabled_and_no_staff_parking.py @@ -1,20 +1,16 @@ import logging -from django import db from django.conf import settings from django.contrib.gis.gdal import DataSource as GDALDataSource from django.contrib.gis.geos import GEOSGeometry from munigeo.models import Municipality from mobility_data.importers.utils import ( - delete_mobile_units, FieldTypes, get_file_name_from_data_source, - get_or_create_content_type_from_config, get_root_dir, - set_translated_field, + MobileUnitDataBase, ) -from mobility_data.models import MobileUnit logger = logging.getLogger("mobility_data") @@ -27,7 +23,7 @@ DISABLED_PARKING_CONTENT_TYPE_NAME = "DisabledParking" -class NoStaffParking: +class Parking(MobileUnitDataBase): PARKING_PLACES = "paikkoja_y" DISABLED_PARKING_PLACES = "invapaikkoja" extra_field_mappings = { @@ -74,8 +70,7 @@ class NoStaffParking: } def __init__(self, feature): - self.address = {} - self.name = {} + super().__init__() # Addresses are in format e.g.: Kupittaankuja 1, 20520 Turku / Kuppisgränden 1 20520 Åbo # Create a list, where every language is a item where trailing spaces, comma, postalcode # and municipality are removed. e.g. ["Kupittaankuja 1", "Kuppisgränden 1"] @@ -142,6 +137,7 @@ def __init__(self, feature): def get_no_staff_parking_objects(geojson_file=None): no_staff_parkings = [] + disabled_parkings = [] file_name = None if not geojson_file: @@ -153,45 +149,11 @@ def get_no_staff_parking_objects(geojson_file=None): file_name = f"{get_root_dir()}/mobility_data/tests/data/{geojson_file}" data_layer = GDALDataSource(file_name)[0] - for feature in data_layer: - no_staff_parkings.append(NoStaffParking(feature)) - return no_staff_parkings - - -@db.transaction.atomic -def delete_no_staff_parkings(): - delete_mobile_units(NO_STAFF_PARKING_CONTENT_TYPE_NAME) - - -@db.transaction.atomic -def delete_disabled_parkings(): - delete_mobile_units(DISABLED_PARKING_CONTENT_TYPE_NAME) - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_no_staff_parkings() - delete_disabled_parkings() - - no_staff_parking_content_type = get_or_create_content_type_from_config( - NO_STAFF_PARKING_CONTENT_TYPE_NAME - ) - disabled_parking_content_type = get_or_create_content_type_from_config( - DISABLED_PARKING_CONTENT_TYPE_NAME - ) - - for object in objects: - mobile_unit = MobileUnit.objects.create( - extra=object.extra, - geometry=object.geometry, - ) - if object.content_type == NO_STAFF_PARKING_CONTENT_TYPE_NAME: - mobile_unit.content_types.add(no_staff_parking_content_type) + for feature in data_layer: + parking = Parking(feature) + if parking.content_type == NO_STAFF_PARKING_CONTENT_TYPE_NAME: + no_staff_parkings.append(parking) else: - mobile_unit.content_types.add(disabled_parking_content_type) - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.address_zip = object.address_zip - mobile_unit.municipality = object.municipality - mobile_unit.save() + disabled_parkings.append(parking) + return no_staff_parkings, disabled_parkings diff --git a/mobility_data/importers/foli_parkandride_stop.py b/mobility_data/importers/foli_parkandride_stop.py index ac912f6fc..1193f54a9 100644 --- a/mobility_data/importers/foli_parkandride_stop.py +++ b/mobility_data/importers/foli_parkandride_stop.py @@ -1,16 +1,8 @@ -from django import db from django.conf import settings from django.contrib.gis.geos import Point from munigeo.models import Municipality -from mobility_data.models import MobileUnit - -from .utils import ( - delete_mobile_units, - fetch_json, - get_or_create_content_type_from_config, - set_translated_field, -) +from .utils import fetch_json, MobileUnitDataBase URL = "https://data.foli.fi/geojson/poi" FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME = "FoliParkAndRideCarsStop" @@ -20,8 +12,9 @@ SOURCE_DATA_SRID = 4326 -class ParkAndRideStop: +class ParkAndRideStop(MobileUnitDataBase): def __init__(self, feature): + super().__init__() properties = feature["properties"] self.name = { "fi": properties["name_fi"], @@ -34,7 +27,7 @@ def __init__(self, feature): "en": properties["address_fi"], } self.address_zip = properties["text"].split(" ")[-1] - self.description = properties["text"] + self.description["fi"] = properties["text"] try: self.municipality = Municipality.objects.get(name=properties["city"]) except Municipality.DoesNotExist: @@ -48,40 +41,19 @@ def __init__(self, feature): self.geometry.transform(settings.DEFAULT_SRID) -def get_parkandride_stop_objects(): +def get_parkandride_car_stop_objects(): json_data = fetch_json(URL) car_stops = [] - bike_stops = [] for feature in json_data["features"]: if feature["properties"]["category"] == PARKANDRIDE_CARS: car_stops.append(ParkAndRideStop(feature)) - elif feature["properties"]["category"] == PARKANDRIDE_BIKES: - bike_stops.append(ParkAndRideStop(feature)) - - return car_stops, bike_stops - - -@db.transaction.atomic -def save_to_database(objects, content_type_name, delete_tables=True): - assert ( - content_type_name == FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME - or content_type_name == FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME - ) - if delete_tables: - delete_mobile_units(content_type_name) + return car_stops - content_type = get_or_create_content_type_from_config(content_type_name) - for object in objects: - mobile_unit = MobileUnit.objects.create( - geometry=object.geometry, - address_zip=object.address_zip, - description=object.description, - municipality=object.municipality, - ) - mobile_unit.content_types.add(content_type) - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() - - return len(objects) +def get_parkandride_bike_stop_objects(): + json_data = fetch_json(URL) + bike_stops = [] + for feature in json_data["features"]: + if feature["properties"]["category"] == PARKANDRIDE_BIKES: + bike_stops.append(ParkAndRideStop(feature)) + return bike_stops diff --git a/mobility_data/importers/foli_stops.py b/mobility_data/importers/foli_stops.py index 8b5f02ca0..5c2891b82 100644 --- a/mobility_data/importers/foli_stops.py +++ b/mobility_data/importers/foli_stops.py @@ -1,16 +1,9 @@ import logging -from django import db from django.conf import settings from django.contrib.gis.geos import Point -from mobility_data.models import MobileUnit - -from .utils import ( - delete_mobile_units, - fetch_json, - get_or_create_content_type_from_config, -) +from .utils import fetch_json, MobileUnitDataBase URL = "http://data.foli.fi/gtfs/stops" CONTENT_TYPE_NAME = "FoliStop" @@ -19,10 +12,10 @@ SOURCE_DATA_SRID = 4326 -class FoliStop: +class FoliStop(MobileUnitDataBase): def __init__(self, stop_data): - self.extra = {} - self.name = stop_data["stop_name"] + super().__init__() + self.name["fi"] = stop_data["stop_name"] lon = stop_data["stop_lon"] lat = stop_data["stop_lat"] self.geometry = Point(lon, lat, srid=SOURCE_DATA_SRID) @@ -34,19 +27,3 @@ def __init__(self, stop_data): def get_foli_stops(): json_data = fetch_json(URL) return [FoliStop(json_data[stop_code]) for stop_code in json_data] - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_mobile_units(CONTENT_TYPE_NAME) - - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - for object in objects: - mobile_unit = MobileUnit.objects.create( - name=object.name, - geometry=object.geometry, - extra=object.extra, - ) - mobile_unit.content_types.add(content_type) - logger.info(f"Saved {len(objects)} Föli stops to database.") diff --git a/mobility_data/importers/loading_unloading_places.py b/mobility_data/importers/loading_unloading_places.py index 9e58a8155..9932d9eae 100644 --- a/mobility_data/importers/loading_unloading_places.py +++ b/mobility_data/importers/loading_unloading_places.py @@ -1,21 +1,17 @@ import logging import re -from django import db from django.conf import settings from django.contrib.gis.gdal import DataSource as GDALDataSource from django.contrib.gis.geos import GEOSGeometry from munigeo.models import Municipality from mobility_data.importers.utils import ( - delete_mobile_units, FieldTypes, get_file_name_from_data_source, - get_or_create_content_type_from_config, get_root_dir, - set_translated_field, + MobileUnitDataBase, ) -from mobility_data.models import MobileUnit logger = logging.getLogger("mobility_data") @@ -25,7 +21,7 @@ CONTENT_TYPE_NAME = "LoadingUnloadingPlace" -class LoadingPlace: +class LoadingPlace(MobileUnitDataBase): extra_field_mappings = { "Saavutettavuus": { @@ -44,10 +40,8 @@ class LoadingPlace: } def __init__(self, feature): - self.address = {} - self.name = {} + super().__init__() municipality = None - self.address_zip = None addresses = [ " ".join(a.strip().split(" ")[:-2]).rstrip(",") for a in feature["Osoite"].as_string().split("/") @@ -80,14 +74,12 @@ def __init__(self, feature): self.name[lang] = names[i] else: self.name[lang] = names[0] - if municipality: - try: - municipality = Municipality.objects.get(name=municipality) - self.municipality = municipality - except Municipality.DoesNotExist: - self.municipality = None - else: + + try: + self.municipality = Municipality.objects.get(name=municipality) + except Municipality.DoesNotExist: self.municipality = None + self.geometry = GEOSGeometry(feature.geom.wkt, srid=SOURCE_DATA_SRID) self.geometry.transform(settings.DEFAULT_SRID) self.extra = {} @@ -129,27 +121,3 @@ def get_loading_and_unloading_objects(geojson_file=None): for feature in data_layer: objects.append(LoadingPlace(feature)) return objects - - -@db.transaction.atomic -def delete_loading_and_unloading_places(): - delete_mobile_units(CONTENT_TYPE_NAME) - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_loading_and_unloading_places() - - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - for object in objects: - mobile_unit = MobileUnit.objects.create( - extra=object.extra, - geometry=object.geometry, - ) - mobile_unit.content_types.add(content_type) - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.address_zip = object.address_zip - mobile_unit.municipality = object.municipality - mobile_unit.save() diff --git a/mobility_data/importers/lounaistieto_shapefiles.py b/mobility_data/importers/lounaistieto_shapefiles.py index 3987a2994..14ce4fd73 100644 --- a/mobility_data/importers/lounaistieto_shapefiles.py +++ b/mobility_data/importers/lounaistieto_shapefiles.py @@ -1,17 +1,16 @@ import logging import shapefile -from django import db from django.conf import settings from django.contrib.gis.geos import GEOSGeometry, LineString, Point from munigeo.models import Municipality from mobility_data.importers.utils import ( - delete_mobile_units, get_or_create_content_type_from_config, - set_translated_field, + log_imported_message, + MobileUnitDataBase, + save_to_database, ) -from mobility_data.models import MobileUnit from .constants import SOUTHWEST_FINLAND_GEOMETRY @@ -20,14 +19,9 @@ DEFAULT_ENCODING = "utf-8" -class MobilityData: +class MobilityData(MobileUnitDataBase): def __init__(self): - self.extra = {} - self.name = {} - self.name = {"fi": None, "sv": None, "en": None} - self.address = {"fi": None, "sv": None, "en": None} - self.geometry = None - self.municipality = None + super().__init__() def validate_coords(self, coords): for coord in coords: @@ -99,27 +93,6 @@ def add_feature(self, feature, config, srid): return True -@db.transaction.atomic -def delete_content_type(config): - delete_mobile_units(config["content_type_name"]) - - -@db.transaction.atomic -def save_to_database(objects, config): - content_type = get_or_create_content_type_from_config(config["content_type_name"]) - if not content_type: - return - for object in objects: - mobile_unit = MobileUnit.objects.create( - extra=object.extra, geometry=object.geometry - ) - mobile_unit.content_types.add(content_type) - mobile_unit.municipality = object.municipality - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() - - def import_lounaistieto_data_source(config): if "content_type_name" not in config: logger.warning( @@ -145,10 +118,10 @@ def import_lounaistieto_data_source(config): ) objects = [] sf = shapefile.Reader(config["data_url"], encoding=encoding) - delete_content_type(config) for feature in sf.shapeRecords(): obj = MobilityData() if obj.add_feature(feature, config, srid): objects.append(obj) - save_to_database(objects, config) - logger.info(f"Saved {len(objects)} {config['content_type_name']} objects.") + content_type = get_or_create_content_type_from_config(config["content_type_name"]) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/importers/parking_machines.py b/mobility_data/importers/parking_machines.py index 917230dd0..3c5529bb6 100644 --- a/mobility_data/importers/parking_machines.py +++ b/mobility_data/importers/parking_machines.py @@ -1,16 +1,11 @@ -from django import db from django.conf import settings from django.contrib.gis.geos import Point -from mobility_data.models import MobileUnit - from .utils import ( - delete_mobile_units, FieldTypes, get_file_name_from_data_source, - get_or_create_content_type_from_config, get_root_dir, - set_translated_field, + MobileUnitDataBase, ) SOURCE_DATA_SRID = 4326 @@ -19,7 +14,7 @@ LANGUAGES = ["fi", "sv", "en"] -class ParkingMachine: +class ParkingMachine(MobileUnitDataBase): extra_field_mappings = { "Sijainti": { @@ -71,9 +66,9 @@ class ParkingMachine: } def __init__(self, feature): + super().__init__() properties = feature["properties"] geometry = feature["geometry"] - self.extra = {} self.address = {"fi": properties["Osoite"]} self.address["sv"] = properties["Adress"] self.address["en"] = properties["Address"] @@ -127,18 +122,3 @@ def get_json_data(): def get_parking_machine_objects(): json_data = get_json_data()["features"] return [ParkingMachine(feature) for feature in json_data] - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_mobile_units(CONTENT_TYPE_NAME) - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - for object in objects: - mobile_unit = MobileUnit.objects.create( - geometry=object.geometry, - extra=object.extra, - ) - mobile_unit.content_types.add(content_type) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() diff --git a/mobility_data/importers/share_car_parking_places.py b/mobility_data/importers/share_car_parking_places.py index 31c7b26ab..eb31dd927 100644 --- a/mobility_data/importers/share_car_parking_places.py +++ b/mobility_data/importers/share_car_parking_places.py @@ -1,18 +1,14 @@ import logging -from django import db from django.conf import settings from django.contrib.gis.gdal import DataSource as GDALDataSource from django.contrib.gis.geos import GEOSGeometry from mobility_data.importers.utils import ( - delete_mobile_units, get_file_name_from_data_source, - get_or_create_content_type_from_config, get_root_dir, - set_translated_field, + MobileUnitDataBase, ) -from mobility_data.models import MobileUnit logger = logging.getLogger("bicycle_network") SOURCE_DATA_SRID = 3877 @@ -22,7 +18,7 @@ CONTENT_TYPE_NAME = "ShareCarParkingPlace" -class CarShareParkingPlace: +class CarShareParkingPlace(MobileUnitDataBase): RESTRICTION_FIELD = "Rajoit_lis" EXCLUDE_FIELDS = ["id", "Osoite", RESTRICTION_FIELD] CAR_PARKING_NAME = { @@ -32,9 +28,7 @@ class CarShareParkingPlace: } def __init__(self, feature): - self.name = {} - self.extra = {} - self.address = {} + super().__init__() street_name = {} self.geometry = GEOSGeometry(feature.geom.wkt, srid=SOURCE_DATA_SRID) self.geometry.transform(settings.DEFAULT_SRID) @@ -75,21 +69,3 @@ def get_car_share_parking_place_objects(geojson_file=None): for feature in data_layer: car_share_parking_places.append(CarShareParkingPlace(feature)) return car_share_parking_places - - -@db.transaction.atomic -def delete_car_share_parking_places(): - delete_mobile_units(CONTENT_TYPE_NAME) - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_car_share_parking_places() - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - for object in objects: - mobile_unit = MobileUnit.objects.create(extra=object.extra) - mobile_unit.content_types.add(content_type) - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() diff --git a/mobility_data/importers/wfs.py b/mobility_data/importers/wfs.py index 1c5f402e7..15cf5bd25 100644 --- a/mobility_data/importers/wfs.py +++ b/mobility_data/importers/wfs.py @@ -11,9 +11,10 @@ delete_mobile_units, get_or_create_content_type_from_config, locates_in_turku, - set_translated_field, + log_imported_message, + MobileUnitDataBase, + save_to_database, ) -from mobility_data.models import MobileUnit DEFAULT_SOURCE_DATA_SRID = 3877 DEFAULT_MAX_FEATURES = 1000 @@ -29,30 +30,9 @@ def delete_content_type_using_yaml_config(config): delete_mobile_units(content_type_name) -@db.transaction.atomic -def save_to_database_using_yaml_config(objects, config): - content_type = get_or_create_content_type_from_config(config["content_type_name"]) - if not content_type: - return - for object in objects: - mobile_unit = MobileUnit.objects.create( - extra=object.extra, geometry=object.geometry - ) - mobile_unit.content_types.add(content_type) - mobile_unit.municipality = object.municipality - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() - - -class MobilityData: +class MobilityData(MobileUnitDataBase): def __init__(self): - self.extra = {} - self.name = {} - self.name = {"fi": None, "sv": None, "en": None} - self.address = {"fi": None, "sv": None, "en": None} - self.geometry = None - self.municipality = None + super().__init__() def add_feature(self, feature, config): create_multipolygon = False @@ -160,9 +140,7 @@ def import_wfs_feature(config, data_file=None): if "max_features" in config: max_features = config["max_features"] wfs_layer = config["wfs_layer"] - delete_content_type_using_yaml_config(config) objects = [] - if data_file: ds = DataSource(data_file) else: @@ -178,5 +156,6 @@ def import_wfs_feature(config, data_file=None): object = MobilityData() if object.add_feature(feature, config): objects.append(object) - save_to_database_using_yaml_config(objects, config) - logger.info(f"Saved {len(objects)} {config['content_type_name']} objects.") + content_type = get_or_create_content_type_from_config(config["content_type_name"]) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) From 54a1eb100cdd56579ff30f696c1e4067737bfc33 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Mon, 13 Mar 2023 13:03:23 +0200 Subject: [PATCH 36/97] Make choices from ContentType type_name field --- mobility_data/management/commands/delete_mobility_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/management/commands/delete_mobility_data.py b/mobility_data/management/commands/delete_mobility_data.py index fef818c4f..78558ed95 100644 --- a/mobility_data/management/commands/delete_mobility_data.py +++ b/mobility_data/management/commands/delete_mobility_data.py @@ -10,7 +10,7 @@ class Command(BaseCommand): def add_arguments(self, parser): - choices = ContentType.objects.all().values_list("name", flat=True) + choices = ContentType.objects.all().values_list("type_name", flat=True) parser.add_argument( "content_type_names", nargs="*", From e94d00e051290e98f8257103f03ae53d91657114 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Mon, 13 Mar 2023 13:05:39 +0200 Subject: [PATCH 37/97] Test that obsolete mobile units are deleted and duplicates are not created --- .../test_import_foli_parkandride_stops.py | 46 ++++++++++++++++--- mobility_data/tests/test_import_foli_stops.py | 24 ++++++++-- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/mobility_data/tests/test_import_foli_parkandride_stops.py b/mobility_data/tests/test_import_foli_parkandride_stops.py index 09d70079f..6f6ac9e64 100644 --- a/mobility_data/tests/test_import_foli_parkandride_stops.py +++ b/mobility_data/tests/test_import_foli_parkandride_stops.py @@ -2,7 +2,11 @@ import pytest -from mobility_data.importers.utils import get_content_type_config +from mobility_data.importers.utils import ( + get_content_type_config, + get_or_create_content_type_from_config, + save_to_database, +) from mobility_data.models import ContentType, MobileUnit from .utils import get_test_fixture_json_data @@ -14,17 +18,28 @@ def test_import_foli_stops(fetch_json_mock, municipalities): from mobility_data.importers.foli_parkandride_stop import ( FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME, FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME, - get_parkandride_stop_objects, - save_to_database, + get_parkandride_bike_stop_objects, + get_parkandride_car_stop_objects, ) fetch_json_mock.return_value = get_test_fixture_json_data( "foli_parkandride_stops.json" ) - car_stops, bike_stops = get_parkandride_stop_objects() - save_to_database(car_stops, FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME) - save_to_database(bike_stops, FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME) + car_stops = get_parkandride_car_stop_objects() + content_type = get_or_create_content_type_from_config( + FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME + ) + num_created, num_deleted = save_to_database(car_stops, content_type) + assert num_created == 2 + assert num_deleted == 0 + bike_stops = get_parkandride_bike_stop_objects() + content_type = get_or_create_content_type_from_config( + FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME + ) + num_created, num_deleted = save_to_database(bike_stops, content_type) + assert num_created == 2 + assert num_deleted == 0 cars_stops_content_type = ContentType.objects.get( type_name=FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME ) @@ -36,6 +51,10 @@ def test_import_foli_stops(fetch_json_mock, municipalities): bikes_stops_content_type = ContentType.objects.get( type_name=FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME ) + config = get_content_type_config(FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME) + bikes_stops_content_type.name_fi = config["name"]["fi"] + bikes_stops_content_type.name_sv = config["name"]["sv"] + bikes_stops_content_type.name_en = config["name"]["en"] # Fixture data contains two park and ride stops for cars and bikes. assert MobileUnit.objects.filter(content_types=cars_stops_content_type).count() == 2 assert ( @@ -56,6 +75,7 @@ def test_import_foli_stops(fetch_json_mock, municipalities): assert lieto_centre.address_sv == "Hyvättyläntie 2" assert lieto_centre.address_en == "Hyvättyläntie 2" assert lieto_centre.municipality.name == "Lieto" + # Test Föli park and ride bikes stop raisio_st1 = MobileUnit.objects.get(name_en="St1 Raisio") assert raisio_st1.content_types.first() == bikes_stops_content_type @@ -67,3 +87,17 @@ def test_import_foli_stops(fetch_json_mock, municipalities): assert raisio_st1.address_fi == "Kirkkoväärtinkuja 2" assert raisio_st1.address_sv == "Kirkkoväärtinkuja 2" assert raisio_st1.address_en == "Kirkkoväärtinkuja 2" + + json_data = get_test_fixture_json_data("foli_parkandride_stops.json") + # Add only One cars parkandride stop + json_data["features"] = [json_data["features"][0]] + fetch_json_mock.return_value = json_data + car_stops = get_parkandride_car_stop_objects() + content_type = get_or_create_content_type_from_config( + FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME + ) + # Test that obsolete mobile units are deleted and duplicates are not created + num_created, num_deleted = save_to_database(car_stops, content_type) + assert num_created == 0 + assert num_deleted == 1 + assert MobileUnit.objects.filter(content_types=cars_stops_content_type).count() == 1 diff --git a/mobility_data/tests/test_import_foli_stops.py b/mobility_data/tests/test_import_foli_stops.py index b0ba3a4be..83c65e7b2 100644 --- a/mobility_data/tests/test_import_foli_stops.py +++ b/mobility_data/tests/test_import_foli_stops.py @@ -3,6 +3,10 @@ import pytest from django.contrib.gis.geos import Point +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + save_to_database, +) from mobility_data.models import ContentType, MobileUnit from .utils import get_test_fixture_json_data @@ -11,13 +15,16 @@ @pytest.mark.django_db @patch("mobility_data.importers.utils.fetch_json") def test_import_foli_stops(fetch_json_mock): - from mobility_data.importers import foli_stops + from mobility_data.importers.foli_stops import CONTENT_TYPE_NAME, get_foli_stops fetch_json_mock.return_value = get_test_fixture_json_data("foli_stops.json") - objects = foli_stops.get_foli_stops() - foli_stops.save_to_database(objects) + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + objects = get_foli_stops() + num_created, num_deleted = save_to_database(objects, content_type) + assert num_created == 3 + assert num_deleted == 0 assert ContentType.objects.count() == 1 - assert ContentType.objects.first().type_name == foli_stops.CONTENT_TYPE_NAME + assert ContentType.objects.first().type_name == CONTENT_TYPE_NAME assert MobileUnit.objects.count() == 3 turun_satama = MobileUnit.objects.get(name="Turun satama (Silja)") assert turun_satama.content_types.all().count() == 1 @@ -27,3 +34,12 @@ def test_import_foli_stops(fetch_json_mock): point_turun_satama = turun_satama.geometry point_turun_satama.transform(4326) assert point_turun_satama == Point(22.21966, 60.43497, srid=4326) + json_data = get_test_fixture_json_data("foli_stops.json") + del json_data["100"] + fetch_json_mock.return_value = json_data + objects = get_foli_stops() + # Test that obsolete mobile units are deleted and duplicates are not created + num_created, num_deleted = save_to_database(objects, content_type) + assert num_created == 0 + assert num_deleted == 1 + assert MobileUnit.objects.count() == 2 From 1b272e61dc77713fc79518021909318caf2f2e51 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Mon, 13 Mar 2023 13:11:31 +0200 Subject: [PATCH 38/97] Test num_created and num_deleted --- .../tests/test_import_parking_machines.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/mobility_data/tests/test_import_parking_machines.py b/mobility_data/tests/test_import_parking_machines.py index 7c835669d..33e50ff36 100644 --- a/mobility_data/tests/test_import_parking_machines.py +++ b/mobility_data/tests/test_import_parking_machines.py @@ -2,6 +2,10 @@ import pytest +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + save_to_database, +) from mobility_data.models import ContentType, MobileUnit from .utils import get_test_fixture_json_data @@ -10,14 +14,20 @@ @pytest.mark.django_db @patch("mobility_data.importers.parking_machines.get_json_data") def test_import_parking_machines(get_json_data_mock): - from mobility_data.importers import parking_machines + from mobility_data.importers.parking_machines import ( + CONTENT_TYPE_NAME, + get_parking_machine_objects, + ) get_json_data_mock.return_value = get_test_fixture_json_data( "parking_machines.geojson" ) - objects = parking_machines.get_parking_machine_objects() - parking_machines.save_to_database(objects) - assert ContentType.objects.first().type_name == parking_machines.CONTENT_TYPE_NAME + objects = get_parking_machine_objects() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_created, num_deleted = save_to_database(objects, content_type) + assert num_created == 3 + assert num_deleted == 0 + assert ContentType.objects.first().type_name == CONTENT_TYPE_NAME assert MobileUnit.objects.count() == 3 satamakatu = MobileUnit.objects.first() assert satamakatu.content_types.all().count() == 1 From 448b1d2573769c9d2faf10e00e2f1d8bfd6ab157 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Mon, 13 Mar 2023 13:12:34 +0200 Subject: [PATCH 39/97] Use generic function to save objects and display logging info --- .../commands/import_bicycle_stands.py | 10 ++++- .../commands/import_bike_service_stations.py | 10 ++++- .../commands/import_charging_stations.py | 9 ++++- .../import_disabled_and_no_staff_parkings.py | 28 +++++++++---- .../commands/import_foli_parkandride_stops.py | 24 +++++++---- .../management/commands/import_foli_stops.py | 11 ++++- .../commands/import_gas_filling_stations.py | 40 ++++++------------- .../import_loading_and_unloading_places.py | 10 ++++- .../commands/import_parking_machines.py | 10 ++++- .../import_share_car_parking_places.py | 10 ++++- 10 files changed, 106 insertions(+), 56 deletions(-) diff --git a/mobility_data/management/commands/import_bicycle_stands.py b/mobility_data/management/commands/import_bicycle_stands.py index bc8debece..ca324392e 100644 --- a/mobility_data/management/commands/import_bicycle_stands.py +++ b/mobility_data/management/commands/import_bicycle_stands.py @@ -2,7 +2,12 @@ from mobility_data.importers.bicycle_stands import ( BICYCLE_STANDS_URL, + CONTENT_TYPE_NAME, get_bicycle_stand_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -30,4 +35,7 @@ def handle(self, *args, **options): else: logger.info("Fetching bicycle stands from: {}".format(BICYCLE_STANDS_URL)) objects = get_bicycle_stand_objects() - save_to_database(objects) + + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_bike_service_stations.py b/mobility_data/management/commands/import_bike_service_stations.py index e2af6487f..9bcce1976 100644 --- a/mobility_data/management/commands/import_bike_service_stations.py +++ b/mobility_data/management/commands/import_bike_service_stations.py @@ -1,7 +1,12 @@ import logging from mobility_data.importers.bike_service_stations import ( + CONTENT_TYPE_NAME, get_bike_service_station_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -18,5 +23,6 @@ def handle(self, *args, **options): geojson_file = options["test_mode"] objects = get_bike_service_station_objects(geojson_file=geojson_file) - save_to_database(objects) - logger.info(f"Imported {len(objects)} bike service stations.") + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_charging_stations.py b/mobility_data/management/commands/import_charging_stations.py index b6c31316c..acd22e813 100644 --- a/mobility_data/management/commands/import_charging_stations.py +++ b/mobility_data/management/commands/import_charging_stations.py @@ -1,7 +1,12 @@ import logging from mobility_data.importers.charging_stations import ( + CONTENT_TYPE_NAME, get_charging_station_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -18,4 +23,6 @@ def handle(self, *args, **options): logger.info("Running charging_station_importer in test mode.") csv_file = options["test_mode"] objects = get_charging_station_objects(csv_file=csv_file) - save_to_database(objects) + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_created, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_created, num_deleted) diff --git a/mobility_data/management/commands/import_disabled_and_no_staff_parkings.py b/mobility_data/management/commands/import_disabled_and_no_staff_parkings.py index b65fbd476..b4c21b859 100644 --- a/mobility_data/management/commands/import_disabled_and_no_staff_parkings.py +++ b/mobility_data/management/commands/import_disabled_and_no_staff_parkings.py @@ -4,6 +4,10 @@ DISABLED_PARKING_CONTENT_TYPE_NAME, get_no_staff_parking_objects, NO_STAFF_PARKING_CONTENT_TYPE_NAME, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -18,13 +22,21 @@ def handle(self, *args, **options): geojson_file = None if options["test_mode"]: geojson_file = options["test_mode"] - objects = get_no_staff_parking_objects(geojson_file=geojson_file) - save_to_database(objects) - num_no_staff_parkings = len( - [x for x in objects if x.content_type == NO_STAFF_PARKING_CONTENT_TYPE_NAME] + ( + no_stuff_parking_objects, + disabled_parking_objects, + ) = get_no_staff_parking_objects(geojson_file=geojson_file) + content_type = get_or_create_content_type_from_config( + NO_STAFF_PARKING_CONTENT_TYPE_NAME + ) + num_ceated, num_deleted = save_to_database( + no_stuff_parking_objects, content_type + ) + log_imported_message(logger, content_type, num_ceated, num_deleted) + content_type = get_or_create_content_type_from_config( + DISABLED_PARKING_CONTENT_TYPE_NAME ) - num_disabled_parkings = len( - [x for x in objects if x.content_type == DISABLED_PARKING_CONTENT_TYPE_NAME] + num_ceated, num_deleted = save_to_database( + disabled_parking_objects, content_type ) - logger.info(f"Imorted {num_no_staff_parkings} no staff parkings") - logger.info(f"Imorted {num_disabled_parkings} disabled parkings") + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_foli_parkandride_stops.py b/mobility_data/management/commands/import_foli_parkandride_stops.py index f237fd912..f2917c459 100644 --- a/mobility_data/management/commands/import_foli_parkandride_stops.py +++ b/mobility_data/management/commands/import_foli_parkandride_stops.py @@ -5,7 +5,12 @@ from mobility_data.importers.foli_parkandride_stop import ( FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME, FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME, - get_parkandride_stop_objects, + get_parkandride_bike_stop_objects, + get_parkandride_car_stop_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -14,12 +19,15 @@ class Command(BaseCommand): def handle(self, *args, **options): - car_stops, bike_stops = get_parkandride_stop_objects() - logger.info( - f"Saved {save_to_database(car_stops, FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME)} " - "Föli park and ride car stops to database" + car_stops = get_parkandride_car_stop_objects() + content_type = get_or_create_content_type_from_config( + FOLI_PARKANDRIDE_CARS_STOP_CONTENT_TYPE_NAME ) - logger.info( - f"Saved {save_to_database(bike_stops, FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME)} " - "Föli park and ride bike stops to database" + num_ceated, num_deleted = save_to_database(car_stops, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) + content_type = get_or_create_content_type_from_config( + FOLI_PARKANDRIDE_BIKES_STOP_CONTENT_TYPE_NAME ) + bike_stops = get_parkandride_bike_stop_objects() + num_ceated, num_deleted = save_to_database(bike_stops, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_foli_stops.py b/mobility_data/management/commands/import_foli_stops.py index ebefee2d5..022f08281 100644 --- a/mobility_data/management/commands/import_foli_stops.py +++ b/mobility_data/management/commands/import_foli_stops.py @@ -2,7 +2,12 @@ from django.core.management import BaseCommand -from mobility_data.importers.foli_stops import get_foli_stops, save_to_database +from mobility_data.importers.foli_stops import CONTENT_TYPE_NAME, get_foli_stops +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, + save_to_database, +) logger = logging.getLogger("mobility_data") @@ -11,4 +16,6 @@ class Command(BaseCommand): def handle(self, *args, **options): logger.info("Importing Föli stops") objects = get_foli_stops() - save_to_database(objects) + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_gas_filling_stations.py b/mobility_data/management/commands/import_gas_filling_stations.py index cfac76538..455a8db6b 100644 --- a/mobility_data/management/commands/import_gas_filling_stations.py +++ b/mobility_data/management/commands/import_gas_filling_stations.py @@ -1,39 +1,23 @@ -import json import logging -import os + +from django.core.management import BaseCommand from mobility_data.importers.gas_filling_station import ( - GAS_FILLING_STATIONS_URL, + CONTENT_TYPE_NAME, get_filtered_gas_filling_station_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) -from mobility_data.models import ContentType - -from ._base_import_command import BaseImportCommand logger = logging.getLogger("mobility_data") -class Command(BaseImportCommand): +class Command(BaseCommand): def handle(self, *args, **options): - logger.info("Importing gas filling stations.") - if options["test_mode"]: - logger.info("Running gas filling station_importer in test mode.") - f = open( - os.getcwd() - + "/" - + ContentType._meta.app_label - + "/tests/data/" - + options["test_mode"], - "r", - ) - json_data = json.load(f) - objects = get_filtered_gas_filling_station_objects(json_data=json_data) - else: - logger.info( - "Fetching gas filling stations from: {}".format( - GAS_FILLING_STATIONS_URL - ) - ) - objects = get_filtered_gas_filling_station_objects() - save_to_database(objects) + objects = get_filtered_gas_filling_station_objects() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_created, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_created, num_deleted) diff --git a/mobility_data/management/commands/import_loading_and_unloading_places.py b/mobility_data/management/commands/import_loading_and_unloading_places.py index a45717d7d..7bef1af63 100644 --- a/mobility_data/management/commands/import_loading_and_unloading_places.py +++ b/mobility_data/management/commands/import_loading_and_unloading_places.py @@ -1,7 +1,12 @@ import logging from mobility_data.importers.loading_unloading_places import ( + CONTENT_TYPE_NAME, get_loading_and_unloading_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -17,5 +22,6 @@ def handle(self, *args, **options): if options["test_mode"]: geojson_file = options["test_mode"] objects = get_loading_and_unloading_objects(geojson_file=geojson_file) - save_to_database(objects) - logger.info(f"Imported {len(objects)} loading and unloading places.") + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_parking_machines.py b/mobility_data/management/commands/import_parking_machines.py index fdc52bda3..8ffadd6b7 100644 --- a/mobility_data/management/commands/import_parking_machines.py +++ b/mobility_data/management/commands/import_parking_machines.py @@ -3,7 +3,12 @@ from django.core.management import BaseCommand from mobility_data.importers.parking_machines import ( + CONTENT_TYPE_NAME, get_parking_machine_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -13,5 +18,6 @@ class Command(BaseCommand): def handle(self, *args, **options): objects = get_parking_machine_objects() - save_to_database(objects) - logger.info(f"Saved {len(objects)} parking machines to database.") + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_share_car_parking_places.py b/mobility_data/management/commands/import_share_car_parking_places.py index d0e003ffc..13e1e6388 100644 --- a/mobility_data/management/commands/import_share_car_parking_places.py +++ b/mobility_data/management/commands/import_share_car_parking_places.py @@ -1,7 +1,12 @@ import logging from mobility_data.importers.share_car_parking_places import ( + CONTENT_TYPE_NAME, get_car_share_parking_place_objects, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, save_to_database, ) @@ -18,5 +23,6 @@ def handle(self, *args, **options): geojson_file = options["test_mode"] objects = get_car_share_parking_place_objects(geojson_file=geojson_file) - save_to_database(objects) - logger.info(f"Imported {len(objects)} car share parking places.") + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) From c5587a287fde557bbed2f2b36225f07d0187cc70 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 09:31:22 +0200 Subject: [PATCH 40/97] Test that id does not change --- mobility_data/tests/test_import_foli_parkandride_stops.py | 1 + mobility_data/tests/test_import_foli_stops.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mobility_data/tests/test_import_foli_parkandride_stops.py b/mobility_data/tests/test_import_foli_parkandride_stops.py index 6f6ac9e64..69dd730fe 100644 --- a/mobility_data/tests/test_import_foli_parkandride_stops.py +++ b/mobility_data/tests/test_import_foli_parkandride_stops.py @@ -101,3 +101,4 @@ def test_import_foli_stops(fetch_json_mock, municipalities): assert num_created == 0 assert num_deleted == 1 assert MobileUnit.objects.filter(content_types=cars_stops_content_type).count() == 1 + assert raisio_st1.id == MobileUnit.objects.get(name_en="St1 Raisio").id diff --git a/mobility_data/tests/test_import_foli_stops.py b/mobility_data/tests/test_import_foli_stops.py index 83c65e7b2..42a30ebc4 100644 --- a/mobility_data/tests/test_import_foli_stops.py +++ b/mobility_data/tests/test_import_foli_stops.py @@ -43,3 +43,5 @@ def test_import_foli_stops(fetch_json_mock): assert num_created == 0 assert num_deleted == 1 assert MobileUnit.objects.count() == 2 + # Test that id is preserved + assert turun_satama.id == MobileUnit.objects.get(name="Turun satama (Silja)").id From 9515d247cc8fca0533fab7dc58926f9e3f7c640e Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 10:26:34 +0200 Subject: [PATCH 41/97] Change to generic format --- mobility_data/importers/marinas.py | 88 +++++++++++------------------- 1 file changed, 33 insertions(+), 55 deletions(-) diff --git a/mobility_data/importers/marinas.py b/mobility_data/importers/marinas.py index 9cab1ee97..e712edd7a 100644 --- a/mobility_data/importers/marinas.py +++ b/mobility_data/importers/marinas.py @@ -5,15 +5,12 @@ """ import logging -from django import db from django.conf import settings from django.contrib.gis.gdal import DataSource from django.contrib.gis.geos import GEOSGeometry -from mobility_data.models import MobileUnit - from .berths import get_berths -from .utils import delete_mobile_units, get_or_create_content_type_from_config +from .utils import MobileUnitDataBase MARINA_URL = "{}{}".format( settings.TURKU_WFS_URL, @@ -33,73 +30,54 @@ logger = logging.getLogger("mobility_data") -class Marina: +class MarinaBase(MobileUnitDataBase): def __init__(self, feature): + super().__init__() self.geometry = GEOSGeometry(feature.geom.wkt, srid=SOURCE_DATA_SRID) self.geometry.transform(settings.DEFAULT_SRID) - self.name = feature["Venesatamat"].as_string() - berths = get_berths(self.name) - self.extra = {"berths": berths} -@db.transaction.atomic -def delete_guest_marina(): - delete_mobile_units(GUEST_MARINA_CONTENT_TYPE_NAME) +class Marina(MarinaBase): + def __init__(self, feature): + super().__init__(feature) + self.name["fi"] = feature["Venesatamat"].as_string() + berths = get_berths(self.name) + self.extra = {"berths": berths} -@db.transaction.atomic -def delete_boat_parking(): - delete_mobile_units(BOAT_PARKING_CONTENT_TYPE_NAME) +class GuestMarina(MarinaBase): + def __init__(self, feature): + super().__init__(feature) -@db.transaction.atomic -def delete_marinas(): - delete_mobile_units(MARINA_CONTENT_TYPE_NAME) +class BoatParking(MarinaBase): + def __init__(self, feature): + super().__init__(feature) -@db.transaction.atomic -def import_marinas(delete=True): +def get_marinas(): marinas = [] - if delete: - delete_marinas() - ds = DataSource(MARINA_URL) for feature in ds[0]: marinas.append(Marina(feature)) - content_type = get_or_create_content_type_from_config(MARINA_CONTENT_TYPE_NAME) - for marina in marinas: - mobile_unit = MobileUnit.objects.create( - geometry=marina.geometry, - name=marina.name, - extra=marina.extra, - ) - mobile_unit.content_types.add(content_type) - return len(marinas) - - -@db.transaction.atomic -def import_guest_marina_and_boat_parking(delete=True): - """ - The data for the guest marina and the boat parking comes from the same - WFS feature. They are recognized by the value of the field "Muu_venesatama". - """ - if delete: - delete_guest_marina() - delete_boat_parking() + return marinas + + +def get_guest_marinas(): + guest_marinas = [] ds = DataSource(GUEST_MARINA_BOAT_PARKING_URL) for feature in ds[0]: - geometry = GEOSGeometry(feature.geom.wkt, srid=SOURCE_DATA_SRID) - geometry.transform(settings.DEFAULT_SRID) type_name = feature["Muu_venesatama"].as_string() - content_type = None if type_name == GUEST_MARINA: - content_type = get_or_create_content_type_from_config( - GUEST_MARINA_CONTENT_TYPE_NAME - ) - elif type_name == BOAT_PARKING: - content_type = get_or_create_content_type_from_config( - BOAT_PARKING_CONTENT_TYPE_NAME - ) - - mobile_unit = MobileUnit.objects.create(geometry=geometry) - mobile_unit.content_types.add(content_type) + guest_marinas.append(GuestMarina(feature)) + return guest_marinas + + +def get_boat_parkings(): + boat_parkings = [] + ds = DataSource(GUEST_MARINA_BOAT_PARKING_URL) + for feature in ds[0]: + type_name = feature["Muu_venesatama"].as_string() + if type_name == BOAT_PARKING: + boat_parkings.append(BoatParking(feature)) + return boat_parkings From 9e867f19c262f1571db84e3ba555f4828e7a9d3c Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 10:38:12 +0200 Subject: [PATCH 42/97] Add functionality for generic saving of MobileUnit instances --- mobility_data/importers/utils.py | 85 +++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/mobility_data/importers/utils.py b/mobility_data/importers/utils.py index 28c1286c6..82dff096b 100644 --- a/mobility_data/importers/utils.py +++ b/mobility_data/importers/utils.py @@ -1,4 +1,5 @@ import io +import logging import os import re import tempfile @@ -7,6 +8,7 @@ import requests import yaml +from django import db from django.conf import settings from django.contrib.gis.db.models.functions import Distance from django.contrib.gis.gdal import DataSource as GDALDataSource @@ -22,6 +24,8 @@ from mobility_data.models import ContentType, DataSource, MobileUnit +logger = logging.getLogger("mobility_data") + # 11 = Southwest Finland GEOMETRY_ID = 11 GEOMETRY_URL = ( @@ -30,6 +34,23 @@ ) +class MobileUnitDataBase: + # Base class for mobile data importers data objects. + # By inheriting this class the generic save_to_database function can be used + # to save a list containing objects. + LANGUAGES = ["fi", "sv", "en"] + + def __init__(self): + self.name = {lang: None for lang in LANGUAGES} + self.address = {lang: None for lang in LANGUAGES} + self.description = {lang: None for lang in LANGUAGES} + self.municipality = None + self.address_zip = None + self.geometry = None + self.extra = {} + self.unit_id = None + + def get_root_dir(): """ Returns the root directory of the project. @@ -85,6 +106,7 @@ def fetch_json(url): return response.json() +@db.transaction.atomic def delete_mobile_units(type_name): MobileUnit.objects.filter(content_types__type_name=type_name).delete() @@ -172,7 +194,7 @@ def get_street_name_translations(name, municipality): names = {} default_attr_name = "name_fi" try: - street = Street.objects.get(name=name, municipality=municipality.lower()) + street = Street.objects.get(name=name, municipality=municipality) for lang in LANGUAGES: attr_name = "name_" + lang name = getattr(street, attr_name) @@ -274,6 +296,7 @@ def get_content_type_config(type_name): return None +@db.transaction.atomic def get_or_create_content_type_from_config(type_name): config = get_content_type_config(type_name) if config is None: @@ -292,3 +315,63 @@ def get_or_create_content_type_from_config(type_name): ) content_type.save() return content_type + + +def log_imported_message(logger, content_type, num_created, num_deleted): + total_num = MobileUnit.objects.filter(content_types=content_type).count() + name = content_type.name_en if content_type.name_en else content_type.name_fi + logger.info(f"Imported {num_created} {name} items of total {total_num}") + logger.info(f"Deleted {num_deleted} obsolete {name} items") + + +@db.transaction.atomic +def save_to_database(objects, content_types, logger=logger): + if type(content_types) != list: + content_types = [content_types] + + content_types_ids = [ct.id for ct in content_types] + num_created = 0 + objs_to_delete = list( + MobileUnit.objects.filter(content_types__in=content_types).values_list( + "id", flat=True + ) + ) + for object in objects: + filter = { + "name": object.name["fi"], + "name_sv": object.name["sv"], + "name_en": object.name["en"], + "description": object.description["fi"], + "description_sv": object.description["sv"], + "description_en": object.description["en"], + "geometry": object.geometry, + "address": object.address["fi"], + "address_sv": object.address["sv"], + "address_en": object.address["en"], + "municipality": object.municipality, + "address_zip": object.address_zip, + "extra": object.extra, + "unit_id": object.unit_id, + } + queryset = MobileUnit.objects.filter(**filter) + + if queryset.count() == 0: + mobile_unit = MobileUnit.objects.create(**filter) + num_created += 1 + else: + if queryset.count() > 1: + logger.warning(f"Found duplicate {filter}") + + mobile_unit = queryset.first() + id = queryset.first().id + if id in objs_to_delete: + objs_to_delete.remove(id) + + if set(mobile_unit.content_types.all()) != set( + ContentType.objects.filter(id__in=content_types_ids).all() + ): + for content_type in content_types: + mobile_unit.content_types.add(content_type) + + MobileUnit.objects.filter(id__in=objs_to_delete).delete() + return num_created, len(objs_to_delete) From 37109b4c686b49dfd072b04f87c77584c73b6674 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 10:39:05 +0200 Subject: [PATCH 43/97] Fix importing berth bug --- mobility_data/importers/marinas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobility_data/importers/marinas.py b/mobility_data/importers/marinas.py index e712edd7a..a2be3283a 100644 --- a/mobility_data/importers/marinas.py +++ b/mobility_data/importers/marinas.py @@ -41,8 +41,8 @@ class Marina(MarinaBase): def __init__(self, feature): super().__init__(feature) self.name["fi"] = feature["Venesatamat"].as_string() - berths = get_berths(self.name) - self.extra = {"berths": berths} + berths = get_berths(self.name["fi"]) + self.extra["berths"] = berths class GuestMarina(MarinaBase): From 4c80a203c428add6830834ce127e5d03f7c3dc96 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 10:54:47 +0200 Subject: [PATCH 44/97] Use generic functions to save and display logging message --- .../management/commands/import_marinas.py | 34 ++++++++++++++++--- .../commands/import_outdoor_gym_devices.py | 15 ++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/mobility_data/management/commands/import_marinas.py b/mobility_data/management/commands/import_marinas.py index 05be097ad..c5bd2afd5 100644 --- a/mobility_data/management/commands/import_marinas.py +++ b/mobility_data/management/commands/import_marinas.py @@ -1,8 +1,17 @@ import logging from mobility_data.importers.marinas import ( - import_guest_marina_and_boat_parking, - import_marinas, + BOAT_PARKING_CONTENT_TYPE_NAME, + get_boat_parkings, + get_guest_marinas, + get_marinas, + GUEST_MARINA_CONTENT_TYPE_NAME, + MARINA_CONTENT_TYPE_NAME, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, + save_to_database, ) from ._base_import_command import BaseImportCommand @@ -12,6 +21,21 @@ class Command(BaseImportCommand): def handle(self, *args, **options): - logger.info(f"Imported {import_marinas()} marinas.") - import_guest_marina_and_boat_parking() - logger.info("Imported guest marina and boat parking.") + objects = get_marinas() + content_type = get_or_create_content_type_from_config(MARINA_CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) + + objects = get_boat_parkings() + content_type = get_or_create_content_type_from_config( + BOAT_PARKING_CONTENT_TYPE_NAME + ) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) + + objects = get_guest_marinas() + content_type = get_or_create_content_type_from_config( + GUEST_MARINA_CONTENT_TYPE_NAME + ) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) diff --git a/mobility_data/management/commands/import_outdoor_gym_devices.py b/mobility_data/management/commands/import_outdoor_gym_devices.py index af7d74571..aa1232eb2 100644 --- a/mobility_data/management/commands/import_outdoor_gym_devices.py +++ b/mobility_data/management/commands/import_outdoor_gym_devices.py @@ -1,6 +1,14 @@ import logging -from mobility_data.importers.outdoor_gym_devices import save_outdoor_gym_devices +from mobility_data.importers.outdoor_gym_devices import ( + CONTENT_TYPE_NAME, + get_oudoor_gym_devices, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, + save_to_database, +) from ._base_import_command import BaseImportCommand @@ -9,4 +17,7 @@ class Command(BaseImportCommand): def handle(self, *args, **options): - logger.info(f"Imported {save_outdoor_gym_devices()} outdoor gym devices") + objects = get_oudoor_gym_devices() + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_ceated, num_deleted = save_to_database(objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) From 2c227a0f292bae0b03939af32f85b16b4cd149be Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 13:07:53 +0200 Subject: [PATCH 45/97] Add task to delete obsolete data --- mobility_data/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mobility_data/tasks.py b/mobility_data/tasks.py index 9ad42bb02..6c9c62337 100644 --- a/mobility_data/tasks.py +++ b/mobility_data/tasks.py @@ -1,5 +1,6 @@ from django.core import management +from mobility_data.models import ContentType, MobileUnit from smbackend.utils import shared_task_email @@ -150,6 +151,12 @@ def import_parking_machines(name="import_parking_machines"): management.call_command("import_parking_machines") +@shared_task_email +def delete_obsolete_data(name="delete_obsolete_data"): + MobileUnit.objects.filter(content_types__isnull=True).delete() + ContentType.objects.filter(mobile_units__content_types__isnull=True).delete() + + @shared_task_email def delete_deprecated_units(name="delete_deprecated_units"): management.call_command("delete_deprecated_units") From 52d7b59af2648172209f36667de089cc67590fa6 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 13:08:32 +0200 Subject: [PATCH 46/97] Make the importer into generic format --- .../importers/outdoor_gym_devices.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/mobility_data/importers/outdoor_gym_devices.py b/mobility_data/importers/outdoor_gym_devices.py index ce7f9f84d..f6acffefb 100644 --- a/mobility_data/importers/outdoor_gym_devices.py +++ b/mobility_data/importers/outdoor_gym_devices.py @@ -1,35 +1,32 @@ import logging -from django import db - -from mobility_data.models import MobileUnit from services.models import Service, Unit -from .utils import delete_mobile_units, get_or_create_content_type_from_config +from .utils import MobileUnitDataBase logger = logging.getLogger("mobility_data") SERVICE_NAME = "Outdoor Gym Devices" CONTENT_TYPE_NAME = "OutdoorGymDevice" -db.transaction.atomic +class OutdoorGymDevice(MobileUnitDataBase): + def __init__(self, unit_id): + super().__init__() + self.unit_id = unit_id -def save_outdoor_gym_devices(): +def get_oudoor_gym_devices(): """ Save only the ID of the Unit. The data will be serialized from the Unit table using this ID. """ - delete_mobile_units(CONTENT_TYPE_NAME) + outdoor_gym_devices = [] try: service = Service.objects.get(name_en=SERVICE_NAME) except Service.DoesNotExist: - return 0 + return None - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) units_qs = Unit.objects.filter(services=service) for unit in units_qs: - mobile_unit = MobileUnit.objects.create(unit_id=unit.id) - mobile_unit.content_types.add(content_type) - mobile_unit.save() - return units_qs.count() + outdoor_gym_devices.append(OutdoorGymDevice(unit.id)) + return outdoor_gym_devices From d486c83f755bf03710e431637a5db2ebf826a295 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 13:10:55 +0200 Subject: [PATCH 47/97] Add content_type_name MarinaSouthwestFinland --- mobility_data/importers/data/content_types.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index 32b090d66..9882a8935 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -70,6 +70,12 @@ content_types: sv: Busshållplatser i Egentliga Finland. en: Bus stops in Southwest Finland. + - content_type_name: MarinaSouthwestFinland + name: + fi: Marina in Southwest Finland + sv: Båt plats i Egentliga Finland + en: Marina berth in Southwest Finland + - content_type_name: FerryDock name: fi: Lossi laituri @@ -147,8 +153,8 @@ content_types: - content_type_name: Marina name: fi: Veneparkki - sv: Båtplatser - en: Marina berths + sv: Båtplats + en: Marina berth - content_type_name: OutdoorGymDevice name: @@ -174,6 +180,7 @@ content_types: fi: Liikennemerkki sv: Trafikmärke en: Traffic sign + # WFS importer content types - content_type_name: PlayGround name: From abe5a8285266fbaefe2bcaef3d2813f5ca2264b7 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 13:33:04 +0200 Subject: [PATCH 48/97] Test deletion of mobility data --- mobility_data/tests/test_import_foli_stops.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mobility_data/tests/test_import_foli_stops.py b/mobility_data/tests/test_import_foli_stops.py index 42a30ebc4..4f106902e 100644 --- a/mobility_data/tests/test_import_foli_stops.py +++ b/mobility_data/tests/test_import_foli_stops.py @@ -4,6 +4,7 @@ from django.contrib.gis.geos import Point from mobility_data.importers.utils import ( + delete_mobile_units, get_or_create_content_type_from_config, save_to_database, ) @@ -45,3 +46,18 @@ def test_import_foli_stops(fetch_json_mock): assert MobileUnit.objects.count() == 2 # Test that id is preserved assert turun_satama.id == MobileUnit.objects.get(name="Turun satama (Silja)").id + # Test deletion + delete_mobile_units(content_type) + assert ( + MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() + == 0 + ) + fetch_json_mock.return_value = get_test_fixture_json_data("foli_stops.json") + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + objects = get_foli_stops() + num_created, num_deleted = save_to_database(objects, content_type) + assert num_created == 3 + assert num_deleted == 0 + assert ContentType.objects.count() == 1 + assert ContentType.objects.first().type_name == CONTENT_TYPE_NAME + assert MobileUnit.objects.count() == 3 From bd209b176fa98a79e252cd942faf45894bc5a10f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 15:50:00 +0200 Subject: [PATCH 49/97] Delete obsolete fixture --- .../tests/data/gas_filling_stations.json | 38 ------------------- 1 file changed, 38 deletions(-) delete mode 100644 smbackend_turku/tests/data/gas_filling_stations.json diff --git a/smbackend_turku/tests/data/gas_filling_stations.json b/smbackend_turku/tests/data/gas_filling_stations.json deleted file mode 100644 index a923f0f8b..000000000 --- a/smbackend_turku/tests/data/gas_filling_stations.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "features": [{ - "attributes": { - "OPERATOR": "Gasum", - "LAT": 60.49397, - "LON": 22.22558, - "STATION_NAME": "Raisio Kuninkoja", - "ADDRESS": "Itäniityntie 15", - "CITY": "Raisio", - "ZIP_CODE": "21280", - "LNG_CNG": "CNG", - "ObjectId": 38 - }, - "geometry": { - "x": 2474140.2481851652, - "y": 8510545.6499780715 - } - }, - { - "attributes": { - "OPERATOR": "Gasum", - "LAT": 60.4473, - "LON": 22.21296, - "STATION_NAME": "Turku Satama", - "ADDRESS": "Tuontiväylä 2", - "CITY": "Turku", - "ZIP_CODE": "20200", - "LNG_CNG": "LNG/CNG", - "ObjectId": 44 - }, - "geometry": { - "x": 2472735.3962113541, - "y": 8500004.76446491 - } - } - - ] -} \ No newline at end of file From 85565682b6daeba0266c8ab25a89af1dd0bf9bff Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 15:54:04 +0200 Subject: [PATCH 50/97] Make into generic format --- .../importers/gas_filling_station.py | 46 ++----------------- 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/mobility_data/importers/gas_filling_station.py b/mobility_data/importers/gas_filling_station.py index 15bbfcdd5..4624c12e8 100644 --- a/mobility_data/importers/gas_filling_station.py +++ b/mobility_data/importers/gas_filling_station.py @@ -1,21 +1,16 @@ import logging -from django import db from django.conf import settings from django.contrib.gis.geos import Point, Polygon from munigeo.models import Municipality -from mobility_data.models import MobileUnit - from .constants import SOUTHWEST_FINLAND_BOUNDARY, SOUTHWEST_FINLAND_BOUNDARY_SRID from .utils import ( - delete_mobile_units, fetch_json, - get_or_create_content_type_from_config, get_street_name_and_number, get_street_name_translations, LANGUAGES, - set_translated_field, + MobileUnitDataBase, ) logger = logging.getLogger("mobility_data") @@ -27,15 +22,9 @@ CONTENT_TYPE_NAME = "GasFillingStation" -class GasFillingStation: +class GasFillingStation(MobileUnitDataBase): def __init__(self, elem, srid=settings.DEFAULT_SRID): - # Contains the complete address with zip_code and city - self.address = {} - self.extra = {} - self.name = {} - # Contains Only steet_name and number - self.street_address = {} - self.is_active = True + super().__init__() attributes = elem.get("attributes") x = attributes.get("LON", 0) y = attributes.get("LAT", 0) @@ -82,37 +71,10 @@ def get_filtered_gas_filling_station_objects(json_data=None): # Filter objects by their location # Polygon used the detect if point intersects. i.e. is in the boundaries of SouthWest Finland. polygon = Polygon(SOUTHWEST_FINLAND_BOUNDARY, srid=SOUTHWEST_FINLAND_BOUNDARY_SRID) - filtered_objects = [o for o in objects if polygon.intersects(o.geometry)] + filtered_objects = [o for o in objects if polygon.covers(o.geometry)] logger.info( "Filtered: {} gas filling stations by location to: {}.".format( len(json_data["features"]), len(filtered_objects) ) ) return filtered_objects - - -@db.transaction.atomic -def delete_gas_filling_stations(): - delete_mobile_units(CONTENT_TYPE_NAME) - - -@db.transaction.atomic -def save_to_database(objects, delete_tables=True): - if delete_tables: - delete_gas_filling_stations() - - content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) - for object in objects: - is_active = object.is_active - mobile_unit = MobileUnit.objects.create( - is_active=is_active, - geometry=object.geometry, - extra=object.extra, - address_zip=object.address_zip, - municipality=object.municipality, - ) - mobile_unit.content_types.add(content_type) - set_translated_field(mobile_unit, "name", object.name) - set_translated_field(mobile_unit, "address", object.address) - mobile_unit.save() - logger.info(f"Saved {len(objects)} gas filling stations to database.") From ac1084ff0151190f572f0ef719f718bab32813ae Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 15:56:18 +0200 Subject: [PATCH 51/97] Use fixture from mobility_data app --- smbackend_turku/tests/test_gas_filling_stations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smbackend_turku/tests/test_gas_filling_stations.py b/smbackend_turku/tests/test_gas_filling_stations.py index afdf4b749..95170924c 100644 --- a/smbackend_turku/tests/test_gas_filling_stations.py +++ b/smbackend_turku/tests/test_gas_filling_stations.py @@ -27,7 +27,7 @@ def test_gas_filling_stations_import(): import_gas_filling_stations( logger=logger, config=config, - test_data=get_test_resource(resource_name="gas_filling_stations"), + test_data=get_test_fixture_json_data("gas_filling_stations.json"), ) service = Service.objects.get(name=config["service"]["name"]["fi"]) assert service.id == config["service"]["id"] From 8dfd19ab187d189f7e08ee56db860f76c6c624bc Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 14 Mar 2023 15:57:09 +0200 Subject: [PATCH 52/97] Use generic functions to create test data --- .../tests/test_import_gas_filling_stations.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mobility_data/tests/test_import_gas_filling_stations.py b/mobility_data/tests/test_import_gas_filling_stations.py index fa7a31650..11dd72c4f 100644 --- a/mobility_data/tests/test_import_gas_filling_stations.py +++ b/mobility_data/tests/test_import_gas_filling_stations.py @@ -1,15 +1,29 @@ import pytest -from mobility_data.importers.gas_filling_station import CONTENT_TYPE_NAME +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + save_to_database, +) from mobility_data.models import ContentType, MobileUnit -from .utils import import_command +from .utils import get_test_fixture_json_data @pytest.mark.django_db def test_importer(municipalities): - import_command("import_gas_filling_stations", test_mode="gas_filling_stations.json") + from mobility_data.importers.gas_filling_station import ( + CONTENT_TYPE_NAME, + get_filtered_gas_filling_station_objects, + ) + + json_data = get_test_fixture_json_data("gas_filling_stations.json") + objects = get_filtered_gas_filling_station_objects(json_data=json_data) + content_type = get_or_create_content_type_from_config(CONTENT_TYPE_NAME) + num_created, num_deleted = save_to_database(objects, content_type) + # Two will be created as One item in the fixture data locates outside Southwest Finland + assert num_created == 2 + assert num_deleted == 0 assert ContentType.objects.filter(type_name=CONTENT_TYPE_NAME).count() == 1 assert ( MobileUnit.objects.filter(content_types__type_name=CONTENT_TYPE_NAME).count() From eec1991c399f75f0ab6915120b1b776c93f2057a Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 15 Mar 2023 13:28:17 +0200 Subject: [PATCH 53/97] Import get_test_fixture_json_data --- smbackend_turku/tests/test_gas_filling_stations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/smbackend_turku/tests/test_gas_filling_stations.py b/smbackend_turku/tests/test_gas_filling_stations.py index 95170924c..80032a60b 100644 --- a/smbackend_turku/tests/test_gas_filling_stations.py +++ b/smbackend_turku/tests/test_gas_filling_stations.py @@ -4,10 +4,11 @@ import pytest import pytz +from mobility_data.tests.utils import get_test_fixture_json_data from services.models import Service, ServiceNode, Unit from smbackend_turku.importers.stations import import_gas_filling_stations from smbackend_turku.importers.utils import get_external_source_config -from smbackend_turku.tests.utils import create_municipalities, get_test_resource +from smbackend_turku.tests.utils import create_municipalities @pytest.mark.django_db From 41e9ee9277a10ffa17d3736439ac66921d4b8336 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 15 Mar 2023 13:50:46 +0200 Subject: [PATCH 54/97] Handle duplicate MobileUnits and multiple ContentTypes --- mobility_data/importers/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mobility_data/importers/utils.py b/mobility_data/importers/utils.py index 82dff096b..b03f65fad 100644 --- a/mobility_data/importers/utils.py +++ b/mobility_data/importers/utils.py @@ -292,6 +292,10 @@ def get_content_type_config(type_name): configs = get_yaml_config(CONTENT_TYPES_CONFIG_FILE) for config in configs.get("content_types", None): if type_name == config.get("content_type_name", None): + if "name" not in config: + raise Exception( + f"Missing name field for {type_name} in {CONTENT_TYPES_CONFIG_FILE}" + ) return config return None @@ -303,8 +307,12 @@ def get_or_create_content_type_from_config(type_name): raise Exception( f"Configuration not found for {type_name} in {CONTENT_TYPES_CONFIG_FILE}" ) + queryset = ContentType.objects.filter(type_name=type_name) + if queryset.count() == 0: + content_type = ContentType.objects.create(type_name=type_name) + else: + content_type = queryset.first() - content_type, _ = ContentType.objects.get_or_create(type_name=type_name) for lang in ["fi", "sv", "en"]: setattr(content_type, f"name_{lang}", config["name"].get(lang, None)) if "description" in config: @@ -354,13 +362,14 @@ def save_to_database(objects, content_types, logger=logger): "unit_id": object.unit_id, } queryset = MobileUnit.objects.filter(**filter) + queryset = queryset.filter(content_types__in=content_types_ids) if queryset.count() == 0: mobile_unit = MobileUnit.objects.create(**filter) num_created += 1 else: if queryset.count() > 1: - logger.warning(f"Found duplicate {filter}") + logger.warning(f"Found duplicate MobileUnit {filter}") mobile_unit = queryset.first() id = queryset.first().id From 17c84defc0e2ca17c3b9897541b4eb288609ebed Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 16 Mar 2023 12:47:30 +0200 Subject: [PATCH 55/97] Remove obsolete type_name mappings --- mobility_data/api/views.py | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py index 652ed3a65..8d159aac3 100644 --- a/mobility_data/api/views.py +++ b/mobility_data/api/views.py @@ -24,45 +24,6 @@ FIELD_TYPES.FLOAT = float FIELD_TYPES.INT = int FIELD_TYPES.BOOL = bool -# Mappings, so that deprecated type_names will work. -# These will be removed when the front end is updated. -group_name_mappings = {"CRE": "CultureRoute"} -type_name_mappings = { - "FGS": "FishingSpot", - "FYR": "FerryRoute", - "MSF": "MarinaSouthwestFinland", - "SWY": "SlipwaySouthwestFinland", - "RCR": "RecreationalRoute", - "PTL": "PaddlingTrail", - "HTL": "HikingTrail", - "NTL": "NatureTrail", - "FTL": "FitnessTrail", - "PPU": "PaavonPolku", - "PAZ": "PaymentZone", - "SPG": "ScooterParkingArea", - "SSL": "ScooterSpeedLimitArea", - "SNP": "ScooterNoParkingArea", - "BLB": "BrushSaltedBicycleNetwork", - "BND": "BrushSandedBicycleNetwork", - "SLZ": "SpeedLimitZone", - "APT": "PublicToilet", - "ATE": "PublicTable", - "ABH": "PublicBench", - "AFG": "PublicFurnitureGroup", - "BIS": "BicycleStand", - "BSS": "BikeServiceStation", - "BOK": "BoatParking", - "CGS": "ChargingStation", - "CRG": "CultureRouteGeometry", - "CRU": "CultureRouteUnit", - "DSP": "DisabledParking", - "GFS": "GasFillingStation", - "GMA": "GuestMarina", - "SCP": "ShareCarParkingPlace", - "MAR": "Marina", - "NSP": "NoStaffParking", - "LUP": "LoadingUnloadingPlace", -} def get_srid_and_latlon(filters): @@ -135,9 +96,6 @@ def list(self, request): mobile_units = get_mobile_units(filters) if "type_name" in filters: type_name = filters["type_name"] - # TODO, remove when front end is updated. - if type_name in type_name_mappings: - type_name = group_name_mappings[type_name] if not GroupType.objects.filter(name=type_name).exists(): return Response( "type_name does not exist.", status=status.HTTP_400_BAD_REQUEST @@ -191,9 +149,6 @@ def list(self, request): srid, latlon = get_srid_and_latlon(filters) if "type_name" in filters: type_name = filters["type_name"] - # TODO, remove when front end is updated. - if type_name in type_name_mappings: - type_name = type_name_mappings[type_name] if not ContentType.objects.filter(type_name=type_name).exists(): return Response( "type_name does not exist.", status=status.HTTP_400_BAD_REQUEST From 286743d3198da57491dcb72ea58afb39d921109b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 17 Mar 2023 10:56:10 +0200 Subject: [PATCH 56/97] Add school_district_sv, remove obsolote upper and lower sv districts --- .../importers/data/divisions_config.yml | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/smbackend_turku/importers/data/divisions_config.yml b/smbackend_turku/importers/data/divisions_config.yml index e79b87866..df6a866ce 100644 --- a/smbackend_turku/importers/data/divisions_config.yml +++ b/smbackend_turku/importers/data/divisions_config.yml @@ -67,7 +67,7 @@ divisions: - type: school_district_fi name: "Oppilaaksiottoalue, suomenkielinen" - ocd_id: oppilaaksiottoalue_alakoulu + ocd_id: oppilaaksiottoalue_suomenkielinen wfs_layer: 'GIS:Oppilasalueet_suomi' fields: name: @@ -75,10 +75,10 @@ divisions: origin_id: Oppilasalueen_kuvaus ocd_id: Oppilasalueen_kuvaus - - type: lower_comprehensive_school_district_sv - name: "Oppilaaksiottoalue, ruotsinkielinen alakoulu" - ocd_id: oppilaaksiottoalue_alakoulu_sv - wfs_layer: 'GIS:Oppilasalueet_ruotsi_1-6' + - type: school_district_sv + name: "Oppilaaksiottoalue, ruotsinkielinen" + ocd_id: oppilaaksiottoalue_ruotsinkielinen + wfs_layer: 'GIS:Oppilasalueet_ruotsi_1-9' check_turku_boundary: False fields: name: @@ -86,14 +86,4 @@ divisions: origin_id: Oppilasalueen_kuvaus ocd_id: Oppilasalueen_kuvaus - - type: upper_comprehensive_school_district_sv - name: "Oppilaaksiottoalue, ruotsinkielinen yläkoulu" - ocd_id: oppilaaksiottoalue_ylakoulu_sv - wfs_layer: 'GIS:Oppilasalueet_ruotsi_7-9' - check_turku_boundary: False - fields: - name: - sv: Oppilasalueen_kuvaus - origin_id: Oppilasalueen_kuvaus - ocd_id: Oppilasalueen_kuvaus - + \ No newline at end of file From ace9a2363eced36eddcd46359857872071641f17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Mar 2023 13:41:23 +0000 Subject: [PATCH 57/97] Bump django from 4.1.2 to 4.1.7 Bumps [django](https://github.com/django/django) from 4.1.2 to 4.1.7. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/4.1.2...4.1.7) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index fa5aafdaf..56715a3b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,9 +47,11 @@ click-repl==0.2.0 # via celery coverage==5.5 # via pytest-cov +cron-descriptor==1.2.35 + # via django-celery-beat deprecated==1.2.13 # via redis -django==4.1.2 +django==4.1.7 # via # -r requirements.in # django-celery-beat @@ -233,9 +235,7 @@ toml==0.10.2 # pytest # pytest-cov tomli==1.2.1 - # via - # black - # pep517 + # via pep517 tqdm==4.62.3 # via -r requirements.in tzdata==2022.1 From a687f4e505ec7dfb8949b1fb910474262df25e24 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 24 Mar 2023 11:43:12 +0200 Subject: [PATCH 58/97] Serialize unit_id for Units --- mobility_data/api/serializers/mobile_unit.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mobility_data/api/serializers/mobile_unit.py b/mobility_data/api/serializers/mobile_unit.py index 6708eb514..8a903bf32 100644 --- a/mobility_data/api/serializers/mobile_unit.py +++ b/mobility_data/api/serializers/mobile_unit.py @@ -81,6 +81,7 @@ class Meta: "address_fi": "street_address_fi", "address_sv": "street_address_sv", "address_en": "street_address_en", + "unit_id": "id", } def to_representation(self, obj): @@ -89,7 +90,6 @@ def to_representation(self, obj): # If serializing Unit instance or MobileUnit with unit_id. if self.context.get("services_unit_instances", False) or unit_id: if unit_id: - # When serializing the MobileUnit from the retrieve method try: unit = Unit.objects.get(id=unit_id) except Unit.DoesNotExist: @@ -97,6 +97,7 @@ def to_representation(self, obj): else: # The obj is a Unit instance. unit = obj + mobile_unit = MobileUnit.objects.filter(unit_id=unit.id).first() for field in self.fields: # lookup the field name in the service_unit table, as not all field names that contains # similar data has the same name. @@ -115,11 +116,11 @@ def to_representation(self, obj): # Serialize the MobileUnit id, otherwise would serialize the serivce_unit id. if field == "id": try: - representation["id"] = MobileUnit.objects.get( - unit_id=unit.id - ).id + representation["id"] = mobile_unit.id except MobileUnit.DoesNotExist: representation["id"] = unit.id + if field == "unit_id": + representation["unit_id"] = mobile_unit.unit_id # The location field must be serialized with its wkt value. if unit.location: representation["geometry"] = unit.location.wkt From 9fed556a7d54d2f2336a1c71f0390c0173f14fb8 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 24 Mar 2023 11:45:03 +0200 Subject: [PATCH 59/97] Add generic function that creates MobileUnits with Unit references --- mobility_data/importers/utils.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/mobility_data/importers/utils.py b/mobility_data/importers/utils.py index b03f65fad..f0ad070eb 100644 --- a/mobility_data/importers/utils.py +++ b/mobility_data/importers/utils.py @@ -23,6 +23,7 @@ ) from mobility_data.models import ContentType, DataSource, MobileUnit +from services.models import Unit logger = logging.getLogger("mobility_data") @@ -111,19 +112,6 @@ def delete_mobile_units(type_name): MobileUnit.objects.filter(content_types__type_name=type_name).delete() -def create_mobile_unit_as_unit_reference(unit_id, content_type): - """ - This function is called by turku_services_importers target that imports both - to the services list and mobile view. The created MobileUnit is used to - serialize the data from the services_unit table in the mobile_unit endpoint. - """ - - mobile_unit = MobileUnit.objects.create( - unit_id=unit_id, - ) - mobile_unit.content_types.add(content_type) - - def get_or_create_content_type(name, description): content_type, created = ContentType.objects.get_or_create( name=name, description=description @@ -384,3 +372,19 @@ def save_to_database(objects, content_types, logger=logger): MobileUnit.objects.filter(id__in=objs_to_delete).delete() return num_created, len(objs_to_delete) + + +def create_mobile_units_as_unit_references(service_id, content_type): + """ + This function is called by turku_services_importers targets that imports both + to the services list and mobile view. The created MobileUnits are used to + serialize the data from the services_unit table in the mobile_unit endpoint. + """ + + units = Unit.objects.filter(services__id=service_id) + objects = [] + for unit in units: + obj = MobileUnitDataBase() + obj.unit_id = unit.id + objects.append(obj) + save_to_database(objects, content_type) From f026e4054beb88022d29cba1b3646bbf0275024b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 24 Mar 2023 11:46:42 +0200 Subject: [PATCH 60/97] Add task to delete_obsolete_external_units --- smbackend_turku/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/smbackend_turku/tasks.py b/smbackend_turku/tasks.py index a36ab80eb..d3b0607c4 100644 --- a/smbackend_turku/tasks.py +++ b/smbackend_turku/tasks.py @@ -85,3 +85,10 @@ def import_charging_stations(name="import_charging_stations"): @shared_task_email def import_external_sources(name="import_external_sources"): management.call_command("turku_services_import", "external_sources") + + +@shared_task_email +def delete_obsolete_external_units(name="delete_obsolete_external_units"): + management.call_command("delete_obsolete_external_units") + + From a128d2dbae2c4c0ec0e31987e8dd1ab1be845142 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 24 Mar 2023 11:55:34 +0200 Subject: [PATCH 61/97] Delete by existing id --- smbackend_turku/importers/utils.py | 31 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/smbackend_turku/importers/utils.py b/smbackend_turku/importers/utils.py index 8b7714eb4..037434831 100644 --- a/smbackend_turku/importers/utils.py +++ b/smbackend_turku/importers/utils.py @@ -18,8 +18,7 @@ ) from mobility_data.importers.utils import ( - create_mobile_unit_as_unit_reference, - delete_mobile_units, + create_mobile_units_as_unit_references, ) from services.management.commands.services_import.services import ( update_service_counts, @@ -350,15 +349,13 @@ def create_service(service_id, service_node_id, service_names): def delete_external_source( service_id, service_node_id, - mobile_units_content_type_name, -): + ): """ Deletes the data source from services list and optionally from mobility_data. """ Unit.objects.filter(services__id=service_id).delete() Service.objects.filter(id=service_id).delete() ServiceNode.objects.filter(id=service_node_id).delete() - delete_mobile_units(mobile_units_content_type_name) update_service_node_counts() update_service_counts() @@ -385,11 +382,22 @@ def __init__(self, config): self.config["service"]["name"], ) - def delete_external_source(self): + def delete_external_source(self): + + # Get ID from name, if ID is changed in config correct ID will be used for deletion + try: + service_id = Service.objects.get(name=self.config["service"]["name"]["fi"]).id + except Service.DoesNotExist: + # If not found get ID from config + service_id = self.config["service"]["id"] + try: + service_node_id = ServiceNode.objects.get(name=self.config["service_node"]["name"]["fi"]).id + except ServiceNode.DoesNotExist: + service_node_id = self.config["service_node"]["id"] + delete_external_source( - self.config["service"]["id"], - self.config["service_node"]["id"], - self.config["mobility_data_content_type_name"], + service_id, + service_node_id, ) @db.transaction.atomic @@ -428,8 +436,9 @@ def save_objects_as_units(self, objects, content_type): unit.last_modified_time = datetime.datetime.now(UTC_TIMEZONE) set_service_names_field(unit) unit.save() - if self.config.get("create_mobile_units_with_unit_reference", False): - create_mobile_unit_as_unit_reference(unit_id, content_type) + + if self.config.get("create_mobile_units_with_unit_reference", False): + create_mobile_units_as_unit_references(self.SERVICE_ID, content_type) update_service_node_counts() update_service_counts() self.logger.info(f"Imported {len(objects)} {self.config['name']}...") From 8b12f4cd120aec104f411fdd66e9de2a505e56a1 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 24 Mar 2023 11:57:37 +0200 Subject: [PATCH 62/97] Add command to delete obsolete Units --- .../delete_obsolete_external_units.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 smbackend_turku/management/commands/delete_obsolete_external_units.py diff --git a/smbackend_turku/management/commands/delete_obsolete_external_units.py b/smbackend_turku/management/commands/delete_obsolete_external_units.py new file mode 100644 index 000000000..736b78eeb --- /dev/null +++ b/smbackend_turku/management/commands/delete_obsolete_external_units.py @@ -0,0 +1,32 @@ +SERVICE_NODE = "service_node" +SERVICE = "service" + +GAS_FILLING_STATIONS_IDS={SERVICE_NODE:20000,SERVICE:20000} +CHARGING_STATIONS_IDS={SERVICE_NODE:30000,SERVICE:30000} +BICYCLE_STANDS_IDS={SERVICE_NODE:40000,SERVICE:40000} +BIKE_SERVICE_STATIONS_IDS={SERVICE_NODE:50000, SERVICE:50000} + +DELETE = [ + GAS_FILLING_STATIONS_IDS, + CHARGING_STATIONS_IDS, + BICYCLE_STANDS_IDS, + BIKE_SERVICE_STATIONS_IDS +] + +from django.core.management import BaseCommand +from services.models import Unit, Service, ServiceNode +from services.management.commands.services_import.services import ( + update_service_counts, + update_service_node_counts, +) +# This is a hack script to delete obsolete data as the IDs +# changed. The bug that caused this is fixed and after this is run the +# script is obsolete. +class Command(BaseCommand): + def handle(self, *args, **options): + for ids in DELETE: + Unit.objects.filter(services__id=ids[SERVICE]).delete() + Service.objects.filter(id=ids[SERVICE]).delete() + ServiceNode.objects.filter(id=ids[SERVICE_NODE]).delete() + update_service_node_counts() + update_service_counts() From 0240ea953f5b7303eda4139195c12a337de6e7a6 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 28 Mar 2023 11:27:09 +0300 Subject: [PATCH 63/97] Add EVENT_CHOICES constant --- street_maintenance/management/commands/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/street_maintenance/management/commands/constants.py b/street_maintenance/management/commands/constants.py index 96254bce6..b2a75e862 100644 --- a/street_maintenance/management/commands/constants.py +++ b/street_maintenance/management/commands/constants.py @@ -60,7 +60,7 @@ HIEKANPOISTO = "hiekanpoisto" # MUUT is set to None as the events are not currently displayed. MUUT = None - +EVENT_CHOICES = [AURAUS, LIUKKAUDENTORJUNTA, PUHTAANAPITO, HIEKANPOISTO] # As data providers have different names for their events, they are mapped # with this dict, so that every event that does the same has the same name. # The value is a list, as there can be events that belong to multiple main groups. From f3e5bf5f4e95f19d6b086446dc3f411fa663736f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 28 Mar 2023 11:27:38 +0300 Subject: [PATCH 64/97] make start_date_time aware --- street_maintenance/api/views.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/street_maintenance/api/views.py b/street_maintenance/api/views.py index 75e228330..df9c728c9 100644 --- a/street_maintenance/api/views.py +++ b/street_maintenance/api/views.py @@ -1,7 +1,7 @@ from datetime import datetime from functools import lru_cache -import pytz +from django.utils.timezone import make_aware from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter from rest_framework import mixins, viewsets from rest_framework.exceptions import ParseError @@ -14,12 +14,12 @@ MaintenanceWorkSerializer, ) from street_maintenance.management.commands.constants import ( + EVENT_CHOICES, PROVIDERS, START_DATE_TIME_FORMAT, ) from street_maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork -UTC_TIMEZONE = pytz.timezone("UTC") EXAMPLE_TIME_FORMAT = "YYYY-MM-DD HH:MM:SS" EXAMPLE_TIME = "2022-09-18 10:00:00" EVENT_PARAM = OpenApiParameter( @@ -27,7 +27,7 @@ location=OpenApiParameter.QUERY, description=( "Return objects of given event. " - f'Event choices are: {", ".join(PROVIDERS).lower()}, ' + f'Event choices are: {", ".join(EVENT_CHOICES).lower()}, ' 'E.g. "auraus".' ), required=False, @@ -112,8 +112,8 @@ def get_queryset(self): raise ParseError( f"'start_date_time' must be in format {EXAMPLE_TIME_FORMAT} elem.g.,'{EXAMPLE_TIME}'" ) - start_date_time = start_date_time.replace(tzinfo=UTC_TIMEZONE) - queryset = queryset.filter(timestamp__gte=start_date_time) + + queryset = queryset.filter(timestamp__gte=make_aware(start_date_time)) return queryset def list(self, request): @@ -184,8 +184,7 @@ def get_queryset(self): raise ParseError( f"'start_date_time' must be in format {EXAMPLE_TIME_FORMAT} elem.g.,'{EXAMPLE_TIME}'" ) - start_date_time = start_date_time.replace(tzinfo=UTC_TIMEZONE) - queryset = queryset.filter(timestamp__gte=start_date_time) + queryset = queryset.filter(timestamp__gte=make_aware(start_date_time)) return queryset @lru_cache(maxsize=16) From 4eb7ea7343bf5f76b2c7a7ca01cc7bbdd6c41b49 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 28 Mar 2023 11:36:44 +0300 Subject: [PATCH 65/97] Format code --- smbackend_turku/importers/utils.py | 20 ++++++++------- .../delete_obsolete_external_units.py | 25 +++++++++++-------- smbackend_turku/tasks.py | 2 -- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/smbackend_turku/importers/utils.py b/smbackend_turku/importers/utils.py index 037434831..a42b14feb 100644 --- a/smbackend_turku/importers/utils.py +++ b/smbackend_turku/importers/utils.py @@ -17,9 +17,7 @@ Municipality, ) -from mobility_data.importers.utils import ( - create_mobile_units_as_unit_references, -) +from mobility_data.importers.utils import create_mobile_units_as_unit_references from services.management.commands.services_import.services import ( update_service_counts, update_service_node_counts, @@ -349,7 +347,7 @@ def create_service(service_id, service_node_id, service_names): def delete_external_source( service_id, service_node_id, - ): +): """ Deletes the data source from services list and optionally from mobility_data. """ @@ -382,16 +380,20 @@ def __init__(self, config): self.config["service"]["name"], ) - def delete_external_source(self): - + def delete_external_source(self): + # Get ID from name, if ID is changed in config correct ID will be used for deletion try: - service_id = Service.objects.get(name=self.config["service"]["name"]["fi"]).id + service_id = Service.objects.get( + name=self.config["service"]["name"]["fi"] + ).id except Service.DoesNotExist: # If not found get ID from config service_id = self.config["service"]["id"] try: - service_node_id = ServiceNode.objects.get(name=self.config["service_node"]["name"]["fi"]).id + service_node_id = ServiceNode.objects.get( + name=self.config["service_node"]["name"]["fi"] + ).id except ServiceNode.DoesNotExist: service_node_id = self.config["service_node"]["id"] @@ -436,7 +438,7 @@ def save_objects_as_units(self, objects, content_type): unit.last_modified_time = datetime.datetime.now(UTC_TIMEZONE) set_service_names_field(unit) unit.save() - + if self.config.get("create_mobile_units_with_unit_reference", False): create_mobile_units_as_unit_references(self.SERVICE_ID, content_type) update_service_node_counts() diff --git a/smbackend_turku/management/commands/delete_obsolete_external_units.py b/smbackend_turku/management/commands/delete_obsolete_external_units.py index 736b78eeb..bf048115e 100644 --- a/smbackend_turku/management/commands/delete_obsolete_external_units.py +++ b/smbackend_turku/management/commands/delete_obsolete_external_units.py @@ -1,24 +1,27 @@ +from django.core.management import BaseCommand + +from services.management.commands.services_import.services import ( + update_service_counts, + update_service_node_counts, +) +from services.models import Service, ServiceNode, Unit + SERVICE_NODE = "service_node" SERVICE = "service" -GAS_FILLING_STATIONS_IDS={SERVICE_NODE:20000,SERVICE:20000} -CHARGING_STATIONS_IDS={SERVICE_NODE:30000,SERVICE:30000} -BICYCLE_STANDS_IDS={SERVICE_NODE:40000,SERVICE:40000} -BIKE_SERVICE_STATIONS_IDS={SERVICE_NODE:50000, SERVICE:50000} +GAS_FILLING_STATIONS_IDS = {SERVICE_NODE: 20000, SERVICE: 20000} +CHARGING_STATIONS_IDS = {SERVICE_NODE: 30000, SERVICE: 30000} +BICYCLE_STANDS_IDS = {SERVICE_NODE: 40000, SERVICE: 40000} +BIKE_SERVICE_STATIONS_IDS = {SERVICE_NODE: 50000, SERVICE: 50000} DELETE = [ GAS_FILLING_STATIONS_IDS, CHARGING_STATIONS_IDS, BICYCLE_STANDS_IDS, - BIKE_SERVICE_STATIONS_IDS + BIKE_SERVICE_STATIONS_IDS, ] -from django.core.management import BaseCommand -from services.models import Unit, Service, ServiceNode -from services.management.commands.services_import.services import ( - update_service_counts, - update_service_node_counts, -) + # This is a hack script to delete obsolete data as the IDs # changed. The bug that caused this is fixed and after this is run the # script is obsolete. diff --git a/smbackend_turku/tasks.py b/smbackend_turku/tasks.py index d3b0607c4..6999f01fe 100644 --- a/smbackend_turku/tasks.py +++ b/smbackend_turku/tasks.py @@ -90,5 +90,3 @@ def import_external_sources(name="import_external_sources"): @shared_task_email def delete_obsolete_external_units(name="delete_obsolete_external_units"): management.call_command("delete_obsolete_external_units") - - From 3ce765cb46eab821f791149a7c8dbd2091b7d929 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 11:02:43 +0300 Subject: [PATCH 66/97] Add TURKU_BBOX --- mobility_data/importers/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mobility_data/importers/constants.py b/mobility_data/importers/constants.py index 12f05f266..3d7fa8ee8 100644 --- a/mobility_data/importers/constants.py +++ b/mobility_data/importers/constants.py @@ -1,5 +1,6 @@ from django.contrib.gis.geos import Polygon +TURKU_BBOX = "22.0351,59.8163,23.1614,60.7483" SOUTHWEST_FINLAND_BOUNDARY_SRID = 4236 SOUTHWEST_FINLAND_BOUNDARY = [ [20.377543, 60.876637], From 762134bf17b35d01b3be67c713b2131d97df72a2 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 11:04:52 +0300 Subject: [PATCH 67/97] Add Overpass and Underpass --- mobility_data/importers/data/content_types.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index 9882a8935..477cc16f1 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -405,4 +405,16 @@ content_types: fi: Nopeusrajoitusalue sv: Hastighetsbegränsningszon en: Speed limit zone - # End of WFS importer content types \ No newline at end of file + # End of WFS importer content types + + - content_type_name: Underpass + name: + fi: Alikulku + sv: Underfartsbro + en: Underpass + + - content_type_name: Overpass + name: + fi: Ylikulku + sv: Överfartsbro + en: Overpass \ No newline at end of file From 48bdd506140f98510ad054a2ca8b907c1012130e Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 11:05:41 +0300 Subject: [PATCH 68/97] Add task that imports over- and underpasses --- mobility_data/tasks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mobility_data/tasks.py b/mobility_data/tasks.py index 6c9c62337..7bc4a593e 100644 --- a/mobility_data/tasks.py +++ b/mobility_data/tasks.py @@ -151,6 +151,11 @@ def import_parking_machines(name="import_parking_machines"): management.call_command("import_parking_machines") +@shared_task_email +def import_under_and_overpasses(name="import_under_and_overpasses"): + management.call_command("import_under_and_overpasses") + + @shared_task_email def delete_obsolete_data(name="delete_obsolete_data"): MobileUnit.objects.filter(content_types__isnull=True).delete() From 285ad03b6f752f84d1a5e482526a8850ae9bbb95 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 11:07:03 +0300 Subject: [PATCH 69/97] Add management command that imports over- and underpasses --- .../commands/import_under_and_overpasses.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 mobility_data/management/commands/import_under_and_overpasses.py diff --git a/mobility_data/management/commands/import_under_and_overpasses.py b/mobility_data/management/commands/import_under_and_overpasses.py new file mode 100644 index 000000000..ead258163 --- /dev/null +++ b/mobility_data/management/commands/import_under_and_overpasses.py @@ -0,0 +1,31 @@ +import logging + +from django.core.management import BaseCommand + +from mobility_data.importers.under_and_overpasses import ( + get_under_and_overpass_objects, + OVERPASS_CONTENT_TYPE_NAME, + UNDERPASS_CONTENT_TYPE_NAME, +) +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + log_imported_message, + save_to_database, +) + +logger = logging.getLogger("mobility_data") + + +class Command(BaseCommand): + def handle(self, *args, **options): + underpass_objects, overpass_objects = get_under_and_overpass_objects() + content_type = get_or_create_content_type_from_config( + UNDERPASS_CONTENT_TYPE_NAME + ) + num_ceated, num_deleted = save_to_database(underpass_objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) + content_type = get_or_create_content_type_from_config( + OVERPASS_CONTENT_TYPE_NAME + ) + num_ceated, num_deleted = save_to_database(overpass_objects, content_type) + log_imported_message(logger, content_type, num_ceated, num_deleted) From f28ea74a9322e9e9de3f4746df9a3721a2fee582 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 11:12:37 +0300 Subject: [PATCH 70/97] Add xmltodict --- requirements.in | 3 ++- requirements.txt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index 6a03ecb6b..85f1db3f1 100644 --- a/requirements.in +++ b/requirements.in @@ -41,4 +41,5 @@ libvoikko numpy>=1.22 pyshp polyline -drf-spectacular \ No newline at end of file +drf-spectacular +xmltodict \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index fa5aafdaf..186504f0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,6 +47,8 @@ click-repl==0.2.0 # via celery coverage==5.5 # via pytest-cov +cron-descriptor==1.2.35 + # via django-celery-beat deprecated==1.2.13 # via redis django==4.1.2 @@ -261,6 +263,8 @@ whitenoise==5.3.0 # via -r requirements.in wrapt==1.13.3 # via deprecated +xmltodict==0.13.0 + # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: # pip From 87fa2308a4322a559b0b8ed15207bb7b623817e3 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 13:53:28 +0300 Subject: [PATCH 71/97] Add under_and_overpasses importer --- mobility_data/management/commands/import_mobility_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mobility_data/management/commands/import_mobility_data.py b/mobility_data/management/commands/import_mobility_data.py index c514b3dc8..bba7c77a9 100644 --- a/mobility_data/management/commands/import_mobility_data.py +++ b/mobility_data/management/commands/import_mobility_data.py @@ -25,6 +25,7 @@ "foli_stops", "outdoor_gym_devices", "foli_parkandride_stops", + "under_and_overpasses", ] # Read the content type names to be imported wfs_content_type_names = get_configured_cotent_type_names() From ef3bca9f10bb1db2def8d6dcb6eb23c604b2e290 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 13:53:51 +0300 Subject: [PATCH 72/97] Add importer for under and overpasses --- .../importers/under_and_overpasses.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 mobility_data/importers/under_and_overpasses.py diff --git a/mobility_data/importers/under_and_overpasses.py b/mobility_data/importers/under_and_overpasses.py new file mode 100644 index 000000000..35a7ef3b6 --- /dev/null +++ b/mobility_data/importers/under_and_overpasses.py @@ -0,0 +1,67 @@ +import types + +import requests +import xmltodict +from django.conf import settings +from django.contrib.gis.geos import LineString + +from mobility_data.importers.constants import TURKU_BBOX + +from .utils import MobileUnitDataBase + +URL = ( + "https://julkinen.vayla.fi/inspirepalvelu/digiroad/wfs?service=WFS&request=GetFeature" + f"&typeName=dr_tielinkki_toim_lk&outputFormat=GML3&bbox={TURKU_BBOX},EPSG:4326&srsName=EPSG:4326" +) +UNDERPASS_CONTENT_TYPE_NAME = "Underpass" +OVERPASS_CONTENT_TYPE_NAME = "Overpass" +KUNTAKOODI = "853" +# 8 = kevyenliikenteenväylä +TOIMINN_LK = "8" +PASS_TYPES = types.SimpleNamespace() +PASS_TYPES.OVERPASS = 1 +PASS_TYPES.UNDERPASS = -1 + + +def get_json_data(url): + response = requests.get(URL) + assert response.status_code == 200 + json_data = xmltodict.parse(response.content) + return json_data + + +class Pass(MobileUnitDataBase): + def __init__(self, feature): + super().__init__() + coord_str = feature["digiroad:SHAPE"]["gml:LineString"]["gml:posList"] + coord_list = coord_str.split(" ") + coords = () + i = 0 + while i < len(coord_list): + x = coord_list[i] + y = coord_list[i + 1] + # discard z, i.e. i+2 + i += 3 + coords += ((float(x), float(y)),) + self.geometry = LineString(coords, srid=4326) + self.geometry.transform(settings.DEFAULT_SRID) + + +def get_under_and_overpass_objects(): + json_data = get_json_data(URL) + overpasses = [] + underpasses = [] + for feature in json_data["wfs:FeatureCollection"]["gml:featureMembers"][ + "digiroad:dr_tielinkki_toim_lk" + ]: + if ( + feature.get("digiroad:KUNTAKOODI", None) == KUNTAKOODI + and feature.get("digiroad:TOIMINN_LK", None) == TOIMINN_LK + ): + silta_alik = int(feature.get("digiroad:SILTA_ALIK", None)) + match silta_alik: + case PASS_TYPES.UNDERPASS: + underpasses.append(Pass(feature)) + case PASS_TYPES.OVERPASS: + overpasses.append(Pass(feature)) + return underpasses, overpasses From 93f9ff812144ebfc9792994a54c53bcbe4cd38ed Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 13:54:22 +0300 Subject: [PATCH 73/97] Add tests for underpass and overpass importer --- .../tests/test_import_under_and_overpasses.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 mobility_data/tests/test_import_under_and_overpasses.py diff --git a/mobility_data/tests/test_import_under_and_overpasses.py b/mobility_data/tests/test_import_under_and_overpasses.py new file mode 100644 index 000000000..5a0ccb56f --- /dev/null +++ b/mobility_data/tests/test_import_under_and_overpasses.py @@ -0,0 +1,51 @@ +from unittest.mock import patch + +import pytest + +from mobility_data.importers.utils import ( + get_or_create_content_type_from_config, + save_to_database, +) +from mobility_data.models import ContentType, MobileUnit + +from .utils import get_test_fixture_json_data + + +@pytest.mark.django_db +@patch("mobility_data.importers.under_and_overpasses.get_json_data") +def test_import_foli_stops(get_json_data_mock): + from mobility_data.importers.under_and_overpasses import ( + get_under_and_overpass_objects, + OVERPASS_CONTENT_TYPE_NAME, + UNDERPASS_CONTENT_TYPE_NAME, + ) + + get_json_data_mock.return_value = get_test_fixture_json_data( + "under_and_overpasses.json" + ) + underpass_objects, overpass_objects = get_under_and_overpass_objects() + assert len(underpass_objects) == 1 + assert len(overpass_objects) == 1 + underpass_content_type = get_or_create_content_type_from_config( + UNDERPASS_CONTENT_TYPE_NAME + ) + num_created, num_deleted = save_to_database( + underpass_objects, underpass_content_type + ) + assert num_created == 1 + assert num_deleted == 0 + assert ( + ContentType.objects.filter(type_name=UNDERPASS_CONTENT_TYPE_NAME).count() == 1 + ) + assert MobileUnit.objects.filter(content_types=underpass_content_type).count() == 1 + overpass_content_type = get_or_create_content_type_from_config( + OVERPASS_CONTENT_TYPE_NAME + ) + num_created, num_deleted = save_to_database(overpass_objects, overpass_content_type) + assert num_created == 1 + assert num_deleted == 0 + assert ContentType.objects.filter(type_name=OVERPASS_CONTENT_TYPE_NAME).count() == 1 + assert MobileUnit.objects.filter(content_types=overpass_content_type).count() == 1 + + assert MobileUnit.objects.count() == 2 + assert ContentType.objects.count() == 2 From 183555be203dafc6046d04e8883433b770436be9 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 29 Mar 2023 13:55:08 +0300 Subject: [PATCH 74/97] Add fixture data for underpass and overpass importer tests --- .../tests/data/under_and_overpasses.json | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 mobility_data/tests/data/under_and_overpasses.json diff --git a/mobility_data/tests/data/under_and_overpasses.json b/mobility_data/tests/data/under_and_overpasses.json new file mode 100644 index 000000000..c3a32970f --- /dev/null +++ b/mobility_data/tests/data/under_and_overpasses.json @@ -0,0 +1,116 @@ +{ + "wfs:FeatureCollection": { + "@xmlns:xs": "http://www.w3.org/2001/XMLSchema", + "@xmlns:digiroad": "www.livi_digiroad.fi", + "@xmlns:wfs": "http://www.opengis.net/wfs", + "@xmlns:gml": "http://www.opengis.net/gml", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@numberOfFeatures": "139994", + "@timeStamp": "2023-03-29T11:48:44.864+03:00", + "@xsi:schemaLocation": "http://www.opengis.net/wfs https://julkinen.vayla.fi/inspirepalvelu/schemas/wfs/1.1.0/wfs.xsd www.livi_digiroad.fi https://julkinen.vayla.fi/inspirepalvelu/digiroad/wfs?service=WFS&version=2.0.0&request=DescribeFeatureType&typeName=digiroad%3Adr_tielinkki_toim_lk", + "gml:featureMembers": + + { + "digiroad:dr_tielinkki_toim_lk": [{ + "@gml:id": "dr_tielinkki_toim_lk.4364843", + "digiroad:LINK_ID": "4364843", + "digiroad:LINK_MMLID": "71182700", + "digiroad:HALLINN_LK": "3", + "digiroad:TOIMINN_LK": "7", + "digiroad:AJOSUUNTA": "2", + "digiroad:LINKKITYYP": "12", + "digiroad:SILTA_ALIK": "0", + "digiroad:TIENIMI_SU": "Nummen mets\u00e4tie", + "digiroad:ENS_TALO_V": "263", + "digiroad:ENS_TALO_O": "264", + "digiroad:VIIM_TAL_V": "251", + "digiroad:VIIM_TAL_O": "252", + "digiroad:KUNTAKOODI": "636", + "digiroad:MUOKKAUSPV": "25.12.2018 01:00:40", + "digiroad:SIJ_TARK": "3000", + "digiroad:KOR_TARK": "201", + "digiroad:GEOM_FLIP": "1", + "digiroad:ALKU_PAALU": "0", + "digiroad:LOPP_PAALU": "325.688", + "digiroad:GEOM_LAHDE": "1", + "digiroad:SHAPE": { + "gml:LineString": { + "@srsName": "http://www.opengis.net/gml/srs/epsg.xml#4326", + "@srsDimension": "3", + "gml:posList": "22.52851124 60.76362636 73.823 22.52807579 60.76374084 73.908 22.52768355 60.76380312 74.073 22.52719903 60.76398517 74.286 22.52638126 60.76421428 74.051 22.52563276 60.76444342 73.59 22.52485326 60.76468844 74.223 22.52381843 60.76500016 74.74 22.52340543 60.76512753 75.399" + } + }, + "digiroad:ALKUSOLMU": "9304724", + "digiroad:LOPPUSOLMU": "9298108", + "digiroad:MTK_TIE_LK": "12141" + }, + { + "@gml:id": "dr_tielinkki_toim_lk.4364843", + "digiroad:LINK_ID": "644680", + "digiroad:LINK_MMLID": "71182700", + "digiroad:HALLINN_LK": "99", + "digiroad:TOIMINN_LK": "8", + "digiroad:AJOSUUNTA": "2", + "digiroad:LINKKITYYP": "12", + "digiroad:SILTA_ALIK": "-1", + "digiroad:TIENIMI_SU": "Läntinen Rantakatu", + "digiroad:ENS_TALO_V": "263", + "digiroad:ENS_TALO_O": "264", + "digiroad:VIIM_TAL_V": "251", + "digiroad:VIIM_TAL_O": "252", + "digiroad:KUNTAKOODI": "853", + "digiroad:MUOKKAUSPV": "25.12.2018 01:00:40", + "digiroad:SIJ_TARK": "3000", + "digiroad:KOR_TARK": "201", + "digiroad:GEOM_FLIP": "1", + "digiroad:ALKU_PAALU": "0", + "digiroad:LOPP_PAALU": "325.688", + "digiroad:GEOM_LAHDE": "1", + "digiroad:SHAPE": { + "gml:LineString": { + "@srsName": "http://www.opengis.net/gml/srs/epsg.xml#4326", + "@srsDimension": "3", + "gml:posList": "22.32554152 60.53907714 41.976 22.32535151 60.53944881 41.501" + } + }, + "digiroad:ALKUSOLMU": "9304724", + "digiroad:LOPPUSOLMU": "9298108", + "digiroad:MTK_TIE_LK": "12141" + }, + { + "@gml:id": "dr_tielinkki_toim_lk.4364843", + "digiroad:LINK_ID": "644680", + "digiroad:LINK_MMLID": "71182700", + "digiroad:HALLINN_LK": "99", + "digiroad:TOIMINN_LK": "8", + "digiroad:AJOSUUNTA": "2", + "digiroad:LINKKITYYP": "12", + "digiroad:SILTA_ALIK": "1", + "digiroad:TIENIMI_SU": "Läntinen Testikatu", + "digiroad:ENS_TALO_V": "263", + "digiroad:ENS_TALO_O": "264", + "digiroad:VIIM_TAL_V": "251", + "digiroad:VIIM_TAL_O": "252", + "digiroad:KUNTAKOODI": "853", + "digiroad:MUOKKAUSPV": "25.12.2018 01:00:40", + "digiroad:SIJ_TARK": "3000", + "digiroad:KOR_TARK": "201", + "digiroad:GEOM_FLIP": "1", + "digiroad:ALKU_PAALU": "0", + "digiroad:LOPP_PAALU": "325.688", + "digiroad:GEOM_LAHDE": "1", + "digiroad:SHAPE": { + "gml:LineString": { + "@srsName": "http://www.opengis.net/gml/srs/epsg.xml#4326", + "@srsDimension": "3", + "gml:posList": "22.3295715 60.54484321 32.318 22.33085745 60.54539151 33.151" + } + }, + "digiroad:ALKUSOLMU": "9304724", + "digiroad:LOPPUSOLMU": "9298108", + "digiroad:MTK_TIE_LK": "12141" + } + ] + } + } +} \ No newline at end of file From c94aa6e3959eac9fa0024390d343d7edd471ece6 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 30 Mar 2023 14:44:13 +0300 Subject: [PATCH 75/97] Add content_type_name for CultureRouteUnit and Geometry --- mobility_data/importers/data/content_types.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index 9882a8935..e817121f5 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -405,4 +405,16 @@ content_types: fi: Nopeusrajoitusalue sv: Hastighetsbegränsningszon en: Speed limit zone - # End of WFS importer content types \ No newline at end of file + # End of WFS importer content types + + - content_type_name: CultureRouteUnit + name: + fi: Kultuurikuntoilureitti + sv: Kulturväg + en: Culture route + + - content_type_name: CultureRouteGeometry + name: + fi: Kulttuurikuntoilureitin geometria + sv: Geometri för kulturväg + en: Geometry for culture route \ No newline at end of file From f0d5436219f696487393d2c1390b56bc4bc49ff5 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 30 Mar 2023 14:45:10 +0300 Subject: [PATCH 76/97] Use generic content type creation, delete obsolete routes --- mobility_data/importers/culture_routes.py | 105 ++++++++++++++-------- 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/mobility_data/importers/culture_routes.py b/mobility_data/importers/culture_routes.py index 0829b9395..36c156028 100644 --- a/mobility_data/importers/culture_routes.py +++ b/mobility_data/importers/culture_routes.py @@ -10,7 +10,7 @@ from mobility_data.models import GroupType, MobileUnit, MobileUnitGroup -from .utils import get_or_create_content_type, set_translated_field +from .utils import get_or_create_content_type_from_config, MobileUnitDataBase logger = logging.getLogger("mobility_data") # Regexps used to remove html, & tags and css. @@ -96,12 +96,11 @@ def __init__(self, documents, languages, trailing_number=None): self.description[lang] = description -class Placemark: +class Placemark(MobileUnitDataBase): def __init__(self): # Store the name and description for every langue to dictionaries - self.name = {} - self.description = {} - self.geometry = None + super().__init__() + self.content_type = None def set_data(self, placemark, lang, add_geometry=False): """ @@ -253,50 +252,82 @@ def get_routes(): @db.transaction.atomic def save_to_database(routes, delete_tables=False): - if delete_tables: - GroupType.objects.filter(name=GROUP_CONTENT_TYPE_NAME).delete() + routes_created = 0 + units_created = 0 + if delete_tables: + GroupType.objects.filter(type_name=GROUP_CONTENT_TYPE_NAME).delete() + units_to_delete = list( + MobileUnit.objects.filter( + mobile_unit_group__group_type__type_name=GROUP_CONTENT_TYPE_NAME + ).values_list("id", flat=True) + ) + routes_to_delete = list( + MobileUnitGroup.objects.filter( + group_type__type_name=GROUP_CONTENT_TYPE_NAME + ).values_list("id", flat=True) + ) group_type, _ = GroupType.objects.get_or_create( - name=GROUP_CONTENT_TYPE_NAME, + type_name=GROUP_CONTENT_TYPE_NAME, description="Culture Routes in Turku", ) - unit_type, _ = get_or_create_content_type( - name=ROUTE_CONTENT_TYPE_NAME, - description="Contains pointdata, name and description of a place in a Culture Route.", + unit_content_type = get_or_create_content_type_from_config(ROUTE_CONTENT_TYPE_NAME) + geometry_content_type = get_or_create_content_type_from_config( + GEOMETRY_CONTENT_TYPE_NAME ) - geometry_type, _ = get_or_create_content_type( - GEOMETRY_CONTENT_TYPE_NAME, - "Contains the LineString geometry of the Culture Route.", - ) - routes_saved = 0 + # Routes are stored as MobileUnitGroups and Placemarks as MobileUnits for route in routes: - group, created = MobileUnitGroup.objects.get_or_create( - group_type=group_type, name=route.name["fi"] - ) - if created: - set_translated_field(group, "name", route.name) - set_translated_field(group, "description", route.description) - group.save() - routes_saved += 1 + filter = { + "name": route.name.get("fi", None), + "name_sv": route.name.get("sv", None), + "name_en": route.name.get("en", None), + "description": route.description.get("fi", None), + "description_sv": route.description.get("sv", None), + "description_en": route.description.get("en", None), + "group_type": group_type, + } + queryset = MobileUnitGroup.objects.filter(**filter) + if queryset.count() == 0: + group = MobileUnitGroup.objects.create(**filter) + routes_created += 1 + + else: + group = queryset.first() + id = group.id + if id in routes_to_delete: + routes_to_delete.remove(id) for placemark in route.placemarks: content_type = None # If the geometry is a Point the content_type is Culture Route MobileUnit if isinstance(placemark.geometry, Point): - content_type = unit_type + content_type = unit_content_type # If the geometry is a LineString we not the content_Type is Culture Route Geometry elif isinstance(placemark.geometry, LineString): - content_type = geometry_type + content_type = geometry_content_type + + filter = { + "name": placemark.name["fi"], + "name_sv": placemark.name["sv"], + "name_en": placemark.name["en"], + "description": placemark.description["fi"], + "description_sv": placemark.description["sv"], + "description_en": placemark.description["en"], + "geometry": placemark.geometry, + "mobile_unit_group": group, + "is_active": True, + } - mobile_unit, created = MobileUnit.objects.get_or_create( - mobile_unit_group=group, - geometry=placemark.geometry, - ) - mobile_unit.content_types.add(content_type) - if created: - mobile_unit.is_active = True - set_translated_field(mobile_unit, "name", placemark.name) - set_translated_field(mobile_unit, "description", placemark.description) - mobile_unit.save() - return routes_saved + queryset = MobileUnit.objects.filter(**filter) + if queryset.count() == 0: + mobile_unit = MobileUnit.objects.create(**filter) + mobile_unit.content_types.add(content_type) + units_created += 1 + else: + id = queryset.first().id + if id in units_to_delete: + units_to_delete.remove(id) + MobileUnit.objects.filter(id__in=units_to_delete).delete() + MobileUnitGroup.objects.filter(id__in=routes_to_delete).delete() + return routes_created, len(routes_to_delete), units_created, len(units_to_delete) From e66d568ea2d2a0ceed11908496f7ed49dc944c8b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 30 Mar 2023 14:46:20 +0300 Subject: [PATCH 77/97] Display number of saved and deleted routes and units --- .../management/commands/import_culture_routes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mobility_data/management/commands/import_culture_routes.py b/mobility_data/management/commands/import_culture_routes.py index 138c80551..a6e7d85a7 100644 --- a/mobility_data/management/commands/import_culture_routes.py +++ b/mobility_data/management/commands/import_culture_routes.py @@ -19,12 +19,13 @@ def add_arguments(self, parser): def handle(self, *args, **options): logger.info("Importing culture routes...") routes = get_routes() - delete_tables = False - if options["delete"]: - delete_tables = True - num_saved = save_to_database(routes, delete_tables=delete_tables) + delete_tables = options.get("delete", False) + routes_saved, routes_deleted, units_saved, units_deleted = save_to_database( + routes, delete_tables=delete_tables + ) logger.info( - "Fetched {} Culture Routes and saved {} new Culture Routes to database.".format( - len(routes), num_saved + "Fetchet {} Culture Routes. Saved {} routes and deleted {} obsolete routes." + " Saved {} units and deleted {} obsolete units".format( + len(routes), routes_saved, routes_deleted, units_saved, units_deleted ) ) From bdecc2ec761d24fd643c9d05a9d8db896fcc9923 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 31 Mar 2023 13:24:44 +0300 Subject: [PATCH 78/97] Fix typo Fetchet -> Fetched --- mobility_data/management/commands/import_culture_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/management/commands/import_culture_routes.py b/mobility_data/management/commands/import_culture_routes.py index a6e7d85a7..241143c96 100644 --- a/mobility_data/management/commands/import_culture_routes.py +++ b/mobility_data/management/commands/import_culture_routes.py @@ -24,7 +24,7 @@ def handle(self, *args, **options): routes, delete_tables=delete_tables ) logger.info( - "Fetchet {} Culture Routes. Saved {} routes and deleted {} obsolete routes." + "Fetched {} Culture Routes. Saved {} routes and deleted {} obsolete routes." " Saved {} units and deleted {} obsolete units".format( len(routes), routes_saved, routes_deleted, units_saved, units_deleted ) From de09d3f6d698f04b5e714bd20955ca974c4fa018 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 13:05:12 +0300 Subject: [PATCH 79/97] Refactor, add get_queryset and get_serializer_context --- mobility_data/api/views.py | 50 +++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py index 8d159aac3..22e97b88b 100644 --- a/mobility_data/api/views.py +++ b/mobility_data/api/views.py @@ -1,8 +1,10 @@ +import logging import types from distutils.util import strtobool from django.contrib.gis.gdal import SpatialReference from django.core.exceptions import ValidationError +from django.db import connection, reset_queries from django.db.models import Q from munigeo import api as munigeo_api from rest_framework import status, viewsets @@ -25,6 +27,8 @@ FIELD_TYPES.INT = int FIELD_TYPES.BOOL = bool +logger = logging.getLogger("mobility_data") + def get_srid_and_latlon(filters): """ @@ -118,7 +122,7 @@ def list(self, request): class MobileUnitViewSet(viewsets.ReadOnlyModelViewSet): - queryset = MobileUnit.objects.all() + queryset = MobileUnit.objects.filter(is_active=True) serializer_class = MobileUnitSerializer def retrieve(self, request, pk=None): @@ -138,15 +142,19 @@ def retrieve(self, request, pk=None): ) return Response(serializer.data, status=status.HTTP_200_OK) - def list(self, request): - """ - Lists MobileUnits, optionally list by type_name if given - and transforms to given srid. - """ - queryset = None + def get_serializer_context(self): + context = super().get_serializer_context() + context["srid"], context["latlon"] = get_srid_and_latlon( + self.request.query_params + ) + context["self.services_unit_instances"] = self.services_unit_instances + return context + + def get_queryset(self): + queryset = MobileUnit.objects.filter(is_active=True) + queryset = queryset.prefetch_related("content_types") unit_ids = [] filters = self.request.query_params - srid, latlon = get_srid_and_latlon(filters) if "type_name" in filters: type_name = filters["type_name"] if not ContentType.objects.filter(type_name=type_name).exists(): @@ -162,13 +170,15 @@ def list(self, request): else: queryset = MobileUnit.objects.all() - services_unit_instances = True if len(unit_ids) > 0 else False - if services_unit_instances: + self.services_unit_instances = True if len(unit_ids) > 0 else False + if self.services_unit_instances: queryset = Unit.objects.filter(id__in=unit_ids) if "bbox" in filters: val = filters.get("bbox", None) - geometry_field_name = "location" if services_unit_instances else "geometry" + geometry_field_name = ( + "location" if self.services_unit_instances else "geometry" + ) if val: ref = SpatialReference(filters.get("bbox_srid", 4326)) bbox_geometry_filter = munigeo_api.build_bbox_filter( @@ -209,16 +219,18 @@ def list(self, request): queryset = queryset.filter(**{filter: value}) + return queryset + + def list(self, request): + queryset = self.get_queryset() page = self.paginate_queryset(queryset) - serializer = MobileUnitSerializer( - page, - many=True, - context={ - "srid": srid, - "latlon": latlon, - "services_unit_instances": services_unit_instances, - }, + logger.debug(connection.queries) + queries_time = sum([float(s["time"]) for s in connection.queries]) + logger.debug( + f"Search queries total execution time: {queries_time} Num queries: {len(connection.queries)}" ) + reset_queries() + serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) From 821feaeff307eae29063700e315e27421269f038 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 13:08:44 +0300 Subject: [PATCH 80/97] Make benchark output conditional, fix output --- mobility_data/api/views.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py index 22e97b88b..a0493b5e4 100644 --- a/mobility_data/api/views.py +++ b/mobility_data/api/views.py @@ -224,12 +224,13 @@ def get_queryset(self): def list(self, request): queryset = self.get_queryset() page = self.paginate_queryset(queryset) - logger.debug(connection.queries) - queries_time = sum([float(s["time"]) for s in connection.queries]) - logger.debug( - f"Search queries total execution time: {queries_time} Num queries: {len(connection.queries)}" - ) - reset_queries() + if logger.level <= logging.DEBUG: + logger.debug(connection.queries) + queries_time = sum([float(s["time"]) for s in connection.queries]) + logger.debug( + f"MobileUnit list queries total execution time: {queries_time} Num queries: {len(connection.queries)}" + ) + reset_queries() serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) From 914d4f2702e8493a343e9bb6e3bccccb1052c4ee Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 13:59:32 +0300 Subject: [PATCH 81/97] Add only param test --- mobility_data/tests/test_api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index 7861f862b..ef9fa121f 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -122,6 +122,13 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): assert result["geometry"] == "POINT (24.24 62.22)" assert result["geometry_coords"]["lon"] == 24.24 assert result["geometry_coords"]["lat"] == 62.22 + # Test only param + url = reverse("mobility_data:mobile_units-list") + "?only=id,name" + response = api_client.get(url) + assert len(response.json()["results"][0]) == 2 + assert len(response.json()["results"][1]) == 2 + + breakpoint() @pytest.mark.django_db From 0a565dcf68b37deef8550fdfffef4779d746e138 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 13:59:58 +0300 Subject: [PATCH 82/97] Serialize only fields in only param --- mobility_data/api/serializers/mobile_unit.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mobility_data/api/serializers/mobile_unit.py b/mobility_data/api/serializers/mobile_unit.py index 8a903bf32..b73a72b25 100644 --- a/mobility_data/api/serializers/mobile_unit.py +++ b/mobility_data/api/serializers/mobile_unit.py @@ -84,6 +84,16 @@ class Meta: "unit_id": "id", } + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + context = kwargs.get("context", {}) + if "only" in context: + self.keep_fields = set(context["only"] + ["id"]) + for field_name in list(self.fields.keys()): + if field_name in self.keep_fields: + continue + del self.fields[field_name] + def to_representation(self, obj): representation = super().to_representation(obj) unit_id = getattr(obj, "unit_id", None) @@ -122,7 +132,7 @@ def to_representation(self, obj): if field == "unit_id": representation["unit_id"] = mobile_unit.unit_id # The location field must be serialized with its wkt value. - if unit.location: + if unit.location and "geometry" in representation: representation["geometry"] = unit.location.wkt return representation From e3d96bf673ce9fa710c7f05dc3f33bd793488ead Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 14:02:34 +0300 Subject: [PATCH 83/97] Get only serializer context variable --- mobility_data/api/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py index a0493b5e4..ac25801e8 100644 --- a/mobility_data/api/views.py +++ b/mobility_data/api/views.py @@ -144,6 +144,10 @@ def retrieve(self, request, pk=None): def get_serializer_context(self): context = super().get_serializer_context() + only = self.request.query_params.get("only", "") + if only: + context["only"] = [x.strip() for x in only.split(",") if x] + context["srid"], context["latlon"] = get_srid_and_latlon( self.request.query_params ) From 5462db29ea103f06b531839c36e1a9296b1f03bc Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 14:04:14 +0300 Subject: [PATCH 84/97] Remove breakpoint --- mobility_data/tests/test_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index ef9fa121f..7690d8138 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -128,8 +128,6 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): assert len(response.json()["results"][0]) == 2 assert len(response.json()["results"][1]) == 2 - breakpoint() - @pytest.mark.django_db def test_mobile_unit_group(api_client, mobile_unit_group, group_type): From 7b766a5dbd33a579024511b5138367dba09080f2 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 14:12:19 +0300 Subject: [PATCH 85/97] Add only_param --- mobility_data/specification.swagger2.0.yaml | 25 ++++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/mobility_data/specification.swagger2.0.yaml b/mobility_data/specification.swagger2.0.yaml index 3c46b7c78..f5297c80c 100644 --- a/mobility_data/specification.swagger2.0.yaml +++ b/mobility_data/specification.swagger2.0.yaml @@ -136,6 +136,7 @@ paths: - $ref: "#/components/parameters/extra_param" - $ref: "#/components/parameters/bbox_param" - $ref: "#/components/parameters/bbox_srid_param" + - $ref: "#/components/parameters/only_param" responses: 200: @@ -305,9 +306,21 @@ components: bbox_srid_param: name: bbox_srid - in: query - description: An SRID coordinate reference system identifier which specifies the - coordinate system used in the bbox parameter. - schema: - type: integer - example: 3046 \ No newline at end of file + in: query + description: An SRID coordinate reference system identifier which specifies the + coordinate system used in the bbox parameter. + schema: + type: integer + example: 3046 + + only_param: + name: only + in: query + style: form + explode: false + description: Restrict the field returned in the results. Separate field names by + commas. + schema: + type: array + items: + type: string \ No newline at end of file From bd7209448d38b112f318615e170fba4b2c01f2eb Mon Sep 17 00:00:00 2001 From: juuso-j Date: Tue, 4 Apr 2023 14:39:25 +0300 Subject: [PATCH 86/97] Remove id from only args --- mobility_data/tests/test_api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index 7690d8138..bdde2aa6a 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -123,10 +123,11 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): assert result["geometry_coords"]["lon"] == 24.24 assert result["geometry_coords"]["lat"] == 62.22 # Test only param - url = reverse("mobility_data:mobile_units-list") + "?only=id,name" + url = reverse("mobility_data:mobile_units-list") + "?only=name,geometry" response = api_client.get(url) - assert len(response.json()["results"][0]) == 2 - assert len(response.json()["results"][1]) == 2 + # 'id' is always serialized, so the length will be 3 + assert len(response.json()["results"][0]) == 3 + assert len(response.json()["results"][1]) == 3 @pytest.mark.django_db From 179ef41765236603d13c16b9fa0e5b57de6885cf Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 5 Apr 2023 11:43:56 +0300 Subject: [PATCH 87/97] Add type_names param --- mobility_data/api/views.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py index ac25801e8..a1545b79c 100644 --- a/mobility_data/api/views.py +++ b/mobility_data/api/views.py @@ -151,7 +151,7 @@ def get_serializer_context(self): context["srid"], context["latlon"] = get_srid_and_latlon( self.request.query_params ) - context["self.services_unit_instances"] = self.services_unit_instances + context["services_unit_instances"] = self.services_unit_instances return context def get_queryset(self): @@ -159,20 +159,22 @@ def get_queryset(self): queryset = queryset.prefetch_related("content_types") unit_ids = [] filters = self.request.query_params - if "type_name" in filters: - type_name = filters["type_name"] - if not ContentType.objects.filter(type_name=type_name).exists(): - return Response( - "type_name does not exist.", status=status.HTTP_400_BAD_REQUEST - ) - queryset = MobileUnit.objects.filter(content_types__type_name=type_name) + if "type_name" in filters or "type_names" in filters: + type_name = filters.get("type_name", None) + if type_name: + queryset = queryset.filter(content_types__type_name=type_name) + else: + type_names = [ + t.strip() for t in filters.get("type_names", "").split(",") + ] + queryset = queryset.filter( + content_types__type_name__in=type_names + ).distinct() # If the data locates in the services_unit table (i.e., MobileUnit has a unit_id) # get the unit_ids to retrieve the Units for filtering(bbox and extra) unit_ids = list( queryset.filter(unit_id__isnull=False).values_list("unit_id", flat=True) ) - else: - queryset = MobileUnit.objects.all() self.services_unit_instances = True if len(unit_ids) > 0 else False if self.services_unit_instances: From 599e0e99c585adc3573d10fd9d0ec7a4a3882480 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 5 Apr 2023 11:44:25 +0300 Subject: [PATCH 88/97] Test type_names param --- mobility_data/tests/test_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index bdde2aa6a..1e79997a4 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -118,7 +118,7 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): result = response.json() assert result["name"] == "Test unit" assert result["description"] == "desc" - assert result["content_types"][0]["type_name"] == "Test unit" + assert result["content_types"][0]["type_name"] == "TestUnit" assert result["geometry"] == "POINT (24.24 62.22)" assert result["geometry_coords"]["lon"] == 24.24 assert result["geometry_coords"]["lat"] == 62.22 @@ -128,6 +128,10 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): # 'id' is always serialized, so the length will be 3 assert len(response.json()["results"][0]) == 3 assert len(response.json()["results"][1]) == 3 + # Test retrieving multiple content types + url = reverse("mobility_data:mobile_units-list") + "?type_names=Test,Test2" + response = api_client.get(url) + assert len(response.json()["results"]) == 2 @pytest.mark.django_db From 3b57ddf300ba94b3f291ac46358a7f5fe009b21b Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 5 Apr 2023 12:40:30 +0300 Subject: [PATCH 89/97] Add type_names param --- mobility_data/specification.swagger2.0.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/mobility_data/specification.swagger2.0.yaml b/mobility_data/specification.swagger2.0.yaml index f5297c80c..6e08eec01 100644 --- a/mobility_data/specification.swagger2.0.yaml +++ b/mobility_data/specification.swagger2.0.yaml @@ -133,6 +133,7 @@ paths: - $ref: "#/components/parameters/srid_param" - $ref: "#/components/parameters/latlon_param" - $ref: "#/components/parameters/type_name_param" + - $ref: "#/components/parameters/type_names_param" - $ref: "#/components/parameters/extra_param" - $ref: "#/components/parameters/bbox_param" - $ref: "#/components/parameters/bbox_srid_param" @@ -323,4 +324,19 @@ components: schema: type: array items: - type: string \ No newline at end of file + type: string + example: name,geometry + + type_names_param: + name: type_names + in: query + style: form + explode: false + description: Return MobileUnits from multiple content types. Separate field names by + commas. Note, Filtering MobileUnits with ContentTypes containing MobileUnits and MobileUnits + that contains references to services_unit table is not possible. + schema: + type: array + items: + type: string + example: PublicToilet,HikingTrail \ No newline at end of file From b035151b5aae14ec5d4d189d99d3f55b474fdfff Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 5 Apr 2023 12:41:52 +0300 Subject: [PATCH 90/97] Refactor --- mobility_data/tests/conftest.py | 112 ++++++++++++-------------------- 1 file changed, 43 insertions(+), 69 deletions(-) diff --git a/mobility_data/tests/conftest.py b/mobility_data/tests/conftest.py index 91eff2084..7cbb8c523 100644 --- a/mobility_data/tests/conftest.py +++ b/mobility_data/tests/conftest.py @@ -50,31 +50,25 @@ def api_client(): @pytest.mark.django_db @pytest.fixture def content_types(): - content_types = [ - ContentType.objects.create( - id="aa6c2903-d36f-4c61-b828-19084fc7a64b", - type_name="Test", - name_fi="fi", - name_sv="sv", - name_en="en", - description="test content type", - ) - ] - content_types.append( - ContentType.objects.create( - id="ba6c2903-d36f-4c61-b828-19084fc7a64b", - type_name="Test2", - description="test content type2", - ) + ContentType.objects.create( + id="aa6c2903-d36f-4c61-b828-19084fc7a64b", + type_name="Test", + name_fi="fi", + name_sv="sv", + name_en="en", + description="test content type", + ) + ContentType.objects.create( + id="ba6c2903-d36f-4c61-b828-19084fc7a64b", + type_name="Test2", + description="test content type2", ) - content_types.append( - ContentType.objects.create( - id="ca6c2903-d36f-4c61-b828-19084fc7a64b", - type_name="Test unit", - description="test content type3", - ) + ContentType.objects.create( + id="ca6c2903-d36f-4c61-b828-19084fc7a64b", + type_name="TestUnit", + description="test content type3", ) - return content_types + return ContentType.objects.all() @pytest.mark.django_db @@ -89,7 +83,6 @@ def group_type(): @pytest.mark.django_db @pytest.fixture def mobile_units(content_types): - mobile_units = [] extra = { "test_int": 4242, "test_float": 42.42, @@ -105,8 +98,7 @@ def mobile_units(content_types): geometry=geometry, extra=extra, ) - mobile_unit.content_types.add(content_types[0]) - mobile_units.append(mobile_units) + mobile_unit.content_types.add(ContentType.objects.get(type_name="Test")) extra = { "test_int": 14, "test_float": 2.4, @@ -120,15 +112,13 @@ def mobile_units(content_types): geometry=Point(23.43, 62.22, srid=settings.DEFAULT_SRID), extra=extra, ) - mobile_unit.content_types.add(content_types[0]) - mobile_unit.content_types.add(content_types[1]) - mobile_units.append(mobile_units) + mobile_unit.content_types.add(ContentType.objects.get(type_name="Test")) + mobile_unit.content_types.add(ContentType.objects.get(type_name="Test2")) mobile_unit = MobileUnit.objects.create( id="ca6c2903-d36f-4c61-b828-19084fc7a64b", unit_id=1 ) - mobile_unit.content_types.add(content_types[2]) - mobile_units.append(mobile_units) - return mobile_units + mobile_unit.content_types.add(ContentType.objects.get(type_name="TestUnit")) + return MobileUnit.objects.all() @pytest.mark.django_db @@ -166,11 +156,10 @@ def mobile_unit_group(group_type): @pytest.mark.django_db @pytest.fixture def municipalities(): - munis = [] - munis.append(Municipality.objects.create(id="turku", name="Turku")) - munis.append(Municipality.objects.create(id="lieto", name="Lieto")) - munis.append(Municipality.objects.create(id="raisio", name="Raisio")) - return munis + Municipality.objects.create(id="turku", name="Turku") + Municipality.objects.create(id="lieto", name="Lieto") + Municipality.objects.create(id="raisio", name="Raisio") + return Municipality.objects.all() @pytest.mark.django_db @@ -204,66 +193,57 @@ def administrative_division_geometry(administrative_division): @pytest.mark.django_db @pytest.fixture def streets(): - streets = [] - street = Street.objects.create( + Street.objects.create( name="Test Street", name_fi="Test Street", name_sv="Test StreetSV", municipality_id="turku", ) - streets.append(street) - street = Street.objects.create( + Street.objects.create( name="Linnanpuisto", name_fi="Linnanpuisto", name_sv="Slottsparken", municipality_id="turku", ) - streets.append(street) - street = Street.objects.create( + Street.objects.create( name="Kristiinankatu", name_fi="Kristiinankatu", name_sv="Kristinegatan", municipality_id="turku", ) - streets.append(street) - street = Street.objects.create( + Street.objects.create( name="Pitkäpellonkatu", name_fi="Pitkäpellonkatu", name_sv="Långåkersgatan", municipality_id="turku", ) - streets.append(street) - street = Street.objects.create( + Street.objects.create( name="Kupittaankatu", name_fi="Kupittaankatu", name_sv="Kuppisgatan", municipality_id="turku", ) - streets.append(street) - street = Street.objects.create( + Street.objects.create( name="Yliopistonkatu", name_fi="Yliopistonkatu", name_sv="Universitetsgatan", municipality_id="turku", ) - streets.append(street) - street = Street.objects.create( + Street.objects.create( name="Ratapihankatu", name_fi="Ratapihankatu", name_sv="Bangårdsgatan", municipality_id="turku", ) - streets.append(street) - return streets + return Street.objects.all() @pytest.mark.django_db @pytest.fixture def address(streets, municipalities): - turku_muni = municipalities[0] - addresses = [] + turku_muni = Municipality.objects.get(id="turku") location = Point(22.244, 60.4, srid=4326) - address = Address.objects.create( + Address.objects.create( municipality_id=turku_muni.id, id=100, location=location, @@ -272,9 +252,8 @@ def address(streets, municipalities): full_name_fi="Test Street 42", full_name_sv="Test StreetSV 42", ) - addresses.append(address) location = Point(22.227168, 60.4350612, srid=4326) - address = Address.objects.create( + Address.objects.create( municipality_id=turku_muni.id, id=101, location=location, @@ -282,9 +261,8 @@ def address(streets, municipalities): full_name_fi="Linnanpuisto", full_name_sv="Slottsparken", ) - addresses.append(address) location = Point(22.264457, 60.448905, srid=4326) - address = Address.objects.create( + Address.objects.create( municipality_id=turku_muni.id, id=102, location=location, @@ -293,9 +271,8 @@ def address(streets, municipalities): full_name_fi="Kristiinankatu 4", full_name_sv="Kristinegata 4", ) - addresses.append(address) location = Point(22.2383, 60.411726, srid=4326) - address = Address.objects.create( + Address.objects.create( municipality_id=turku_muni.id, id=103, location=location, @@ -304,9 +281,8 @@ def address(streets, municipalities): full_name_fi="Pitkäpellonkatu 7", full_name_sv="Långåkersgatan 7", ) - addresses.append(address) location = Point(22.2871092678621, 60.44677715747775, srid=4326) - address = Address.objects.create( + Address.objects.create( municipality_id=turku_muni.id, id=104, location=location, @@ -315,9 +291,8 @@ def address(streets, municipalities): full_name_fi="Kupittaankatu 8", full_name_sv="Kuppisgatan 8", ) - addresses.append(address) location = Point(22.26097246971352, 60.45055294118857, srid=4326) - address = Address.objects.create( + Address.objects.create( municipality_id=turku_muni.id, id=105, location=location, @@ -326,9 +301,8 @@ def address(streets, municipalities): full_name_fi="Yliopistonkatu 29", full_name_sv="Universitetsgatan 29", ) - addresses.append(address) location = Point(22.247047171564706, 60.45159033848499, srid=4326) - address = Address.objects.create( + Address.objects.create( municipality_id=turku_muni.id, id=106, location=location, @@ -337,4 +311,4 @@ def address(streets, municipalities): full_name_fi="Ratapihankatu 53", full_name_sv="Bangårdsgatan 53", ) - addresses.append(address) + return Address.objects.all() From 4251ac904166db029fb4a3097d69f8964d95d914 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 5 Apr 2023 12:42:14 +0300 Subject: [PATCH 91/97] Fix bbox coordinates --- mobility_data/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/tests/test_api.py b/mobility_data/tests/test_api.py index 1e79997a4..34b95695d 100644 --- a/mobility_data/tests/test_api.py +++ b/mobility_data/tests/test_api.py @@ -105,7 +105,7 @@ def test_mobile_unit(api_client, mobile_units, content_types, unit): response = api_client.get(url) assert len(response.json()["results"]) == 1 # Test bbox where no mobile units are inside. - url = reverse("mobility_data:mobile_units-list") + "?bbox=22.1,60.2,2.3,60.4" + url = reverse("mobility_data:mobile_units-list") + "?bbox=22.3,61.4,23,62.4" response = api_client.get(url) assert len(response.json()["results"]) == 0 # Test data serialization from services_unit model From 041a534cd19ec6881f384faf97cb6450f7c26255 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 5 Apr 2023 12:42:39 +0300 Subject: [PATCH 92/97] Raise exception if incompatible type_names are used --- mobility_data/api/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py index a1545b79c..4a83f79fc 100644 --- a/mobility_data/api/views.py +++ b/mobility_data/api/views.py @@ -159,6 +159,8 @@ def get_queryset(self): queryset = queryset.prefetch_related("content_types") unit_ids = [] filters = self.request.query_params + type_names = None + if "type_name" in filters or "type_names" in filters: type_name = filters.get("type_name", None) if type_name: @@ -170,11 +172,19 @@ def get_queryset(self): queryset = queryset.filter( content_types__type_name__in=type_names ).distinct() + # If the data locates in the services_unit table (i.e., MobileUnit has a unit_id) # get the unit_ids to retrieve the Units for filtering(bbox and extra) unit_ids = list( queryset.filter(unit_id__isnull=False).values_list("unit_id", flat=True) ) + if type_names: + mobile_units_qs = queryset.exclude(id__in=unit_ids) + if mobile_units_qs.count() > 0 and unit_ids: + raise Exception( + "Filtering MobileUnits with ContentTypes containing MobileUnits and MobileUnits that contains" + " references to services_unit table is not possible." + ) self.services_unit_instances = True if len(unit_ids) > 0 else False if self.services_unit_instances: From ea90e331bcf74c6dfe0b5e0255eccc9e930e525f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 12 Apr 2023 08:53:25 +0300 Subject: [PATCH 93/97] Return empty dict if json fetch fails --- street_maintenance/management/commands/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/street_maintenance/management/commands/utils.py b/street_maintenance/management/commands/utils.py index f920e14b5..0a7adec3e 100644 --- a/street_maintenance/management/commands/utils.py +++ b/street_maintenance/management/commands/utils.py @@ -54,9 +54,11 @@ def get_turku_boundary(): def get_json_data(url): response = requests.get(url) - assert ( - response.status_code == 200 - ), "Fetching Maintenance Unit {} status code: {}".format(url, response.status_code) + if response.status_code != 200: + logger.warning( + f"Fetching Maintenance Unit {url} status code: {response.status_code} response: {response.content}" + ) + return {} return response.json() From 92b9378606c1471aa4acb28f8d362ba2263d6ed9 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Wed, 12 Apr 2023 14:27:41 +0300 Subject: [PATCH 94/97] Get service id from config --- mobility_data/importers/bicycle_stands.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mobility_data/importers/bicycle_stands.py b/mobility_data/importers/bicycle_stands.py index fa79658e6..5b82c0949 100644 --- a/mobility_data/importers/bicycle_stands.py +++ b/mobility_data/importers/bicycle_stands.py @@ -15,6 +15,7 @@ ) from services.models import Unit +from smbackend_turku.importers.utils import get_external_source_config from .utils import ( get_closest_address_full_name, @@ -34,7 +35,6 @@ SV_KEY: "Cykelparkering", EN_KEY: "Bicycle parking", } -BICYCLE_STANDS_SERVICE_ID = settings.BICYCLE_STANDS_IDS["service"] BICYCLE_STANDS_URL = "{}{}".format( settings.TURKU_WFS_URL, "?service=WFS&request=GetFeature&typeName=GIS:Polkupyoraparkki&outputFormat=GML3", @@ -70,6 +70,8 @@ def __init__(self): self.extra = {f: None for f in self.EXTRA_FIELDS} def set_geojson_feature(self, feature): + config = get_external_source_config("bicycle_stands") + bicycle_stands_service_id = config["service"]["id"] name = feature["kohde"].as_string().strip() unit_name = name.split(",")[0] self.geometry = GEOSGeometry(feature.geom.wkt, srid=GEOJSON_SOURCE_DATA_SRID) @@ -78,7 +80,7 @@ def set_geojson_feature(self, feature): # Make first unit with same name that is not a Bicycle Stand the related_unit for unit in units_qs: # Ensure we do not connect to a Bicycle stand unit - if not unit.services.filter(id=BICYCLE_STANDS_SERVICE_ID): + if not unit.services.filter(id=bicycle_stands_service_id): self.related_unit = unit break From 23f947dd2f33ebe5f10b2ba733601137304dd827 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Apr 2023 11:40:27 +0000 Subject: [PATCH 95/97] Bump redis from 4.1.3 to 4.4.4 Bumps [redis](https://github.com/redis/redis-py) from 4.1.3 to 4.4.4. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v4.1.3...v4.4.4) --- updated-dependencies: - dependency-name: redis dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0e7908aed..c5816cae9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,8 @@ appdirs==1.4.4 # via requests-cache asgiref==3.5.2 # via django +async-timeout==4.0.2 + # via redis attrs==21.2.0 # via # cattrs @@ -49,8 +51,6 @@ coverage==5.5 # via pytest-cov cron-descriptor==1.2.35 # via django-celery-beat -deprecated==1.2.13 - # via redis django==4.1.7 # via # -r requirements.in @@ -137,9 +137,7 @@ numpy==1.23.0 # -r requirements.in # pandas packaging==21.0 - # via - # pytest - # redis + # via pytest pandas==1.4.3 # via -r requirements.in parso==0.8.2 @@ -205,7 +203,7 @@ pyyaml==5.4.1 # drf-spectacular raven==6.10.0 # via -r requirements.in -redis==4.1.3 +redis==4.4.4 # via -r requirements.in requests==2.26.0 # via @@ -259,8 +257,6 @@ wheel==0.38.1 # via pip-tools whitenoise==5.3.0 # via -r requirements.in -wrapt==1.13.3 - # via deprecated xmltodict==0.13.0 # via -r requirements.in From f4c3e972c81d55bd5de760e37e38d4d42ee2850f Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 14 Apr 2023 09:00:50 +0300 Subject: [PATCH 96/97] Add event mappings --- street_maintenance/management/commands/constants.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/street_maintenance/management/commands/constants.py b/street_maintenance/management/commands/constants.py index b2a75e862..b82e03a97 100644 --- a/street_maintenance/management/commands/constants.py +++ b/street_maintenance/management/commands/constants.py @@ -104,6 +104,7 @@ "pesu": [PUHTAANAPITO], "harjaus ja sohjonpoisto": [PUHTAANAPITO], "pölynsidonta": [PUHTAANAPITO], + "Imulakaisu": [PUHTAANAPITO], "hiekanpoisto": [HIEKANPOISTO], "lakaisu": [HIEKANPOISTO], "muu": [MUUT], @@ -139,6 +140,11 @@ "siirtoajo": [MUUT], "Kelintarkastus": [MUUT], "Sulamisvesien hallinta / höyrytys": [MUUT], + "Sorateiden kunnossapito": [MUUT], + "Äkillinen hoitotyö": [MUUT], + "KT-valu": [MUUT], + "Pintakelirikko": [MUUT], + "Liikennemerkkityö": [MUUT], } TIMESTAMP_FORMATS = { INFRAROAD: "%Y-%m-%d %H:%M:%S", From 92d22dcdf8d8f378e5b924957aef69f8aaa252d7 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Thu, 20 Apr 2023 12:36:57 +0300 Subject: [PATCH 97/97] Remove atomic transaction from importing divisions --- smbackend_turku/management/commands/turku_services_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smbackend_turku/management/commands/turku_services_import.py b/smbackend_turku/management/commands/turku_services_import.py index d3bff40eb..984d7aaa4 100644 --- a/smbackend_turku/management/commands/turku_services_import.py +++ b/smbackend_turku/management/commands/turku_services_import.py @@ -142,7 +142,6 @@ def import_geo_search_addresses(self): def import_enriched_addresses(self): return import_enriched_addresses(logger=self.logger) - @db.transaction.atomic def import_divisions(self): return import_divisions(logger=self.logger)