Feature/mobility data importers tests use mock fixtures

README.md

@@ -141,8 +141,12 @@ For Turku specific imports see smbackend_turku/README.md.
 ./manage.py geo_import helsinki --divisions
 ./manage.py index_search_columns
 ```
-
+Import exclude rules fixtures used by the search:
+```
+./manage.py loaddata services/fixtures/exclusion_rules.json
+```
 7. Redis
+
 Redis is used for caching and as a message broker for Celery.
 Install Redis. Ubuntu: `sudo apt-get install redis-server`
 
@@ -198,3 +202,6 @@ psql template1 -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 Mobility platform
 -----------------
 The mobility data platform of the service map is being developed as part of European Union Horizon 2020 programme funded SCALE-UP project (grant agreement no. 955332).
+
+For more information see: mobility_data/README.mk
+

mobility_data/importers/bicycle_stands.py

@@ -198,24 +198,24 @@
         self.prefix_name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()}
 
 
-def get_bicycle_stand_objects(data_source=None):
+def get_data_sources():
+    data_sources = []
+    # Add the WFS datasource that is in GML format
+    ds = DataSource(BICYCLE_STANDS_URL)
+    data_sources.append(("gml", ds))
+    # Add the GEOJSON datasource which is a file
+    data_path = os.path.join(get_root_dir(), "mobility_data/data")
+    file_path = os.path.join(data_path, GEOJSON_FILENAME)
+    ds = DataSource(file_path)
+    data_sources.append(("geojson", ds))
+    return data_sources
+
+
+def get_bicycle_stand_objects():
     """
     Returns a list containg instances of BicycleStand class.
     """
-    data_sources = []
-
-    if data_source:
-        data_sources.append(data_source)
-    else:
-        # Add the WFS datasource that is in GML format
-        ds = DataSource(BICYCLE_STANDS_URL)
-        data_sources.append(("gml", ds))
-        # Add the GEOJSON datasource which is a file
-        data_path = os.path.join(get_root_dir(), "mobility_data/data")
-        file_path = os.path.join(data_path, GEOJSON_FILENAME)
-        ds = DataSource(file_path)
-        data_sources.append(("geojson", ds))
-
+    data_sources = get_data_sources()
     bicycle_stands = []
     """
     external_stands dict is used to keep track of the names of imported external stands

mobility_data/importers/bike_service_stations.py

@@ -64,20 +64,18 @@
         self.extra["in_terrain"] = feature["Maastossa"].as_string()
 
 
-def get_bike_service_station_objects(geojson_file=None):
-    bicycle_repair_points = []
-    file_name = None
-    if not geojson_file:
-        file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
-        if not file_name:
-            file_name = f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
-        else:
-            file_name = f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
+def get_data_layer():
+    file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
+    if not file_name:
+        file_name = f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
     else:
-        # Use the test data file
-        file_name = f"{get_root_dir()}/mobility_data/tests/data/{geojson_file}"
-
+        file_name = f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
     data_layer = GDALDataSource(file_name)[0]
-    for feature in data_layer:
+    return data_layer
+
+
+def get_bike_service_station_objects():
+    bicycle_repair_points = []
+    for feature in get_data_layer():
         bicycle_repair_points.append(BikeServiceStation(feature))
     return bicycle_repair_points

mobility_data/importers/charging_stations.py

@@ -111,18 +111,18 @@
     return number_of_rows
 
 
-def get_charging_station_objects(csv_file=None):
+def get_csv_file_name():
+    file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
+    if file_name:
+        return file_name
+    return f"{get_root_dir()}/mobility_data/data/{SOURCE_DATA_FILE_NAME}"
+
+
+def get_charging_station_objects():
     # Store the imported stations to dict, the index is the key.
+    file_name = get_csv_file_name()
     charging_stations = {}
     column_mappings = {}
-    if not csv_file:
-        file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
-        if not file_name:
-            file_name = f"{get_root_dir()}/mobility_data/data/{SOURCE_DATA_FILE_NAME}"
-    else:
-        # Use the test data file
-        file_name = f"{get_root_dir()}/mobility_data/tests/data/{csv_file}"
-
     number_of_rows = get_number_of_rows(file_name)
     with open(file_name, encoding="utf-8-sig") as csv_file:
         csv_reader = csv.reader(csv_file, delimiter=";")

mobility_data/importers/disabled_and_no_staff_parking.py

@@ -135,19 +135,17 @@
                         self.extra[field_name] = feature[field].as_double()
 
 
-def get_no_staff_parking_objects(geojson_file=None):
-    no_staff_parkings = []
-    disabled_parkings = []
-    file_name = None
+def get_geojson_file_name():
+    file_name = get_file_name_from_data_source(NO_STAFF_PARKING_CONTENT_TYPE_NAME)
+    if file_name:
+        return file_name
+    return f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
 
-    if not geojson_file:
-        file_name = get_file_name_from_data_source(NO_STAFF_PARKING_CONTENT_TYPE_NAME)
-        if not file_name:
-            file_name = f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
-    else:
-        # Use the test data file
-        file_name = f"{get_root_dir()}/mobility_data/tests/data/{geojson_file}"
 
+def get_no_staff_parking_objects():
+    no_staff_parkings = []
+    disabled_parkings = []
+    file_name = get_geojson_file_name()
     data_layer = GDALDataSource(file_name)[0]
 
     for feature in data_layer:

mobility_data/importers/gas_filling_station.py

@@ -1,12 +1,12 @@
 import logging
 
+import requests
 from django.conf import settings
 from django.contrib.gis.geos import Point, Polygon
 from munigeo.models import Municipality
 
 from .constants import SOUTHWEST_FINLAND_BOUNDARY, SOUTHWEST_FINLAND_BOUNDARY_SRID
 from .utils import (
-    fetch_json,
     get_street_name_and_number,
     get_street_name_translations,
     LANGUAGES,
@@ -54,15 +54,21 @@
         self.extra["lng_cng"] = self.lng_cng
 
 
-def get_filtered_gas_filling_station_objects(json_data=None):
+def get_json_data(url):
+    response = requests.get(url)
+    assert response.status_code == 200, "Fetching {} status code: {}".format(
+        url, response.status_code
+    )
+    return response.json()
+
+
+def get_filtered_gas_filling_station_objects():
     """
     Returns a list of GasFillingStation objects that are filtered by location.
     Stations inside boundarys of Southwest Finland are included, the rest
     are discarded.
     """
-
-    if not json_data:
-        json_data = fetch_json(GAS_FILLING_STATIONS_URL)
+    json_data = get_json_data(GAS_FILLING_STATIONS_URL)
     # srid = json_data["spatialReference"]["wkid"]
     # NOTE, hack to fix srid 102100 in source data causes "crs not found"
     srid = 4326

mobility_data/importers/loading_unloading_places.py

@@ -104,18 +104,16 @@
                         self.extra[field_name] = feature[field].as_int()
 
 
-def get_loading_and_unloading_objects(geojson_file=None):
-    objects = []
-    file_name = None
+def get_geojson_file_name():
+    file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
+    if file_name:
+        return file_name
+    return f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
 
-    if not geojson_file:
-        file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
-        if not file_name:
-            file_name = f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
-    else:
-        # Use the test data file
-        file_name = f"{get_root_dir()}/mobility_data/tests/data/{geojson_file}"
 
+def get_loading_and_unloading_objects():
+    objects = []
+    file_name = get_geojson_file_name()
     data_layer = GDALDataSource(file_name)[0]
     for feature in data_layer:
         objects.append(LoadingPlace(feature))

mobility_data/importers/lounaistieto_shapefiles.py

@@ -123,5 +123,5 @@
         if obj.add_feature(feature, config, srid):
             objects.append(obj)
     content_type = get_or_create_content_type_from_config(config["content_type_name"])
-    num_ceated, num_deleted = save_to_database(objects, content_type)
-    log_imported_message(logger, content_type, num_ceated, num_deleted)
+    num_created, num_deleted = save_to_database(objects, content_type)
+    log_imported_message(logger, content_type, num_created, num_deleted)

mobility_data/importers/share_car_parking_places.py

@@ -54,16 +54,16 @@
             self.extra[self.RESTRICTION_FIELD][language] = restrictions[i].strip()
 
 
+def get_geojson_file_name():
+    file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
+    if file_name:
+        return file_name
+    return f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
+
+
 def get_car_share_parking_place_objects(geojson_file=None):
     car_share_parking_places = []
-    file_name = None
-    if not geojson_file:
-        file_name = get_file_name_from_data_source(CONTENT_TYPE_NAME)
-        if not file_name:
-            file_name = f"{get_root_dir()}/mobility_data/data/{GEOJSON_FILENAME}"
-    else:
-        # Use the test data file
-        file_name = f"{get_root_dir()}/mobility_data/tests/data/{geojson_file}"
+    file_name = get_geojson_file_name()
 
     data_layer = GDALDataSource(file_name)[0]
     for feature in data_layer:

mobility_data/importers/wfs.py

@@ -4,6 +4,7 @@
 from django import db
 from django.conf import settings
 from django.contrib.gis.gdal import DataSource
+from django.contrib.gis.gdal.error import GDALException
 from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Polygon
 from munigeo.models import Municipality
 
@@ -67,22 +68,24 @@
         if config.get("locates_in_turku", False):
             if not locates_in_turku(feature, source_srid):
                 return False
-
         # If geometry contains multiple polygons and create_multipolygon attribute is True
         # create one multipolygon from the polygons.
-        if (
-            len(feature.geom.coords) > 1
-            and create_multipolygon
-            and isinstance(feature.geom, gdalgeometries.Polygon)
-        ):
-            polygons = []
-            for coords in feature.geom.coords:
-                polygons.append(Polygon(coords, srid=source_srid))
-            self.geometry = MultiPolygon(polygons, srid=source_srid)
-        else:
-            self.geometry = GEOSGeometry(feature.geom.wkt, srid=source_srid)
-        self.geometry.transform(settings.DEFAULT_SRID)
-
+        try:
+            if (
+                len(feature.geom.coords) > 1
+                and create_multipolygon
+                and isinstance(feature.geom, gdalgeometries.Polygon)
+            ):
+                polygons = []
+                for coords in feature.geom.coords:
+                    polygons.append(Polygon(coords, srid=source_srid))
+                self.geometry = MultiPolygon(polygons, srid=source_srid)
+            else:
+                self.geometry = GEOSGeometry(feature.geom.wkt, srid=source_srid)
+            self.geometry.transform(settings.DEFAULT_SRID)
+        except GDALException as ex:
+            logger.error(ex)
+            return False
         if "municipality" in config:
             municipality = feature[config["municipality"]].as_string()
             if municipality:
@@ -129,8 +132,16 @@
         return True
 
 
+def get_data_source(config, max_features):
+    wfs_url = config.get("wfs_url", settings.TURKU_WFS_URL)
+    url = WFS_URL.format(
+        wfs_url=wfs_url, wfs_layer=config["wfs_layer"], max_features=max_features
+    )
+    ds = DataSource(url)
+    return ds
+
+
 def import_wfs_feature(config, data_file=None):
-    max_features = DEFAULT_MAX_FEATURES
     if "content_type_name" not in config:
         logger.warning(f"Skipping feature {config}, 'content_type_name' is required.")
         return False
@@ -139,23 +150,19 @@
         return False
     if "max_features" in config:
         max_features = config["max_features"]
-    wfs_layer = config["wfs_layer"]
+    else:
+        max_features = DEFAULT_MAX_FEATURES
     objects = []
     if data_file:
         ds = DataSource(data_file)
     else:
-        wfs_url = config.get("wfs_url", settings.TURKU_WFS_URL)
-
-        url = WFS_URL.format(
-            wfs_url=wfs_url, wfs_layer=wfs_layer, max_features=max_features
-        )
-        ds = DataSource(url)
+        ds = get_data_source(config, max_features)
     assert len(ds) == 1
     layer = ds[0]
     for feature in layer:
         object = MobilityData()
         if object.add_feature(feature, config):
             objects.append(object)
     content_type = get_or_create_content_type_from_config(config["content_type_name"])
-    num_ceated, num_deleted = save_to_database(objects, content_type)
-    log_imported_message(logger, content_type, num_ceated, num_deleted)
+    num_created, num_deleted = save_to_database(objects, content_type)
+    log_imported_message(logger, content_type, num_created, num_deleted)

mobility_data/management/commands/delete_deprecated_units.py

@@ -5,7 +5,7 @@
 """
 This command removes all units that have a ContentType or
 GroupType where type_name is not Null. This data is deprecated
-as the only the name will be used in future.
+as only the name will be used in future.
 """