Merge pull request #270 from City-of-Turku/develop

Production update

mobility_data/admin.py

@@ -93,12 +93,12 @@ def type_name(self, obj):
 
 
 class ContentTypeAdmin(admin.ModelAdmin):
- # readonly_fields = ("id", "type_name", "name", "description")
- readonly_fields = ("id", "name", "description")
+ def has_change_permission(self, request, obj=None):
+  return False
 
 
 class GroupTypeAdmin(admin.ModelAdmin):
- readonly_fields = ("id", "name", "description")
+ readonly_fields = ("id", "type_name", "description")
 
 
 class DataSourceAdmin(admin.ModelAdmin):

mobility_data/api/serializers/content_type.py

@@ -6,4 +6,13 @@
 class ContentTypeSerializer(serializers.ModelSerializer):
  class Meta:
  model = ContentType
- fields = ["id", "name", "description"]
+ fields = [
+ "id",
+ "type_name",
+ "name",
+ "name_sv",
+ "name_en",
+ "description",
+ "description_sv",
+ "description_en",
+ ]

mobility_data/api/serializers/group_type.py

@@ -6,4 +6,4 @@
 class GroupTypeSerializer(serializers.ModelSerializer):
  class Meta:
  model = GroupType
- fields = ["id", "name", "description"]
+ fields = "__all__"

mobility_data/api/serializers/mobile_unit.py

@@ -81,22 +81,33 @@ class Meta:
  "address_fi": "street_address_fi",
  "address_sv": "street_address_sv",
  "address_en": "street_address_en",
+ "unit_id": "id",
  }
 
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ context = kwargs.get("context", {})
+ if "only" in context:
+ self.keep_fields = set(context["only"] + ["id"])
+ for field_name in list(self.fields.keys()):
+ if field_name in self.keep_fields:
+ continue
+ del self.fields[field_name]
+
  def to_representation(self, obj):
  representation = super().to_representation(obj)
  unit_id = getattr(obj, "unit_id", None)
  # If serializing Unit instance or MobileUnit with unit_id.
  if self.context.get("services_unit_instances", False) or unit_id:
  if unit_id:
- # When serializing the MobileUnit from the retrieve method
  try:
  unit = Unit.objects.get(id=unit_id)
  except Unit.DoesNotExist:
  return representation
  else:
  # The obj is a Unit instance.
  unit = obj
+ mobile_unit = MobileUnit.objects.filter(unit_id=unit.id).first()
  for field in self.fields:
  # lookup the field name in the service_unit table, as not all field names that contains
  # similar data has the same name.
@@ -115,13 +126,13 @@ def to_representation(self, obj):
  # Serialize the MobileUnit id, otherwise would serialize the serivce_unit id.
  if field == "id":
  try:
- representation["id"] = MobileUnit.objects.get(
- unit_id=unit.id
- ).id
+ representation["id"] = mobile_unit.id
  except MobileUnit.DoesNotExist:
  representation["id"] = unit.id
+ if field == "unit_id":
+ representation["unit_id"] = mobile_unit.unit_id
  # The location field must be serialized with its wkt value.
- if unit.location:
+ if unit.location and "geometry" in representation:
  representation["geometry"] = unit.location.wkt
  return representation
 

mobility_data/api/views.py

@@ -1,8 +1,10 @@
+import logging
 import types
 from distutils.util import strtobool
 
 from django.contrib.gis.gdal import SpatialReference
 from django.core.exceptions import ValidationError
+from django.db import connection, reset_queries
 from django.db.models import Q
 from munigeo import api as munigeo_api
 from rest_framework import status, viewsets
@@ -24,45 +26,8 @@
 FIELD_TYPES.FLOAT = float
 FIELD_TYPES.INT = int
 FIELD_TYPES.BOOL = bool
-# Mappings, so that deprecated type_names will work.
-# These will be removed when the front end is updated.
-group_name_mappings = {"CRE": "CultureRoute"}
-type_name_mappings = {
- "FGS": "FishingSpot",
- "FYR": "FerryRoute",
- "MSF": "MarinaSouthwestFinland",
- "SWY": "SlipwaySouthwestFinland",
- "RCR": "RecreationalRoute",
- "PTL": "PaddlingTrail",
- "HTL": "HikingTrail",
- "NTL": "NatureTrail",
- "FTL": "FitnessTrail",
- "PPU": "PaavonPolku",
- "PAZ": "PaymentZone",
- "SPG": "ScooterParkingArea",
- "SSL": "ScooterSpeedLimitArea",
- "SNP": "ScooterNoParkingArea",
- "BLB": "BrushSaltedBicycleNetwork",
- "BND": "BrushSandedBicycleNetwork",
- "SLZ": "SpeedLimitZone",
- "APT": "PublicToilet",
- "ATE": "PublicTable",
- "ABH": "PublicBench",
- "AFG": "PublicFurnitureGroup",
- "BIS": "BicycleStand",
- "BSS": "BikeServiceStation",
- "BOK": "BoatParking",
- "CGS": "ChargingStation",
- "CRG": "CultureRouteGeometry",
- "CRU": "CultureRouteUnit",
- "DSP": "DisabledParking",
- "GFS": "GasFillingStation",
- "GMA": "GuestMarina",
- "SCP": "ShareCarParkingPlace",
- "MAR": "Marina",
- "NSP": "NoStaffParking",
- "LUP": "LoadingUnloadingPlace",
-}
+
+logger = logging.getLogger("mobility_data")
 
 
 def get_srid_and_latlon(filters):
@@ -135,9 +100,6 @@ def list(self, request):
  mobile_units = get_mobile_units(filters)
  if "type_name" in filters:
  type_name = filters["type_name"]
- # TODO, remove when front end is updated.
- if type_name in type_name_mappings:
- type_name = group_name_mappings[type_name]
  if not GroupType.objects.filter(name=type_name).exists():
  return Response(
  "type_name does not exist.", status=status.HTTP_400_BAD_REQUEST
@@ -160,7 +122,7 @@ def list(self, request):
 
 class MobileUnitViewSet(viewsets.ReadOnlyModelViewSet):
 
- queryset = MobileUnit.objects.all()
+ queryset = MobileUnit.objects.filter(is_active=True)
  serializer_class = MobileUnitSerializer
 
  def retrieve(self, request, pk=None):
@@ -180,40 +142,59 @@ def retrieve(self, request, pk=None):
  )
  return Response(serializer.data, status=status.HTTP_200_OK)
 
- def list(self, request):
- """
- Lists MobileUnits, optionally list by type_name if given
- and transforms to given srid.
- """
- queryset = None
+ def get_serializer_context(self):
+ context = super().get_serializer_context()
+ only = self.request.query_params.get("only", "")
+ if only:
+ context["only"] = [x.strip() for x in only.split(",") if x]
+
+ context["srid"], context["latlon"] = get_srid_and_latlon(
+ self.request.query_params
+ )
+ context["services_unit_instances"] = self.services_unit_instances
+ return context
+
+ def get_queryset(self):
+ queryset = MobileUnit.objects.filter(is_active=True)
+ queryset = queryset.prefetch_related("content_types")
  unit_ids = []
  filters = self.request.query_params
- srid, latlon = get_srid_and_latlon(filters)
- if "type_name" in filters:
- type_name = filters["type_name"]
- # TODO, remove when front end is updated.
- if type_name in type_name_mappings:
- type_name = type_name_mappings[type_name]
- if not ContentType.objects.filter(name=type_name).exists():
- return Response(
- "type_name does not exist.", status=status.HTTP_400_BAD_REQUEST
- )
- queryset = MobileUnit.objects.filter(content_types__name=type_name)
+ type_names = None
+
+ if "type_name" in filters or "type_names" in filters:
+ type_name = filters.get("type_name", None)
+ if type_name:
+ queryset = queryset.filter(content_types__type_name=type_name)
+ else:
+ type_names = [
+ t.strip() for t in filters.get("type_names", "").split(",")
+ ]
+ queryset = queryset.filter(
+ content_types__type_name__in=type_names
+ ).distinct()
+
  # If the data locates in the services_unit table (i.e., MobileUnit has a unit_id)
  # get the unit_ids to retrieve the Units for filtering(bbox and extra)
  unit_ids = list(
  queryset.filter(unit_id__isnull=False).values_list("unit_id", flat=True)
  )
- else:
- queryset = MobileUnit.objects.all()
+ if type_names:
+ mobile_units_qs = queryset.exclude(id__in=unit_ids)
+ if mobile_units_qs.count() > 0 and unit_ids:
+ raise Exception(
+ "Filtering MobileUnits with ContentTypes containing MobileUnits and MobileUnits that contains"
+ " references to services_unit table is not possible."
+ )
 
- services_unit_instances = True if len(unit_ids) > 0 else False
- if services_unit_instances:
+ self.services_unit_instances = True if len(unit_ids) > 0 else False
+ if self.services_unit_instances:
  queryset = Unit.objects.filter(id__in=unit_ids)
 
  if "bbox" in filters:
  val = filters.get("bbox", None)
- geometry_field_name = "location" if services_unit_instances else "geometry"
+ geometry_field_name = (
+ "location" if self.services_unit_instances else "geometry"
+ )
  if val:
  ref = SpatialReference(filters.get("bbox_srid", 4326))
  bbox_geometry_filter = munigeo_api.build_bbox_filter(
@@ -254,16 +235,19 @@ def list(self, request):
 
  queryset = queryset.filter(**{filter: value})
 
+ return queryset
+
+ def list(self, request):
+ queryset = self.get_queryset()
  page = self.paginate_queryset(queryset)
- serializer = MobileUnitSerializer(
- page,
- many=True,
- context={
- "srid": srid,
- "latlon": latlon,
- "services_unit_instances": services_unit_instances,
- },
- )
+ if logger.level <= logging.DEBUG:
+ logger.debug(connection.queries)
+ queries_time = sum([float(s["time"]) for s in connection.queries])
+ logger.debug(
+ f"MobileUnit list queries total execution time: {queries_time} Num queries: {len(connection.queries)}"
+ )
+ reset_queries()
+ serializer = self.get_serializer(page, many=True)
  return self.get_paginated_response(serializer.data)
 
 

mobility_data/importers/bicycle_stands.py

@@ -5,7 +5,6 @@
 import logging
 import os
 
-from django import db
 from django.conf import settings
 from django.contrib.gis.gdal import DataSource
 from django.contrib.gis.geos import GEOSGeometry
@@ -15,18 +14,16 @@
  Municipality,
 )
 
-from mobility_data.models import MobileUnit
 from services.models import Unit
+from smbackend_turku.importers.utils import get_external_source_config
 
 from .utils import (
- delete_mobile_units,
  get_closest_address_full_name,
  get_municipality_name,
- get_or_create_content_type,
  get_root_dir,
  get_street_name_translations,
  locates_in_turku,
- set_translated_field,
+ MobileUnitDataBase,
 )
 
 CONTENT_TYPE_NAME = "BicycleStand"
@@ -38,7 +35,6 @@
  SV_KEY: "Cykelparkering",
  EN_KEY: "Bicycle parking",
 }
-BICYCLE_STANDS_SERVICE_ID = settings.BICYCLE_STANDS_IDS["service"]
 BICYCLE_STANDS_URL = "{}{}".format(
  settings.TURKU_WFS_URL,
  "?service=WFS&request=GetFeature&typeName=GIS:Polkupyoraparkki&outputFormat=GML3",
@@ -53,7 +49,7 @@
 ).boundary
 
 
-class BicyleStand:
+class BicyleStand(MobileUnitDataBase):
 
  WFS_HULL_LOCKABLE_STR = "runkolukitusmahdollisuus"
  GEOJSON_HULL_LOCKABLE_STR = "runkolukittava"
@@ -68,15 +64,14 @@ class BicyleStand:
  ]
 
  def __init__(self):
- self.geometry = None
- self.municipality = None
- self.name = {}
+ super().__init__()
  self.prefix_name = {}
- self.address = {}
  self.related_unit = None
  self.extra = {f: None for f in self.EXTRA_FIELDS}
 
  def set_geojson_feature(self, feature):
+ config = get_external_source_config("bicycle_stands")
+ bicycle_stands_service_id = config["service"]["id"]
  name = feature["kohde"].as_string().strip()
  unit_name = name.split(",")[0]
  self.geometry = GEOSGeometry(feature.geom.wkt, srid=GEOJSON_SOURCE_DATA_SRID)
@@ -85,7 +80,7 @@ def set_geojson_feature(self, feature):
  # Make first unit with same name that is not a Bicycle Stand the related_unit
  for unit in units_qs:
  # Ensure we do not connect to a Bicycle stand unit
- if not unit.services.filter(id=BICYCLE_STANDS_SERVICE_ID):
+ if not unit.services.filter(id=bicycle_stands_service_id):
  self.related_unit = unit
  break
 
@@ -140,7 +135,7 @@ def set_geojson_feature(self, feature):
  # The last part is always the number
  address_number = address[-1]
  translated_street_names = get_street_name_translations(
- street_name, municipality_name
+ street_name, self.municipality
  )
  self.address["fi"] = f"{translated_street_names['fi']} {address_number}"
  self.address["sv"] = f"{translated_street_names['sv']} {address_number}"
@@ -253,34 +248,3 @@ def get_bicycle_stand_objects(data_source=None):
 
  logger.info(f"Retrieved {len(bicycle_stands)} bicycle stands.")
  return bicycle_stands
-
-
-@db.transaction.atomic
-def delete_bicycle_stands():
- delete_mobile_units(CONTENT_TYPE_NAME)
-
-
-@db.transaction.atomic
-def create_bicycle_stand_content_type():
- description = "Bicycle stands in The Turku Region."
- content_type, _ = get_or_create_content_type(CONTENT_TYPE_NAME, description)
- return content_type
-
-
-@db.transaction.atomic
-def save_to_database(objects, delete_tables=True):
- if delete_tables:
- delete_bicycle_stands()
-
- content_type = create_bicycle_stand_content_type()
- for object in objects:
- mobile_unit = MobileUnit.objects.create(
- extra=object.extra,
- municipality=object.municipality,
- )
- mobile_unit.content_types.add(content_type)
- mobile_unit.geometry = object.geometry
- set_translated_field(mobile_unit, "name", object.name)
- if object.address:
- set_translated_field(mobile_unit, "address", object.address)
- mobile_unit.save()