Merge pull request #252 from City-of-Turku/develop

Production update

bicycle_network/api/views.py

@@ -82,11 +82,13 @@ def list(self, request):
  )
  # Return elements that are inside bbox
  if "bbox" in filters:
- ref = SpatialReference(4326)
- val = self.request.query_params.get("bbox", None)
- bbox_filter = munigeo_api.build_bbox_filter(ref, val, "geometry")
- bbox_geometry_filter = munigeo_api.build_bbox_filter(ref, val, "geometry")
- queryset = queryset.filter(Q(**bbox_filter) | Q(**bbox_geometry_filter))
+ val = filters.get("bbox", None)
+ if val:
+ ref = SpatialReference(filters.get("bbox_srid", 4326))
+ bbox_geometry_filter = munigeo_api.build_bbox_filter(
+ ref, val, "geometry"
+ )
+ queryset = queryset.filter(Q(**bbox_geometry_filter))
 
  page = self.paginate_queryset(queryset)
  if only_coords:

config_dev.env.example

@@ -138,6 +138,13 @@ CELERY_BROKER_URL=redis://localhost:6379/0
 # Cache location, e.g. redis on localhost using default port and database 0
 CACHE_LOCATION=redis://localhost:6379/0
 
+# Email settings
+EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
+EMAIL_HOST=smtp.example.com
+EMAIL_HOST_USER=example@example.com
+EMAIL_PORT=25
+EMAIL_USE_TLS=True
+
 # Settings needed for enabling Turku area:
 #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv
 #TURKU_API_KEY=secret
@@ -161,10 +168,6 @@ TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL=https://data.turku.fi/2yxpk2imqi2mzxpa6e6k
 LAM_COUNTER_STATIONS_URL=https://tie.digitraffic.fi/api/v3/metadata/tms-stations
 LAM_COUNTER_API_BASE_URL=https://tie-lam-test.digitraffic.fi
 
-GAS_FILLING_STATIONS_IDS=service_node=200000,service=200000,units_offset=200000
-CHARGING_STATIONS_IDS=service_node=300000,service=300000,units_offset=300000
-BICYCLE_STANDS_IDS=service_node=400000,service=400000,units_offset=400000
-BIKE_SERVICE_STATIONS_IDS=service_node=500000,service=500000,units_offset=500000
 MOBILITY_DATA_CHARGING_STATIONS_URL=https://services1.arcgis.com/rhs5fjYxdOG1Et61/ArcGIS/rest/services/GasFillingStations/FeatureServer/0/query?f=json&where=1%3D1&outFields=OPERATOR%2CLAT%2CLON%2CSTATION_NAME%2CADDRESS%2CCITY%2CZIP_CODE%2CLNG_CNG%2CObjectId
 MOBILITY_DATA_GAS_FILLING_STATIONS_URL=https://services1.arcgis.com/rhs5fjYxdOG1Et61/ArcGIS/rest/services/ChargingStations/FeatureServer/0/query?f=json&where=1%20%3D%201%20OR%201%20%3D%201&returnGeometry=true&spatialRel=esriSpatialRelIntersects&outFields=LOCATION_ID%2CNAME%2CADDRESS%2CURL%2COBJECTID%2CTYPE
 MOBILITY_DATA_GEOMETRY_URL=https://tie.digitraffic.fi/api/v3/data/traffic-messages/area-geometries?id=11&lastUpdated=false

eco_counter/api/serializers.py

@@ -31,6 +31,7 @@ class StationSerializer(serializers.ModelSerializer):
  y = serializers.SerializerMethodField()
  lon = serializers.SerializerMethodField()
  lat = serializers.SerializerMethodField()
+ sensor_types = serializers.SerializerMethodField()
 
  class Meta:
  model = Station
@@ -47,6 +48,7 @@ class Meta:
  "y",
  "lon",
  "lat",
+ "sensor_types",
  ]
 
  def get_y(self, obj):
@@ -63,6 +65,17 @@ def get_lon(self, obj):
  obj.geom.transform(4326)
  return obj.geom.x
 
+ def get_sensor_types(self, obj):
+ # Return the sensor types(car, bike etc) that has a total year value >0.
+ # i.e., there are sensors for counting the type of data.
+ types = ["at", "pt", "jt", "bt"]
+ result = []
+ for type in types:
+ filter = {"station": obj, f"value_{type}__gt": 0}
+ if YearData.objects.filter(**filter).count() > 0:
+ result.append(type)
+ return result
+
 
 class YearSerializer(serializers.ModelSerializer):
  station_name = serializers.PrimaryKeyRelatedField(

eco_counter/tasks.py

@@ -1,12 +1,13 @@
-from celery import shared_task
 from django.core import management
 
+from smbackend.utils import shared_task_email
 
-@shared_task
+
+@shared_task_email
 def import_counter_data(args, name="import_counter_data"):
  management.call_command("import_counter_data", "--counters", args)
 
 
-@shared_task
+@shared_task_email
 def initial_import_counter_data(args, name="initial_import_counter_data"):
  management.call_command("import_counter_data", "--init", args)

eco_counter/tests/test_api.py

@@ -263,8 +263,9 @@ def test__months_multiple_years(api_client, years, test_timestamp):
 
 
 @pytest.mark.django_db
-def test__station(api_client, station):
+def test__station(api_client, station, year_datas):
  url = reverse("eco_counter:stations-list")
  response = api_client.get(url)
  assert response.status_code == 200
  assert response.json()["results"][0]["name"] == station.name
+ assert response.json()["results"][0]["sensor_types"] == ["at"]

iot/tasks.py

@@ -1,7 +1,8 @@
-from celery import shared_task
 from django.core import management
 
+from smbackend.utils import shared_task_email
 
-@shared_task
+
+@shared_task_email
 def import_iot_data(source_name, name="Import iot data"):
  management.call_command("import_iot_data", source_name)

mobility_data/README.md

@@ -160,12 +160,34 @@ To import data type:
 ./manage.py import_foli_stops
 ```
 
+### Barbecue places
+```
+./manage.py import_wfs BarbecuePlace
+```
+
+
+### Playgrounds
+```
+./manage.py import_wfs PlayGround
+```
+
+### Föli park and ride stop
+Imports park and ride stops for bikes and cars.
+```
+./manage.py import_foli_parkandride_stops
+```
+
 ### Outdoor gym devices
 Imports the outdoor gym devices from the services.unit model. i.e., sets references by id to the services.unit model. The data is then serialized from the services.unit model.
 ```
 ./manage.py import_outdoor_gym_devices
 ```
 
+### Parking machines
+```
+./manage.py import_parking_machines
+```
+
 ## Deletion
 To delete mobile units for a content type.
 ```

mobility_data/api/serializers/mobile_unit.py

@@ -42,7 +42,7 @@ class Meta:
 
 class MobileUnitSerializer(serializers.ModelSerializer):
 
- content_type = ContentTypeSerializer(many=False, read_only=True)
+ content_types = ContentTypeSerializer(many=True, read_only=True)
  mobile_unit_group = MobileUnitGroupBasicInfoSerializer(many=False, read_only=True)
  geometry_coords = serializers.SerializerMethodField(read_only=True)
 
@@ -64,13 +64,14 @@ class Meta:
  "description_fi",
  "description_sv",
  "description_en",
- "content_type",
+ "content_types",
  "mobile_unit_group",
  "is_active",
  "created_time",
  "geometry",
  "geometry_coords",
  "extra",
+ "unit_id",
  ]
 
  # Contains the corresponding field names of the MobileUnit model if they differs
@@ -84,10 +85,18 @@ class Meta:
 
  def to_representation(self, obj):
  representation = super().to_representation(obj)
- # If mobile_unit has a unit_id we serialize the data from the services_unit table.
- unit_id = obj.unit_id
- if unit_id:
- unit = Unit.objects.get(id=unit_id)
+ unit_id = getattr(obj, "unit_id", None)
+ # If serializing Unit instance or MobileUnit with unit_id.
+ if self.context.get("services_unit_instances", False) or unit_id:
+ if unit_id:
+ # When serializing the MobileUnit from the retrieve method
+ try:
+ unit = Unit.objects.get(id=unit_id)
+ except Unit.DoesNotExist:
+ return representation
+ else:
+ # The obj is a Unit instance.
+ unit = obj
  for field in self.fields:
  # lookup the field name in the service_unit table, as not all field names that contains
  # similar data has the same name.
@@ -99,19 +108,34 @@ def to_representation(self, obj):
  if hasattr(unit, key):
  # unit.municipality is of type munigeo.models.Municipality and not serializable
  if key == "municipality":
- representation[field] = unit.municipality.id
+ muni = getattr(unit, key, None)
+ representation[field] = muni.id if muni else None
  else:
- representation[field] = getattr(unit, key)
-
+ representation[field] = getattr(unit, key, None)
+ # Serialize the MobileUnit id, otherwise would serialize the serivce_unit id.
+ if field == "id":
+ try:
+ representation["id"] = MobileUnit.objects.get(
+ unit_id=unit.id
+ ).id
+ except MobileUnit.DoesNotExist:
+ representation["id"] = unit.id
  # The location field must be serialized with its wkt value.
  if unit.location:
  representation["geometry"] = unit.location.wkt
  return representation
 
  def get_geometry_coords(self, obj):
- # If stored to Unit table, retrieve geometry from there.
- if obj.unit_id:
- geometry = Unit.objects.get(id=obj.unit_id).location
+ unit_id = getattr(obj, "unit_id", None)
+ if unit_id:
+ # If stored to Unit table, retrieve geometry from there.
+ try:
+ geometry = Unit.objects.get(id=unit_id).location
+ except Unit.DoesNotExist:
+ return None
+ # If serializing Unit object retrieved from the view.
+ elif self.context.get("services_unit_instances", False):
+ geometry = obj.location
  else:
  geometry = obj.geometry
  if isinstance(geometry, GEOSGeometry):
@@ -130,7 +154,6 @@ def get_geometry_coords(self, obj):
  pos["lon"] = geometry.x
  pos["lat"] = geometry.y
  return pos
-
  elif isinstance(geometry, LineString):
  if self.context["latlon"]:
  # Return LineString coordinates in (lat,lon) format
@@ -142,7 +165,6 @@ def get_geometry_coords(self, obj):
  return coords
  else:
  return geometry.coords
-
  elif isinstance(geometry, Polygon):
  if self.context["latlon"]:
  # Return Polygon coordinates in (lat,lon) format
@@ -189,6 +211,5 @@ def get_geometry_coords(self, obj):
  return coords
  else:
  return geometry.coords
-
  else:
  return ""

mobility_data/api/views.py

@@ -1,10 +1,16 @@
+import types
 from distutils.util import strtobool
 
+from django.contrib.gis.gdal import SpatialReference
 from django.core.exceptions import ValidationError
+from django.db.models import Q
+from munigeo import api as munigeo_api
 from rest_framework import status, viewsets
 from rest_framework.exceptions import ParseError
 from rest_framework.response import Response
 
+from services.models import Unit
+
 from ..models import ContentType, GroupType, MobileUnit, MobileUnitGroup
 from .serializers import (
  ContentTypeSerializer,
@@ -14,6 +20,10 @@
  MobileUnitSerializer,
 )
 
+FIELD_TYPES = types.SimpleNamespace()
+FIELD_TYPES.FLOAT = float
+FIELD_TYPES.INT = int
+FIELD_TYPES.BOOL = bool
 # Mappings, so that deprecated type_names will work.
 # These will be removed when the front end is updated.
 group_name_mappings = {"CRE": "CultureRoute"}
@@ -176,6 +186,7 @@ def list(self, request):
  and transforms to given srid.
  """
  queryset = None
+ unit_ids = []
  filters = self.request.query_params
  srid, latlon = get_srid_and_latlon(filters)
  if "type_name" in filters:
@@ -187,10 +198,29 @@ def list(self, request):
  return Response(
  "type_name does not exist.", status=status.HTTP_400_BAD_REQUEST
  )
- queryset = MobileUnit.objects.filter(content_type__name=type_name)
+ queryset = MobileUnit.objects.filter(content_types__name=type_name)
+ # If the data locates in the services_unit table (i.e., MobileUnit has a unit_id)
+ # get the unit_ids to retrieve the Units for filtering(bbox and extra)
+ unit_ids = list(
+ queryset.filter(unit_id__isnull=False).values_list("unit_id", flat=True)
+ )
  else:
  queryset = MobileUnit.objects.all()
 
+ services_unit_instances = True if len(unit_ids) > 0 else False
+ if services_unit_instances:
+ queryset = Unit.objects.filter(id__in=unit_ids)
+
+ if "bbox" in filters:
+ val = filters.get("bbox", None)
+ geometry_field_name = "location" if services_unit_instances else "geometry"
+ if val:
+ ref = SpatialReference(filters.get("bbox_srid", 4326))
+ bbox_geometry_filter = munigeo_api.build_bbox_filter(
+ ref, val, geometry_field_name
+ )
+ queryset = queryset.filter(Q(**bbox_geometry_filter))
+
  for filter in filters:
  if filter.startswith("extra__"):
  if "type_name" not in filters:
@@ -212,15 +242,27 @@ def list(self, request):
  f"extra field '{key}' does not exist",
  status=status.HTTP_400_BAD_REQUEST,
  )
- if field_type == float:
- value = float(value)
- elif field_type == int:
- value = int(value)
+
+ match field_type:
+ case FIELD_TYPES.FLOAT:
+ value = float(value)
+ case FIELD_TYPES.INT:
+ value = int(value)
+ case FIELD_TYPES.BOOL:
+ value = strtobool(value)
+ value = bool(value)
+
  queryset = queryset.filter(**{filter: value})
 
  page = self.paginate_queryset(queryset)
  serializer = MobileUnitSerializer(
- page, many=True, context={"srid": srid, "latlon": latlon}
+ page,
+ many=True,
+ context={
+ "srid": srid,
+ "latlon": latlon,
+ "services_unit_instances": services_unit_instances,
+ },
  )
  return self.get_paginated_response(serializer.data)
 

mobility_data/constants.py

@@ -12,6 +12,9 @@
 from mobility_data.importers.loading_unloading_places import (
  CONTENT_TYPE_NAME as LOADING_UNLOADING_PLACE,
 )
+from mobility_data.importers.parking_machines import (
+ CONTENT_TYPE_NAME as PARKING_MACHINE,
+)
 from mobility_data.importers.share_car_parking_places import (
  CONTENT_TYPE_NAME as SHARE_CAR_PARKING_PLACE,
 )
@@ -48,4 +51,8 @@
  "display_name": "berths",
  "to_services_list": False,
  },
+ PARKING_MACHINE: {
+ "importer_name": "parking_machines",
+ "to_services_list": False,
+ },
 }