Merge pull request #215 from City-of-Turku/develop

Production update

eco_counter/api/views.py

@@ -228,6 +228,25 @@ def get_year_data(self, request):
         serializer = YearDataSerializer(queryset, many=False)
         return Response(serializer.data, status=status.HTTP_200_OK)
 
+    @action(detail=False, methods=["get"])
+    def get_year_datas(self, request):
+        start_year_number = request.query_params.get("start_year_number", None)
+        end_year_number = request.query_params.get("end_year_number", None)
+        station_id = request.query_params.get("station_id", None)
+        if start_year_number is None or end_year_number is None or station_id is None:
+            return Response(status=status.HTTP_400_BAD_REQUEST)
+        try:
+            queryset = YearData.objects.filter(
+                station_id=station_id,
+                year__year_number__gte=start_year_number,
+                year__year_number__lte=end_year_number,
+            ).order_by("year__year_number")
+        except YearData.DoesNotExist:
+            return Response(NOT_FOUND_RESPONSE_MSG, status=status.HTTP_400_BAD_REQUEST)
+
+        serializer = YearDataSerializer(queryset, many=True)
+        return Response(serializer.data, status=status.HTTP_200_OK)
+
 
 class DayViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Day.objects.all()

eco_counter/migrations/0012_set_static_default_year_year_number.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.2 on 2023-01-11 05:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("eco_counter", "0011_add_lam_id_to_station"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="year",
+            name="year_number",
+            field=models.PositiveSmallIntegerField(default=2023),
+        ),
+    ]

eco_counter/models.py

@@ -1,5 +1,3 @@
-from datetime import datetime
-
 from django.conf import settings
 from django.contrib.gis.db import models
 from django.contrib.postgres.fields import ArrayField
@@ -86,7 +84,7 @@ class Year(models.Model):
     station = models.ForeignKey(
         "Station", on_delete=models.CASCADE, related_name="years", null=True
     )
-    year_number = models.PositiveSmallIntegerField(default=datetime.now().year)
+    year_number = models.PositiveSmallIntegerField(default=2023)
 
     @property
     def num_days(self):

eco_counter/specification.swagger2.0.yaml

@@ -619,5 +619,32 @@ paths:
             $ref: "#/definitions/year_data"
         400:
           description: "Invalid year number or station_id." 
-
+  /year_data/get_year_datas:
+    get:
+      summary: "Returns year_datas for  span of year numbers and station_id"
+      parameters:     
+      - in: query
+        name: start_year_number
+        type: integer
+        required: true
+      - in: query
+        name: end_year_number
+        type: integer
+        required: true
+      - in: query
+        name: station_id
+        type: integer
+        required: true
+      responses:
+        200:
+          description: "List of year_datas for the given span."
+          schema:
+            type: object
+            properties:
+              results:
+                type: array
+                items:
+                  $ref: "#/definitions/year_data"
+        400:
+          description: "Invalid start_year_number, end_year_number or station_id."        
 

eco_counter/tests/conftest.py

@@ -19,6 +19,7 @@
 )
 
 TEST_TIMESTAMP = dateutil.parser.parse("2020-01-01 00:00:00")
+
 TEST_STATION_NAME = "Auransilta"
 
 
@@ -212,6 +213,7 @@ def year_datas(station, years):
         year_data = YearData.objects.create(station=station, year=years[i])
         year_data.value_ak = 42 + i
         year_data.value_ap = 43 + i
+        year_data.value_at = year_data.value_ak + year_data.value_ap
         year_data.save()
         year_datas.append(year_data)
     return year_datas

eco_counter/tests/test_api.py

@@ -145,6 +145,35 @@ def test__get_month_data(api_client, month_datas, station_id, test_timestamp):
     assert response.json()["value_pk"] == month_datas[0].value_pk
 
 
+@pytest.mark.django_db
+def test__get_year_datas(api_client, year_datas, station_id, test_timestamp):
+    end_year_number = test_timestamp.replace(year=test_timestamp.year + 1).year
+    url = reverse(
+        "eco_counter:year_data-get-year-datas"
+    ) + "?station_id={}&start_year_number={}&end_year_number={}".format(
+        station_id, test_timestamp.year, end_year_number
+    )
+    response = api_client.get(url)
+    assert response.status_code == 200
+    json_data = response.json()
+    index = 0
+    assert json_data[index]["year_info"]["year_number"] == 2020
+    assert json_data[index]["value_ak"] == 42
+    assert json_data[index]["value_ap"] == 43
+    assert (
+        json_data[index]["value_at"]
+        == json_data[index]["value_ak"] + json_data[index]["value_ap"]
+    )
+    index = 1
+    assert json_data[index]["year_info"]["year_number"] == 2021
+    assert json_data[index]["value_ak"] == 43
+    assert json_data[index]["value_ap"] == 44
+    assert (
+        json_data[index]["value_at"]
+        == json_data[index]["value_ak"] + json_data[index]["value_ap"]
+    )
+
+
 @pytest.mark.django_db
 def test__get_month_datas(api_client, month_datas, station_id, test_timestamp):
     url = reverse(

requirements.txt

@@ -26,7 +26,7 @@ celery==5.2.3
     #   -r requirements.in
     #   django-celery-beat
     #   django-celery-results
-certifi==2021.5.30
+certifi==2022.12.7
     # via requests
 charset-normalizer==2.0.6
     # via requests
@@ -219,9 +219,7 @@ toml==0.10.2
     #   pytest
     #   pytest-cov
 tomli==1.2.1
-    # via
-    #   black
-    #   pep517
+    # via pep517
 tqdm==4.62.3
     # via -r requirements.in
 tzdata==2022.1
@@ -239,7 +237,7 @@ vine==5.0.0
     #   kombu
 wcwidth==0.2.5
     # via prompt-toolkit
-wheel==0.37.0
+wheel==0.38.1
     # via pip-tools
 whitenoise==5.3.0
     # via -r requirements.in

street_maintenance/README.md

@@ -16,6 +16,12 @@ Django app for importing and serving street maintenance data.
 ```
 ./manage.py import_kuntec_street_maintenance_history
 ```
+
+### Destia
+```
+./manage.py import_destia_street_maintenance_history
+```
+
 ### Periodically imorting
 To periodically import data use Celery, for more information [see](https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#street-maintenance-history-street_maintenancetasksimport_street_maintenance_history).
 
@@ -27,6 +33,8 @@ e.g., would import the Autori data for the last 30 days.
 ```
 ### Infraroad
 The default history size for a infraroad maintenance unit is 10000. That is works per unit. A work contains the timestamp, point data and events.
+### Destia
+The default history size for a Destia maintenance unit is 10000. 
 ### Autori
 The history size is in days. The default is 5.
 Note, the max size for Autori history is 31 days.

street_maintenance/api/serializers.py

@@ -24,13 +24,6 @@ def get_geometry_type(self, obj):
         return obj.geometry.geom_type
 
 
-class HistoryGeometrySerializer(serializers.Serializer):
-    def to_representation(self, obj):
-        representation = super().to_representation(obj)
-        representation["geometry"] = obj
-        return representation
-
-
 class ActiveEventSerializer(serializers.Serializer):
     events = serializers.CharField(max_length=64)
 

street_maintenance/api/views.py

@@ -2,27 +2,21 @@
 from functools import lru_cache
 
 import pytz
-from django.contrib.gis.geos import LineString
 from rest_framework import mixins, viewsets
-from rest_framework.decorators import action
 from rest_framework.exceptions import ParseError
 from rest_framework.pagination import PageNumberPagination
-from rest_framework.response import Response
 
 from street_maintenance.api.serializers import (
     ActiveEventSerializer,
     GeometryHistorySerializer,
-    HistoryGeometrySerializer,
     MaintenanceUnitSerializer,
     MaintenanceWorkSerializer,
 )
-from street_maintenance.management.commands.constants import PROVIDERS
-from street_maintenance.models import (
-    DEFAULT_SRID,
-    GeometryHistory,
-    MaintenanceUnit,
-    MaintenanceWork,
+from street_maintenance.management.commands.constants import (
+    PROVIDERS,
+    START_DATE_TIME_FORMAT,
 )
+from street_maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork
 
 UTC_TIMEZONE = pytz.timezone("UTC")
 
@@ -59,7 +53,7 @@ def get_queryset(self):
             start_date_time = filters["start_date_time"]
             try:
                 start_date_time = datetime.strptime(
-                    start_date_time, "%Y-%m-%d %H:%M:%S"
+                    start_date_time, START_DATE_TIME_FORMAT
                 )
             except ValueError:
                 raise ParseError(
@@ -80,84 +74,6 @@ def list(self, request):
         serializer = self.serializer_class(page, many=True)
         return self.get_paginated_response(serializer.data)
 
-    @action(detail=False, methods=["get"])
-    def get_geometry_history(self, request):
-        """
-        Returns linestrings(if two or more points(works) can be determined to belong to the same uniform work)
-        and/or points for works that can not be determined to belong to a uniform work(linestring).
-        """
-        queryset = self.get_queryset()
-        filters = self.request.query_params
-
-        if "event" not in filters:
-            raise ParseError("'get_geometry_history' requires the 'event' argument.")
-
-        if "max_work_length" in filters:
-            try:
-                max_work_length = int(filters.get("max_work_length"))
-            except ValueError:
-                raise ParseError("'max_work_length' needs to be of type integer.")
-        else:
-            max_work_length = DEFAULT_MAX_WORK_LENGTH
-        queryset = self.get_queryset()
-        geometries = []
-        elements_to_remove = []
-        # Add works that are linestrings,
-        for elem in queryset:
-            if isinstance(elem.geometry, LineString):
-                geometries.append(elem.geometry)
-                elements_to_remove.append(elem.id)
-        # Remove the linestring elements, as they are not needed when generaintg
-        # linestrings from point data
-        queryset = queryset.exclude(id__in=elements_to_remove)
-        unit_ids = (
-            queryset.order_by("maintenance_unit_id")
-            .values_list("maintenance_unit_id", flat=True)
-            .distinct("maintenance_unit_id")
-        )
-        for unit_id in unit_ids:
-            # Temporary store points to list for LineString creation
-            points = []
-            qs = queryset.filter(maintenance_unit_id=unit_id).order_by("timestamp")
-            prev_timestamp = None
-            for elem in qs:
-
-                if prev_timestamp:
-                    delta_time = elem.timestamp - prev_timestamp
-                    # If delta_time is bigger than the max_work_length, then we can assume
-                    # that the work should not be in the same linestring/point .
-                    if delta_time.seconds > max_work_length:
-                        if len(points) > 1:
-                            geometries.append(LineString(points, srid=DEFAULT_SRID))
-                        else:
-                            geometries.append(elem.geometry)
-                        points = []
-                points.append(elem.geometry)
-                prev_timestamp = elem.timestamp
-
-            if len(points) > 1:
-                geometries.append(LineString(points, srid=DEFAULT_SRID))
-            else:
-                geometries.append(elem.geometry)
-
-        # Create objects for every geometry to the serializer
-        if geometries:
-            data = []
-            for geometry in geometries:
-                elem = {}
-                elem["event"] = request.query_params["event"]
-                if isinstance(geometry, LineString):
-                    elem["name"] = "LineString"
-                else:
-                    elem["name"] = "Point"
-                elem["coordinates"] = geometry.coords
-                data.append(elem)
-        else:
-            data = []
-
-        results = HistoryGeometrySerializer(data, many=True).data
-        return Response(results)
-
 
 class MaintenanceUnitViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = MaintenanceUnit.objects.all()
@@ -171,17 +87,17 @@ class GeometryHitoryViewSet(viewsets.ReadOnlyModelViewSet):
     def get_queryset(self):
         queryset = GeometryHistory.objects.all()
         filters = self.request.query_params
+
         if "provider" in filters:
             provider = filters["provider"].upper()
+            queryset = queryset.filter(provider=provider)
             if provider in PROVIDERS:
-                queryset = GeometryHistory.objects.filter(provider=provider)
+                queryset = queryset.filter(provider=provider)
             else:
                 raise ParseError(f"Providers are: {', '.join(PROVIDERS)}")
 
         if "event" in filters:
-            queryset = GeometryHistory.objects.filter(
-                events__contains=[filters["event"]]
-            )
+            queryset = queryset.filter(events__contains=[filters["event"]])
         if "start_date_time" in filters:
             start_date_time = filters["start_date_time"]
             try:

street_maintenance/management/commands/base_import_command.py

@@ -0,0 +1,16 @@
+import logging
+from datetime import datetime
+
+from django.core.management import BaseCommand
+
+logger = logging.getLogger("street_maintenance")
+
+
+class BaseImportCommand(BaseCommand):
+    def __init__(self):
+        self.start_time = datetime.now()
+
+    def display_duration(self, provider):
+        end_time = datetime.now()
+        duration = end_time - self.start_time
+        logger.info(f"Imported {provider} street maintenance history in: {duration}")