Merge pull request #319 from City-of-Turku/develop

Production update

.github/workflows/run-tests.yml

@@ -17,6 +17,9 @@ jobs:
       ADDITIONAL_INSTALLED_APPS: smbackend_turku,ptv
       PTV_ID_OFFSET: 10000000
       LAM_COUNTER_API_BASE_URL: https://tie.digitraffic.fi/api/tms/v1/history
+      ECO_COUNTER_STATIONS_URL: https://dev.turku.fi/datasets/ecocounter/liikennelaskimet.geojson
+      ECO_COUNTER_OBSERVATIONS_URL: https://data.turku.fi/cjtv3brqr7gectdv7rfttc/counters-15min.csv
+      TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL: https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/
 
     steps:
     - uses: actions/checkout@v3

eco_counter/README.md

@@ -34,6 +34,13 @@ e.g. ./manage.py import_counter_data --counters EC TC
 Counter names are: EC (Eco Counter), TC (Traffic Counter), LC (Lam Counter) and TR (Telraam Counter).
 Note, Traffic Counter data is updated once a week and Lam Counter data once a day.
 
+## Deleting data
+To delete data use the delete_counter_data management command.
+e.g. to delete all Lam Counter data type:
+```
+./manage.py delete_counter_data --counters LC
+```
+
 ### Importing Telraam raw data
 In order to import Telraam data into the database the raw data has to be imported. The raw data is imported with the _import_telraam_to_csv_ management command.
 The imported should be set to be run once a hour (see: https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#telraam-to-csv-eco_countertasksimport_telraam_to_csv )

eco_counter/api/serializers.py

@@ -1,3 +1,5 @@
+from datetime import date, timedelta
+
 from django.db.models import Q
 from rest_framework import serializers
 
@@ -28,6 +30,7 @@
     "value_bp",
     "value_bt",
 ]
+Q_EXP = Q(value_at__gt=0) | Q(value_pt__gt=0) | Q(value_jt__gt=0) | Q(value_bt__gt=0)
 
 
 class StationSerializer(serializers.ModelSerializer):
@@ -36,7 +39,9 @@ class StationSerializer(serializers.ModelSerializer):
     lon = serializers.SerializerMethodField()
     lat = serializers.SerializerMethodField()
     sensor_types = serializers.SerializerMethodField()
-    data_from_year = serializers.SerializerMethodField()
+    data_until_date = serializers.SerializerMethodField()
+    data_from_date = serializers.SerializerMethodField()
+    is_active = serializers.SerializerMethodField()
 
     class Meta:
         model = Station
@@ -54,7 +59,9 @@ class Meta:
             "lon",
             "lat",
             "sensor_types",
-            "data_from_year",
+            "data_until_date",
+            "data_from_date",
+            "is_active",
         ]
 
     def get_y(self, obj):
@@ -82,17 +89,35 @@ def get_sensor_types(self, obj):
                 result.append(type)
         return result
 
-    def get_data_from_year(self, obj):
-        q_exp = (
-            Q(value_at__gt=0)
-            | Q(value_pt__gt=0)
-            | Q(value_jt__gt=0)
-            | Q(value_bt__gt=0)
-        )
-        qs = YearData.objects.filter(q_exp, station=obj).order_by("year__year_number")
-        if qs.count() > 0:
-            return qs[0].year.year_number
-        else:
+    def get_is_active(self, obj):
+        num_days = [1, 7, 30, 365]
+        res = {}
+        for days in num_days:
+            from_date = date.today() - timedelta(days=days - 1)
+            day_qs = Day.objects.filter(station=obj, date__gte=from_date)
+            day_data_qs = DayData.objects.filter(day__in=day_qs)
+            if day_data_qs.filter(Q_EXP).count() > 0:
+                res[days] = True
+            else:
+                res[days] = False
+        return res
+
+    def get_data_until_date(self, obj):
+        try:
+            return (
+                DayData.objects.filter(Q_EXP, station=obj).latest("day__date").day.date
+            )
+        except DayData.DoesNotExist:
+            return None
+
+    def get_data_from_date(self, obj):
+        try:
+            return (
+                DayData.objects.filter(Q_EXP, station=obj)
+                .earliest("day__date")
+                .day.date
+            )
+        except DayData.DoesNotExist:
             return None
 
 

eco_counter/constants.py

@@ -30,6 +30,10 @@
 COUNTERS.LAM_COUNTER = LAM_COUNTER
 COUNTERS.TELRAAM_COUNTER = TELRAAM_COUNTER
 
+COUNTERS_LIST = [ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER, TELRAAM_COUNTER]
+COUNTER_CHOICES_STR = (
+    f"{ECO_COUNTER}, {TRAFFIC_COUNTER}, {TELRAAM_COUNTER} and {LAM_COUNTER}"
+)
 CSV_DATA_SOURCES = (
     (TRAFFIC_COUNTER, "TrafficCounter"),
     (ECO_COUNTER, "EcoCounter"),
@@ -118,12 +122,14 @@
 
 TELRAAM_COUNTER_CSV_FILE_PATH = f"{settings.MEDIA_ROOT}/telraam_data/"
 TELRAAM_COUNTER_CSV_FILE = (
-    TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{id}_{day}_{month}_{year}.csv"
+    TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{mac}_{day}_{month}_{year}.csv"
 )
+TELRAAM_STATION_350457790598039 = 350457790598039
+TELRAAM_STATION_350457790600975 = 350457790600975
 TELRAAM_COUNTER_CAMERAS = {
     # Mac id: Direction flag (True=rgt prefix will be keskustaan päin)
-    350457790598039: False,  # Kristiinanankatu, Joelle katsottaessa vasemmalle
-    350457790600975: True,  # Kristiinanankatu, Joelle katsottaessa oikealle
+    TELRAAM_STATION_350457790598039: False,  # Kristiinanankatu, Joelle katsottaessa vasemmalle
+    TELRAAM_STATION_350457790600975: True,  # Kristiinanankatu, Joelle katsottaessa oikealle
 }
 # For 429 (too many request) TELRAAM need a retry strategy
 retry_strategy = Retry(
@@ -136,3 +142,23 @@
 TELRAAM_HTTP = requests.Session()
 TELRAAM_HTTP.mount("https://", adapter)
 TELRAAM_HTTP.mount("http://", adapter)
+
+
+# Telraam stations initial geometries in WKT format
+# These coordinates are used if CSV files do not include any geometries.
+TELRAAM_STATIONS_INITIAL_WKT_GEOMETRIES = {
+    TELRAAM_STATION_350457790598039: {
+        "location": "POINT (239628.47846388057 6710757.471557152)",
+        "geometry": "MULTILINESTRING ((239565.80107971327 6710861.8209667895, 239572.58901459936 6710850.524818219,"
+        " 239574.73294378238 6710846.950531884, 239628.47846388057 6710757.471557152,"
+        " 239630.0339247121 6710754.923177836, 239635.52748551324 6710745.732077925))",
+    },
+    TELRAAM_STATION_350457790600975: {
+        "location": "POINT (239523.2288977413 6710932.715108742)",
+        "geometry": "MULTILINESTRING ((239490.42663459244 6710989.092283992, 239493.45037993207 6710983.7110835295,"
+        " 239495.88642941663 6710979.3668986475, 239517.9904128411 6710941.530425406,"
+        " 239520.0691194288 6710937.971973339, 239523.2288977413 6710932.715108742,"
+        " 239529.37000273907 6710922.482472116, 239558.08254550528 6710874.681753734,"
+        " 239559.97438753376 6710871.516775628, 239565.80107971327 6710861.8209667895))",
+    },
+}

eco_counter/management/commands/delete_counter_data.py

@@ -0,0 +1,36 @@
+import logging
+
+from django import db
+from django.core.management.base import BaseCommand
+
+from eco_counter.constants import COUNTER_CHOICES_STR
+from eco_counter.management.commands.utils import check_counters_argument
+from eco_counter.models import ImportState, Station
+
+logger = logging.getLogger("eco_counter")
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--counters",
+            type=str,
+            nargs="+",
+            default=False,
+            help=f"Delete given counter data, choices are: {COUNTER_CHOICES_STR}.",
+        )
+
+    @db.transaction.atomic
+    def handle(self, *args, **options):
+        counters = options.get("counters", None)
+        check_counters_argument(counters)
+        if counters:
+            for counter in counters:
+                logger.info(f"Deleting counter data for {counter}")
+                logger.info(
+                    f"{Station.objects.filter(csv_data_source=counter).delete()}"
+                )
+                logger.info(
+                    f"{ImportState.objects.filter(csv_data_source=counter).delete()}"
+                )
+                logger.info("Deleted counter data.")