Production update

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,4 +67,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/run-tests.yml

@@ -20,6 +20,7 @@ jobs:
       ECO_COUNTER_STATIONS_URL: https://dev.turku.fi/datasets/ecocounter/liikennelaskimet.geojson
       ECO_COUNTER_OBSERVATIONS_URL: https://data.turku.fi/cjtv3brqr7gectdv7rfttc/counters-15min.csv
       TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL: https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/
+      CACHE_LOCATION: redis://localhost:6379/0
 
     steps:
     - uses: actions/checkout@v3
@@ -29,9 +30,8 @@ jobs:
         python-version: 3.10.0
     - name: Install required Ubuntu packages
       run: |
-        sudo apt-get install gdal-bin
-        sudo apt-get install voikko-fi
-        sudo apt-get install libvoikko-dev
+        sudo apt-get update && sudo apt-get -y --no-install-recommends install gdal-bin voikko-fi libvoikko-dev
+        sudo apt-get install redis-server
     - name: Create needed postgis extensions
       run: |
         psql -h localhost -U postgres template1 -c 'create extension hstore;'

eco_counter/api/serializers.py

@@ -1,5 +1,3 @@
-from datetime import date, timedelta
-
 from django.db.models import Q
 from rest_framework import serializers
 
@@ -38,10 +36,6 @@ class StationSerializer(serializers.ModelSerializer):
     y = serializers.SerializerMethodField()
     lon = serializers.SerializerMethodField()
     lat = serializers.SerializerMethodField()
-    sensor_types = serializers.SerializerMethodField()
-    data_until_date = serializers.SerializerMethodField()
-    data_from_date = serializers.SerializerMethodField()
-    is_active = serializers.SerializerMethodField()
 
     class Meta:
         model = Station
@@ -78,48 +72,6 @@ def get_lon(self, obj):
         obj.location.transform(4326)
         return obj.location.x
 
-    def get_sensor_types(self, obj):
-        # Return the sensor types(car, bike etc) that has a total year value >0.
-        # i.e., there are sensors for counting the type of data.
-        types = ["at", "pt", "jt", "bt"]
-        result = []
-        for type in types:
-            filter = {"station": obj, f"value_{type}__gt": 0}
-            if YearData.objects.filter(**filter).count() > 0:
-                result.append(type)
-        return result
-
-    def get_is_active(self, obj):
-        num_days = [1, 7, 30, 365]
-        res = {}
-        for days in num_days:
-            from_date = date.today() - timedelta(days=days - 1)
-            day_qs = Day.objects.filter(station=obj, date__gte=from_date)
-            day_data_qs = DayData.objects.filter(day__in=day_qs)
-            if day_data_qs.filter(Q_EXP).count() > 0:
-                res[days] = True
-            else:
-                res[days] = False
-        return res
-
-    def get_data_until_date(self, obj):
-        try:
-            return (
-                DayData.objects.filter(Q_EXP, station=obj).latest("day__date").day.date
-            )
-        except DayData.DoesNotExist:
-            return None
-
-    def get_data_from_date(self, obj):
-        try:
-            return (
-                DayData.objects.filter(Q_EXP, station=obj)
-                .earliest("day__date")
-                .day.date
-            )
-        except DayData.DoesNotExist:
-            return None
-
 
 class YearSerializer(serializers.ModelSerializer):
     station_name = serializers.PrimaryKeyRelatedField(

eco_counter/api/views.py

@@ -1,5 +1,7 @@
 import sys
 
+from django.utils.decorators import method_decorator
+from django.views.decorators.cache import cache_page
 from rest_framework import status, viewsets
 from rest_framework.decorators import action
 from rest_framework.exceptions import ParseError
@@ -54,6 +56,7 @@ class StationViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Station.objects.all()
     serializer_class = StationSerializer
 
+    @method_decorator(cache_page(60 * 60))
     def list(self, request):
         queryset = Station.objects.all()
         filters = self.request.query_params

eco_counter/management/commands/import_counter_data.py

@@ -44,9 +44,13 @@
 from .utils import (
     check_counters_argument,
     gen_eco_counter_test_csv,
+    get_data_from_date,
+    get_data_until_date,
     get_eco_counter_csv,
+    get_is_active,
     get_lam_counter_csv,
     get_or_create_telraam_station,
+    get_sensor_types,
     get_telraam_data_frames,
     get_test_dataframe,
     get_traffic_counter_csv,
@@ -360,6 +364,7 @@ def save_observations(csv_data, start_time, csv_data_source=ECO_COUNTER, station
     import_state.current_month_number = end_date.month
     import_state.current_day_number = end_date.day
     import_state.save()
+    add_additional_data_to_stations(csv_data_source)
     logger.info(f"Imported observations until:{str(end_date)}")
 
 
@@ -458,6 +463,17 @@ def import_data(counters):
             gc.collect()
 
 
+def add_additional_data_to_stations(csv_data_source):
+
+    logger.info(f"Updating {csv_data_source} stations informations...")
+    for station in Station.objects.filter(csv_data_source=csv_data_source):
+        station.data_from_date = get_data_from_date(station)
+        station.data_until_date = get_data_until_date(station)
+        station.sensor_types = get_sensor_types(station)
+        station.is_active = get_is_active(station)
+        station.save()
+
+
 class Command(BaseCommand):
     help = "Imports traffic counter data in the Turku region."
 

eco_counter/management/commands/utils.py

@@ -9,6 +9,7 @@
 from django.contrib.gis.gdal import DataSource
 from django.contrib.gis.geos import GEOSGeometry, LineString, MultiLineString, Point
 from django.core.management.base import CommandError
+from django.db.models import Q
 
 from eco_counter.constants import (
     COUNTER_CHOICES_STR,
@@ -33,11 +34,12 @@
     TRAFFIC_COUNTER_CSV_URLS,
     TRAFFIC_COUNTER_METADATA_GEOJSON,
 )
-from eco_counter.models import Station
+from eco_counter.models import Day, DayData, Station, YearData
 from eco_counter.tests.constants import TEST_COLUMN_NAMES
 from mobility_data.importers.utils import get_root_dir
 
 logger = logging.getLogger("eco_counter")
+Q_EXP = Q(value_at__gt=0) | Q(value_pt__gt=0) | Q(value_jt__gt=0) | Q(value_bt__gt=0)
 
 
 class LAMStation:
@@ -112,6 +114,53 @@
         self.geometry = geometry
 
 
+def get_is_active(station):
+    # Returns if the station has been active during time period defined in num_days
+    num_days = [1, 7, 30, 365]
+    res = {}
+    for days in num_days:
+        from_date = date.today() - timedelta(days=days - 1)
+        day_qs = Day.objects.filter(station=station, date__gte=from_date)
+        day_data_qs = DayData.objects.filter(day__in=day_qs)
+        if day_data_qs.filter(Q_EXP).count() > 0:
+            res[days] = True
+        else:
+            res[days] = False
+    return res
+
+
+def get_sensor_types(station):
+    # Return the sensor types(car, bike etc) that has a total year value >0.
+    # i.e., there are sensors for counting the type of data.
+    types = ["at", "pt", "jt", "bt"]
+    result = []
+    for type in types:
+        filter = {"station": station, f"value_{type}__gt": 0}
+        if YearData.objects.filter(**filter).count() > 0:
+            result.append(type)
+    return result
+
+
+def get_data_until_date(station):
+    try:
+        return (
+            DayData.objects.filter(Q_EXP, station=station).latest("day__date").day.date
+        )
+    except DayData.DoesNotExist:
+        return None
+
+
+def get_data_from_date(station):
+    try:
+        return (
+            DayData.objects.filter(Q_EXP, station=station)
+            .earliest("day__date")
+            .day.date
+        )
+    except DayData.DoesNotExist:
+        return None
+
+
 def check_counters_argument(counters):
     for counter in counters:
         if counter not in COUNTERS_LIST:

eco_counter/migrations/0019_station_data_from_date_station_data_until_date.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.10 on 2024-01-25 11:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("eco_counter", "0018_add_geometry_to_station"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="station",
+            name="data_from_date",
+            field=models.DateField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="station",
+            name="data_until_date",
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]

eco_counter/migrations/0020_station_sensor_types.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.10 on 2024-01-25 11:58
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("eco_counter", "0019_station_data_from_date_station_data_until_date"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="station",
+            name="sensor_types",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=2), default=list, size=None
+            ),
+        ),
+    ]

eco_counter/migrations/0021_station_is_active.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.10 on 2024-01-25 12:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("eco_counter", "0020_station_sensor_types"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="station",
+            name="is_active",
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

eco_counter/models.py

@@ -35,6 +35,10 @@ class Station(models.Model):
     # Optioal id of the station, used when fetching LAM
     # and TELRAAM station data
     station_id = models.CharField(max_length=16, null=True)
+    data_until_date = models.DateField(null=True, blank=True)
+    data_from_date = models.DateField(null=True, blank=True)
+    sensor_types = ArrayField(models.CharField(max_length=2), default=list)
+    is_active = models.JSONField(null=True, blank=True)
 
     def __str__(self):
         return "%s %s" % (self.name, self.location)

eco_counter/tests/conftest.py

@@ -43,6 +43,9 @@ def stations():
             name=TEST_EC_STATION_NAME,
             location="POINT(60.4487578455581 22.269454227550053)",
             csv_data_source=ECO_COUNTER,
+            sensor_types=["at"],
+            data_until_date="2020-01-07",
+            data_from_date="2020-01-01",
         )
     )
     stations.append(
@@ -201,24 +204,28 @@ def is_active_fixtures():
         name="Station with 0 day of data",
         location="POINT(0 0)",
         csv_data_source=LAM_COUNTER,
+        is_active={"1": False, "7": False, "30": False, "365": False},
     )
     station1 = Station.objects.create(
         id=1,
         name="Station with 1 day of data",
         location="POINT(0 0)",
         csv_data_source=LAM_COUNTER,
+        is_active={"1": True, "7": True, "30": True, "365": True},
     )
     station7 = Station.objects.create(
         id=7,
         name="Station with 7 days of data",
         location="POINT(0 0)",
         csv_data_source=LAM_COUNTER,
+        is_active={"1": False, "7": True, "30": True, "365": True},
     )
     station30 = Station.objects.create(
         id=30,
         name="Station with 30 days of data",
         location="POINT(0 0)",
         csv_data_source=LAM_COUNTER,
+        is_active={"1": False, "7": False, "30": True, "365": True},
     )
     start_date = date.today()
     current_date = start_date

eco_counter/tests/test_import_counter_data.py

@@ -7,7 +7,7 @@
 imports and calculates the data correctly.
 """
 import calendar
-from datetime import timedelta
+from datetime import datetime, timedelta
 from io import StringIO
 from unittest.mock import patch
 
@@ -402,7 +402,16 @@
     start_time = dateutil.parser.parse("2020-01-01T00:00")
     end_time = dateutil.parser.parse("2020-02-29T23:45")
     import_command(test_counter=(TRAFFIC_COUNTER, start_time, end_time))
-    num_tc_stations = Station.objects.filter(csv_data_source=TRAFFIC_COUNTER).count()
+    stations_qs = Station.objects.filter(csv_data_source=TRAFFIC_COUNTER)
+    num_tc_stations = stations_qs.count()
+    station1 = stations_qs.first()
+    assert station1.sensor_types == ["at", "pt", "jt", "bt"]
+    assert station1.data_from_date == datetime.strptime("2020-01-01", "%Y-%m-%d").date()
+    assert (
+        station1.data_until_date == datetime.strptime("2020-02-29", "%Y-%m-%d").date()
+    )
+    assert station1.is_active == {"1": False, "7": False, "30": False, "365": False}
+
     state = ImportState.objects.get(csv_data_source=TRAFFIC_COUNTER)
     assert state.current_year_number == 2020
     assert state.current_month_number == 2