Merge pull request #332 from City-of-Turku/develop

Production update

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
  # Initializes the CodeQL tools for scanning.
  - name: Initialize CodeQL
- uses: github/codeql-action/init@v1
+ uses: github/codeql-action/init@v2
  with:
  languages: ${{ matrix.language }}
  # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
  # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
  # If this step fails, then you should remove it and run the build manually (see below)
  - name: Autobuild
- uses: github/codeql-action/autobuild@v1
+ uses: github/codeql-action/autobuild@v2
 
  # ℹ️ Command-line programs to run using the OS shell.
  # 📚 https://git.io/JvXDl
@@ -67,4 +67,4 @@ jobs:
  # make release
 
  - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v1
+ uses: github/codeql-action/analyze@v2

.github/workflows/run-tests.yml

@@ -20,6 +20,7 @@ jobs:
  ECO_COUNTER_STATIONS_URL: https://dev.turku.fi/datasets/ecocounter/liikennelaskimet.geojson
  ECO_COUNTER_OBSERVATIONS_URL: https://data.turku.fi/cjtv3brqr7gectdv7rfttc/counters-15min.csv
  TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL: https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/
+ CACHE_LOCATION: redis://localhost:6379/0
 
  steps:
  - uses: actions/checkout@v3
@@ -29,9 +30,8 @@ jobs:
  python-version: 3.10.0
  - name: Install required Ubuntu packages
  run: |
- sudo apt-get install gdal-bin
- sudo apt-get install voikko-fi
- sudo apt-get install libvoikko-dev
+ sudo apt-get update && sudo apt-get -y --no-install-recommends install gdal-bin voikko-fi libvoikko-dev
+ sudo apt-get install redis-server
  - name: Create needed postgis extensions
  run: |
  psql -h localhost -U postgres template1 -c 'create extension hstore;'

eco_counter/api/serializers.py

@@ -1,5 +1,3 @@
-from datetime import date, timedelta
-
 from django.db.models import Q
 from rest_framework import serializers
 
@@ -38,10 +36,6 @@ class StationSerializer(serializers.ModelSerializer):
  y = serializers.SerializerMethodField()
  lon = serializers.SerializerMethodField()
  lat = serializers.SerializerMethodField()
- sensor_types = serializers.SerializerMethodField()
- data_until_date = serializers.SerializerMethodField()
- data_from_date = serializers.SerializerMethodField()
- is_active = serializers.SerializerMethodField()
 
  class Meta:
  model = Station
@@ -78,48 +72,6 @@ def get_lon(self, obj):
  obj.location.transform(4326)
  return obj.location.x
 
- def get_sensor_types(self, obj):
- # Return the sensor types(car, bike etc) that has a total year value >0.
- # i.e., there are sensors for counting the type of data.
- types = ["at", "pt", "jt", "bt"]
- result = []
- for type in types:
- filter = {"station": obj, f"value_{type}__gt": 0}
- if YearData.objects.filter(**filter).count() > 0:
- result.append(type)
- return result
-
- def get_is_active(self, obj):
- num_days = [1, 7, 30, 365]
- res = {}
- for days in num_days:
- from_date = date.today() - timedelta(days=days - 1)
- day_qs = Day.objects.filter(station=obj, date__gte=from_date)
- day_data_qs = DayData.objects.filter(day__in=day_qs)
- if day_data_qs.filter(Q_EXP).count() > 0:
- res[days] = True
- else:
- res[days] = False
- return res
-
- def get_data_until_date(self, obj):
- try:
- return (
- DayData.objects.filter(Q_EXP, station=obj).latest("day__date").day.date
- )
- except DayData.DoesNotExist:
- return None
-
- def get_data_from_date(self, obj):
- try:
- return (
- DayData.objects.filter(Q_EXP, station=obj)
- .earliest("day__date")
- .day.date
- )
- except DayData.DoesNotExist:
- return None
-
 
 class YearSerializer(serializers.ModelSerializer):
  station_name = serializers.PrimaryKeyRelatedField(

eco_counter/api/views.py

@@ -1,5 +1,7 @@
 import sys
 
+from django.utils.decorators import method_decorator
+from django.views.decorators.cache import cache_page
 from rest_framework import status, viewsets
 from rest_framework.decorators import action
 from rest_framework.exceptions import ParseError
@@ -54,6 +56,7 @@ class StationViewSet(viewsets.ReadOnlyModelViewSet):
  queryset = Station.objects.all()
  serializer_class = StationSerializer
 
+ @method_decorator(cache_page(60 * 60))
  def list(self, request):
  queryset = Station.objects.all()
  filters = self.request.query_params

eco_counter/management/commands/import_counter_data.py

@@ -44,9 +44,13 @@
 from .utils import (
  check_counters_argument,
  gen_eco_counter_test_csv,
+ get_data_from_date,
+ get_data_until_date,
  get_eco_counter_csv,
+ get_is_active,
  get_lam_counter_csv,
  get_or_create_telraam_station,
+ get_sensor_types,
  get_telraam_data_frames,
  get_test_dataframe,
  get_traffic_counter_csv,
@@ -360,6 +364,7 @@ def save_observations(csv_data, start_time, csv_data_source=ECO_COUNTER, station
  import_state.current_month_number = end_date.month
  import_state.current_day_number = end_date.day
  import_state.save()
+ add_additional_data_to_stations(csv_data_source)
  logger.info(f"Imported observations until:{str(end_date)}")
 
 
@@ -458,6 +463,17 @@ def import_data(counters):
  gc.collect()
 
 
+def add_additional_data_to_stations(csv_data_source):
+
+ logger.info(f"Updating {csv_data_source} stations informations...")
+ for station in Station.objects.filter(csv_data_source=csv_data_source):
+ station.data_from_date = get_data_from_date(station)
+ station.data_until_date = get_data_until_date(station)
+ station.sensor_types = get_sensor_types(station)
+ station.is_active = get_is_active(station)
+ station.save()
+
+
 class Command(BaseCommand):
  help = "Imports traffic counter data in the Turku region."
 

eco_counter/management/commands/utils.py

@@ -9,6 +9,7 @@
 from django.contrib.gis.gdal import DataSource
 from django.contrib.gis.geos import GEOSGeometry, LineString, MultiLineString, Point
 from django.core.management.base import CommandError
+from django.db.models import Q
 
 from eco_counter.constants import (
  COUNTER_CHOICES_STR,
@@ -33,11 +34,12 @@
  TRAFFIC_COUNTER_CSV_URLS,
  TRAFFIC_COUNTER_METADATA_GEOJSON,
 )
-from eco_counter.models import Station
+from eco_counter.models import Day, DayData, Station, YearData
 from eco_counter.tests.constants import TEST_COLUMN_NAMES
 from mobility_data.importers.utils import get_root_dir
 
 logger = logging.getLogger("eco_counter")
+Q_EXP = Q(value_at__gt=0) | Q(value_pt__gt=0) | Q(value_jt__gt=0) | Q(value_bt__gt=0)
 
 
 class LAMStation:
@@ -112,6 +114,53 @@ def __init__(self, mac, location, geometry):
  self.geometry = geometry
 
 
+def get_is_active(station):
+ # Returns if the station has been active during time period defined in num_days
+ num_days = [1, 7, 30, 365]
+ res = {}
+ for days in num_days:
+ from_date = date.today() - timedelta(days=days - 1)
+ day_qs = Day.objects.filter(station=station, date__gte=from_date)
+ day_data_qs = DayData.objects.filter(day__in=day_qs)
+ if day_data_qs.filter(Q_EXP).count() > 0:
+ res[days] = True
+ else:
+ res[days] = False
+ return res
+
+
+def get_sensor_types(station):
+ # Return the sensor types(car, bike etc) that has a total year value >0.
+ # i.e., there are sensors for counting the type of data.
+ types = ["at", "pt", "jt", "bt"]
+ result = []
+ for type in types:
+ filter = {"station": station, f"value_{type}__gt": 0}
+ if YearData.objects.filter(**filter).count() > 0:
+ result.append(type)
+ return result
+
+
+def get_data_until_date(station):
+ try:
+ return (
+ DayData.objects.filter(Q_EXP, station=station).latest("day__date").day.date
+ )
+ except DayData.DoesNotExist:
+ return None
+
+
+def get_data_from_date(station):
+ try:
+ return (
+ DayData.objects.filter(Q_EXP, station=station)
+ .earliest("day__date")
+ .day.date
+ )
+ except DayData.DoesNotExist:
+ return None
+
+
 def check_counters_argument(counters):
  for counter in counters:
  if counter not in COUNTERS_LIST:

eco_counter/migrations/0019_station_data_from_date_station_data_until_date.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.10 on 2024-01-25 11:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("eco_counter", "0018_add_geometry_to_station"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="station",
+ name="data_from_date",
+ field=models.DateField(blank=True, null=True),
+ ),
+ migrations.AddField(
+ model_name="station",
+ name="data_until_date",
+ field=models.DateField(blank=True, null=True),
+ ),
+ ]

eco_counter/migrations/0020_station_sensor_types.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.10 on 2024-01-25 11:58
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("eco_counter", "0019_station_data_from_date_station_data_until_date"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="station",
+ name="sensor_types",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.CharField(max_length=2), default=list, size=None
+ ),
+ ),
+ ]

eco_counter/migrations/0021_station_is_active.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.10 on 2024-01-25 12:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("eco_counter", "0020_station_sensor_types"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="station",
+ name="is_active",
+ field=models.JSONField(blank=True, null=True),
+ ),
+ ]

eco_counter/models.py

@@ -35,6 +35,10 @@ class Station(models.Model):
  # Optioal id of the station, used when fetching LAM
  # and TELRAAM station data
  station_id = models.CharField(max_length=16, null=True)
+ data_until_date = models.DateField(null=True, blank=True)
+ data_from_date = models.DateField(null=True, blank=True)
+ sensor_types = ArrayField(models.CharField(max_length=2), default=list)
+ is_active = models.JSONField(null=True, blank=True)
 
  def __str__(self):
  return "%s %s" % (self.name, self.location)

eco_counter/tests/conftest.py

@@ -43,6 +43,9 @@ def stations():
  name=TEST_EC_STATION_NAME,
  location="POINT(60.4487578455581 22.269454227550053)",
  csv_data_source=ECO_COUNTER,
+ sensor_types=["at"],
+ data_until_date="2020-01-07",
+ data_from_date="2020-01-01",
  )
  )
  stations.append(
@@ -201,24 +204,28 @@ def is_active_fixtures():
  name="Station with 0 day of data",
  location="POINT(0 0)",
  csv_data_source=LAM_COUNTER,
+ is_active={"1": False, "7": False, "30": False, "365": False},
  )
  station1 = Station.objects.create(
  id=1,
  name="Station with 1 day of data",
  location="POINT(0 0)",
  csv_data_source=LAM_COUNTER,
+ is_active={"1": True, "7": True, "30": True, "365": True},
  )
  station7 = Station.objects.create(
  id=7,
  name="Station with 7 days of data",
  location="POINT(0 0)",
  csv_data_source=LAM_COUNTER,
+ is_active={"1": False, "7": True, "30": True, "365": True},
  )
  station30 = Station.objects.create(
  id=30,
  name="Station with 30 days of data",
  location="POINT(0 0)",
  csv_data_source=LAM_COUNTER,
+ is_active={"1": False, "7": False, "30": True, "365": True},
  )
  start_date = date.today()
  current_date = start_date

eco_counter/tests/test_import_counter_data.py

@@ -7,7 +7,7 @@
 imports and calculates the data correctly.
 """
 import calendar
-from datetime import timedelta
+from datetime import datetime, timedelta
 from io import StringIO
 from unittest.mock import patch
 
@@ -402,7 +402,16 @@ def test_import_traffic_counter_data(stations):
  start_time = dateutil.parser.parse("2020-01-01T00:00")
  end_time = dateutil.parser.parse("2020-02-29T23:45")
  import_command(test_counter=(TRAFFIC_COUNTER, start_time, end_time))
- num_tc_stations = Station.objects.filter(csv_data_source=TRAFFIC_COUNTER).count()
+ stations_qs = Station.objects.filter(csv_data_source=TRAFFIC_COUNTER)
+ num_tc_stations = stations_qs.count()
+ station1 = stations_qs.first()
+ assert station1.sensor_types == ["at", "pt", "jt", "bt"]
+ assert station1.data_from_date == datetime.strptime("2020-01-01", "%Y-%m-%d").date()
+ assert (
+ station1.data_until_date == datetime.strptime("2020-02-29", "%Y-%m-%d").date()
+ )
+ assert station1.is_active == {"1": False, "7": False, "30": False, "365": False}
+
  state = ImportState.objects.get(csv_data_source=TRAFFIC_COUNTER)
  assert state.current_year_number == 2020
  assert state.current_month_number == 2