Production update

.github/workflows/run-tests.yml

@@ -17,6 +17,9 @@ jobs:
  ADDITIONAL_INSTALLED_APPS: smbackend_turku,ptv
  PTV_ID_OFFSET: 10000000
  LAM_COUNTER_API_BASE_URL: https://tie.digitraffic.fi/api/tms/v1/history
+ ECO_COUNTER_STATIONS_URL: https://dev.turku.fi/datasets/ecocounter/liikennelaskimet.geojson
+ ECO_COUNTER_OBSERVATIONS_URL: https://data.turku.fi/cjtv3brqr7gectdv7rfttc/counters-15min.csv
+ TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL: https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/
 
  steps:
  - uses: actions/checkout@v3

eco_counter/README.md

@@ -34,6 +34,13 @@ e.g. ./manage.py import_counter_data --counters EC TC
 Counter names are: EC (Eco Counter), TC (Traffic Counter), LC (Lam Counter) and TR (Telraam Counter).
 Note, Traffic Counter data is updated once a week and Lam Counter data once a day.
 
+## Deleting data
+To delete data use the delete_counter_data management command.
+e.g. to delete all Lam Counter data type:
+```
+./manage.py delete_counter_data --counters LC
+```
+
 ### Importing Telraam raw data
 In order to import Telraam data into the database the raw data has to be imported. The raw data is imported with the _import_telraam_to_csv_ management command.
 The imported should be set to be run once a hour (see: https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#telraam-to-csv-eco_countertasksimport_telraam_to_csv )

eco_counter/api/serializers.py

@@ -1,3 +1,5 @@
+from datetime import date, timedelta
+
 from django.db.models import Q
 from rest_framework import serializers
 
@@ -28,6 +30,7 @@
  "value_bp",
  "value_bt",
 ]
+Q_EXP = Q(value_at__gt=0) | Q(value_pt__gt=0) | Q(value_jt__gt=0) | Q(value_bt__gt=0)
 
 
 class StationSerializer(serializers.ModelSerializer):
@@ -36,7 +39,9 @@ class StationSerializer(serializers.ModelSerializer):
  lon = serializers.SerializerMethodField()
  lat = serializers.SerializerMethodField()
  sensor_types = serializers.SerializerMethodField()
- data_from_year = serializers.SerializerMethodField()
+ data_until_date = serializers.SerializerMethodField()
+ data_from_date = serializers.SerializerMethodField()
+ is_active = serializers.SerializerMethodField()
 
  class Meta:
  model = Station
@@ -54,7 +59,9 @@ class Meta:
  "lon",
  "lat",
  "sensor_types",
- "data_from_year",
+ "data_until_date",
+ "data_from_date",
+ "is_active",
  ]
 
  def get_y(self, obj):
@@ -82,17 +89,35 @@ def get_sensor_types(self, obj):
  result.append(type)
  return result
 
- def get_data_from_year(self, obj):
- q_exp = (
- Q(value_at__gt=0)
- | Q(value_pt__gt=0)
- | Q(value_jt__gt=0)
- | Q(value_bt__gt=0)
- )
- qs = YearData.objects.filter(q_exp, station=obj).order_by("year__year_number")
- if qs.count() > 0:
- return qs[0].year.year_number
- else:
+ def get_is_active(self, obj):
+ num_days = [1, 7, 30, 365]
+ res = {}
+ for days in num_days:
+ from_date = date.today() - timedelta(days=days - 1)
+ day_qs = Day.objects.filter(station=obj, date__gte=from_date)
+ day_data_qs = DayData.objects.filter(day__in=day_qs)
+ if day_data_qs.filter(Q_EXP).count() > 0:
+ res[days] = True
+ else:
+ res[days] = False
+ return res
+
+ def get_data_until_date(self, obj):
+ try:
+ return (
+ DayData.objects.filter(Q_EXP, station=obj).latest("day__date").day.date
+ )
+ except DayData.DoesNotExist:
+ return None
+
+ def get_data_from_date(self, obj):
+ try:
+ return (
+ DayData.objects.filter(Q_EXP, station=obj)
+ .earliest("day__date")
+ .day.date
+ )
+ except DayData.DoesNotExist:
  return None
 
 

eco_counter/constants.py

@@ -30,6 +30,10 @@
 COUNTERS.LAM_COUNTER = LAM_COUNTER
 COUNTERS.TELRAAM_COUNTER = TELRAAM_COUNTER
 
+COUNTERS_LIST = [ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER, TELRAAM_COUNTER]
+COUNTER_CHOICES_STR = (
+ f"{ECO_COUNTER}, {TRAFFIC_COUNTER}, {TELRAAM_COUNTER} and {LAM_COUNTER}"
+)
 CSV_DATA_SOURCES = (
  (TRAFFIC_COUNTER, "TrafficCounter"),
  (ECO_COUNTER, "EcoCounter"),
@@ -118,12 +122,14 @@
 
 TELRAAM_COUNTER_CSV_FILE_PATH = f"{settings.MEDIA_ROOT}/telraam_data/"
 TELRAAM_COUNTER_CSV_FILE = (
- TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{id}_{day}_{month}_{year}.csv"
+ TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{mac}_{day}_{month}_{year}.csv"
 )
+TELRAAM_STATION_350457790598039 = 350457790598039
+TELRAAM_STATION_350457790600975 = 350457790600975
 TELRAAM_COUNTER_CAMERAS = {
  # Mac id: Direction flag (True=rgt prefix will be keskustaan päin)
- 350457790598039: False, # Kristiinanankatu, Joelle katsottaessa vasemmalle
- 350457790600975: True, # Kristiinanankatu, Joelle katsottaessa oikealle
+ TELRAAM_STATION_350457790598039: False, # Kristiinanankatu, Joelle katsottaessa vasemmalle
+ TELRAAM_STATION_350457790600975: True, # Kristiinanankatu, Joelle katsottaessa oikealle
 }
 # For 429 (too many request) TELRAAM need a retry strategy
 retry_strategy = Retry(
@@ -136,3 +142,23 @@
 TELRAAM_HTTP = requests.Session()
 TELRAAM_HTTP.mount("https://", adapter)
 TELRAAM_HTTP.mount("http://", adapter)
+
+
+# Telraam stations initial geometries in WKT format
+# These coordinates are used if CSV files do not include any geometries.
+TELRAAM_STATIONS_INITIAL_WKT_GEOMETRIES = {
+ TELRAAM_STATION_350457790598039: {
+ "location": "POINT (239628.47846388057 6710757.471557152)",
+ "geometry": "MULTILINESTRING ((239565.80107971327 6710861.8209667895, 239572.58901459936 6710850.524818219,"
+ " 239574.73294378238 6710846.950531884, 239628.47846388057 6710757.471557152,"
+ " 239630.0339247121 6710754.923177836, 239635.52748551324 6710745.732077925))",
+ },
+ TELRAAM_STATION_350457790600975: {
+ "location": "POINT (239523.2288977413 6710932.715108742)",
+ "geometry": "MULTILINESTRING ((239490.42663459244 6710989.092283992, 239493.45037993207 6710983.7110835295,"
+ " 239495.88642941663 6710979.3668986475, 239517.9904128411 6710941.530425406,"
+ " 239520.0691194288 6710937.971973339, 239523.2288977413 6710932.715108742,"
+ " 239529.37000273907 6710922.482472116, 239558.08254550528 6710874.681753734,"
+ " 239559.97438753376 6710871.516775628, 239565.80107971327 6710861.8209667895))",
+ },
+}

eco_counter/management/commands/delete_counter_data.py

@@ -0,0 +1,36 @@
+import logging
+
+from django import db
+from django.core.management.base import BaseCommand
+
+from eco_counter.constants import COUNTER_CHOICES_STR
+from eco_counter.management.commands.utils import check_counters_argument
+from eco_counter.models import ImportState, Station
+
+logger = logging.getLogger("eco_counter")
+
+
+class Command(BaseCommand):
+ def add_arguments(self, parser):
+ parser.add_argument(
+ "--counters",
+ type=str,
+ nargs="+",
+ default=False,
+ help=f"Delete given counter data, choices are: {COUNTER_CHOICES_STR}.",
+ )
+
+ @db.transaction.atomic
+ def handle(self, *args, **options):
+ counters = options.get("counters", None)
+ check_counters_argument(counters)
+ if counters:
+ for counter in counters:
+ logger.info(f"Deleting counter data for {counter}")
+ logger.info(
+ f"{Station.objects.filter(csv_data_source=counter).delete()}"
+ )
+ logger.info(
+ f"{ImportState.objects.filter(csv_data_source=counter).delete()}"
+ )
+ logger.info("Deleted counter data.")