Merge pull request #312 from City-of-Turku/develop

Production update

.github/workflows/run-tests.yml

@@ -19,9 +19,9 @@ jobs:
  LAM_COUNTER_API_BASE_URL: https://tie.digitraffic.fi/api/tms/v1/history
 
  steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
  - name: Set up Python 3.10.0
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v3
  with:
  python-version: 3.10.0
  - name: Install required Ubuntu packages
@@ -50,7 +50,7 @@ jobs:
  pip install coverage
  coverage report -m
  - name: Upload Coverage to Codecov
- uses: codecov/codecov-action@v2 
+ uses: codecov/codecov-action@v3 
  # Majority of the tests require database
  services:
  # Label used to access the service container

README.md

@@ -141,8 +141,12 @@ For Turku specific imports see smbackend_turku/README.md.
 ./manage.py geo_import helsinki --divisions
 ./manage.py index_search_columns
 ```
-
+Import exclude rules fixtures used by the search:
+```
+./manage.py loaddata services/fixtures/exclusion_rules.json
+```
 7. Redis
+
 Redis is used for caching and as a message broker for Celery.
 Install Redis. Ubuntu: `sudo apt-get install redis-server`
 
@@ -198,3 +202,6 @@ psql template1 -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 Mobility platform
 -----------------
 The mobility data platform of the service map is being developed as part of European Union Horizon 2020 programme funded SCALE-UP project (grant agreement no. 955332).
+
+For more information see: mobility_data/README.mk
+

bicycle_network/api/views.py

@@ -1,5 +1,3 @@
-from distutils.util import strtobool
-
 from django.contrib.gis.db.models.functions import Distance
 from django.contrib.gis.gdal import SpatialReference
 from django.contrib.gis.geos import Point
@@ -10,6 +8,7 @@
 from rest_framework.exceptions import ParseError
 
 from services.api_pagination import Pagination
+from services.utils import strtobool
 
 from ..models import BicycleNetwork, BicycleNetworkPart
 from .serializers import (

config_dev.env.example

@@ -145,6 +145,23 @@ EMAIL_HOST_USER=example@example.com
 EMAIL_PORT=25
 EMAIL_USE_TLS=True
 
+# Django project log level, default INFO
+DJANGO_LOG_LEVEL=
+# Turku services importers log level, default DEBUG
+TURKU_SERVICES_IMPORT_LOG_LEVEL=
+# Search log level, default INFO
+SEARCH_LOG_LEVEL=
+# IoT APP, default INFO 
+IOT_LOG_LEVEL=
+# Eco counter, default INFO
+ECO_COUNTER_LOG_LEVEL=
+# Mobility data (includes importers), default INFO
+MOBILITY_DATA_LOG_LEVEL=
+# Bicycle networks APP, default INFO
+BICYCLE_NETWORK_LOG_LEVEL=
+# Street maintenance, default INFO
+STREET_MAINTENANCE_LOG_LEVEL=
+
 # Settings needed for enabling Turku area:
 #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv
 #TURKU_API_KEY=secret
@@ -184,4 +201,4 @@ YIT_TOKEN_URL=https://login.microsoftonline.com/86792d09-0d81-4899-8d66-95dfc96c
 KUNTEC_KEY=
 # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py)
 # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api
-TELRAAM_TOKEN=
+TELRAAM_TOKEN=

eco_counter/api/serializers.py

@@ -1,3 +1,4 @@
+from django.db.models import Q
 from rest_framework import serializers
 
 from ..models import (
@@ -23,6 +24,9 @@
  "value_jk",
  "value_jp",
  "value_jt",
+ "value_bk",
+ "value_bp",
+ "value_bt",
 ]
 
 
@@ -32,6 +36,7 @@ class StationSerializer(serializers.ModelSerializer):
  lon = serializers.SerializerMethodField()
  lat = serializers.SerializerMethodField()
  sensor_types = serializers.SerializerMethodField()
+ data_from_year = serializers.SerializerMethodField()
 
  class Meta:
  model = Station
@@ -49,6 +54,7 @@ class Meta:
  "lon",
  "lat",
  "sensor_types",
+ "data_from_year",
  ]
 
  def get_y(self, obj):
@@ -76,6 +82,19 @@ def get_sensor_types(self, obj):
  result.append(type)
  return result
 
+ def get_data_from_year(self, obj):
+ q_exp = (
+ Q(value_at__gt=0)
+ | Q(value_pt__gt=0)
+ | Q(value_jt__gt=0)
+ | Q(value_bt__gt=0)
+ )
+ qs = YearData.objects.filter(q_exp, station=obj).order_by("year__year_number")
+ if qs.count() > 0:
+ return qs[0].year.year_number
+ else:
+ return None
+
 
 class YearSerializer(serializers.ModelSerializer):
  station_name = serializers.PrimaryKeyRelatedField(

eco_counter/api/views.py

@@ -62,7 +62,24 @@ def list(self, request):
  if counter_type in str(CSV_DATA_SOURCES):
  queryset = Station.objects.filter(csv_data_source=counter_type)
  else:
- raise ParseError("Valid 'counter_type' choices are: 'EC','TC' or 'LC'.")
+ raise ParseError(
+ "Valid 'counter_type' choices are: 'EC', 'TC', 'TR' or 'LC'."
+ )
+ if "data_type" in filters:
+ data_type = filters["data_type"].lower()
+ data_types = ["a", "j", "b", "p"]
+ if data_type not in data_types:
+ raise ParseError(
+ f"Valid 'data_type' choices are: {', '.join(data_types)}"
+ )
+ ids = []
+ data_type = data_type + "t"
+ for station in Station.objects.all():
+ filter = {"station": station, f"value_{data_type}__gt": 0}
+ if YearData.objects.filter(**filter).count() > 0:
+ ids.append(station.id)
+ queryset = Station.objects.filter(id__in=ids)
+
  page = self.paginate_queryset(queryset)
  serializer = StationSerializer(page, many=True)
  return self.get_paginated_response(serializer.data)

eco_counter/constants.py

@@ -114,6 +114,8 @@
 # from the beginning of the start tear
 TELRAAM_COUNTER_START_MONTH = 5
 TELRAAM_COUNTER_API_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+TELRAAM_COUNTER_DATA_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
 TELRAAM_COUNTER_CSV_FILE_PATH = f"{settings.MEDIA_ROOT}/telraam_data/"
 TELRAAM_COUNTER_CSV_FILE = (
  TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{id}_{day}_{month}_{year}.csv"
@@ -127,7 +129,7 @@
 retry_strategy = Retry(
  total=10,
  status_forcelist=[429],
- method_whitelist=["GET", "POST"],
+ allowed_methods=["GET", "POST"],
  backoff_factor=30, # 30, 60, 120 , 240, ..seconds
 )
 adapter = HTTPAdapter(max_retries=retry_strategy)

eco_counter/data/traffic_counter_metadata.geojson

@@ -108,7 +108,20 @@
  { "type": "Feature", "properties": { "fid": 235, "ID": "9", "Osoite_fi": "Uudenmaantie 5", "Osoite_sv": "Nylandsvägen 5", "Osoite_en": "Uudenmaantie 5", "Liittymänumero": "609", "Ilmaisimet": "208L", "Mittauspisteiden_ID": "5537", "Tyyppi": "B", "Suunta": "K" }, "geometry": { "type": "Point", "coordinates": [ 23460746.775335978716612, 6703683.553299261257052 ] } },
  { "type": "Feature", "properties": { "fid": 80, "ID": "15", "Osoite_fi": "Ispoisten puistotie 2", "Osoite_sv": "Ispois parväg 2", "Osoite_en": "Ispoisten puistotie 2", "Liittymänumero": "727", "Ilmaisimet": "206L", "Mittauspisteiden_ID": "8746", "Tyyppi": "B", "Suunta": "K" }, "geometry": { "type": "Point", "coordinates": [ 23459625.991249438375235, 6702405.184565890580416 ] } },
  { "type": "Feature", "properties": { "fid": 128, "ID": "5", "Osoite_fi": "Skarppakullantie 37", "Osoite_sv": "Skarppakullav 37", "Osoite_en": "Skarppakullantie 37", "Liittymänumero": "652", "Ilmaisimet": "108L", "Mittauspisteiden_ID": "8388", "Tyyppi": "B", "Suunta": "P" }, "geometry": { "type": "Point", "coordinates": [ 23462435.035431213676929, 6702584.968190263025463 ] } },
- { "type": "Feature", "properties": { "fid": 140, "ID": "9", "Osoite_fi": "Skarppakullantie/Skanssinkatu", "Osoite_sv": "Skarppakullavägen/Skansgatan", "Osoite_en": "Skarppakullantie/Skanssinkatu", "Liittymänumero": "653", "Ilmaisimet": "208L", "Mittauspisteiden_ID": "8461", "Tyyppi": "B", "Suunta": "K" }, "geometry": { "type": "Point", "coordinates": [ 23462437.22350587323308, 6702359.768247644416988 ] } }
+ { "type": "Feature", "properties": { "fid": 140, "ID": "9", "Osoite_fi": "Skarppakullantie/Skanssinkatu", "Osoite_sv": "Skarppakullavägen/Skansgatan", "Osoite_en": "Skarppakullantie/Skanssinkatu", "Liittymänumero": "653", "Ilmaisimet": "208L", "Mittauspisteiden_ID": "8461", "Tyyppi": "B", "Suunta": "K" }, "geometry": { "type": "Point", "coordinates": [ 23462437.22350587323308, 6702359.768247644416988 ] } },
+ { "type": "Feature", "properties": { "fid": 3, "ID": null, "Osoite_fi": "Itäinen rantakatu 8", "Osoite_sv": "Östra strandgatan 8", "Osoite_en": "Itäinen rantakatu 8", "Liittymänumero": "701", "Ilmaisimet": "X3", "Mittauspisteiden_ID": "903", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23459837.50562446564436, 6704244.290738201700151 ] } },
+ { "type": "Feature", "properties": { "fid": 4, "ID": null, "Osoite_fi": "Itäinen rantakatu 43", "Osoite_sv": "Östra strandgatan 43", "Osoite_en": "Itäinen rantakatu 43", "Liittymänumero": "710", "Ilmaisimet": "pp1", "Mittauspisteiden_ID": "2273", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23459072.372346129268408, 6703798.638464853167534 ] } },
+ { "type": "Feature", "properties": { "fid": 5, "ID": null, "Osoite_fi": "Itäinen rantakatu 43", "Osoite_sv": "Östra strandgatan 43", "Osoite_en": "Itäinen rantakatu 43", "Liittymänumero": "710", "Ilmaisimet": "pp2", "Mittauspisteiden_ID": "2274", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23459073.968076642602682, 6703799.561500884592533 ] } },
+ { "type": "Feature", "properties": { "fid": 6, "ID": null, "Osoite_fi": "Itäinen rantakatu 43", "Osoite_sv": "Östra strandgatan 43", "Osoite_en": "Itäinen rantakatu 43", "Liittymänumero": "710", "Ilmaisimet": "pp3", "Mittauspisteiden_ID": "2275", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23459076.346651934087276, 6703801.192924783565104 ] } },
+ { "type": "Feature", "properties": { "fid": 7, "ID": null, "Osoite_fi": "Itäinen rantakatu 43", "Osoite_sv": "Östra strandgatan 43", "Osoite_en": "Itäinen rantakatu 43", "Liittymänumero": "710", "Ilmaisimet": "pp4", "Mittauspisteiden_ID": "2276", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23459077.927361972630024, 6703802.362830685451627 ] } },
+ { "type": "Feature", "properties": { "fid": 8, "ID": null, "Osoite_fi": "Myllysilta 1", "Osoite_sv": "Kvarnbron 1", "Osoite_en": "Myllysilta 1", "Liittymänumero": "108", "Ilmaisimet": "2PP", "Mittauspisteiden_ID": "1921", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23459138.651998423039913, 6703933.748812982812524 ] } },
+ { "type": "Feature", "properties": { "fid": 9, "ID": "", "Osoite_fi": "Myllysilta 2", "Osoite_sv": "Kvarnbron 2", "Osoite_en": "Myllysilta 2", "Liittymänumero": "108", "Ilmaisimet": "1PP", "Mittauspisteiden_ID": "1920", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23459121.444739304482937, 6703958.63655239995569 ] } },
+ { "type": "Feature", "properties": { "fid": 10, "ID": null, "Osoite_fi": "Martinsilta", "Osoite_sv": "Martinsbron", "Osoite_en": "Martinsilta", "Liittymänumero": "101", "Ilmaisimet": "1PP", "Mittauspisteiden_ID": "1570", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23458979.291542522609234, 6703870.531346249394119 ] } },
+ { "type": "Feature", "properties": { "fid": 11, "ID": null, "Osoite_fi": "Kalevantie 17", "Osoite_sv": "Kalevavägen 17", "Osoite_en": "Kalevantie 17", "Liittymänumero": "518", "Ilmaisimet": "ZELT_pp", "Mittauspisteiden_ID": "6816", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23461917.734735164791346, 6704427.855136526748538 ] } },
+ { "type": "Feature", "properties": { "fid": 12, "ID": null, "Osoite_fi": "Lemminkäisenkatu 35", "Osoite_sv": "Lemminkäinengatan 35", "Osoite_en": "Lemminkäisenkatu 35", "Liittymänumero": "642", "Ilmaisimet": "PP2", "Mittauspisteiden_ID": "7172", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23461714.611965991556644, 6703777.939228449948132 ] } },
+ { "type": "Feature", "properties": { "fid": 13, "ID": "", "Osoite_fi": "Lemminkäisenkatu 36", "Osoite_sv": "Lemminkäinengatan 36", "Osoite_en": "Lemminkäisenkatu 36", "Liittymänumero": "642", "Ilmaisimet": "ZELT_pp", "Mittauspisteiden_ID": "7171", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23461637.359737996011972, 6703819.079618069343269 ] } },
+ { "type": "Feature", "properties": { "fid": 35, "ID": null, "Osoite_fi": "Helsinginkatu 7", "Osoite_sv": "Helsingforsgatan 7", "Osoite_en": "Helsinginkatu 7", "Liittymänumero": "619", "Ilmaisimet": "X4", "Mittauspisteiden_ID": "4083", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23460447.87659065425396, 6705474.187256957404315 ] } },
+ { "type": "Feature", "properties": { "fid": 36, "ID": null, "Osoite_fi": "Helsinginkatu 7", "Osoite_sv": "Helsingforsgatan 7", "Osoite_en": "Helsinginkatu 7", "Liittymänumero": "619", "Ilmaisimet": "X3", "Mittauspisteiden_ID": "4080", "Tyyppi": "P", "Suunta": "T" }, "geometry": { "type": "Point", "coordinates": [ 23460433.286612428724766, 6705502.207770394161344 ] } }
  ]
  }