Merge pull request #280 from City-of-Turku/feature/lam-stations-impor…

…ting-from-new-api

Feature/lam stations importing from new api

eco_counter/constants.py

@@ -1,3 +1,4 @@
+import platform
 import types
 
 import requests
@@ -44,31 +45,54 @@
 }
 
 TRAFFIC_COUNTER_METADATA_GEOJSON = "traffic_counter_metadata.geojson"
-# LAM stations located in the municipalities list are included.
-LAM_STATION_MUNICIPALITIES = ["Turku", "Raisio", "Kaarina", "Lieto"]
-
 LAM_STATIONS_API_FETCH_URL = (
  settings.LAM_COUNTER_API_BASE_URL
  + "?api=liikennemaara&tyyppi=h&pvm={start_date}&loppu={end_date}"
  + "&lam_type=option1&piste={id}&luokka=kaikki&suunta={direction}&sisallytakaistat=0"
 )
-# Maps the direction of the traffic of station, (P)oispäin or (K)eskustaan päin)
+# LAM stations in the locations list are included.
+LAM_STATION_LOCATIONS = ["Turku", "Raisio", "Kaarina", "Lieto", "Hauninen", "Oriketo"]
+# Header that is added to the request that fetches the LAM data.
+LAM_STATION_USER_HEADER = {
+ "Digitraffic-User": f"{platform.uname()[1]}/Turun Palvelukartta"
+}
+# Mappings are derived by the 'suunta' and the 'suuntaselite' columns in the source data.
+# (P)oispäin or (K)eskustaan päin)
 LAM_STATIONS_DIRECTION_MAPPINGS = {
- "1_Piikkiö": "P",
- "1_Naantali": "P",
- "2_Naantali": "K",
- "1_Turku": "K",
+ # vt8_Raisio
+ "1_Vaasa": "P",
  "2_Turku": "K",
+ # vt1_Kaarina_Kirismäki
+ "1_Turku": "K",
  "2_Helsinki": "P",
- "1_Suikkila.": "K",
- "2_Artukainen.": "P",
- "1_Vaasa": "P",
- "1_Kuusisto": "P",
- "2_Kaarina": "K",
- "1_Tampere": "P",
+ # vt10_Lieto
  "1_Hämeenlinna": "P",
+ # "2_Turku": "K", Duplicate
+ # vt1_Turku_Kupittaa
+ # "1_Turku" Duplicate
+ # "2_Helsinki" Duplicate
+ # vt1_Turku_Kurkela_länsi
+ # "1_Turku" Duplicate
+ # "2_Helsinki" Duplicate
+ # vt1_Kaarina_Kurkela_itä
+ # "1_Turku" Duplicate
+ # "2_Helsinki" Duplicate
+ # vt1_Kaarina
+ # "1_Turku" Duplicate
+ # "2_Helsinki" Duplicate
+ # vt1_Kaarina_Piikkiö
+ # "1_Turku" Duplicate
+ # "2_Helsinki" Duplicate
+ # yt1851_Turku_Härkämäki
+ "1_Suikkila": "K",
+ "2_Artukainen": "P",
+ # kt40_Hauninen
+ "1_Piikkiö": "K",
+ "2_Naantali": "P",
+ # kt40_Oriketo
+ # "1_Piikkiö": "K", duplicate
+ # "2_Naantali": "P", dupicate
 }
-
 keys = [k for k in range(TRAFFIC_COUNTER_START_YEAR, TRAFFIC_COUNTER_END_YEAR + 1)]
 # Create a dict where the years to be importer are keys and the value is the url of the csv data.
 # e.g. {2015, "https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/2015_laskenta_juha.csv"}

eco_counter/management/commands/utils.py

@@ -1,5 +1,4 @@
 import io
-import json
 import logging
 from datetime import date, timedelta
 
@@ -15,7 +14,8 @@
  ECO_COUNTER,
  INDEX_COLUMN_NAME,
  LAM_COUNTER,
- LAM_STATION_MUNICIPALITIES,
+ LAM_STATION_LOCATIONS,
+ LAM_STATION_USER_HEADER,
  LAM_STATIONS_API_FETCH_URL,
  LAM_STATIONS_DIRECTION_MAPPINGS,
  TELRAAM_COUNTER,
@@ -39,13 +39,8 @@
 
 class LAMStation:
  def __init__(self, feature):
- if feature["municipality"].as_string() not in LAM_STATION_MUNICIPALITIES:
- self.active = False
  self.station_id = feature["tmsNumber"].as_int()
- names = json.loads(feature["names"].as_string())
- self.name = names["fi"]
- self.name_sv = names["sv"]
- self.name_en = names["en"]
+ self.name = self.name_sv = self.name_en = feature["name"].as_string()
  # The source data has a obsolete Z dimension with value 0, remove it.
  geom = feature.geom.clone()
  geom.coord_dim = 2
@@ -217,7 +212,7 @@ def get_traffic_counter_csv(start_year=2015):
 
 
 def get_lam_dataframe(csv_url):
- response = requests.get(csv_url)
+ response = requests.get(csv_url, headers=LAM_STATION_USER_HEADER)
  string_data = response.content
  csv_data = pd.read_csv(io.StringIO(string_data.decode("utf-8")), delimiter=";")
  return csv_data
@@ -276,7 +271,7 @@ def get_lam_counter_csv(start_date):
  df = get_lam_station_dataframe(
  station.station_id, direction, start_date, today
  )
- # Read the direction
+ # Read the direction, e.g., Vaasa
  direction_name = df["suuntaselite"].iloc[0]
  # From the mappings determine the 'keskustaan päin' or 'poispäin keskustasta' direction.
  try:
@@ -337,11 +332,20 @@ def get_lam_counter_csv(start_date):
  return data_frame
 
 
+def has_list_elements_in_string(elements, string):
+ for element in elements:
+ if element in string:
+ return True
+ return False
+
+
 def get_lam_counter_stations():
  stations = []
  data_layer = DataSource(settings.LAM_COUNTER_STATIONS_URL)[0]
  for feature in data_layer:
- if feature["municipality"].as_string() in LAM_STATION_MUNICIPALITIES:
+ if has_list_elements_in_string(
+ LAM_STATION_LOCATIONS, feature["name"].as_string()
+ ):
  stations.append(ObservationStation(LAM_COUNTER, feature))
  return stations
 
@@ -377,7 +381,8 @@ def fetch_telraam_camera(mac_id):
  response = TELRAAM_HTTP.get(url, headers=headers)
  cameras = response.json().get("camera", None)
  if cameras:
- # Return first camera
+ # Return first camera, as currently only one camera is
+ # returned in Turku by mac_id
  return cameras[0]
  else:
  return None

eco_counter/models.py

@@ -11,7 +11,7 @@ class ImportState(models.Model):
  current_year_number = models.PositiveSmallIntegerField(null=True)
  current_month_number = models.PositiveSmallIntegerField(
  validators=[MinValueValidator(1), MaxValueValidator(12)],
- null=True, # , default=1
+ null=True,
  )
  current_day_number = models.PositiveSmallIntegerField(
  null=True, blank=True, validators=[MinValueValidator(1), MaxValueValidator(31)]
@@ -37,7 +37,7 @@ class Station(models.Model):
  station_id = models.CharField(max_length=16, null=True)
 
  def __str__(self):
- return "%s %s" % (self.name, self.geom)
+ return "%s %s" % (self.name, self.location)
 
  class Meta:
  ordering = ["id"]