diff --git a/eco_counter/constants.py b/eco_counter/constants.py index 11ebf577b..7744b4229 100644 --- a/eco_counter/constants.py +++ b/eco_counter/constants.py @@ -1,3 +1,4 @@ +import platform import types import requests @@ -44,31 +45,54 @@ } TRAFFIC_COUNTER_METADATA_GEOJSON = "traffic_counter_metadata.geojson" -# LAM stations located in the municipalities list are included. -LAM_STATION_MUNICIPALITIES = ["Turku", "Raisio", "Kaarina", "Lieto"] - LAM_STATIONS_API_FETCH_URL = ( settings.LAM_COUNTER_API_BASE_URL + "?api=liikennemaara&tyyppi=h&pvm={start_date}&loppu={end_date}" + "&lam_type=option1&piste={id}&luokka=kaikki&suunta={direction}&sisallytakaistat=0" ) -# Maps the direction of the traffic of station, (P)oispäin or (K)eskustaan päin) +# LAM stations in the locations list are included. +LAM_STATION_LOCATIONS = ["Turku", "Raisio", "Kaarina", "Lieto", "Hauninen", "Oriketo"] +# Header that is added to the request that fetches the LAM data. +LAM_STATION_USER_HEADER = { + "Digitraffic-User": f"{platform.uname()[1]}/Turun Palvelukartta" +} +# Mappings are derived by the 'suunta' and the 'suuntaselite' columns in the source data. +# (P)oispäin or (K)eskustaan päin) LAM_STATIONS_DIRECTION_MAPPINGS = { - "1_Piikkiö": "P", - "1_Naantali": "P", - "2_Naantali": "K", - "1_Turku": "K", + # vt8_Raisio + "1_Vaasa": "P", "2_Turku": "K", + # vt1_Kaarina_Kirismäki + "1_Turku": "K", "2_Helsinki": "P", - "1_Suikkila.": "K", - "2_Artukainen.": "P", - "1_Vaasa": "P", - "1_Kuusisto": "P", - "2_Kaarina": "K", - "1_Tampere": "P", + # vt10_Lieto "1_Hämeenlinna": "P", + # "2_Turku": "K", Duplicate + # vt1_Turku_Kupittaa + # "1_Turku" Duplicate + # "2_Helsinki" Duplicate + # vt1_Turku_Kurkela_länsi + # "1_Turku" Duplicate + # "2_Helsinki" Duplicate + # vt1_Kaarina_Kurkela_itä + # "1_Turku" Duplicate + # "2_Helsinki" Duplicate + # vt1_Kaarina + # "1_Turku" Duplicate + # "2_Helsinki" Duplicate + # vt1_Kaarina_Piikkiö + # "1_Turku" Duplicate + # "2_Helsinki" Duplicate + # yt1851_Turku_Härkämäki + "1_Suikkila": "K", + "2_Artukainen": "P", + # kt40_Hauninen + "1_Piikkiö": "K", + "2_Naantali": "P", + # kt40_Oriketo + # "1_Piikkiö": "K", duplicate + # "2_Naantali": "P", dupicate } - keys = [k for k in range(TRAFFIC_COUNTER_START_YEAR, TRAFFIC_COUNTER_END_YEAR + 1)] # Create a dict where the years to be importer are keys and the value is the url of the csv data. # e.g. {2015, "https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/2015_laskenta_juha.csv"} diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py index 02de99ec9..695eefb81 100644 --- a/eco_counter/management/commands/utils.py +++ b/eco_counter/management/commands/utils.py @@ -1,5 +1,4 @@ import io -import json import logging from datetime import date, timedelta @@ -15,7 +14,8 @@ ECO_COUNTER, INDEX_COLUMN_NAME, LAM_COUNTER, - LAM_STATION_MUNICIPALITIES, + LAM_STATION_LOCATIONS, + LAM_STATION_USER_HEADER, LAM_STATIONS_API_FETCH_URL, LAM_STATIONS_DIRECTION_MAPPINGS, TELRAAM_COUNTER, @@ -39,13 +39,8 @@ class LAMStation: def __init__(self, feature): - if feature["municipality"].as_string() not in LAM_STATION_MUNICIPALITIES: - self.active = False self.station_id = feature["tmsNumber"].as_int() - names = json.loads(feature["names"].as_string()) - self.name = names["fi"] - self.name_sv = names["sv"] - self.name_en = names["en"] + self.name = self.name_sv = self.name_en = feature["name"].as_string() # The source data has a obsolete Z dimension with value 0, remove it. geom = feature.geom.clone() geom.coord_dim = 2 @@ -217,7 +212,7 @@ def get_traffic_counter_csv(start_year=2015): def get_lam_dataframe(csv_url): - response = requests.get(csv_url) + response = requests.get(csv_url, headers=LAM_STATION_USER_HEADER) string_data = response.content csv_data = pd.read_csv(io.StringIO(string_data.decode("utf-8")), delimiter=";") return csv_data @@ -276,7 +271,7 @@ def get_lam_counter_csv(start_date): df = get_lam_station_dataframe( station.station_id, direction, start_date, today ) - # Read the direction + # Read the direction, e.g., Vaasa direction_name = df["suuntaselite"].iloc[0] # From the mappings determine the 'keskustaan päin' or 'poispäin keskustasta' direction. try: @@ -337,11 +332,20 @@ def get_lam_counter_csv(start_date): return data_frame +def has_list_elements_in_string(elements, string): + for element in elements: + if element in string: + return True + return False + + def get_lam_counter_stations(): stations = [] data_layer = DataSource(settings.LAM_COUNTER_STATIONS_URL)[0] for feature in data_layer: - if feature["municipality"].as_string() in LAM_STATION_MUNICIPALITIES: + if has_list_elements_in_string( + LAM_STATION_LOCATIONS, feature["name"].as_string() + ): stations.append(ObservationStation(LAM_COUNTER, feature)) return stations @@ -377,7 +381,8 @@ def fetch_telraam_camera(mac_id): response = TELRAAM_HTTP.get(url, headers=headers) cameras = response.json().get("camera", None) if cameras: - # Return first camera + # Return first camera, as currently only one camera is + # returned in Turku by mac_id return cameras[0] else: return None diff --git a/eco_counter/models.py b/eco_counter/models.py index a96b5d74c..a58565184 100644 --- a/eco_counter/models.py +++ b/eco_counter/models.py @@ -11,7 +11,7 @@ class ImportState(models.Model): current_year_number = models.PositiveSmallIntegerField(null=True) current_month_number = models.PositiveSmallIntegerField( validators=[MinValueValidator(1), MaxValueValidator(12)], - null=True, # , default=1 + null=True, ) current_day_number = models.PositiveSmallIntegerField( null=True, blank=True, validators=[MinValueValidator(1), MaxValueValidator(31)] @@ -37,7 +37,7 @@ class Station(models.Model): station_id = models.CharField(max_length=16, null=True) def __str__(self): - return "%s %s" % (self.name, self.geom) + return "%s %s" % (self.name, self.location) class Meta: ordering = ["id"]