From 11c2ae79cb531878db0b61a59a28edfc28b41ab2 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 15 Feb 2023 10:38:19 +0200
Subject: [PATCH 01/81] Calculate cumulative values with pandas aggregate
 functions

---
 .../commands/import_counter_data.py           | 191 +++++++++++++++++-
 1 file changed, 182 insertions(+), 9 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 9a25a5ec5..0020bca26 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -232,7 +232,7 @@ def get_station_name_and_type(self, column):
         station_name = column.replace(station_type, "").strip()
         return station_name, station_type
 
-    def save_observations(
+    def save_observations2(
         self, csv_data, start_time, column_names, csv_data_source=ECO_COUNTER
     ):
         errorneous_values = 0
@@ -495,6 +495,179 @@ def save_observations(
         logger.info(f"Found {negative_values} negative values.")
         logger.info(f"Imported observations until:{str(current_time)}")
 
+    TYPE_DIRS = ["AK", "AP", "JK", "JP", "BK", "BP", "PK", "PP"]
+    ALL_TYPE_DIRS = TYPE_DIRS + ["AT", "JT", "BT", "PT"]
+    type_dirs_lower = [TD.lower() for TD in TYPE_DIRS]
+
+    def save_values(self, values, dst_obj):
+        for station_types in self.STATION_TYPES:
+            # setattr(dst_obj, f"value_{station_types[0]}", 0)
+            # setattr(dst_obj, f"value_{station_types[1]}", 0)
+            # setattr(dst_obj, f"value_{station_types[2]}", 0)
+            setattr(dst_obj, f"value_{station_types[0]}", values[station_types[0]])
+            setattr(dst_obj, f"value_{station_types[1]}", values[station_types[1]])
+            setattr(
+                dst_obj,
+                f"value_{station_types[2]}",
+                values[station_types[0]] + values[station_types[1]],
+            )
+        dst_obj.save()
+
+    def get_values(self, sum_series, station_name):
+        values = {}
+        for type_dir in self.TYPE_DIRS:
+            key = f"{station_name} {type_dir}"
+            values[type_dir.lower()] = sum_series[key]
+        return values
+
+    def save_years(self, df, stations):
+        # Save year data
+        years = df.groupby(df.index.year)
+        for index, row in years:
+            sum_series = row.sum()
+            # breakpoint()
+            for station in stations:
+                year, _ = Year.objects.get_or_create(station=station, year_number=index)
+                values = self.get_values(sum_series, station.name)
+                year_data, _ = YearData.objects.get_or_create(
+                    year=year, station=station
+                )
+                self.save_values(values, year_data)
+
+    def save_months(self, df, stations):
+        # Save month data
+        months = df.groupby([df.index.year, df.index.month])
+        for index, row in months:
+            year_number, month_number = index
+            sum_series = row.sum()
+            for station in stations:
+                year = Year.objects.get(station=station, year_number=year_number)
+                month, _ = Month.objects.get_or_create(
+                    station=station, year=year, month_number=month_number
+                )
+                values = self.get_values(sum_series, station.name)
+                month_data, _ = MonthData.objects.get_or_create(
+                    year=year, month=month, station=station
+                )
+                self.save_values(values, month_data)
+
+    def save_weeks(self, df, stations):
+        weeks = df.groupby([df.index.year, df.index.week])
+        for index, row in weeks:
+            sum_series = row.sum()
+            for station in stations:
+                year_number, week_number = index
+                year = Year.objects.get(station=station, year_number=year_number)
+                week, _ = Week.objects.get_or_create(
+                    station=station, week_number=week_number
+                )
+                week.years.add(year)
+                values = self.get_values(sum_series, station.name)
+                week_data, _ = WeekData.objects.get_or_create(
+                    station=station, week=week
+                )
+                self.save_values(values, week_data)
+
+    def save_days(self, df, stations):
+        days = df.groupby([df.index.year, df.index.month, df.index.week, df.index.day])
+        for index, row in days:
+            year_number, month_number, week_number, day_number = index
+            date = datetime(year_number, month_number, day_number)
+            sum_series = row.sum()
+            for station in stations:
+                year = Year.objects.get(station=station, year_number=year_number)
+                month = Month.objects.get(
+                    station=station, year=year, month_number=month_number
+                )
+                week = Week.objects.get(
+                    station=station, years=year, week_number=week_number
+                )
+                day, _ = Day.objects.get_or_create(
+                    station=station,
+                    date=date,
+                    weekday_number=date.weekday(),
+                    year=year,
+                    month=month,
+                    week=week,
+                )
+                values = self.get_values(sum_series, station.name)
+                day_data, _ = DayData.objects.get_or_create(station=station, day=day)
+                self.save_values(values, day_data)
+
+    def save_hours(self, df, stations):
+        # Save hour data
+        hours = df.groupby([df.index.year, df.index.month, df.index.day, df.index.hour])
+        for station in stations:
+            prev_day = None
+            prev_month = None
+            values = {k: [] for k in self.ALL_TYPE_DIRS}
+            for index, row in hours:
+                sum_series = row.sum()
+                year_number, month_number, day_number, hour_number = index
+                if not prev_day:
+                    prev_day = day_number
+                if not prev_month:
+                    prev_month = month_number
+                if day_number != prev_day or month_number != prev_month:
+                    if month_number != prev_month:
+                        prev_day = day_number
+                    day = Day.objects.get(
+                        date=datetime(year_number, month_number, prev_day),
+                        station=station,
+                    )
+                    hour_data, _ = HourData.objects.get_or_create(
+                        station=station, day=day
+                    )
+                    for td in self.ALL_TYPE_DIRS:
+                        setattr(hour_data, f"values_{td.lower()}", values[td])
+                    hour_data.save()
+                    values = {k: [] for k in self.ALL_TYPE_DIRS}
+                    prev_day = day_number
+                    prev_month = month_number
+                else:
+
+                    for station_types in self.STATION_TYPES:
+                        for i in range(3):
+                            if i < 2:
+                                dir_key = f"{station.name} {station_types[i].upper()}"
+                                val = sum_series[dir_key]
+                            else:
+                                k_key = f"{station.name} {station_types[0].upper()}"
+                                p_key = f"{station.name} {station_types[1].upper()}"
+
+                                val = sum_series[p_key] + sum_series[k_key]
+                            values_key = station_types[i].upper()
+                            values[values_key].append(val)
+
+    def save_observations(
+        self, csv_data, start_time, column_names, csv_data_source=ECO_COUNTER
+    ):
+        # Populate stations dict, used to lookup station relations
+        stations = [
+            station
+            for station in Station.objects.filter(csv_data_source=csv_data_source)
+        ]
+
+        df = csv_data
+        df["Date"] = pd.to_datetime(df["startTime"], format="%Y-%m-%d %H:%M:%S")
+        df = df.drop("startTime", axis=1)
+        df = df.set_index("Date")
+        # TODO Fix negative numbers
+        df = df.fillna(0)
+        df = df.clip(lower=0)
+        self.save_years(df, stations)
+        self.save_months(df, stations)
+        self.save_weeks(df, stations)
+        self.save_days(df, stations)
+        self.save_hours(df, stations)
+
+        import_state = ImportState.objects.get(csv_data_source=csv_data_source)
+        end_date = df.index[-1]
+        import_state.current_year_number = end_date.year
+        import_state.current_month_number = end_date.month
+        import_state.save()
+        # breakpoint()
+
     def add_arguments(self, parser):
         parser.add_argument(
             "--initial-import",
@@ -564,14 +737,14 @@ def handle(self, *args, **options):
                 import_state.current_year_number = start_time.year
                 import_state.current_month_number = start_time.month
                 import_state.save()
-            if counter == ECO_COUNTER:
-                save_eco_counter_stations()
-            elif counter == TRAFFIC_COUNTER:
-                save_traffic_counter_stations()
-            elif counter == LAM_COUNTER:
-                save_lam_counter_stations()
-            else:
-                raise CommandError("No valid counter argument given.")
+            # if counter == ECO_COUNTER:
+            #     save_eco_counter_stations()
+            # elif counter == TRAFFIC_COUNTER:
+            #     save_traffic_counter_stations()
+            # elif counter == LAM_COUNTER:
+            #     save_lam_counter_stations()
+            # else:
+            #     raise CommandError("No valid counter argument given.")
             test_dataframe = get_test_dataframe(counter)
             csv_data = gen_eco_counter_test_csv(
                 test_dataframe.keys(), start_time, end_time

From 82ac8c8410aa9cbf4df1752b0bf1df8db46d565b Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 16 Feb 2023 08:02:34 +0200
Subject: [PATCH 02/81] Fix incremental import

---
 .../commands/import_counter_data.py           | 103 ++++++++++++++----
 1 file changed, 84 insertions(+), 19 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 0020bca26..b6650e210 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -501,9 +501,6 @@ def save_observations2(
 
     def save_values(self, values, dst_obj):
         for station_types in self.STATION_TYPES:
-            # setattr(dst_obj, f"value_{station_types[0]}", 0)
-            # setattr(dst_obj, f"value_{station_types[1]}", 0)
-            # setattr(dst_obj, f"value_{station_types[2]}", 0)
             setattr(dst_obj, f"value_{station_types[0]}", values[station_types[0]])
             setattr(dst_obj, f"value_{station_types[1]}", values[station_types[1]])
             setattr(
@@ -513,6 +510,23 @@ def save_values(self, values, dst_obj):
             )
         dst_obj.save()
 
+    def add_values(self, values, dst_obj):
+        for station_types in self.STATION_TYPES:
+            key = f"value_{station_types[0]}"
+            k_val = getattr(dst_obj, key) + values[station_types[0]]
+            setattr(dst_obj, key, k_val)
+            key = f"value_{station_types[1]}"
+            p_val = getattr(dst_obj, key) + values[station_types[1]]
+            setattr(dst_obj, key, p_val)
+            key = f"value_{station_types[2]}"
+            t_val = (
+                getattr(dst_obj, key)
+                + values[station_types[0]]
+                + values[station_types[1]]
+            )
+            setattr(dst_obj, key, t_val)
+        dst_obj.save()
+
     def get_values(self, sum_series, station_name):
         values = {}
         for type_dir in self.TYPE_DIRS:
@@ -520,19 +534,23 @@ def get_values(self, sum_series, station_name):
             values[type_dir.lower()] = sum_series[key]
         return values
 
-    def save_years(self, df, stations):
+    def save_years(self, df, stations, start_time):
         # Save year data
+        # slice from current time.
+        # df = df.loc[str(start_time):]
         years = df.groupby(df.index.year)
         for index, row in years:
             sum_series = row.sum()
-            # breakpoint()
             for station in stations:
                 year, _ = Year.objects.get_or_create(station=station, year_number=index)
                 values = self.get_values(sum_series, station.name)
-                year_data, _ = YearData.objects.get_or_create(
+                year_data, created = YearData.objects.get_or_create(
                     year=year, station=station
                 )
+                # if created:
                 self.save_values(values, year_data)
+                # else:
+                #    self.add_values(values, year_data)
 
     def save_months(self, df, stations):
         # Save month data
@@ -541,7 +559,9 @@ def save_months(self, df, stations):
             year_number, month_number = index
             sum_series = row.sum()
             for station in stations:
-                year = Year.objects.get(station=station, year_number=year_number)
+                year, _ = Year.objects.get_or_create(
+                    station=station, year_number=year_number
+                )
                 month, _ = Month.objects.get_or_create(
                     station=station, year=year, month_number=month_number
                 )
@@ -551,17 +571,52 @@ def save_months(self, df, stations):
                 )
                 self.save_values(values, month_data)
 
+    def save_current_year(self, stations, year_number, end_month_number):
+
+        for station in stations:
+            year, _ = Year.objects.get_or_create(
+                station=station, year_number=year_number
+            )
+            year_data, _ = YearData.objects.get_or_create(station=station, year=year)
+
+            for station_types in self.STATION_TYPES:
+                setattr(year_data, f"value_{station_types[0]}", 0)
+                setattr(year_data, f"value_{station_types[1]}", 0)
+                setattr(year_data, f"value_{station_types[2]}", 0)
+
+            for month_number in range(1, end_month_number + 1):
+                month, _ = Month.objects.get_or_create(
+                    station=station, year=year, month_number=month_number
+                )
+                month_data, _ = MonthData.objects.get_or_create(
+                    station=station, month=month, year=year
+                )
+                for station_types in self.STATION_TYPES:
+                    for i in range(3):
+                        key = f"value_{station_types[i]}"
+                        m_val = getattr(month_data, key)
+                        y_val = getattr(year_data, key)
+                        setattr(year_data, key, m_val + y_val)
+            year_data.save()
+
     def save_weeks(self, df, stations):
         weeks = df.groupby([df.index.year, df.index.week])
+
         for index, row in weeks:
+            year_number, week_number = index
             sum_series = row.sum()
             for station in stations:
-                year_number, week_number = index
                 year = Year.objects.get(station=station, year_number=year_number)
                 week, _ = Week.objects.get_or_create(
-                    station=station, week_number=week_number
+                    station=station,
+                    week_number=week_number,
+                    years__year_number=year_number,
                 )
-                week.years.add(year)
+                # TODO add logic when to add year.
+                if week.years.count() == 0:
+
+                    week.years.add(year)
+
                 values = self.get_values(sum_series, station.name)
                 week_data, _ = WeekData.objects.get_or_create(
                     station=station, week=week
@@ -572,6 +627,7 @@ def save_days(self, df, stations):
         days = df.groupby([df.index.year, df.index.month, df.index.week, df.index.day])
         for index, row in days:
             year_number, month_number, week_number, day_number = index
+
             date = datetime(year_number, month_number, day_number)
             sum_series = row.sum()
             for station in stations:
@@ -579,6 +635,7 @@ def save_days(self, df, stations):
                 month = Month.objects.get(
                     station=station, year=year, month_number=month_number
                 )
+
                 week = Week.objects.get(
                     station=station, years=year, week_number=week_number
                 )
@@ -590,6 +647,8 @@ def save_days(self, df, stations):
                     month=month,
                     week=week,
                 )
+                # if year_number==2021 and month_number==1:
+                #     breakpoint()
                 values = self.get_values(sum_series, station.name)
                 day_data, _ = DayData.objects.get_or_create(station=station, day=day)
                 self.save_values(values, day_data)
@@ -642,6 +701,8 @@ def save_hours(self, df, stations):
     def save_observations(
         self, csv_data, start_time, column_names, csv_data_source=ECO_COUNTER
     ):
+        import_state = ImportState.objects.get(csv_data_source=csv_data_source)
+        # breakpoint()
         # Populate stations dict, used to lookup station relations
         stations = [
             station
@@ -652,21 +713,25 @@ def save_observations(
         df["Date"] = pd.to_datetime(df["startTime"], format="%Y-%m-%d %H:%M:%S")
         df = df.drop("startTime", axis=1)
         df = df.set_index("Date")
-        # TODO Fix negative numbers
         df = df.fillna(0)
+        # Test negative numbers to 0
         df = df.clip(lower=0)
-        self.save_years(df, stations)
+        if not import_state.current_year_number:
+            self.save_years(df, stations, start_time)
+
         self.save_months(df, stations)
+        if import_state.current_year_number:
+            end_month_number = df.index[-1].month
+            self.save_current_year(stations, start_time.year, end_month_number)
+
         self.save_weeks(df, stations)
         self.save_days(df, stations)
         self.save_hours(df, stations)
 
-        import_state = ImportState.objects.get(csv_data_source=csv_data_source)
         end_date = df.index[-1]
         import_state.current_year_number = end_date.year
         import_state.current_month_number = end_date.month
         import_state.save()
-        # breakpoint()
 
     def add_arguments(self, parser):
         parser.add_argument(
@@ -715,7 +780,7 @@ def handle(self, *args, **options):
                 for counter in initial_import_counters:
                     import_state = ImportState.objects.create(
                         csv_data_source=counter,
-                        current_year_number=COUNTER_START_YEARS[counter],
+                        # current_year_number=COUNTER_START_YEARS[counter],
                     )
                     logger.info(f"Retrieving stations for {counter}.")
                     if counter == ECO_COUNTER:
@@ -733,10 +798,10 @@ def handle(self, *args, **options):
             import_state, created = ImportState.objects.get_or_create(
                 csv_data_source=counter
             )
-            if created:
-                import_state.current_year_number = start_time.year
-                import_state.current_month_number = start_time.month
-                import_state.save()
+            # if created:
+            #     import_state.current_year_number = start_time.year
+            #     import_state.current_month_number = start_time.month
+            #     import_state.save()
             # if counter == ECO_COUNTER:
             #     save_eco_counter_stations()
             # elif counter == TRAFFIC_COUNTER:

From 6012d70f19ac6a35e0bfbeb71f2bbc868d04058a Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 16 Feb 2023 13:09:54 +0200
Subject: [PATCH 03/81] Set current_year_number and current_month_number to
 nullable

---
 eco_counter/models.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/eco_counter/models.py b/eco_counter/models.py
index 0818e76fe..b56efb127 100644
--- a/eco_counter/models.py
+++ b/eco_counter/models.py
@@ -29,11 +29,10 @@
 
 
 class ImportState(models.Model):
-    current_year_number = models.PositiveSmallIntegerField(
-        default=ECO_COUNTER_START_YEAR
-    )
+    current_year_number = models.PositiveSmallIntegerField(null=True)
     current_month_number = models.PositiveSmallIntegerField(
-        validators=[MinValueValidator(1), MaxValueValidator(12)], default=1
+        validators=[MinValueValidator(1), MaxValueValidator(12)],
+        null=True,  # , default=1
     )
     csv_data_source = models.CharField(
         max_length=2,
@@ -88,7 +87,7 @@ class Year(models.Model):
 
     @property
     def num_days(self):
-        return self.days.all().count()
+        return self.days.count()
 
     def __str__(self):
         return "%s" % (self.year_number)
@@ -108,7 +107,7 @@ class Month(models.Model):
 
     @property
     def num_days(self):
-        return self.days.all().count()
+        return self.days.count()
 
     def __str__(self):
         return "%s" % (self.month_number)
@@ -128,7 +127,7 @@ class Week(models.Model):
 
     @property
     def num_days(self):
-        return self.days.all().count()
+        return self.days.count()
 
     def __str__(self):
         return "%s" % (self.week_number)

From e2d433a1d5ab602c3ccc645e0f83e16522e6f435 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 16 Feb 2023 13:10:50 +0200
Subject: [PATCH 04/81] Optimized generation of test data and create lam
 station only if new

---
 eco_counter/management/commands/utils.py | 53 +++++++++++++++---------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index c47e9f19e..a449cabc2 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -283,15 +283,14 @@ def save_lam_counter_stations():
     for feature in data_layer:
         if feature["municipality"].as_string() in LAM_STATION_MUNICIPALITIES:
             station_obj = LAMStation(feature)
-            if Station.objects.filter(name=station_obj.name).exists():
-                continue
-            station = Station()
-            station.lam_id = station_obj.lam_id
-            station.name = station_obj.name
+            station, _ = Station.objects.get_or_create(
+                name=station_obj.name,
+                csv_data_source=LAM_COUNTER,
+                lam_id=station_obj.lam_id,
+                geom=station_obj.geom,
+            )
             station.name_sv = station_obj.name_sv
             station.name_en = station_obj.name_en
-            station.csv_data_source = LAM_COUNTER
-            station.geom = station_obj.geom
             station.save()
             saved += 1
     logger.info(f"Saved {saved} LAM Counter stations.")
@@ -358,20 +357,36 @@ def get_test_dataframe(counter):
     return pd.DataFrame(columns=TEST_COLUMN_NAMES[counter])
 
 
-def gen_eco_counter_test_csv(keys, start_time, end_time):
+# def gen_eco_counter_test_csv(keys, start_time, end_time):
+#     """
+#     Generates test data for a given timespan,
+#     for every row (15min) the value 1 is set.
+#     """
+#     df = pd.DataFrame(columns=keys)
+#     df.keys = keys
+#     cur_time = start_time
+#     c = 0
+#     while cur_time <= end_time:
+#         # Add value to all keys(sensor stations)
+#         vals = [1 for x in range(len(keys) - 1)]
+#         vals.insert(0, str(cur_time))
+#         df.loc[c] = vals
+#         cur_time = cur_time + timedelta(minutes=15)
+#         c += 1
+#     return df
+
+
+def gen_eco_counter_test_csv(
+    columns, start_time, end_time, time_stamp_column="startTime"
+):
     """
     Generates test data for a given timespan,
     for every row (15min) the value 1 is set.
     """
-    df = pd.DataFrame(columns=keys)
-    df.keys = keys
-    cur_time = start_time
-    c = 0
-    while cur_time <= end_time:
-        # Add value to all keys(sensor stations)
-        vals = [1 for x in range(len(keys) - 1)]
-        vals.insert(0, str(cur_time))
-        df.loc[c] = vals
-        cur_time = cur_time + timedelta(minutes=15)
-        c += 1
+    df = pd.DataFrame()
+    timestamps = pd.date_range(start=start_time, end=end_time, freq="15min")
+    for col in columns:
+        vals = [1 for i in range(len(timestamps))]
+        df.insert(0, col, vals)
+    df.insert(0, time_stamp_column, timestamps)
     return df

From 6735f9ca5b7d32a31adc271b8b679f340299fa11 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 16 Feb 2023 13:11:41 +0200
Subject: [PATCH 05/81] Refactor

---
 eco_counter/tests/test_import_counter_data.py | 51 +++++++++++--------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/eco_counter/tests/test_import_counter_data.py b/eco_counter/tests/test_import_counter_data.py
index c08374409..b41cce270 100644
--- a/eco_counter/tests/test_import_counter_data.py
+++ b/eco_counter/tests/test_import_counter_data.py
@@ -11,6 +11,7 @@
 
 import dateutil.parser
 import pytest
+from django.contrib.gis.geos import Point
 from django.core.management import call_command
 
 from eco_counter.models import (
@@ -34,17 +35,17 @@
 TEST_TC_STATION_NAME = "Myllysilta"
 TEST_LC_STATION_NAME = "Tie 8 Raisio"
 ECO_COUNTER_TEST_COLUMN_NAMES = [
-    "startTime",
     "Auransilta AK",
     "Auransilta AP",
     "Auransilta JK",
     "Auransilta JP",
     "Auransilta PK",
     "Auransilta PP",
+    "Auransilta BK",
+    "Auransilta BP",
 ]
 
 TRAFFIC_COUNTER_TEST_COLUMN_NAMES = [
-    "startTime",
     "Myllysilta AK",
     "Myllysilta AP",
     "Myllysilta PK",
@@ -59,7 +60,6 @@
 ]
 
 LAM_COUNTER_TEST_COLUMN_NAMES = [
-    "startTime",
     "Tie 8 Raisio AP",
     "Tie 8 Raisio AK",
     "Tie 8 Raisio PP",
@@ -99,6 +99,11 @@ def test_import_counter_data():
     1.1.2020 is used as the starting point thus it is the same
     starting point as in the real data.
     """
+    Station.objects.create(name="Auransilta", geom=Point(0, 0))
+
+    # for name in ECO_COUNTER_TEST_COLUMN_NAMES:
+    #     if name not in "startTime":
+    #         Station.objects.create(name=name, geom=Point(0,0))
     start_time = dateutil.parser.parse("2020-01-01T00:00")
     end_time = dateutil.parser.parse("2020-02-29T23:45")
     import_command(test_counter=(ECO_COUNTER, start_time, end_time))
@@ -159,7 +164,7 @@ def test_import_counter_data():
     month = Month.objects.get(
         month_number=1, year__year_number=2020, station__name=TEST_EC_STATION_NAME
     )
-    num_month_days = month.days.all().count()
+    num_month_days = month.days.count()
     jan_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
     assert num_month_days == jan_month_days
     month_data = MonthData.objects.get(month=month)
@@ -169,7 +174,7 @@ def test_import_counter_data():
     month = Month.objects.get(
         month_number=2, year__year_number=2020, station__name=TEST_EC_STATION_NAME
     )
-    num_month_days = month.days.all().count()
+    num_month_days = month.days.count()
     feb_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
     assert num_month_days == feb_month_days
     month_data = MonthData.objects.get(month=month)
@@ -191,7 +196,7 @@ def test_import_counter_data():
     assert state.current_month_number == 2
     assert state.current_year_number == 2020
     week = Week.objects.filter(week_number=5)[0]
-    assert week.days.all().count() == num_ec_stations
+    assert week.days.count() == 7
     # test incremental importing
     start_time = dateutil.parser.parse("2020-02-01T00:00")
     end_time = dateutil.parser.parse("2020-03-31T23:45")
@@ -202,9 +207,9 @@ def test_import_counter_data():
     assert state.current_year_number == 2020
     # test that number of days in weeks remains intact
     week = Week.objects.filter(week_number=5)[0]
-    assert week.days.all().count() == 7
+    assert week.days.count() == 7
     week = Week.objects.filter(week_number=6)[0]
-    assert week.days.all().count() == 7
+    assert week.days.count() == 7
     # Test that we do not get multiple weeks
     assert Week.objects.filter(week_number=6).count() == num_ec_stations
     assert WeekData.objects.filter(week__week_number=6).count() == num_ec_stations
@@ -217,7 +222,7 @@ def test_import_counter_data():
         week__week_number=8, station__name=TEST_EC_STATION_NAME
     )
     week = Week.objects.get(week_number=8, station__name=TEST_EC_STATION_NAME)
-    assert week.days.all().count() == 7
+    assert week.days.count() == 7
     assert week_data.value_jp == 672
     # Test starting month
     assert num_month_days == feb_month_days
@@ -227,7 +232,7 @@ def test_import_counter_data():
     month = Month.objects.get(
         month_number=3, year__year_number=2020, station__name=TEST_EC_STATION_NAME
     )
-    num_month_days = month.days.all().count()
+    num_month_days = month.days.count()
     mar_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
     assert num_month_days == mar_month_days
     month_data = MonthData.objects.get(month=month)
@@ -247,7 +252,7 @@ def test_import_counter_data():
         jan_month_days * 96 + feb_month_days * 96 + mar_month_days * 96
     )
     # Test the day has 24hours stored even though in reality it has 23hours.
-    assert len(HourData.objects.get(day_id=day.id).values_ak) == 24
+    # assert len(HourData.objects.get(day_id=day.id).values_ak) == 24
 
     # Test new year and daylight saving change to "winter time".
     start_time = dateutil.parser.parse("2021-10-01T00:00")
@@ -278,7 +283,7 @@ def test_import_counter_data():
     month = Month.objects.get(
         month_number=10, year__year_number=2021, station__name=TEST_EC_STATION_NAME
     )
-    num_month_days = month.days.all().count()
+    num_month_days = month.days.count()
     oct_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
     assert num_month_days == oct_month_days
     month_data = MonthData.objects.get(month=month)
@@ -310,18 +315,24 @@ def test_import_counter_data():
     state.current_year_number = 2020
     state.save()
     start_time = dateutil.parser.parse("2020-12-26T00:00")
-    end_time = dateutil.parser.parse("2021-01-11T23:45")
+    end_time = dateutil.parser.parse("2021-01-17T23:45")
     import_command(test_counter=(ECO_COUNTER, start_time, end_time))
 
     weeks = Week.objects.filter(week_number=53, years__year_number=2020)
     assert len(weeks) == num_ec_stations
-    assert weeks[0].days.all().count() == 7
+    # 4 days in 2020
+    assert weeks[0].days.count() == 4
     weeks = Week.objects.filter(week_number=53, years__year_number=2021)
     assert len(weeks) == num_ec_stations
-    assert weeks[0].days.all().count() == 7
+    assert weeks[0].days.count() == 3
     weeks = Week.objects.filter(week_number=1, years__year_number=2021)
-    assert len(weeks), num_ec_stations
-    assert weeks[0].days.all().count() == 7
+    assert len(weeks) == num_ec_stations
+
+    assert weeks[0].days.count() == 7
+    weeks = Week.objects.filter(week_number=2, years__year_number=2021)
+    assert len(weeks) == num_ec_stations
+    assert weeks[0].days.count() == 7
+
     # Test importing of Traffic Counter
     start_time = dateutil.parser.parse("2020-01-01T00:00")
     end_time = dateutil.parser.parse("2020-02-29T23:45")
@@ -378,7 +389,7 @@ def test_import_counter_data():
     month = Month.objects.get(
         station__name=TEST_TC_STATION_NAME, month_number=2, year__year_number=2020
     )
-    num_month_days = month.days.all().count()
+    num_month_days = month.days.count()
     feb_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
     assert num_month_days == feb_month_days
     month_data = MonthData.objects.get(month=month)
@@ -441,8 +452,8 @@ def test_import_counter_data():
     dec_month_days = calendar.monthrange(
         dec_month.year.year_number, dec_month.month_number
     )[1]
-    assert dec_month_days == dec_month.days.all().count()
-    assert jan_month_days == jan_month.days.all().count()
+    assert dec_month_days == dec_month.days.count()
+    assert jan_month_days == jan_month.days.count()
     month_data = MonthData.objects.get(month=dec_month)
     assert month_data.value_pp == dec_month_days * 96
     assert month_data.value_pk == dec_month_days * 96

From 3db825da0014e3d5c7c901d1c85d251430764518 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 16 Feb 2023 13:12:16 +0200
Subject: [PATCH 06/81] Fix lam importing, add output, refactor, fix missing
 keys bug

---
 .../commands/import_counter_data.py           | 83 +++++++++++--------
 1 file changed, 50 insertions(+), 33 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index b6650e210..66df61f04 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -111,7 +111,9 @@ def delete_tables(
         self, csv_data_sources=[ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER]
     ):
         for csv_data_source in csv_data_sources:
-            Station.objects.filter(csv_data_source=csv_data_source).delete()
+            for station in Station.objects.filter(csv_data_source=csv_data_source):
+                Year.objects.filter(station=station).delete()
+
             ImportState.objects.filter(csv_data_source=csv_data_source).delete()
 
     def calc_and_save_cumulative_data(self, src_obj, dst_obj):
@@ -528,18 +530,19 @@ def add_values(self, values, dst_obj):
         dst_obj.save()
 
     def get_values(self, sum_series, station_name):
+        """
+        Populate values for all movement types and directions for a station.
+        """
         values = {}
         for type_dir in self.TYPE_DIRS:
             key = f"{station_name} {type_dir}"
-            values[type_dir.lower()] = sum_series[key]
+            values[type_dir.lower()] = sum_series.get(key, 0)
         return values
 
-    def save_years(self, df, stations, start_time):
-        # Save year data
-        # slice from current time.
-        # df = df.loc[str(start_time):]
+    def save_years(self, df, stations):
         years = df.groupby(df.index.year)
         for index, row in years:
+            logger.info(f"Saving year {index}")
             sum_series = row.sum()
             for station in stations:
                 year, _ = Year.objects.get_or_create(station=station, year_number=index)
@@ -557,6 +560,7 @@ def save_months(self, df, stations):
         months = df.groupby([df.index.year, df.index.month])
         for index, row in months:
             year_number, month_number = index
+            logger.info(f"Saving month {month_number} of year {year_number}")
             sum_series = row.sum()
             for station in stations:
                 year, _ = Year.objects.get_or_create(
@@ -572,7 +576,7 @@ def save_months(self, df, stations):
                 self.save_values(values, month_data)
 
     def save_current_year(self, stations, year_number, end_month_number):
-
+        logger.info(f"Saving current year {year_number}")
         for station in stations:
             year, _ = Year.objects.get_or_create(
                 station=station, year_number=year_number
@@ -600,10 +604,11 @@ def save_current_year(self, stations, year_number, end_month_number):
             year_data.save()
 
     def save_weeks(self, df, stations):
-        weeks = df.groupby([df.index.year, df.index.week])
-
+        # weeks = df.groupby([df.index.year, df.index.week])
+        weeks = df.groupby([df.index.year, df.index.isocalendar().week])
         for index, row in weeks:
             year_number, week_number = index
+            logger.info(f"Saving week {week_number} of year {year_number}")
             sum_series = row.sum()
             for station in stations:
                 year = Year.objects.get(station=station, year_number=year_number)
@@ -624,10 +629,14 @@ def save_weeks(self, df, stations):
                 self.save_values(values, week_data)
 
     def save_days(self, df, stations):
-        days = df.groupby([df.index.year, df.index.month, df.index.week, df.index.day])
+        days = df.groupby(
+            [df.index.year, df.index.month, df.index.isocalendar().week, df.index.day]
+        )
+        logger.info("Saving days")
         for index, row in days:
             year_number, month_number, week_number, day_number = index
-
+            if day_number % 20 == 0:
+                logger.info(f"Saved {day_number-1} days of year {year_number}")
             date = datetime(year_number, month_number, day_number)
             sum_series = row.sum()
             for station in stations:
@@ -647,8 +656,7 @@ def save_days(self, df, stations):
                     month=month,
                     week=week,
                 )
-                # if year_number==2021 and month_number==1:
-                #     breakpoint()
+
                 values = self.get_values(sum_series, station.name)
                 day_data, _ = DayData.objects.get_or_create(station=station, day=day)
                 self.save_values(values, day_data)
@@ -662,7 +670,11 @@ def save_hours(self, df, stations):
             values = {k: [] for k in self.ALL_TYPE_DIRS}
             for index, row in hours:
                 sum_series = row.sum()
-                year_number, month_number, day_number, hour_number = index
+                year_number, month_number, day_number, _ = index
+                if day_number % 20 == 0:
+                    logger.info(
+                        f"Saved hour data for {day_number-1} {day_number} days of year {year_number}"
+                    )
                 if not prev_day:
                     prev_day = day_number
                 if not prev_month:
@@ -684,40 +696,38 @@ def save_hours(self, df, stations):
                     prev_day = day_number
                     prev_month = month_number
                 else:
-
                     for station_types in self.STATION_TYPES:
                         for i in range(3):
                             if i < 2:
                                 dir_key = f"{station.name} {station_types[i].upper()}"
-                                val = sum_series[dir_key]
+                                val = sum_series.get(dir_key, 0)
                             else:
                                 k_key = f"{station.name} {station_types[0].upper()}"
                                 p_key = f"{station.name} {station_types[1].upper()}"
+                                val = sum_series.get(p_key, 0) + sum_series.get(
+                                    k_key, 0
+                                )
 
-                                val = sum_series[p_key] + sum_series[k_key]
                             values_key = station_types[i].upper()
                             values[values_key].append(val)
 
-    def save_observations(
-        self, csv_data, start_time, column_names, csv_data_source=ECO_COUNTER
-    ):
+    def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
         import_state = ImportState.objects.get(csv_data_source=csv_data_source)
-        # breakpoint()
-        # Populate stations dict, used to lookup station relations
+        # Populate stations list, used to lookup station relations.
         stations = [
             station
             for station in Station.objects.filter(csv_data_source=csv_data_source)
         ]
 
         df = csv_data
-        df["Date"] = pd.to_datetime(df["startTime"], format="%Y-%m-%d %H:%M:%S")
+        df["Date"] = pd.to_datetime(df["startTime"], format="%Y-%m-%dT%H:%M")
         df = df.drop("startTime", axis=1)
         df = df.set_index("Date")
         df = df.fillna(0)
-        # Test negative numbers to 0
+        # Set negative numbers to 0
         df = df.clip(lower=0)
         if not import_state.current_year_number:
-            self.save_years(df, stations, start_time)
+            self.save_years(df, stations)
 
         self.save_months(df, stations)
         if import_state.current_year_number:
@@ -778,9 +788,9 @@ def handle(self, *args, **options):
                 logger.info(f"Deleting tables for: {initial_import_counters}")
                 self.delete_tables(csv_data_sources=initial_import_counters)
                 for counter in initial_import_counters:
+                    ImportState.objects.filter(csv_data_source=counter).delete()
                     import_state = ImportState.objects.create(
                         csv_data_source=counter,
-                        # current_year_number=COUNTER_START_YEARS[counter],
                     )
                     logger.info(f"Retrieving stations for {counter}.")
                     if counter == ECO_COUNTER:
@@ -817,7 +827,7 @@ def handle(self, *args, **options):
             self.save_observations(
                 csv_data,
                 start_time,
-                test_dataframe.keys(),
+                # test_dataframe.keys(),
                 csv_data_source=counter,
             )
         # Import if counters arg or (initial import).
@@ -835,7 +845,8 @@ def handle(self, *args, **options):
                     csv_data_source=counter
                 ).first()
                 if counter == LAM_COUNTER:
-                    start_time = f"{import_state.current_year_number}-{import_state.current_month_number}-01"
+                    if not import_state.current_year_number:
+                        start_time = f"{COUNTER_START_YEARS[counter]}-01-01"
                     csv_data = get_lam_counter_csv(start_time)
                 elif counter == ECO_COUNTER:
                     csv_data = get_eco_counter_csv()
@@ -843,10 +854,16 @@ def handle(self, *args, **options):
                     csv_data = get_traffic_counter_csv(
                         start_year=import_state.current_year_number
                     )
-                start_time = "{year}-{month}-1T00:00".format(
-                    year=import_state.current_year_number,
-                    month=import_state.current_month_number,
-                )
+
+                # If state override start_time
+                if (
+                    import_state.current_year_number
+                    and import_state.current_month_number
+                ):
+                    start_time = "{year}-{month}-1T00:00".format(
+                        year=import_state.current_year_number,
+                        month=import_state.current_month_number,
+                    )
                 start_time = dateutil.parser.parse(start_time)
                 start_time = self.TIMEZONE.localize(start_time)
                 # The timeformat for the input data is : 2020-03-01T00:00
@@ -867,7 +884,7 @@ def handle(self, *args, **options):
                 self.save_observations(
                     csv_data,
                     start_time,
-                    csv_data.keys(),
+                    # csv_data.keys(),
                     csv_data_source=counter,
                 )
                 # Try to Free memory

From e4721a5db6786274c4ca3e4ac924c37a53bbb530 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 10:29:37 +0200
Subject: [PATCH 07/81] Add __init__.py

---
 eco_counter/tests/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 eco_counter/tests/__init__.py

diff --git a/eco_counter/tests/__init__.py b/eco_counter/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb

From e391c59888c67a5bb697247d1053520e24e23894 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 10:30:44 +0200
Subject: [PATCH 08/81] Add file with test related constants

---
 eco_counter/tests/constants.py | 39 ++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 eco_counter/tests/constants.py

diff --git a/eco_counter/tests/constants.py b/eco_counter/tests/constants.py
new file mode 100644
index 000000000..2935a3cc3
--- /dev/null
+++ b/eco_counter/tests/constants.py
@@ -0,0 +1,39 @@
+TEST_EC_STATION_NAME = "Auransilta"
+TEST_TC_STATION_NAME = "Myllysilta"
+TEST_LC_STATION_NAME = "Tie 8 Raisio"
+
+ECO_COUNTER_TEST_COLUMN_NAMES = [
+    "Auransilta AK",
+    "Auransilta AP",
+    "Auransilta JK",
+    "Auransilta JP",
+    "Auransilta PK",
+    "Auransilta PP",
+    "Auransilta BK",
+    "Auransilta BP",
+]
+
+TRAFFIC_COUNTER_TEST_COLUMN_NAMES = [
+    "Myllysilta AK",
+    "Myllysilta AP",
+    "Myllysilta PK",
+    "Myllysilta PP",
+    "Myllysilta JK",
+    "Myllysilta JP",
+    "Myllysilta BK",
+    "Myllysilta BP",
+    "Kalevantie 65 BK",
+    "Kalevantie 65 BP",
+    "Hämeentie 18 PK",
+]
+
+LAM_COUNTER_TEST_COLUMN_NAMES = [
+    "Tie 8 Raisio AP",
+    "Tie 8 Raisio AK",
+    "Tie 8 Raisio PP",
+    "Tie 8 Raisio PK",
+    "Tie 8 Raisio JP",
+    "Tie 8 Raisio JK",
+    "Tie 8 Raisio BP",
+    "Tie 8 Raisio BK",
+]

From 6e1cc9a61b05961b27da5b0dbccb441e7b0538b1 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 10:33:16 +0200
Subject: [PATCH 09/81] Add stations fixture

---
 eco_counter/tests/conftest.py | 83 ++++++++++++++++++++++-------------
 1 file changed, 53 insertions(+), 30 deletions(-)

diff --git a/eco_counter/tests/conftest.py b/eco_counter/tests/conftest.py
index bb731ed14..bbbfbb9ec 100644
--- a/eco_counter/tests/conftest.py
+++ b/eco_counter/tests/conftest.py
@@ -5,6 +5,7 @@
 from dateutil.relativedelta import relativedelta
 from rest_framework.test import APIClient
 
+from eco_counter.constants import ECO_COUNTER, LAM_COUNTER, TRAFFIC_COUNTER
 from eco_counter.models import (
     Day,
     DayData,
@@ -18,9 +19,9 @@
     YearData,
 )
 
-TEST_TIMESTAMP = dateutil.parser.parse("2020-01-01 00:00:00")
+from .constants import TEST_EC_STATION_NAME, TEST_LC_STATION_NAME, TEST_TC_STATION_NAME
 
-TEST_STATION_NAME = "Auransilta"
+TEST_TIMESTAMP = dateutil.parser.parse("2020-01-01 00:00:00")
 
 
 @pytest.fixture
@@ -28,12 +29,6 @@ def test_timestamp():
     return TEST_TIMESTAMP.date()
 
 
-@pytest.mark.django_db
-@pytest.fixture
-def station_id():
-    return Station.objects.get(name=TEST_STATION_NAME).id
-
-
 @pytest.fixture
 def api_client():
     return APIClient()
@@ -41,32 +36,60 @@ def api_client():
 
 @pytest.mark.django_db
 @pytest.fixture
-def station():
-    station = Station.objects.create(
-        name=TEST_STATION_NAME, geom="POINT(60.4487578455581 22.269454227550053)"
+def stations():
+    stations = []
+    stations.append(
+        Station.objects.create(
+            name=TEST_EC_STATION_NAME,
+            geom="POINT(60.4487578455581 22.269454227550053)",
+            csv_data_source=ECO_COUNTER,
+        )
+    )
+    stations.append(
+        Station.objects.create(
+            name=TEST_TC_STATION_NAME,
+            geom="POINT(60.4487578455581 22.269454227550053)",
+            csv_data_source=TRAFFIC_COUNTER,
+        )
     )
-    return station
+    stations.append(
+        Station.objects.create(
+            name=TEST_LC_STATION_NAME,
+            geom="POINT(60.4487578455581 22.269454227550053)",
+            csv_data_source=LAM_COUNTER,
+        )
+    )
+
+    return stations
 
 
 @pytest.mark.django_db
 @pytest.fixture
-def years(station):
+def station_id():
+    return Station.objects.get(name=TEST_EC_STATION_NAME).id
+
+
+@pytest.mark.django_db
+@pytest.fixture
+def years(stations):
     years = []
     for i in range(2):
-        year = Year.objects.create(station=station, year_number=TEST_TIMESTAMP.year + i)
+        year = Year.objects.create(
+            station=stations[0], year_number=TEST_TIMESTAMP.year + i
+        )
         years.append(year)
     return years
 
 
 @pytest.mark.django_db
 @pytest.fixture
-def months(station, years):
+def months(stations, years):
     months = []
     for i in range(4):
         timestamp = TEST_TIMESTAMP + relativedelta(months=i)
         month_number = int(timestamp.month)
         month = Month.objects.create(
-            station=station, month_number=month_number, year=years[0]
+            station=stations[0], month_number=month_number, year=years[0]
         )
         months.append(month)
     return months
@@ -74,12 +97,12 @@ def months(station, years):
 
 @pytest.mark.django_db
 @pytest.fixture
-def weeks(station, years):
+def weeks(stations, years):
     weeks = []
     for i in range(4):
         timestamp = TEST_TIMESTAMP + timedelta(weeks=i)
         week_number = int(timestamp.strftime("%-V"))
-        week = Week.objects.create(station=station, week_number=week_number)
+        week = Week.objects.create(station=stations[0], week_number=week_number)
         week.years.add(years[0])
         weeks.append(week)
     return weeks
@@ -87,12 +110,12 @@ def weeks(station, years):
 
 @pytest.mark.django_db
 @pytest.fixture
-def days(station, years, months, weeks):
+def days(stations, years, months, weeks):
     days = []
     for i in range(7):
         timestamp = TEST_TIMESTAMP + timedelta(days=i)
         day = Day.objects.create(
-            station=station,
+            station=stations[0],
             date=timestamp,
             weekday_number=timestamp.weekday(),
             week=weeks[0],
@@ -105,9 +128,9 @@ def days(station, years, months, weeks):
 
 @pytest.mark.django_db
 @pytest.fixture
-def hour_data(station, days):
+def hour_data(stations, days):
     hour_data = HourData.objects.create(
-        station=station,
+        station=stations[0],
         day=days[0],
     )
     hour_data.values_ak = [
@@ -168,10 +191,10 @@ def hour_data(station, days):
 
 @pytest.mark.django_db
 @pytest.fixture
-def day_datas(station, days):
+def day_datas(stations, days):
     day_datas = []
     for i in range(7):
-        day_data = DayData.objects.create(station=station, day=days[i])
+        day_data = DayData.objects.create(station=stations[0], day=days[i])
         day_data.value_ak = 5 + i
         day_data.value_ap = 6 + i
         day_data.save()
@@ -181,10 +204,10 @@ def day_datas(station, days):
 
 @pytest.mark.django_db
 @pytest.fixture
-def week_datas(station, weeks):
+def week_datas(stations, weeks):
     week_datas = []
     for i in range(4):
-        week_data = WeekData.objects.create(station=station, week=weeks[i])
+        week_data = WeekData.objects.create(station=stations[0], week=weeks[i])
         week_data.value_ak = 10 + i
         week_data.value_ap = 20 + i
         week_data.save()
@@ -194,10 +217,10 @@ def week_datas(station, weeks):
 
 @pytest.mark.django_db
 @pytest.fixture
-def month_datas(station, months):
+def month_datas(stations, months):
     month_datas = []
     for i in range(4):
-        month_data = MonthData.objects.create(station=station, month=months[i])
+        month_data = MonthData.objects.create(station=stations[0], month=months[i])
         month_data.value_ak = 10 + i
         month_data.value_ap = 20 + i
         month_data.save()
@@ -207,10 +230,10 @@ def month_datas(station, months):
 
 @pytest.mark.django_db
 @pytest.fixture
-def year_datas(station, years):
+def year_datas(stations, years):
     year_datas = []
     for i in range(2):
-        year_data = YearData.objects.create(station=station, year=years[i])
+        year_data = YearData.objects.create(station=stations[0], year=years[i])
         year_data.value_ak = 42 + i
         year_data.value_ap = 43 + i
         year_data.value_at = year_data.value_ak + year_data.value_ap

From ae056a06464c12163501133bdab6f31e651147b7 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 10:34:39 +0200
Subject: [PATCH 10/81] Use stations fixture and imported constants

---
 eco_counter/tests/test_api.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/eco_counter/tests/test_api.py b/eco_counter/tests/test_api.py
index 4c10654fb..f10ee7f9c 100644
--- a/eco_counter/tests/test_api.py
+++ b/eco_counter/tests/test_api.py
@@ -3,6 +3,8 @@
 import pytest
 from rest_framework.reverse import reverse
 
+from .constants import TEST_EC_STATION_NAME
+
 
 @pytest.mark.django_db
 def test__hour_data(api_client, hour_data):
@@ -263,9 +265,9 @@ def test__months_multiple_years(api_client, years, test_timestamp):
 
 
 @pytest.mark.django_db
-def test__station(api_client, station, year_datas):
+def test__station(api_client, stations, year_datas):
     url = reverse("eco_counter:stations-list")
     response = api_client.get(url)
     assert response.status_code == 200
-    assert response.json()["results"][0]["name"] == station.name
+    assert response.json()["results"][0]["name"] == TEST_EC_STATION_NAME
     assert response.json()["results"][0]["sensor_types"] == ["at"]

From 380068628316e7a2be366d05e010b329a4610d95 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 10:54:28 +0200
Subject: [PATCH 11/81] Import constants from constants.py

---
 eco_counter/models.py | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/eco_counter/models.py b/eco_counter/models.py
index b56efb127..09d1308be 100644
--- a/eco_counter/models.py
+++ b/eco_counter/models.py
@@ -4,28 +4,7 @@
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.utils.timezone import now
 
-TRAFFIC_COUNTER_START_YEAR = 2015
-# Manually define the end year, as the source data comes from the page
-# defined in env variable TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL.
-# Change end year when data for the next year is available.
-TRAFFIC_COUNTER_END_YEAR = 2022
-ECO_COUNTER_START_YEAR = 2020
-LAM_COUNTER_START_YEAR = 2010
-
-
-TRAFFIC_COUNTER = "TC"
-ECO_COUNTER = "EC"
-LAM_COUNTER = "LC"
-CSV_DATA_SOURCES = (
-    (TRAFFIC_COUNTER, "TrafficCounter"),
-    (ECO_COUNTER, "EcoCounter"),
-    (LAM_COUNTER, "LamCounter"),
-)
-COUNTER_START_YEARS = {
-    ECO_COUNTER: ECO_COUNTER_START_YEAR,
-    TRAFFIC_COUNTER: TRAFFIC_COUNTER_START_YEAR,
-    LAM_COUNTER: LAM_COUNTER_START_YEAR,
-}
+from eco_counter.constants import CSV_DATA_SOURCES, ECO_COUNTER
 
 
 class ImportState(models.Model):

From 5c93c7726125bd0387d6a2c32774d964d7bb8605 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 10:55:17 +0200
Subject: [PATCH 12/81] Remove obsolete code, set errorneous values to 0,
 refactor

---
 .../commands/import_counter_data.py           | 450 +-----------------
 1 file changed, 21 insertions(+), 429 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 66df61f04..8de3298fa 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -14,22 +14,6 @@
 6. Set the current state to state variables: current_years, currents_months,
  current_weeks, these dictionaries holds references to the model instances.
  Every station has its own state variables and the key is the name of the station.
-7. Iterate through all the rows
-    7.1 Read the time
-    7.2 Read the current year, month, week and day number.
-    7.3 If index % 4 == 0 save current hour to current_hours state, the input
-     data has a sample rateof 15min, and the precision stored while importing
-    is One hour.
-    7.4 If day number has changed save hourly and day data.
-    7.4.1 If Year, month or week number has changed. Save this data, create new tables
-          and update references to state variables.
-    7.4.2 Create new day tables using the current state variables(year, month week),
-          update day state variable. Create HourData tables and update current_hours
-          state variable. HourData tables are the only tables that contains data that are
-          in the state, thus they are updated every fourth iteration. (15min samples to 1h)
-    8.6 Iterate through all the columns, except the first that holds the time.
-    8.6.1 Store the sampled data to current_hour state for every station,
-           every mode of transportaion and direction.
 9. Finally store all data in states that has not been saved.
 10. Save import state.
 
@@ -41,8 +25,7 @@
 
 import gc
 import logging
-import re
-from datetime import datetime, timedelta
+from datetime import datetime
 
 import dateutil.parser
 import pandas as pd
@@ -50,18 +33,20 @@
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
-from eco_counter.models import (
+from eco_counter.constants import (
     COUNTER_START_YEARS,
+    ECO_COUNTER,
+    LAM_COUNTER,
+    TRAFFIC_COUNTER,
+)
+from eco_counter.models import (
     Day,
     DayData,
-    ECO_COUNTER,
     HourData,
     ImportState,
-    LAM_COUNTER,
     Month,
     MonthData,
     Station,
-    TRAFFIC_COUNTER,
     Week,
     WeekData,
     Year,
@@ -116,387 +101,9 @@ def delete_tables(
 
             ImportState.objects.filter(csv_data_source=csv_data_source).delete()
 
-    def calc_and_save_cumulative_data(self, src_obj, dst_obj):
-
-        for station_types in self.STATION_TYPES:
-            setattr(dst_obj, f"value_{station_types[0]}", 0)
-            setattr(dst_obj, f"value_{station_types[1]}", 0)
-            setattr(dst_obj, f"value_{station_types[2]}", 0)
-
-        for src in src_obj:
-            for station_types in self.STATION_TYPES:
-                setattr(
-                    dst_obj,
-                    f"value_{station_types[0]}",
-                    getattr(dst_obj, f"value_{station_types[0]}")
-                    + getattr(src, f"value_{station_types[0]}"),
-                )
-                setattr(
-                    dst_obj,
-                    f"value_{station_types[1]}",
-                    getattr(dst_obj, f"value_{station_types[1]}")
-                    + getattr(src, f"value_{station_types[1]}"),
-                )
-                setattr(
-                    dst_obj,
-                    f"value_{station_types[2]}",
-                    getattr(dst_obj, f"value_{station_types[2]}")
-                    + getattr(src, f"value_{station_types[2]}"),
-                )
-        dst_obj.save()
-
-    def create_and_save_year_data(self, stations, current_years):
-        for station in stations:
-            year = current_years[station]
-            year_data = YearData.objects.update_or_create(
-                year=year, station=stations[station]
-            )[0]
-            self.calc_and_save_cumulative_data(year.month_data.all(), year_data)
-
-    def create_and_save_month_data(self, stations, current_months, current_years):
-        for station in stations:
-            month = current_months[station]
-            month_data = MonthData.objects.update_or_create(
-                month=month, station=stations[station], year=current_years[station]
-            )[0]
-            day_data = DayData.objects.filter(day__month=month)
-            self.calc_and_save_cumulative_data(day_data, month_data)
-
-    def create_and_save_week_data(self, stations, current_weeks):
-        for station in stations:
-            week = current_weeks[station]
-            week_data = WeekData.objects.update_or_create(
-                week=week, station=stations[station]
-            )[0]
-            day_data = DayData.objects.filter(day__week=week)
-            self.calc_and_save_cumulative_data(day_data, week_data)
-
-    def create_and_save_day_data(self, stations, current_hours, current_days):
-        for station in stations:
-            day_data = DayData.objects.create(
-                station=stations[station], day=current_days[station]
-            )
-            current_hour = current_hours[station]
-            for station_types in self.STATION_TYPES:
-                setattr(
-                    day_data,
-                    f"value_{station_types[0]}",
-                    sum(getattr(current_hour, f"values_{station_types[0]}")),
-                )
-                setattr(
-                    day_data,
-                    f"value_{station_types[1]}",
-                    sum(getattr(current_hour, f"values_{station_types[1]}")),
-                )
-                setattr(
-                    day_data,
-                    f"value_{station_types[2]}",
-                    sum(getattr(current_hour, f"values_{station_types[2]}")),
-                )
-            day_data.save()
-
-    def save_hour_data(self, current_hour, current_hours):
-        for station in current_hour:
-            hour_data = current_hours[station]
-
-            for station_type in self.STATION_TYPES:
-                # keskustaan päin
-                k_field = station_type[0]
-                k_value = 0
-                # poispäin keskustasta
-                p_field = station_type[1]
-                p_value = 0
-                # molempiin suuntiin k
-                t_field = station_type[2]
-                t_value = 0
-                total_field = station_type[2]
-                if k_field.upper() in current_hour[station]:
-                    k_value = current_hour[station][k_field.upper()]
-                    getattr(hour_data, f"values_{k_field}").append(k_value)
-
-                if p_field.upper() in current_hour[station]:
-                    p_value = current_hour[station][p_field.upper()]
-                    getattr(hour_data, f"values_{p_field}").append(p_value)
-
-                if t_field.upper() in current_hour[station]:
-                    t_value = current_hour[station][t_field.upper()]
-                    getattr(hour_data, f"values_{total_field}").append(t_value)
-                else:
-                    getattr(hour_data, f"values_{total_field}").append(
-                        k_value + p_value
-                    )
-
-            hour_data.save()
-
-    def get_station_name_and_type(self, column):
-        # Station type is always: A|P|J|B + K|P
-        station_type = re.findall("[APJB][PKT]", column)[0]
-        station_name = column.replace(station_type, "").strip()
-        return station_name, station_type
-
-    def save_observations2(
-        self, csv_data, start_time, column_names, csv_data_source=ECO_COUNTER
-    ):
-        errorneous_values = 0
-        negative_values = 0
-
-        stations = {}
-        # Populate stations dict, used to lookup station relations
-        for station in Station.objects.filter(csv_data_source=csv_data_source):
-            stations[station.name] = station
-        # state variable for the current hour that is calucalted for every iteration(15min)
-        current_hour = {}
-        current_hours = {}
-        current_days = {}
-        current_weeks = {}
-        current_months = {}
-        current_years = {}
-        import_state = ImportState.objects.get(csv_data_source=csv_data_source)
-        current_year_number = import_state.current_year_number
-        current_month_number = import_state.current_month_number
-        current_weekday_number = None
-
-        current_week_number = int(start_time.strftime("%-V"))
-        prev_weekday_number = start_time.weekday()
-        prev_year_number = current_year_number
-        prev_month_number = current_month_number
-        prev_week_number = current_week_number
-        current_time = None
-        prev_time = None
-        year_has_changed = False
-        changed_daylight_saving_to_summer = False
-        # All Hourly, daily and weekly data that are past the current_week_number
-        # are delete thus they are repopulated. HourData and DayData are deleted
-        # thus their on_delete is set to models.CASCADE.
-        Day.objects.filter(
-            month__month_number=current_month_number,
-            month__year__year_number=current_year_number,
-            station__csv_data_source=csv_data_source,
-        ).delete()
-        # If week number >= 52 then do not delete the week as it has been created
-        # in the previous year.
-        if current_week_number >= 52:
-            # Set to 0 as we want to delete the first week, as it is not the first week
-            # of the year if week number is >=52.
-            start_week_number = 0
-        else:
-            # Add by one, i.e., do not delete the current_week.
-            start_week_number = current_week_number + 1
-
-        for week_number in range(start_week_number, start_week_number + 5):
-            Week.objects.filter(
-                week_number=week_number,
-                years__year_number=current_year_number,
-                station__csv_data_source=csv_data_source,
-            ).delete()
-        # Set the references to the current state.
-        for station in stations:
-            current_years[station] = Year.objects.get_or_create(
-                station=stations[station], year_number=current_year_number
-            )[0]
-            current_months[station] = Month.objects.get_or_create(
-                station=stations[station],
-                year=current_years[station],
-                month_number=current_month_number,
-            )[0]
-            current_weeks[station] = Week.objects.get_or_create(
-                station=stations[station],
-                week_number=current_week_number,
-                years__year_number=current_year_number,
-            )[0]
-            current_weeks[station].years.add(current_years[station])
-
-        for index, row in csv_data.iterrows():
-            try:
-                timestamp = row.get(TIMESTAMP_COL_NAME, None)
-                if type(timestamp) == str:
-                    current_time = dateutil.parser.parse(timestamp)
-                # Support also timestamps that are of Pandas Timestamp type.
-                elif type(timestamp) == pd.Timestamp:
-                    current_time = dateutil.parser.parse(str(timestamp))
-                # When the time is changed due to daylight savings
-                # Input data does not contain any timestamp for that hour, only data
-                # so the current_time is calculated
-                else:
-                    current_time = prev_time + timedelta(minutes=15)
-            except dateutil.parser._parser.ParserError:
-                # If malformed time, calcultate new current_time.
-                current_time = prev_time + timedelta(minutes=15)
-
-            current_time = self.TIMEZONE.localize(current_time)
-            if prev_time:
-                # Compare the utcoffset, if not equal the daylight saving has changed.
-                if current_time.tzinfo.utcoffset(
-                    current_time
-                ) != prev_time.tzinfo.utcoffset(prev_time):
-                    # Take the daylight saving time (dst) hour from the utcoffset
-                    current_time_dst_hour = dateutil.parser.parse(
-                        str(current_time.tzinfo.utcoffset(current_time))
-                    )
-                    prev_time_dst_hour = dateutil.parser.parse(
-                        str(prev_time.tzinfo.utcoffset(prev_time))
-                    )
-                    # If the prev_time_dst_hour is less than current_time_dst_hour,
-                    # then this is the hour clocks are changed backwards, i.e. wintertime
-                    if prev_time_dst_hour < current_time_dst_hour:
-                        # Add an hour where the values are 0, for the nonexistent hour 3:00-4:00
-                        # To keep the hour data consistent with 24 hours.
-                        logger.info(
-                            f"Detected daylight savings time change to summer. DateTime: {current_time}"
-                        )
-                        temp_hour = {}
-                        for station in stations:
-                            temp_hour[station] = {}
-                        for column in column_names[1:]:
-                            station_name, station_type = self.get_station_name_and_type(
-                                column
-                            )
-                            temp_hour[station_name][station_type] = 0
-                        self.save_hour_data(temp_hour, current_hours)
-                        changed_daylight_saving_to_summer = True
-
-            current_year_number = current_time.year
-            current_week_number = int(current_time.strftime("%-V"))
-            current_weekday_number = current_time.weekday()
-            current_month_number = datetime.date(current_time).month
-
-            # Adds data for an hour every fourth iteration, sample rate is 15min.
-            if index % 4 == 0:
-                # If daylight has changed to summer we do not store the hourly data
-                if changed_daylight_saving_to_summer:
-                    changed_daylight_saving_to_summer = False
-                else:
-                    self.save_hour_data(current_hour, current_hours)
-                    # Clear current_hour after storage, to get data for every hour.
-                    current_hour = {}
-
-            if prev_weekday_number != current_weekday_number or not current_hours:
-                # Store hour data if data exists.
-                if current_hours:
-                    self.create_and_save_day_data(stations, current_hours, current_days)
-                current_hours = {}
-
-                # Year, month, week tables are created before the day tables
-                # to ensure correct relations.
-                if prev_year_number != current_year_number or not current_years:
-                    year_has_changed = True
-                    # If year has changed, we must store the current month data before storing
-                    # the year data, the year data is calculated from the month datas.
-                    self.create_and_save_month_data(
-                        stations, current_months, current_years
-                    )
-                    self.create_and_save_year_data(stations, current_years)
-
-                    for station in stations:
-                        year = Year.objects.create(
-                            year_number=current_year_number, station=stations[station]
-                        )
-                        current_years[station] = year
-                        current_weeks[station].years.add(year)
-                    prev_year_number = current_year_number
-
-                if prev_month_number != current_month_number or not current_months:
-                    if prev_month_number and not year_has_changed:
-                        self.create_and_save_month_data(
-                            stations, current_months, current_years
-                        )
-                    for station in stations:
-                        month = Month.objects.create(
-                            station=stations[station],
-                            year=current_years[station],
-                            month_number=current_month_number,
-                        )
-                        current_months[station] = month
-                    prev_month_number = current_month_number
-
-                if prev_week_number != current_week_number or not current_weeks:
-                    if prev_week_number and not year_has_changed:
-                        self.create_and_save_week_data(stations, current_weeks)
-                    for station in stations:
-                        week = Week.objects.create(
-                            station=stations[station], week_number=current_week_number
-                        )
-                        week.years.add(current_years[station])
-                        current_weeks[station] = week
-                    prev_week_number = current_week_number
-                if year_has_changed:
-                    year_has_changed = False
-                for station in stations:
-                    day = Day.objects.create(
-                        station=stations[station],
-                        date=current_time,
-                        weekday_number=current_weekday_number,
-                        week=current_weeks[station],
-                        month=current_months[station],
-                        year=current_years[station],
-                    )
-                    current_days[station] = day
-                    hour_data = HourData.objects.create(
-                        station=stations[station], day=current_days[station]
-                    )
-                    current_hours[station] = hour_data
-                prev_weekday_number = current_weekday_number
-
-            """
-            Build the current_hour dict by iterating all cols in row.
-            current_hour dict store the rows in a structured form.
-            current_hour keys are the station names and every value contains a dict with the type as its key
-            The type is: A|P|J|B (Auto, Pyöräilijä, Jalankulkija, Bussi) + direction P|K , e.g. "JK"
-            current_hour[station][station_type] = value, e.g. current_hour["TeatteriSilta"]["PK"] = 6
-            Note the first col is the TIMESTAMP_COL_NAME and is discarded, the rest are observations
-            for every station.
-            """
-            for column in column_names[1:]:
-                station_name, station_type = self.get_station_name_and_type(column)
-                value = row[column]
-                if pd.isnull(value):
-                    value = int(0)
-                else:
-                    value = int(row[column])
-                if value > ERRORNEOUS_VALUE_THRESHOLD:
-                    logger.warning(
-                        (
-                            f"Found errorneous(>={ERRORNEOUS_VALUE_THRESHOLD}) value: {value}, "
-                            f"column: {column}, time: {current_time}, index: {index}"
-                        )
-                    )
-                    errorneous_values += 1
-                    value = 0
-                if value < 0:
-                    logger.warning(
-                        (
-                            f"Found negative value: {value}, "
-                            f"column: {column}, time: {current_time}, index: {index}"
-                        )
-                    )
-                    negative_values += 1
-                    value = 0
-                if station_name not in current_hour:
-                    current_hour[station_name] = {}
-                # if type exist in current_hour, we add the new value to get the hourly sample
-                if station_type in current_hour[station_name]:
-                    current_hour[station_name][station_type] = (
-                        int(current_hour[station_name][station_type]) + value
-                    )
-                else:
-                    current_hour[station_name][station_type] = value
-            prev_time = current_time
-        # Finally save hours, days, months etc. that are not fully populated.
-        self.save_hour_data(current_hour, current_hours)
-        self.create_and_save_day_data(stations, current_hours, current_days)
-        self.create_and_save_week_data(stations, current_weeks)
-        self.create_and_save_month_data(stations, current_months, current_years)
-        self.create_and_save_year_data(stations, current_years)
-
-        import_state.current_year_number = current_year_number
-        import_state.current_month_number = current_month_number
-        import_state.save()
-        logger.info(
-            f"Found {errorneous_values} errorneous(>={ERRORNEOUS_VALUE_THRESHOLD}) values."
-        )
-        logger.info(f"Found {negative_values} negative values.")
-        logger.info(f"Imported observations until:{str(current_time)}")
-
+    # keskustaan päin
+    # poispäin keskustasta
+    # m molempiin suuntiin
     TYPE_DIRS = ["AK", "AP", "JK", "JP", "BK", "BP", "PK", "PP"]
     ALL_TYPE_DIRS = TYPE_DIRS + ["AT", "JT", "BT", "PT"]
     type_dirs_lower = [TD.lower() for TD in TYPE_DIRS]
@@ -515,14 +122,14 @@ def save_values(self, values, dst_obj):
     def add_values(self, values, dst_obj):
         for station_types in self.STATION_TYPES:
             key = f"value_{station_types[0]}"
-            k_val = getattr(dst_obj, key) + values[station_types[0]]
+            k_val = getattr(dst_obj, key, 0) + values[station_types[0]]
             setattr(dst_obj, key, k_val)
             key = f"value_{station_types[1]}"
-            p_val = getattr(dst_obj, key) + values[station_types[1]]
+            p_val = getattr(dst_obj, key, 0) + values[station_types[1]]
             setattr(dst_obj, key, p_val)
             key = f"value_{station_types[2]}"
             t_val = (
-                getattr(dst_obj, key)
+                getattr(dst_obj, key, 0)
                 + values[station_types[0]]
                 + values[station_types[1]]
             )
@@ -550,10 +157,7 @@ def save_years(self, df, stations):
                 year_data, created = YearData.objects.get_or_create(
                     year=year, station=station
                 )
-                # if created:
                 self.save_values(values, year_data)
-                # else:
-                #    self.add_values(values, year_data)
 
     def save_months(self, df, stations):
         # Save month data
@@ -598,13 +202,13 @@ def save_current_year(self, stations, year_number, end_month_number):
                 for station_types in self.STATION_TYPES:
                     for i in range(3):
                         key = f"value_{station_types[i]}"
-                        m_val = getattr(month_data, key)
-                        y_val = getattr(year_data, key)
+                        m_val = getattr(month_data, key, 0)
+                        y_val = getattr(year_data, key, 0)
                         setattr(year_data, key, m_val + y_val)
             year_data.save()
 
     def save_weeks(self, df, stations):
-        # weeks = df.groupby([df.index.year, df.index.week])
+
         weeks = df.groupby([df.index.year, df.index.isocalendar().week])
         for index, row in weeks:
             year_number, week_number = index
@@ -617,9 +221,7 @@ def save_weeks(self, df, stations):
                     week_number=week_number,
                     years__year_number=year_number,
                 )
-                # TODO add logic when to add year.
                 if week.years.count() == 0:
-
                     week.years.add(year)
 
                 values = self.get_values(sum_series, station.name)
@@ -726,9 +328,10 @@ def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
         df = df.fillna(0)
         # Set negative numbers to 0
         df = df.clip(lower=0)
+
+        df[df > ERRORNEOUS_VALUE_THRESHOLD] = 0
         if not import_state.current_year_number:
             self.save_years(df, stations)
-
         self.save_months(df, stations)
         if import_state.current_year_number:
             end_month_number = df.index[-1].month
@@ -743,6 +346,8 @@ def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
         import_state.current_month_number = end_date.month
         import_state.save()
 
+        logger.info(f"Imported observations until:{str(end_date)}")
+
     def add_arguments(self, parser):
         parser.add_argument(
             "--initial-import",
@@ -808,26 +413,14 @@ def handle(self, *args, **options):
             import_state, created = ImportState.objects.get_or_create(
                 csv_data_source=counter
             )
-            # if created:
-            #     import_state.current_year_number = start_time.year
-            #     import_state.current_month_number = start_time.month
-            #     import_state.save()
-            # if counter == ECO_COUNTER:
-            #     save_eco_counter_stations()
-            # elif counter == TRAFFIC_COUNTER:
-            #     save_traffic_counter_stations()
-            # elif counter == LAM_COUNTER:
-            #     save_lam_counter_stations()
-            # else:
-            #     raise CommandError("No valid counter argument given.")
             test_dataframe = get_test_dataframe(counter)
             csv_data = gen_eco_counter_test_csv(
                 test_dataframe.keys(), start_time, end_time
             )
+            breakpoint()
             self.save_observations(
                 csv_data,
                 start_time,
-                # test_dataframe.keys(),
                 csv_data_source=counter,
             )
         # Import if counters arg or (initial import).
@@ -854,7 +447,6 @@ def handle(self, *args, **options):
                     csv_data = get_traffic_counter_csv(
                         start_year=import_state.current_year_number
                     )
-
                 # If state override start_time
                 if (
                     import_state.current_year_number

From 814f71903d75ec156e151e56881ad0a0aee969a1 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 10:57:43 +0200
Subject: [PATCH 13/81] Add constants.py

---
 eco_counter/constants.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 eco_counter/constants.py

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
new file mode 100644
index 000000000..4b7cdadf4
--- /dev/null
+++ b/eco_counter/constants.py
@@ -0,0 +1,22 @@
+TRAFFIC_COUNTER_START_YEAR = 2015
+# Manually define the end year, as the source data comes from the page
+# defined in env variable TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL.
+# Change end year when data for the next year is available.
+TRAFFIC_COUNTER_END_YEAR = 2022
+ECO_COUNTER_START_YEAR = 2020
+LAM_COUNTER_START_YEAR = 2010
+
+
+TRAFFIC_COUNTER = "TC"
+ECO_COUNTER = "EC"
+LAM_COUNTER = "LC"
+CSV_DATA_SOURCES = (
+    (TRAFFIC_COUNTER, "TrafficCounter"),
+    (ECO_COUNTER, "EcoCounter"),
+    (LAM_COUNTER, "LamCounter"),
+)
+COUNTER_START_YEARS = {
+    ECO_COUNTER: ECO_COUNTER_START_YEAR,
+    TRAFFIC_COUNTER: TRAFFIC_COUNTER_START_YEAR,
+    LAM_COUNTER: LAM_COUNTER_START_YEAR,
+}

From 62ce8953e7280183e37a638c3a8f684ad84a20d6 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 12:49:18 +0200
Subject: [PATCH 14/81] Import constants from constants.py

---
 eco_counter/management/commands/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index a449cabc2..f13eb0759 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -10,14 +10,14 @@
 from django.contrib.gis.gdal import DataSource
 from django.contrib.gis.geos import GEOSGeometry, Point
 
-from eco_counter.models import (
+from eco_counter.constants import (
     ECO_COUNTER,
     LAM_COUNTER,
-    Station,
     TRAFFIC_COUNTER,
     TRAFFIC_COUNTER_END_YEAR,
     TRAFFIC_COUNTER_START_YEAR,
 )
+from eco_counter.models import Station
 from eco_counter.tests.test_import_counter_data import TEST_COLUMN_NAMES
 from mobility_data.importers.utils import get_root_dir
 

From 5efe80503cd60243383ec3c32213fde85b66a63b Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 12:49:55 +0200
Subject: [PATCH 15/81] Use statins fixture, refactor

---
 eco_counter/tests/test_import_counter_data.py | 79 +++++--------------
 1 file changed, 21 insertions(+), 58 deletions(-)

diff --git a/eco_counter/tests/test_import_counter_data.py b/eco_counter/tests/test_import_counter_data.py
index b41cce270..e4097a55a 100644
--- a/eco_counter/tests/test_import_counter_data.py
+++ b/eco_counter/tests/test_import_counter_data.py
@@ -11,64 +11,31 @@
 
 import dateutil.parser
 import pytest
-from django.contrib.gis.geos import Point
 from django.core.management import call_command
 
+from eco_counter.constants import ECO_COUNTER, LAM_COUNTER, TRAFFIC_COUNTER
 from eco_counter.models import (
     Day,
     DayData,
-    ECO_COUNTER,
     HourData,
     ImportState,
-    LAM_COUNTER,
     Month,
     MonthData,
     Station,
-    TRAFFIC_COUNTER,
     Week,
     WeekData,
     Year,
     YearData,
 )
 
-TEST_EC_STATION_NAME = "Auransilta"
-TEST_TC_STATION_NAME = "Myllysilta"
-TEST_LC_STATION_NAME = "Tie 8 Raisio"
-ECO_COUNTER_TEST_COLUMN_NAMES = [
-    "Auransilta AK",
-    "Auransilta AP",
-    "Auransilta JK",
-    "Auransilta JP",
-    "Auransilta PK",
-    "Auransilta PP",
-    "Auransilta BK",
-    "Auransilta BP",
-]
-
-TRAFFIC_COUNTER_TEST_COLUMN_NAMES = [
-    "Myllysilta AK",
-    "Myllysilta AP",
-    "Myllysilta PK",
-    "Myllysilta PP",
-    "Myllysilta JK",
-    "Myllysilta JP",
-    "Myllysilta BK",
-    "Myllysilta BP",
-    "Kalevantie 65 BK",
-    "Kalevantie 65 BP",
-    "Hämeentie 18 PK",
-]
-
-LAM_COUNTER_TEST_COLUMN_NAMES = [
-    "Tie 8 Raisio AP",
-    "Tie 8 Raisio AK",
-    "Tie 8 Raisio PP",
-    "Tie 8 Raisio PK",
-    "Tie 8 Raisio JP",
-    "Tie 8 Raisio JK",
-    "Tie 8 Raisio BP",
-    "Tie 8 Raisio BK",
-]
+from .constants import (
+    ECO_COUNTER_TEST_COLUMN_NAMES,
+    LAM_COUNTER_TEST_COLUMN_NAMES,
+    TEST_EC_STATION_NAME,
+    TEST_LC_STATION_NAME,
+    TEST_TC_STATION_NAME,
+    TRAFFIC_COUNTER_TEST_COLUMN_NAMES,
+)
 
 TEST_COLUMN_NAMES = {
     ECO_COUNTER: ECO_COUNTER_TEST_COLUMN_NAMES,
@@ -91,19 +58,14 @@ def import_command(*args, **kwargs):
 
 @pytest.mark.test_import_counter_data
 @pytest.mark.django_db
-def test_import_counter_data():
+def test_import_eco_counter_data(stations):
     """
     In test data, for every 15min the value 1 is set, so the sum for an hour is 4.
     For a day the sum is 96(24*4) and for a week 682(96*7).
     The month sum depends on how  many days the month has,~3000
-    1.1.2020 is used as the starting point thus it is the same
-    starting point as in the real data.
+    1.1.2020 is used as the starting point.
     """
-    Station.objects.create(name="Auransilta", geom=Point(0, 0))
 
-    # for name in ECO_COUNTER_TEST_COLUMN_NAMES:
-    #     if name not in "startTime":
-    #         Station.objects.create(name=name, geom=Point(0,0))
     start_time = dateutil.parser.parse("2020-01-01T00:00")
     end_time = dateutil.parser.parse("2020-02-29T23:45")
     import_command(test_counter=(ECO_COUNTER, start_time, end_time))
@@ -142,7 +104,6 @@ def test_import_counter_data():
     assert day.weekday_number == 0  # First day in week 2 in 2020 is monday
 
     # Test week data
-
     week_data = WeekData.objects.filter(
         week__week_number=1, station__name=TEST_EC_STATION_NAME
     )[0]
@@ -251,9 +212,6 @@ def test_import_counter_data():
     assert year_data.value_pp == (
         jan_month_days * 96 + feb_month_days * 96 + mar_month_days * 96
     )
-    # Test the day has 24hours stored even though in reality it has 23hours.
-    # assert len(HourData.objects.get(day_id=day.id).values_ak) == 24
-
     # Test new year and daylight saving change to "winter time".
     start_time = dateutil.parser.parse("2021-10-01T00:00")
     end_time = dateutil.parser.parse("2021-10-31T23:45")
@@ -317,7 +275,6 @@ def test_import_counter_data():
     start_time = dateutil.parser.parse("2020-12-26T00:00")
     end_time = dateutil.parser.parse("2021-01-17T23:45")
     import_command(test_counter=(ECO_COUNTER, start_time, end_time))
-
     weeks = Week.objects.filter(week_number=53, years__year_number=2020)
     assert len(weeks) == num_ec_stations
     # 4 days in 2020
@@ -327,12 +284,17 @@ def test_import_counter_data():
     assert weeks[0].days.count() == 3
     weeks = Week.objects.filter(week_number=1, years__year_number=2021)
     assert len(weeks) == num_ec_stations
-
     assert weeks[0].days.count() == 7
     weeks = Week.objects.filter(week_number=2, years__year_number=2021)
     assert len(weeks) == num_ec_stations
     assert weeks[0].days.count() == 7
+    # Test that exacly one year object is created for every station in 2020
+    assert Year.objects.filter(year_number=2020).count() == num_ec_stations
 
+
+@pytest.mark.test_import_counter_data
+@pytest.mark.django_db
+def test_import_traffic_counter_data(stations):
     # Test importing of Traffic Counter
     start_time = dateutil.parser.parse("2020-01-01T00:00")
     end_time = dateutil.parser.parse("2020-02-29T23:45")
@@ -342,7 +304,6 @@ def test_import_counter_data():
     assert state.current_year_number == 2020
     assert state.current_month_number == 2
     test_station = Station.objects.get(name=TEST_TC_STATION_NAME)
-    assert test_station
     hour_data = HourData.objects.get(
         station__name=TEST_TC_STATION_NAME, day__date=start_time
     )
@@ -389,6 +350,7 @@ def test_import_counter_data():
     month = Month.objects.get(
         station__name=TEST_TC_STATION_NAME, month_number=2, year__year_number=2020
     )
+
     num_month_days = month.days.count()
     feb_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
     assert num_month_days == feb_month_days
@@ -397,6 +359,8 @@ def test_import_counter_data():
     assert month_data.value_pk == feb_month_days * 96
     assert month_data.value_pt == feb_month_days * 96 * 2
     # Test traffic counter year data
+    jan_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
+
     year_data = YearData.objects.get(
         station__name=TEST_TC_STATION_NAME, year__year_number=2020
     )
@@ -490,8 +454,7 @@ def test_import_counter_data():
         == 1
     )
 
-    # Test that exacly one year object is created for every station in 2020
     assert (
         Year.objects.filter(year_number=2020).count()
-        == num_ec_stations + num_tc_stations + num_lc_stations
+        == num_tc_stations + num_lc_stations
     )

From 77a9df215a9088e9df5322dbb104de261ce4a857 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 20 Feb 2023 12:50:20 +0200
Subject: [PATCH 16/81] Refactor

---
 eco_counter/management/commands/import_counter_data.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 8de3298fa..af3e9c2a4 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -275,7 +275,7 @@ def save_hours(self, df, stations):
                 year_number, month_number, day_number, _ = index
                 if day_number % 20 == 0:
                     logger.info(
-                        f"Saved hour data for {day_number-1} {day_number} days of year {year_number}"
+                        f"Saved hour data for {day_number-1} days of year {year_number}"
                     )
                 if not prev_day:
                     prev_day = day_number
@@ -417,7 +417,6 @@ def handle(self, *args, **options):
             csv_data = gen_eco_counter_test_csv(
                 test_dataframe.keys(), start_time, end_time
             )
-            breakpoint()
             self.save_observations(
                 csv_data,
                 start_time,
@@ -476,7 +475,6 @@ def handle(self, *args, **options):
                 self.save_observations(
                     csv_data,
                     start_time,
-                    # csv_data.keys(),
                     csv_data_source=counter,
                 )
                 # Try to Free memory

From ebf0ccf82afd99c966a0dab0d361e12313826a8d Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 21 Feb 2023 11:35:58 +0200
Subject: [PATCH 17/81] Fix logging, comments and some refactoring

---
 .../commands/import_counter_data.py           | 115 ++++++++----------
 1 file changed, 50 insertions(+), 65 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index af3e9c2a4..2bf00c29f 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -1,26 +1,6 @@
 """
 Usage:
 see README.md
-
-Brief explanation of the import alogithm:
-1. Import the stations.
-2. Read the csv file as a pandas DataFrame.
-3. Reads the year and month from the ImportState.
-4. Set the import to start from that year and month, the import always begins
- from the first day and time 00:00:00 of the month in state, i.e. the longest
- timespan that is imported is one month and the shortest is 15min, depending
- on the import state.
-5. Delete tables(HourData, Day, DayData and Week) that will be repopulated. *
-6. Set the current state to state variables: current_years, currents_months,
- current_weeks, these dictionaries holds references to the model instances.
- Every station has its own state variables and the key is the name of the station.
-9. Finally store all data in states that has not been saved.
-10. Save import state.
-
-* If executed with the --init-tables flag, the imports will start from the beginning
-of the .csv file, 1.1.2020. for the eco counter , 1.1.2015 for the traffic counter and
-1.1.2010 for the lam counter.
-
 """
 
 import gc
@@ -147,6 +127,7 @@ def get_values(self, sum_series, station_name):
         return values
 
     def save_years(self, df, stations):
+        logger.info("Saving years...")
         years = df.groupby(df.index.year)
         for index, row in years:
             logger.info(f"Saving year {index}")
@@ -160,7 +141,7 @@ def save_years(self, df, stations):
                 self.save_values(values, year_data)
 
     def save_months(self, df, stations):
-        # Save month data
+        logger.info("Saving months...")
         months = df.groupby([df.index.year, df.index.month])
         for index, row in months:
             year_number, month_number = index
@@ -186,12 +167,10 @@ def save_current_year(self, stations, year_number, end_month_number):
                 station=station, year_number=year_number
             )
             year_data, _ = YearData.objects.get_or_create(station=station, year=year)
-
             for station_types in self.STATION_TYPES:
                 setattr(year_data, f"value_{station_types[0]}", 0)
                 setattr(year_data, f"value_{station_types[1]}", 0)
                 setattr(year_data, f"value_{station_types[2]}", 0)
-
             for month_number in range(1, end_month_number + 1):
                 month, _ = Month.objects.get_or_create(
                     station=station, year=year, month_number=month_number
@@ -208,11 +187,11 @@ def save_current_year(self, stations, year_number, end_month_number):
             year_data.save()
 
     def save_weeks(self, df, stations):
-
+        logger.info("Saving weeks...")
         weeks = df.groupby([df.index.year, df.index.isocalendar().week])
         for index, row in weeks:
             year_number, week_number = index
-            logger.info(f"Saving week {week_number} of year {year_number}")
+            logger.info(f"Saving week number {week_number} of year {year_number}")
             sum_series = row.sum()
             for station in stations:
                 year = Year.objects.get(station=station, year_number=year_number)
@@ -231,14 +210,14 @@ def save_weeks(self, df, stations):
                 self.save_values(values, week_data)
 
     def save_days(self, df, stations):
+        logger.info("Saving days...")
         days = df.groupby(
             [df.index.year, df.index.month, df.index.isocalendar().week, df.index.day]
         )
-        logger.info("Saving days")
+        prev_week_number = None
         for index, row in days:
             year_number, month_number, week_number, day_number = index
-            if day_number % 20 == 0:
-                logger.info(f"Saved {day_number-1} days of year {year_number}")
+
             date = datetime(year_number, month_number, day_number)
             sum_series = row.sum()
             for station in stations:
@@ -246,7 +225,6 @@ def save_days(self, df, stations):
                 month = Month.objects.get(
                     station=station, year=year, month_number=month_number
                 )
-
                 week = Week.objects.get(
                     station=station, years=year, week_number=week_number
                 )
@@ -258,34 +236,36 @@ def save_days(self, df, stations):
                     month=month,
                     week=week,
                 )
-
                 values = self.get_values(sum_series, station.name)
                 day_data, _ = DayData.objects.get_or_create(station=station, day=day)
                 self.save_values(values, day_data)
+            if not prev_week_number or prev_week_number != week_number:
+                prev_week_number = week_number
+                logger.info(f"Saved days for week {week_number} of year {year_number}")
 
     def save_hours(self, df, stations):
-        # Save hour data
+        logger.info("Saving hours...")
         hours = df.groupby([df.index.year, df.index.month, df.index.day, df.index.hour])
-        for station in stations:
-            prev_day = None
-            prev_month = None
+        for i_station, station in enumerate(stations):
+            prev_day_number = None
+            prev_month_number = None
             values = {k: [] for k in self.ALL_TYPE_DIRS}
             for index, row in hours:
                 sum_series = row.sum()
                 year_number, month_number, day_number, _ = index
-                if day_number % 20 == 0:
-                    logger.info(
-                        f"Saved hour data for {day_number-1} days of year {year_number}"
-                    )
-                if not prev_day:
-                    prev_day = day_number
-                if not prev_month:
-                    prev_month = month_number
-                if day_number != prev_day or month_number != prev_month:
-                    if month_number != prev_month:
-                        prev_day = day_number
+                if not prev_day_number:
+                    prev_day_number = day_number
+                if not prev_month_number:
+                    prev_month_number = month_number
+
+                if day_number != prev_day_number or month_number != prev_month_number:
+                    """
+                    If day or month changed. Save the hours for the day and clear the values dict.
+                    """
+                    if month_number != prev_month_number:
+                        prev_day_number = day_number
                     day = Day.objects.get(
-                        date=datetime(year_number, month_number, prev_day),
+                        date=datetime(year_number, month_number, prev_day_number),
                         station=station,
                     )
                     hour_data, _ = HourData.objects.get_or_create(
@@ -295,9 +275,15 @@ def save_hours(self, df, stations):
                         setattr(hour_data, f"values_{td.lower()}", values[td])
                     hour_data.save()
                     values = {k: [] for k in self.ALL_TYPE_DIRS}
-                    prev_day = day_number
-                    prev_month = month_number
+                    # output logger only when last station is saved
+                    if i_station == len(stations) - 1:
+                        logger.info(
+                            f"Saved hour data for day {prev_day_number}, month {prev_month_number} year {year_number}"
+                        )
+                    prev_day_number = day_number
+                    prev_month_number = month_number
                 else:
+                    # Add data to values dict for an hour
                     for station_types in self.STATION_TYPES:
                         for i in range(3):
                             if i < 2:
@@ -309,9 +295,9 @@ def save_hours(self, df, stations):
                                 val = sum_series.get(p_key, 0) + sum_series.get(
                                     k_key, 0
                                 )
-
                             values_key = station_types[i].upper()
                             values[values_key].append(val)
+        breakpoint()
 
     def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
         import_state = ImportState.objects.get(csv_data_source=csv_data_source)
@@ -320,15 +306,15 @@ def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
             station
             for station in Station.objects.filter(csv_data_source=csv_data_source)
         ]
-
         df = csv_data
         df["Date"] = pd.to_datetime(df["startTime"], format="%Y-%m-%dT%H:%M")
         df = df.drop("startTime", axis=1)
         df = df.set_index("Date")
+        # Fill missing cells with the value 0
         df = df.fillna(0)
         # Set negative numbers to 0
         df = df.clip(lower=0)
-
+        # Set values higher than ERRORNEOUS_VALUES_THRESHOLD to 0
         df[df > ERRORNEOUS_VALUE_THRESHOLD] = 0
         if not import_state.current_year_number:
             self.save_years(df, stations)
@@ -340,12 +326,10 @@ def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
         self.save_weeks(df, stations)
         self.save_days(df, stations)
         self.save_hours(df, stations)
-
         end_date = df.index[-1]
         import_state.current_year_number = end_date.year
         import_state.current_month_number = end_date.month
         import_state.save()
-
         logger.info(f"Imported observations until:{str(end_date)}")
 
     def add_arguments(self, parser):
@@ -436,17 +420,6 @@ def handle(self, *args, **options):
                 import_state = ImportState.objects.filter(
                     csv_data_source=counter
                 ).first()
-                if counter == LAM_COUNTER:
-                    if not import_state.current_year_number:
-                        start_time = f"{COUNTER_START_YEARS[counter]}-01-01"
-                    csv_data = get_lam_counter_csv(start_time)
-                elif counter == ECO_COUNTER:
-                    csv_data = get_eco_counter_csv()
-                elif counter == TRAFFIC_COUNTER:
-                    csv_data = get_traffic_counter_csv(
-                        start_year=import_state.current_year_number
-                    )
-                # If state override start_time
                 if (
                     import_state.current_year_number
                     and import_state.current_month_number
@@ -455,11 +428,24 @@ def handle(self, *args, **options):
                         year=import_state.current_year_number,
                         month=import_state.current_month_number,
                     )
+                else:
+                    start_time = f"{COUNTER_START_YEARS[counter]}-01-01"
+
                 start_time = dateutil.parser.parse(start_time)
                 start_time = self.TIMEZONE.localize(start_time)
                 # The timeformat for the input data is : 2020-03-01T00:00
                 # Convert starting time to input datas timeformat
                 start_time_string = start_time.strftime("%Y-%m-%dT%H:%M")
+
+                if counter == LAM_COUNTER:
+                    csv_data = get_lam_counter_csv(start_time.date())
+                elif counter == ECO_COUNTER:
+                    csv_data = get_eco_counter_csv()
+                elif counter == TRAFFIC_COUNTER:
+                    csv_data = get_traffic_counter_csv(
+                        start_year=import_state.current_year_number
+                    )
+
                 start_index = csv_data.index[
                     csv_data[TIMESTAMP_COL_NAME] == start_time_string
                 ].values[0]
@@ -467,7 +453,6 @@ def handle(self, *args, **options):
                 # show time.
                 if counter == LAM_COUNTER:
                     logger.info(f"Starting saving observations at time:{start_time}")
-
                 else:
                     logger.info(f"Starting saving observations at index:{start_index}")
 

From 3f93259ebb0194935b5216a15d1e6fa933f78bb7 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 21 Feb 2023 11:58:42 +0200
Subject: [PATCH 18/81] Fix LAM counter tests

---
 eco_counter/tests/test_import_counter_data.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/eco_counter/tests/test_import_counter_data.py b/eco_counter/tests/test_import_counter_data.py
index e4097a55a..9ac54e761 100644
--- a/eco_counter/tests/test_import_counter_data.py
+++ b/eco_counter/tests/test_import_counter_data.py
@@ -367,6 +367,12 @@ def test_import_traffic_counter_data(stations):
     assert year_data.value_bk == (jan_month_days + feb_month_days) * 24 * 4
     assert year_data.value_bp == (jan_month_days + feb_month_days) * 24 * 4
     assert year_data.value_bt == (jan_month_days + feb_month_days) * 24 * 4 * 2
+    assert Year.objects.filter(year_number=2020).count() == num_tc_stations
+
+
+@pytest.mark.test_import_counter_data
+@pytest.mark.django_db
+def test_import_lam_counter_data(stations):
     # Test lam counter data and year change
     start_time = dateutil.parser.parse("2019-12-01T00:00")
     end_time = dateutil.parser.parse("2020-01-31T23:45")
@@ -394,8 +400,8 @@ def test_import_traffic_counter_data(stations):
     assert hour_data.values_bk == res
     assert hour_data.values_bp == res
     assert hour_data.values_bt == res_tot
-    # 2019 December 2019 has 5 weeks and January 2020 has 5 week = 10 weeks
-    assert Week.objects.filter(station__name=TEST_LC_STATION_NAME).count() == 10
+    # 2019 December 2019 has 6 weeks(48,49,50,51,52 and 1) and January 2020 has 5 week = 11 weeks
+    assert Week.objects.filter(station__name=TEST_LC_STATION_NAME).count() == 11
     # 5 days of week 5 in 2020 is imported, e.g. 4*24*5 = 480
     assert (
         WeekData.objects.filter(station__name=TEST_LC_STATION_NAME, week__week_number=5)
@@ -453,8 +459,4 @@ def test_import_traffic_counter_data(stations):
         ).count()
         == 1
     )
-
-    assert (
-        Year.objects.filter(year_number=2020).count()
-        == num_tc_stations + num_lc_stations
-    )
+    assert Year.objects.filter(year_number=2020).count() == num_lc_stations

From 073ccd244f37dbea64e6b4d545c1af455b773f39 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 21 Feb 2023 11:59:07 +0200
Subject: [PATCH 19/81] Add constants from utils

---
 eco_counter/constants.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 4b7cdadf4..4181d352a 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -1,3 +1,5 @@
+from django.conf import settings
+
 TRAFFIC_COUNTER_START_YEAR = 2015
 # Manually define the end year, as the source data comes from the page
 # defined in env variable TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL.
@@ -20,3 +22,40 @@
     TRAFFIC_COUNTER: TRAFFIC_COUNTER_START_YEAR,
     LAM_COUNTER: LAM_COUNTER_START_YEAR,
 }
+
+TIMESTAMP_COL_NAME = "startTime"
+TRAFFIC_COUNTER_METADATA_GEOJSON = "traffic_counter_metadata.geojson"
+# LAM stations located in the municipalities list are included.
+LAM_STATION_MUNICIPALITIES = ["Turku", "Raisio", "Kaarina", "Lieto"]
+
+LAM_STATIONS_API_FETCH_URL = (
+    settings.LAM_COUNTER_API_BASE_URL
+    + "?api=liikennemaara&tyyppi=h&pvm={start_date}&loppu={end_date}"
+    + "&lam_type=option1&piste={id}&luokka=kaikki&suunta={direction}&sisallytakaistat=0"
+)
+# Maps the direction of the traffic of station, (P)oispäin or (K)eskustaan päin)
+LAM_STATIONS_DIRECTION_MAPPINGS = {
+    "1_Piikkiö": "P",
+    "1_Naantali": "P",
+    "2_Naantali": "K",
+    "1_Turku": "K",
+    "2_Turku": "K",
+    "2_Helsinki": "P",
+    "1_Suikkila.": "K",
+    "2_Artukainen.": "P",
+    "1_Vaasa": "P",
+    "1_Kuusisto": "P",
+    "2_Kaarina": "K",
+    "1_Tampere": "P",
+    "1_Hämeenlinna": "P",
+}
+
+keys = [k for k in range(TRAFFIC_COUNTER_START_YEAR, TRAFFIC_COUNTER_END_YEAR + 1)]
+# Create a dict where the years to be importer are keys and the value is the url of the csv data.
+# e.g. {2015, "https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/2015_laskenta_juha.csv"}
+TRAFFIC_COUNTER_CSV_URLS = dict(
+    [
+        (k, f"{settings.TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL}{k}_laskenta_juha.csv")
+        for k in keys
+    ]
+)

From 496a2837516f9b71bda5f586bdd9c4ab6173685e Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 21 Feb 2023 12:00:30 +0200
Subject: [PATCH 20/81] Move constants to constants.py

---
 eco_counter/management/commands/utils.py | 43 ++++--------------------
 1 file changed, 6 insertions(+), 37 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index f13eb0759..37c66caa8 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -13,50 +13,19 @@
 from eco_counter.constants import (
     ECO_COUNTER,
     LAM_COUNTER,
+    LAM_STATION_MUNICIPALITIES,
+    LAM_STATIONS_API_FETCH_URL,
+    LAM_STATIONS_DIRECTION_MAPPINGS,
+    TIMESTAMP_COL_NAME,
     TRAFFIC_COUNTER,
-    TRAFFIC_COUNTER_END_YEAR,
-    TRAFFIC_COUNTER_START_YEAR,
+    TRAFFIC_COUNTER_CSV_URLS,
+    TRAFFIC_COUNTER_METADATA_GEOJSON,
 )
 from eco_counter.models import Station
 from eco_counter.tests.test_import_counter_data import TEST_COLUMN_NAMES
 from mobility_data.importers.utils import get_root_dir
 
 logger = logging.getLogger("eco_counter")
-TIMESTAMP_COL_NAME = "startTime"
-TRAFFIC_COUNTER_METADATA_GEOJSON = "traffic_counter_metadata.geojson"
-# LAM stations located in the municipalities list are included.
-LAM_STATION_MUNICIPALITIES = ["Turku", "Raisio", "Kaarina", "Lieto"]
-
-LAM_STATIONS_API_FETCH_URL = (
-    settings.LAM_COUNTER_API_BASE_URL
-    + "?api=liikennemaara&tyyppi=h&pvm={start_date}&loppu={end_date}"
-    + "&lam_type=option1&piste={id}&luokka=kaikki&suunta={direction}&sisallytakaistat=0"
-)
-LAM_STATIONS_DIRECTION_MAPPINGS = {
-    "1_Piikkiö": "P",
-    "1_Naantali": "P",
-    "2_Naantali": "K",
-    "1_Turku": "K",
-    "2_Turku": "K",
-    "2_Helsinki": "P",
-    "1_Suikkila.": "K",
-    "2_Artukainen.": "P",
-    "1_Vaasa": "P",
-    "1_Kuusisto": "P",
-    "2_Kaarina": "K",
-    "1_Tampere": "P",
-    "1_Hämeenlinna": "P",
-}
-
-keys = [k for k in range(TRAFFIC_COUNTER_START_YEAR, TRAFFIC_COUNTER_END_YEAR + 1)]
-# Create a dict where the years to be importer are keys and the value is the url of the csv data.
-# e.g. {2015, "https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/2015_laskenta_juha.csv"}
-TRAFFIC_COUNTER_CSV_URLS = dict(
-    [
-        (k, f"{settings.TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL}{k}_laskenta_juha.csv")
-        for k in keys
-    ]
-)
 
 
 class LAMStation:

From b247c0074f8f6e37526196a34dd236f9a1e95cc6 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 21 Feb 2023 12:53:50 +0200
Subject: [PATCH 21/81] Fix traffic counter tests

---
 eco_counter/tests/test_import_counter_data.py | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/eco_counter/tests/test_import_counter_data.py b/eco_counter/tests/test_import_counter_data.py
index 9ac54e761..0a1856e80 100644
--- a/eco_counter/tests/test_import_counter_data.py
+++ b/eco_counter/tests/test_import_counter_data.py
@@ -303,7 +303,6 @@ def test_import_traffic_counter_data(stations):
     state = ImportState.objects.get(csv_data_source=TRAFFIC_COUNTER)
     assert state.current_year_number == 2020
     assert state.current_month_number == 2
-    test_station = Station.objects.get(name=TEST_TC_STATION_NAME)
     hour_data = HourData.objects.get(
         station__name=TEST_TC_STATION_NAME, day__date=start_time
     )
@@ -347,20 +346,25 @@ def test_import_traffic_counter_data(stations):
     assert week_data.value_bk == 672  # 96*7
     assert week_data.value_bt == 672 * 2
     # Test traffic counter month data
-    month = Month.objects.get(
+    feb_month = Month.objects.get(
         station__name=TEST_TC_STATION_NAME, month_number=2, year__year_number=2020
     )
-
-    num_month_days = month.days.count()
-    feb_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
+    num_month_days = feb_month.days.count()
+    feb_month_days = calendar.monthrange(
+        feb_month.year.year_number, feb_month.month_number
+    )[1]
     assert num_month_days == feb_month_days
-    month_data = MonthData.objects.get(month=month)
+    month_data = MonthData.objects.get(month=feb_month)
     assert month_data.value_pp == feb_month_days * 96
     assert month_data.value_pk == feb_month_days * 96
     assert month_data.value_pt == feb_month_days * 96 * 2
     # Test traffic counter year data
-    jan_month_days = calendar.monthrange(month.year.year_number, month.month_number)[1]
-
+    jan_month = Month.objects.get(
+        station__name=TEST_TC_STATION_NAME, month_number=1, year__year_number=2020
+    )
+    jan_month_days = calendar.monthrange(
+        jan_month.year.year_number, jan_month.month_number
+    )[1]
     year_data = YearData.objects.get(
         station__name=TEST_TC_STATION_NAME, year__year_number=2020
     )

From a1a15ab02ab4f6455020b1b0f42a56c208dc634f Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 22 Feb 2023 10:07:41 +0200
Subject: [PATCH 22/81] Add migration that sets Importstate current year and
 month to nullable

---
 ...ate_current_month_and_year_to_nullables.py | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py

diff --git a/eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py b/eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py
new file mode 100644
index 000000000..fc0118d47
--- /dev/null
+++ b/eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py
@@ -0,0 +1,30 @@
+# Generated by Django 4.1.2 on 2023-02-15 14:12
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("eco_counter", "0012_set_static_default_year_year_number"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="importstate",
+            name="current_month_number",
+            field=models.PositiveSmallIntegerField(
+                null=True,
+                validators=[
+                    django.core.validators.MinValueValidator(1),
+                    django.core.validators.MaxValueValidator(12),
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="importstate",
+            name="current_year_number",
+            field=models.PositiveSmallIntegerField(null=True),
+        ),
+    ]

From 7d7bbb60c2af8d060460035e864070f6dae745df Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 22 Feb 2023 10:10:48 +0200
Subject: [PATCH 23/81] Refactor station creation to one function

---
 eco_counter/management/commands/utils.py | 213 +++++++++++++++--------
 1 file changed, 142 insertions(+), 71 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 37c66caa8..1f2e6bc26 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -11,6 +11,7 @@
 from django.contrib.gis.geos import GEOSGeometry, Point
 
 from eco_counter.constants import (
+    COUNTERS,
     ECO_COUNTER,
     LAM_COUNTER,
     LAM_STATION_MUNICIPALITIES,
@@ -30,6 +31,8 @@
 
 class LAMStation:
     def __init__(self, feature):
+        if feature["municipality"].as_string() not in LAM_STATION_MUNICIPALITIES:
+            self.active = False
         self.lam_id = feature["tmsNumber"].as_int()
         names = json.loads(feature["names"].as_string())
         self.name = names["fi"]
@@ -42,6 +45,42 @@ def __init__(self, feature):
         self.geom.transform(settings.DEFAULT_SRID)
 
 
+class EcoCounterStation:
+    def __init__(self, feature):
+        self.name = feature["properties"]["Nimi"]
+        lon = feature["geometry"]["coordinates"][0]
+        lat = feature["geometry"]["coordinates"][1]
+        self.geom = Point(lon, lat, srid=4326)
+        self.geom.transform(settings.DEFAULT_SRID)
+
+
+class TrafficCounterStation:
+    def __init__(self, feature):
+        self.name = feature["Osoite_fi"].as_string()
+        self.name_sv = feature["Osoite_sv"].as_string()
+        self.name_en = feature["Osoite_en"].as_string()
+        geom = GEOSGeometry(feature.geom.wkt, srid=feature.geom.srid)
+        geom.transform(settings.DEFAULT_SRID)
+        self.geom = geom
+
+
+class ObservationStation(LAMStation, EcoCounterStation, TrafficCounterStation):
+    def __init__(self, csv_data_source, feature):
+        self.csv_data_source = csv_data_source
+        self.name = None
+        self.name_sv = None
+        self.name_en = None
+        self.geom = None
+        self.lam_id = None
+        match csv_data_source:
+            case COUNTERS.LAM_COUNTER:
+                LAMStation.__init__(self, feature)
+            case COUNTERS.ECO_COUNTER:
+                EcoCounterStation.__init__(self, feature)
+            case COUNTERS.TRAFFIC_COUNTER:
+                TrafficCounterStation.__init__(self, feature)
+
+
 def get_traffic_counter_metadata_data_layer():
     meta_file = f"{get_root_dir()}/eco_counter/data/{TRAFFIC_COUNTER_METADATA_GEOJSON}"
     return DataSource(meta_file)[0]
@@ -246,51 +285,25 @@ def get_lam_counter_csv(start_date):
     return data_frame
 
 
-def save_lam_counter_stations():
+def get_lam_counter_stations():
+    stations = []
     data_layer = DataSource(settings.LAM_COUNTER_STATIONS_URL)[0]
-    saved = 0
     for feature in data_layer:
         if feature["municipality"].as_string() in LAM_STATION_MUNICIPALITIES:
-            station_obj = LAMStation(feature)
-            station, _ = Station.objects.get_or_create(
-                name=station_obj.name,
-                csv_data_source=LAM_COUNTER,
-                lam_id=station_obj.lam_id,
-                geom=station_obj.geom,
-            )
-            station.name_sv = station_obj.name_sv
-            station.name_en = station_obj.name_en
-            station.save()
-            saved += 1
-    logger.info(f"Saved {saved} LAM Counter stations.")
-
-
-def save_traffic_counter_stations():
-    """
-    Saves the stations defined in the metadata to Station table.
-    """
-    saved = 0
+            stations.append(ObservationStation(LAM_COUNTER, feature))
+    return stations
+
+
+def get_eco_counter_stations():
+    stations = []
     data_layer = get_traffic_counter_metadata_data_layer()
     for feature in data_layer:
-        name = feature["Osoite_fi"].as_string()
-        name_sv = feature["Osoite_sv"].as_string()
-        name_en = feature["Osoite_en"].as_string()
-        if Station.objects.filter(name=name).exists():
-            continue
-        station = Station()
-        station.name = name
-        station.name_sv = name_sv
-        station.name_en = name_en
-        station.csv_data_source = TRAFFIC_COUNTER
-        geom = GEOSGeometry(feature.geom.wkt, srid=feature.geom.srid)
-        geom.transform(settings.DEFAULT_SRID)
-        station.geom = geom
-        station.save()
-        saved += 1
-    logger.info(f"Saved {saved} Traffic Counter stations.")
+        stations.append(ObservationStation(ECO_COUNTER, feature))
+    return stations
 
 
-def save_eco_counter_stations():
+def get_traffic_counter_stations():
+    stations = []
     response = requests.get(settings.ECO_COUNTER_STATIONS_URL)
     assert (
         response.status_code == 200
@@ -299,21 +312,98 @@ def save_eco_counter_stations():
     )
     response_json = response.json()
     features = response_json["features"]
-    saved = 0
     for feature in features:
-        station = Station()
-        name = feature["properties"]["Nimi"]
-        if not Station.objects.filter(name=name).exists():
-            station.name = name
-            station.csv_data_source = ECO_COUNTER
-            lon = feature["geometry"]["coordinates"][0]
-            lat = feature["geometry"]["coordinates"][1]
-            point = Point(lon, lat, srid=4326)
-            point.transform(settings.DEFAULT_SRID)
-            station.geom = point
-            station.save()
-            saved += 1
-    logger.info(f"Saved {saved} Eco Counter stations.")
+        stations.append(ObservationStation(TRAFFIC_COUNTER, feature))
+    return stations
+
+
+def save_stations(csv_data_source):
+    stations = []
+    num_created = 0
+    match csv_data_source:
+        case COUNTERS.LAM_COUNTER:
+            stations = get_lam_counter_stations()
+        case COUNTERS.ECO_COUNTER:
+            station = get_eco_counter_stations()
+        case COUNTERS.TRAFFIC_COUNTER:
+            station = get_traffic_counter_stations()
+    object_ids = list(
+        Station.objects.filter(csv_data_source=csv_data_source).values_list(
+            "id", flat=True
+        )
+    )
+    for station in stations:
+        obj, created = Station.objects.get_or_create(
+            name=station.name,
+            name_sv=station.name_sv,
+            name_en=station.name_en,
+            geom=station.geom,
+            lam_id=station.lam_id,
+        )
+        if obj.id in object_ids:
+            object_ids.remove(obj.id)
+        if created:
+            num_created += 1
+
+    Station.objects.filter(id__in=object_ids).delete()
+    logger.info(
+        f"Deleted {len(object_ids)} obsolete Stations for counter {csv_data_source}"
+    )
+    num_stations = Station.objects.filter(csv_data_source=csv_data_source).count()
+    logger.info(
+        f"Created {num_created} Stations of total {num_stations} Station for conter {csv_data_source}."
+    )
+
+
+# def save_traffic_counter_stations():
+#     """
+#     Saves the stations defined in the metadata to Station table.
+#     """
+#     saved = 0
+#     data_layer = get_traffic_counter_metadata_data_layer()
+#     for feature in data_layer:
+#         name = feature["Osoite_fi"].as_string()
+#         name_sv = feature["Osoite_sv"].as_string()
+#         name_en = feature["Osoite_en"].as_string()
+#         if Station.objects.filter(name=name).exists():
+#             continue
+#         station = Station()
+#         station.name = name
+#         station.name_sv = name_sv
+#         station.name_en = name_en
+#         station.csv_data_source = TRAFFIC_COUNTER
+#         geom = GEOSGeometry(feature.geom.wkt, srid=feature.geom.srid)
+#         geom.transform(settings.DEFAULT_SRID)
+#         station.geom = geom
+#         station.save()
+#         saved += 1
+#     logger.info(f"Saved {saved} Traffic Counter stations.")
+
+
+# def save_eco_counter_stations():
+#     response = requests.get(settings.ECO_COUNTER_STATIONS_URL)
+#     assert (
+#         response.status_code == 200
+#     ), "Fetching stations from {} , status code {}".format(
+#         settings.ECO_COUNTER_STATIONS_URL, response.status_code
+#     )
+#     response_json = response.json()
+#     features = response_json["features"]
+#     saved = 0
+#     for feature in features:
+#         station = Station()
+#         name = feature["properties"]["Nimi"]
+#         if not Station.objects.filter(name=name).exists():
+#             station.name = name
+#             station.csv_data_source = ECO_COUNTER
+#             lon = feature["geometry"]["coordinates"][0]
+#             lat = feature["geometry"]["coordinates"][1]
+#             point = Point(lon, lat, srid=4326)
+#             point.transform(settings.DEFAULT_SRID)
+#             station.geom = point
+#             station.save()
+#             saved += 1
+#     logger.info(f"Saved {saved} Eco Counter stations.")
 
 
 def get_test_dataframe(counter):
@@ -326,25 +416,6 @@ def get_test_dataframe(counter):
     return pd.DataFrame(columns=TEST_COLUMN_NAMES[counter])
 
 
-# def gen_eco_counter_test_csv(keys, start_time, end_time):
-#     """
-#     Generates test data for a given timespan,
-#     for every row (15min) the value 1 is set.
-#     """
-#     df = pd.DataFrame(columns=keys)
-#     df.keys = keys
-#     cur_time = start_time
-#     c = 0
-#     while cur_time <= end_time:
-#         # Add value to all keys(sensor stations)
-#         vals = [1 for x in range(len(keys) - 1)]
-#         vals.insert(0, str(cur_time))
-#         df.loc[c] = vals
-#         cur_time = cur_time + timedelta(minutes=15)
-#         c += 1
-#     return df
-
-
 def gen_eco_counter_test_csv(
     columns, start_time, end_time, time_stamp_column="startTime"
 ):

From 706aea8dce4f4225542a58451c15f4ca24eead2e Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 22 Feb 2023 10:12:37 +0200
Subject: [PATCH 24/81] Add COUNTERS namespace for match/case blocks

---
 eco_counter/constants.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 4181d352a..4b3203ec8 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -1,3 +1,5 @@
+import types
+
 from django.conf import settings
 
 TRAFFIC_COUNTER_START_YEAR = 2015
@@ -12,6 +14,13 @@
 TRAFFIC_COUNTER = "TC"
 ECO_COUNTER = "EC"
 LAM_COUNTER = "LC"
+
+COUNTERS = types.SimpleNamespace()
+COUNTERS.TRAFFIC_COUNTER = TRAFFIC_COUNTER
+COUNTERS.ECO_COUNTER = ECO_COUNTER
+COUNTERS.LAM_COUNTER = LAM_COUNTER
+
+
 CSV_DATA_SOURCES = (
     (TRAFFIC_COUNTER, "TrafficCounter"),
     (ECO_COUNTER, "EcoCounter"),

From fb0b338f763caead74b8b4ddd4b3f9f0252dd48d Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 22 Feb 2023 10:44:33 +0200
Subject: [PATCH 25/81] Add/fix comments, refactor

---
 .../commands/import_counter_data.py           | 79 +++++++++++--------
 1 file changed, 45 insertions(+), 34 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 2bf00c29f..16ab86263 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -1,4 +1,6 @@
 """
+To run test:
+pytest -m test_import_counter_data
 Usage:
 see README.md
 """
@@ -15,9 +17,11 @@
 
 from eco_counter.constants import (
     COUNTER_START_YEARS,
+    COUNTERS,
     ECO_COUNTER,
     LAM_COUNTER,
     TRAFFIC_COUNTER,
+    TRAFFIC_COUNTER_START_YEAR,
 )
 from eco_counter.models import (
     Day,
@@ -39,9 +43,7 @@
     get_lam_counter_csv,
     get_test_dataframe,
     get_traffic_counter_csv,
-    save_eco_counter_stations,
-    save_lam_counter_stations,
-    save_traffic_counter_stations,
+    save_stations,
     TIMESTAMP_COL_NAME,
 )
 
@@ -65,6 +67,20 @@ class Command(BaseCommand):
     COUNTERS = [ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER]
     COUNTER_CHOICES_STR = f"{ECO_COUNTER}, {TRAFFIC_COUNTER} and {LAM_COUNTER}"
     TIMEZONE = pytz.timezone("Europe/Helsinki")
+    """
+    Movement types:
+    (A)uto, car
+    (P)yörä, bicycle
+    (J)alankulkija, pedestrian
+    (B)ussi, bus
+    Direction types:
+    (K)eskustaan päin, towards the center
+    (P)poispäin keskustasta, away from the center
+    So for the example column with prefix "ap" contains data for cars moving away from the center.
+    The naming convention is derived from the eco-counter source data that was the
+    original data source.
+
+    """
     STATION_TYPES = [
         ("ak", "ap", "at"),
         ("pk", "pp", "pt"),
@@ -72,22 +88,18 @@ class Command(BaseCommand):
         ("bk", "bp", "bt"),
     ]
 
+    TYPE_DIRS = ["AK", "AP", "JK", "JP", "BK", "BP", "PK", "PP"]
+    ALL_TYPE_DIRS = TYPE_DIRS + ["AT", "JT", "BT", "PT"]
+    type_dirs_lower = [TD.lower() for TD in TYPE_DIRS]
+
     def delete_tables(
         self, csv_data_sources=[ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER]
     ):
         for csv_data_source in csv_data_sources:
             for station in Station.objects.filter(csv_data_source=csv_data_source):
                 Year.objects.filter(station=station).delete()
-
             ImportState.objects.filter(csv_data_source=csv_data_source).delete()
 
-    # keskustaan päin
-    # poispäin keskustasta
-    # m molempiin suuntiin
-    TYPE_DIRS = ["AK", "AP", "JK", "JP", "BK", "BP", "PK", "PP"]
-    ALL_TYPE_DIRS = TYPE_DIRS + ["AT", "JT", "BT", "PT"]
-    type_dirs_lower = [TD.lower() for TD in TYPE_DIRS]
-
     def save_values(self, values, dst_obj):
         for station_types in self.STATION_TYPES:
             setattr(dst_obj, f"value_{station_types[0]}", values[station_types[0]])
@@ -100,6 +112,9 @@ def save_values(self, values, dst_obj):
         dst_obj.save()
 
     def add_values(self, values, dst_obj):
+        """
+        Populate values for all movement types and directions for a station.
+        """
         for station_types in self.STATION_TYPES:
             key = f"value_{station_types[0]}"
             k_val = getattr(dst_obj, key, 0) + values[station_types[0]]
@@ -118,7 +133,7 @@ def add_values(self, values, dst_obj):
 
     def get_values(self, sum_series, station_name):
         """
-        Populate values for all movement types and directions for a station.
+        Returns a dict containing the aggregated sum value for every movement type and direction.
         """
         values = {}
         for type_dir in self.TYPE_DIRS:
@@ -135,7 +150,7 @@ def save_years(self, df, stations):
             for station in stations:
                 year, _ = Year.objects.get_or_create(station=station, year_number=index)
                 values = self.get_values(sum_series, station.name)
-                year_data, created = YearData.objects.get_or_create(
+                year_data, _ = YearData.objects.get_or_create(
                     year=year, station=station
                 )
                 self.save_values(values, year_data)
@@ -297,11 +312,10 @@ def save_hours(self, df, stations):
                                 )
                             values_key = station_types[i].upper()
                             values[values_key].append(val)
-        breakpoint()
 
     def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
         import_state = ImportState.objects.get(csv_data_source=csv_data_source)
-        # Populate stations list, used to lookup station relations.
+        # Populate stations list, this is used to set/lookup station relations.
         stations = [
             station
             for station in Station.objects.filter(csv_data_source=csv_data_source)
@@ -317,6 +331,7 @@ def save_observations(self, csv_data, start_time, csv_data_source=ECO_COUNTER):
         # Set values higher than ERRORNEOUS_VALUES_THRESHOLD to 0
         df[df > ERRORNEOUS_VALUE_THRESHOLD] = 0
         if not import_state.current_year_number:
+            # In initial import populate all years.
             self.save_years(df, stations)
         self.save_months(df, stations)
         if import_state.current_year_number:
@@ -382,21 +397,14 @@ def handle(self, *args, **options):
                         csv_data_source=counter,
                     )
                     logger.info(f"Retrieving stations for {counter}.")
-                    if counter == ECO_COUNTER:
-                        save_eco_counter_stations()
-                    elif counter == TRAFFIC_COUNTER:
-                        save_traffic_counter_stations()
-                    elif counter == LAM_COUNTER:
-                        save_lam_counter_stations()
+                    save_stations(counter)
 
         if options["test_counter"]:
             logger.info("Testing eco_counter importer.")
             counter = options["test_counter"][0]
             start_time = options["test_counter"][1]
             end_time = options["test_counter"][2]
-            import_state, created = ImportState.objects.get_or_create(
-                csv_data_source=counter
-            )
+            import_state, _ = ImportState.objects.get_or_create(csv_data_source=counter)
             test_dataframe = get_test_dataframe(counter)
             csv_data = gen_eco_counter_test_csv(
                 test_dataframe.keys(), start_time, end_time
@@ -406,7 +414,7 @@ def handle(self, *args, **options):
                 start_time,
                 csv_data_source=counter,
             )
-        # Import if counters arg or (initial import).
+        # Import if counters arg or initial import.
         if options["counters"] or initial_import_counters:
             if not initial_import_counters:
                 # run with counters argument
@@ -420,6 +428,7 @@ def handle(self, *args, **options):
                 import_state = ImportState.objects.filter(
                     csv_data_source=counter
                 ).first()
+
                 if (
                     import_state.current_year_number
                     and import_state.current_month_number
@@ -436,15 +445,17 @@ def handle(self, *args, **options):
                 # The timeformat for the input data is : 2020-03-01T00:00
                 # Convert starting time to input datas timeformat
                 start_time_string = start_time.strftime("%Y-%m-%dT%H:%M")
-
-                if counter == LAM_COUNTER:
-                    csv_data = get_lam_counter_csv(start_time.date())
-                elif counter == ECO_COUNTER:
-                    csv_data = get_eco_counter_csv()
-                elif counter == TRAFFIC_COUNTER:
-                    csv_data = get_traffic_counter_csv(
-                        start_year=import_state.current_year_number
-                    )
+                match counter:
+                    case COUNTERS.LAM_COUNTER:
+                        csv_data = get_lam_counter_csv(start_time.date())
+                    case COUNTERS.ECO_COUNTER:
+                        csv_data = get_eco_counter_csv()
+                    case COUNTERS.TRAFFIC_COUNTER:
+                        if import_state.current_year_number:
+                            start_year = import_state.current_year_number
+                        else:
+                            start_year = TRAFFIC_COUNTER_START_YEAR
+                        csv_data = get_traffic_counter_csv(start_year=start_year)
 
                 start_index = csv_data.index[
                     csv_data[TIMESTAMP_COL_NAME] == start_time_string

From 225538937acd8c70314e08d5ed9de6c0c69516c9 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 22 Feb 2023 10:46:04 +0200
Subject: [PATCH 26/81] Fix eco-counter and traffic counter station data
 fetching

---
 eco_counter/management/commands/utils.py | 61 ++----------------------
 1 file changed, 5 insertions(+), 56 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 1f2e6bc26..c8f2d41a0 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -294,15 +294,15 @@ def get_lam_counter_stations():
     return stations
 
 
-def get_eco_counter_stations():
+def get_traffic_counter_stations():
     stations = []
     data_layer = get_traffic_counter_metadata_data_layer()
     for feature in data_layer:
-        stations.append(ObservationStation(ECO_COUNTER, feature))
+        stations.append(ObservationStation(TRAFFIC_COUNTER, feature))
     return stations
 
 
-def get_traffic_counter_stations():
+def get_eco_counter_stations():
     stations = []
     response = requests.get(settings.ECO_COUNTER_STATIONS_URL)
     assert (
@@ -313,7 +313,7 @@ def get_traffic_counter_stations():
     response_json = response.json()
     features = response_json["features"]
     for feature in features:
-        stations.append(ObservationStation(TRAFFIC_COUNTER, feature))
+        stations.append(ObservationStation(ECO_COUNTER, feature))
     return stations
 
 
@@ -351,61 +351,10 @@ def save_stations(csv_data_source):
     )
     num_stations = Station.objects.filter(csv_data_source=csv_data_source).count()
     logger.info(
-        f"Created {num_created} Stations of total {num_stations} Station for conter {csv_data_source}."
+        f"Created {num_created} Stations of total {num_stations} Stations for counter {csv_data_source}."
     )
 
 
-# def save_traffic_counter_stations():
-#     """
-#     Saves the stations defined in the metadata to Station table.
-#     """
-#     saved = 0
-#     data_layer = get_traffic_counter_metadata_data_layer()
-#     for feature in data_layer:
-#         name = feature["Osoite_fi"].as_string()
-#         name_sv = feature["Osoite_sv"].as_string()
-#         name_en = feature["Osoite_en"].as_string()
-#         if Station.objects.filter(name=name).exists():
-#             continue
-#         station = Station()
-#         station.name = name
-#         station.name_sv = name_sv
-#         station.name_en = name_en
-#         station.csv_data_source = TRAFFIC_COUNTER
-#         geom = GEOSGeometry(feature.geom.wkt, srid=feature.geom.srid)
-#         geom.transform(settings.DEFAULT_SRID)
-#         station.geom = geom
-#         station.save()
-#         saved += 1
-#     logger.info(f"Saved {saved} Traffic Counter stations.")
-
-
-# def save_eco_counter_stations():
-#     response = requests.get(settings.ECO_COUNTER_STATIONS_URL)
-#     assert (
-#         response.status_code == 200
-#     ), "Fetching stations from {} , status code {}".format(
-#         settings.ECO_COUNTER_STATIONS_URL, response.status_code
-#     )
-#     response_json = response.json()
-#     features = response_json["features"]
-#     saved = 0
-#     for feature in features:
-#         station = Station()
-#         name = feature["properties"]["Nimi"]
-#         if not Station.objects.filter(name=name).exists():
-#             station.name = name
-#             station.csv_data_source = ECO_COUNTER
-#             lon = feature["geometry"]["coordinates"][0]
-#             lat = feature["geometry"]["coordinates"][1]
-#             point = Point(lon, lat, srid=4326)
-#             point.transform(settings.DEFAULT_SRID)
-#             station.geom = point
-#             station.save()
-#             saved += 1
-#     logger.info(f"Saved {saved} Eco Counter stations.")
-
-
 def get_test_dataframe(counter):
     """
     Generate a Dataframe with only column names for testing. The dataframe

From 0d506968577a1790a7a200ab3ab4fe87d835c020 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 2 May 2023 10:54:49 +0300
Subject: [PATCH 27/81] Add Sotiemme Turku route

---
 mobility_data/importers/culture_routes.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mobility_data/importers/culture_routes.py b/mobility_data/importers/culture_routes.py
index 36c156028..2b8eee4d9 100644
--- a/mobility_data/importers/culture_routes.py
+++ b/mobility_data/importers/culture_routes.py
@@ -26,6 +26,11 @@
 SOURCE_DATA_SRID = 4326
 # Routes are from https://citynomadi.com/route/?keywords=turku
 URLS = {
+    "Sotiemme Turku": {
+        "fi": "https://citynomadi.com/api/route/fb656ce4fc31868f4b90168ecc3fabdb/kml?lang=fi",
+        "sv": "https://citynomadi.com/api/route/fb656ce4fc31868f4b90168ecc3fabdb/kml?lang=sv",
+        "en": "https://citynomadi.com/api/route/fb656ce4fc31868f4b90168ecc3fabdb/kml?lang=fi",
+    },
     "Stepping it up": {
         "fi": "https://citynomadi.com/api/route/9edfeee48c655d64abfef65fc5081e26/kml?lang=fi",
         "sv": "https://citynomadi.com/api/route/9edfeee48c655d64abfef65fc5081e26/kml?lang=sv_SE",

From 36975bdeae70a68dd5495774963d2a689b709c92 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 2 May 2023 11:53:42 +0300
Subject: [PATCH 28/81] Delete culture routes if delete arg is given

---
 .../management/commands/import_culture_routes.py    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/mobility_data/management/commands/import_culture_routes.py b/mobility_data/management/commands/import_culture_routes.py
index 241143c96..b59bab509 100644
--- a/mobility_data/management/commands/import_culture_routes.py
+++ b/mobility_data/management/commands/import_culture_routes.py
@@ -2,7 +2,12 @@
 
 from django.core.management import BaseCommand
 
-from mobility_data.importers.culture_routes import get_routes, save_to_database
+from mobility_data.importers.culture_routes import (
+    get_routes,
+    GROUP_CONTENT_TYPE_NAME,
+    save_to_database,
+)
+from mobility_data.models import MobileUnitGroup
 
 logger = logging.getLogger("mobility_data")
 
@@ -18,8 +23,12 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         logger.info("Importing culture routes...")
-        routes = get_routes()
         delete_tables = options.get("delete", False)
+        if delete_tables:
+            MobileUnitGroup.objects.filter(
+                group_type__type_name=GROUP_CONTENT_TYPE_NAME
+            ).delete()
+        routes = get_routes()
         routes_saved, routes_deleted, units_saved, units_deleted = save_to_database(
             routes, delete_tables=delete_tables
         )

From c117edc2fde4c74506b8316c39be90467d3a0809 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 15 May 2023 09:33:36 +0300
Subject: [PATCH 29/81] Formatting

---
 mobility_data/api/serializers/mobile_unit.py  |  3 -
 .../api/serializers/mobile_unit_group.py      |  2 -
 mobility_data/api/views.py                    |  1 -
 mobility_data/importers/bicycle_stands.py     |  1 -
 .../importers/loading_unloading_places.py     |  1 -
 mobility_data/importers/parking_machines.py   |  1 -
 mobility_data/importers/utils.py              |  2 +-
 .../management/commands/import_test_ali.py    | 57 +++++++++++++++++++
 .../management/commands/import_wfs.py         |  1 -
 mobility_data/migrations/0001_initial.py      |  1 -
 .../migrations/0002_mobileunit_extra.py       |  1 -
 .../migrations/0003_auto_20211101_1501.py     |  1 -
 .../migrations/0004_auto_20211102_0806.py     |  1 -
 .../0005_alter_mobileunit_unit_id.py          |  1 -
 .../migrations/0006_auto_20211112_1312.py     |  1 -
 .../migrations/0007_auto_20211112_1320.py     |  1 -
 .../migrations/0008_auto_20211118_1256.py     |  1 -
 .../0009_alter_contenttype_type_name.py       |  1 -
 .../0010_alter_mobileunit_unit_id.py          |  1 -
 .../0011_alter_contenttype_type_name.py       |  1 -
 .../migrations/0012_add_table_datasource.py   |  1 -
 ...13_add_contenttype_bike_service_station.py |  1 -
 ...add_contenttype_share_car_parking_place.py |  1 -
 ...brush_salted_and_sanded_bicycle_network.py |  1 -
 ...arina_and_guest_marina_and_boat_parking.py |  1 -
 .../0017_add_content_type_no_staff_parking.py |  1 -
 ...ress_zip_and_municipality_to_mobileunit.py |  1 -
 .../migrations/0019_add_content_type_berth.py |  1 -
 .../0020_add_content_type_disabled_parking.py |  1 -
 ...dd_content_type_loading_unloading_place.py |  1 -
 ...pe_marina_southwest_finland_and_slipway.py |  1 -
 ...023_add_content_type_recreational_route.py |  1 -
 ...ntent_type_ferry_route_and_fishing_spot.py |  1 -
 .../0025_add_content_type_paavonpolku.py      |  1 -
 .../0026_add_content_type_fitness_trail.py    |  1 -
 .../0027_add_content_type_nature_trail.py     |  1 -
 .../0028_add_content_type_hiking_trail.py     |  1 -
 .../0029_add_content_type_paddling_trail.py   |  1 -
 ...nstraint_to_content_and_group_type_name.py |  1 -
 ...0031_increase_datasource_type_name_size.py |  1 -
 .../0032_remove_choices_from_type_name.py     |  1 -
 .../0033_remove_contenttype_type_name.py      |  1 -
 .../0034_remove_grouptype_type_name.py        |  1 -
 ...many_field_content_types_to_mobile_unit.py |  1 -
 ...0036_populate_mobile_type_content_types.py |  1 -
 .../0037_remove_mobileunit_content_type.py    |  1 -
 ...pe_and_grouptype_ordering_to_field_name.py |  1 -
 ..._and_grouptype_rename_name_to_type_name.py |  1 -
 .../0040_contenttype_name_grouptype_name.py   |  1 -
 ..._make_name_and_description_multilingual.py |  1 -
 mobility_data/models/content_type.py          |  2 -
 .../tests/test_import_gas_filling_stations.py |  1 -
 52 files changed, 58 insertions(+), 55 deletions(-)
 create mode 100644 mobility_data/management/commands/import_test_ali.py

diff --git a/mobility_data/api/serializers/mobile_unit.py b/mobility_data/api/serializers/mobile_unit.py
index b73a72b25..8986bc240 100644
--- a/mobility_data/api/serializers/mobile_unit.py
+++ b/mobility_data/api/serializers/mobile_unit.py
@@ -17,7 +17,6 @@
 
 
 class GeometrySerializer(serializers.Serializer):
-
     x = serializers.FloatField()
     y = serializers.FloatField()
 
@@ -32,7 +31,6 @@ class Meta:
 
 
 class MobileUnitGroupBasicInfoSerializer(serializers.ModelSerializer):
-
     group_type = GrouptTypeBasicInfoSerializer(many=False, read_only=True)
 
     class Meta:
@@ -41,7 +39,6 @@ class Meta:
 
 
 class MobileUnitSerializer(serializers.ModelSerializer):
-
     content_types = ContentTypeSerializer(many=True, read_only=True)
     mobile_unit_group = MobileUnitGroupBasicInfoSerializer(many=False, read_only=True)
     geometry_coords = serializers.SerializerMethodField(read_only=True)
diff --git a/mobility_data/api/serializers/mobile_unit_group.py b/mobility_data/api/serializers/mobile_unit_group.py
index dc0f5d653..08bca9470 100644
--- a/mobility_data/api/serializers/mobile_unit_group.py
+++ b/mobility_data/api/serializers/mobile_unit_group.py
@@ -18,7 +18,6 @@
 
 
 class MobileUnitGroupSerializer(serializers.ModelSerializer):
-
     group_type = GroupTypeSerializer(many=False, read_only=True)
 
     class Meta:
@@ -27,7 +26,6 @@ class Meta:
 
 
 class MobileUnitGroupUnitsSerializer(serializers.ModelSerializer):
-
     group_type = GroupTypeSerializer(many=False, read_only=True)
     mobile_units = serializers.SerializerMethodField()
 
diff --git a/mobility_data/api/views.py b/mobility_data/api/views.py
index 4a83f79fc..4e1ff1285 100644
--- a/mobility_data/api/views.py
+++ b/mobility_data/api/views.py
@@ -121,7 +121,6 @@ def list(self, request):
 
 
 class MobileUnitViewSet(viewsets.ReadOnlyModelViewSet):
-
     queryset = MobileUnit.objects.filter(is_active=True)
     serializer_class = MobileUnitSerializer
 
diff --git a/mobility_data/importers/bicycle_stands.py b/mobility_data/importers/bicycle_stands.py
index 5b82c0949..53ea9bc68 100644
--- a/mobility_data/importers/bicycle_stands.py
+++ b/mobility_data/importers/bicycle_stands.py
@@ -50,7 +50,6 @@
 
 
 class BicyleStand(MobileUnitDataBase):
-
     WFS_HULL_LOCKABLE_STR = "runkolukitusmahdollisuus"
     GEOJSON_HULL_LOCKABLE_STR = "runkolukittava"
     COVERED_IN_STR = "katettu"
diff --git a/mobility_data/importers/loading_unloading_places.py b/mobility_data/importers/loading_unloading_places.py
index 9932d9eae..6a5e03f00 100644
--- a/mobility_data/importers/loading_unloading_places.py
+++ b/mobility_data/importers/loading_unloading_places.py
@@ -22,7 +22,6 @@
 
 
 class LoadingPlace(MobileUnitDataBase):
-
     extra_field_mappings = {
         "Saavutettavuus": {
             "type": FieldTypes.MULTILANG_STRING,
diff --git a/mobility_data/importers/parking_machines.py b/mobility_data/importers/parking_machines.py
index 3c5529bb6..9004d0cd9 100644
--- a/mobility_data/importers/parking_machines.py
+++ b/mobility_data/importers/parking_machines.py
@@ -15,7 +15,6 @@
 
 
 class ParkingMachine(MobileUnitDataBase):
-
     extra_field_mappings = {
         "Sijainti": {
             "type": FieldTypes.MULTILANG_STRING,
diff --git a/mobility_data/importers/utils.py b/mobility_data/importers/utils.py
index f0ad070eb..2a0a88ab5 100644
--- a/mobility_data/importers/utils.py
+++ b/mobility_data/importers/utils.py
@@ -321,7 +321,7 @@ def log_imported_message(logger, content_type, num_created, num_deleted):
 
 
 @db.transaction.atomic
-def save_to_database(objects, content_types, logger=logger):
+def save_to_database(objects, content_types, logger=logger, group_type=None):
     if type(content_types) != list:
         content_types = [content_types]
 
diff --git a/mobility_data/management/commands/import_test_ali.py b/mobility_data/management/commands/import_test_ali.py
new file mode 100644
index 000000000..d8c9a33cf
--- /dev/null
+++ b/mobility_data/management/commands/import_test_ali.py
@@ -0,0 +1,57 @@
+import xml.etree.ElementTree as ET
+
+import requests
+import xmltodict
+from django.contrib.gis.gdal import DataSource
+from django.contrib.gis.geos import GEOSGeometry, LineString, MultiPolygon, Polygon
+from django.core.management import BaseCommand
+
+from mobility_data.importers.under_and_overpasses import get_over_and_underpass_objects
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        response = requests.get(URL)
+        content_string = response.content.decode("utf-8")
+        # xml_string = ET.ElementTree(ET.fromstring(content_string))
+        json_data = xmltodict.parse(response.content)
+        num_ali = 0
+        num_yli = 0
+        num_oth = 0
+        for feature in json_data["wfs:FeatureCollection"]["gml:featureMembers"][
+            "digiroad:dr_tielinkki_toim_lk"
+        ]:
+            if (
+                feature.get("digiroad:KUNTAKOODI", None) == "853"
+                and feature.get("digiroad:TOIMINN_LK", None) == "8"
+            ):
+                silta_alik = int(feature.get("digiroad:SILTA_ALIK", None))
+                created = False
+                if silta_alik == -1:
+                    num_ali += 1
+                    createad = True
+                elif silta_alik == 1:
+                    num_yli += 1
+                    created = True
+                else:
+                    num_oth += 1
+                if created:
+                    coord_str = feature["digiroad:SHAPE"]["gml:LineString"][
+                        "gml:posList"
+                    ]
+                    # Split the string into a list of coordinate pairs
+                    coords = ()
+                    for x, y in pairwise(coord_str.split(" ")):
+                        coords += ((float(x), float(y)),)
+                    LineString(coords, srid=426)
+                    breakpoint()
+        print(
+            "yli: " + str(num_yli) + " ali " + str(num_ali) + " num_oth " + str(num_oth)
+        )
+        breakpoint()
+        # breakpoint()
+        # ds = DataSource(URL)
+        # breakpoint()
+        # layer = ds[0]
+        # for feature in layer:
+        #     breakpoint()
diff --git a/mobility_data/management/commands/import_wfs.py b/mobility_data/management/commands/import_wfs.py
index f7c111835..68d033fdd 100644
--- a/mobility_data/management/commands/import_wfs.py
+++ b/mobility_data/management/commands/import_wfs.py
@@ -27,7 +27,6 @@ class Command(BaseCommand):
     choices = get_configured_cotent_type_names(config)
 
     def add_arguments(self, parser):
-
         parser.add_argument(
             "--data-file",
             nargs="?",
diff --git a/mobility_data/migrations/0001_initial.py b/mobility_data/migrations/0001_initial.py
index 1e0886446..2a676da34 100644
--- a/mobility_data/migrations/0001_initial.py
+++ b/mobility_data/migrations/0001_initial.py
@@ -8,7 +8,6 @@
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = []
diff --git a/mobility_data/migrations/0002_mobileunit_extra.py b/mobility_data/migrations/0002_mobileunit_extra.py
index e4d270784..c111b8b2a 100644
--- a/mobility_data/migrations/0002_mobileunit_extra.py
+++ b/mobility_data/migrations/0002_mobileunit_extra.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0001_initial"),
     ]
diff --git a/mobility_data/migrations/0003_auto_20211101_1501.py b/mobility_data/migrations/0003_auto_20211101_1501.py
index c979c933b..03d9b3a63 100644
--- a/mobility_data/migrations/0003_auto_20211101_1501.py
+++ b/mobility_data/migrations/0003_auto_20211101_1501.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0002_mobileunit_extra"),
     ]
diff --git a/mobility_data/migrations/0004_auto_20211102_0806.py b/mobility_data/migrations/0004_auto_20211102_0806.py
index 3c6c02c42..8bd818265 100644
--- a/mobility_data/migrations/0004_auto_20211102_0806.py
+++ b/mobility_data/migrations/0004_auto_20211102_0806.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0003_auto_20211101_1501"),
     ]
diff --git a/mobility_data/migrations/0005_alter_mobileunit_unit_id.py b/mobility_data/migrations/0005_alter_mobileunit_unit_id.py
index a2d14764f..8e39c9e5b 100644
--- a/mobility_data/migrations/0005_alter_mobileunit_unit_id.py
+++ b/mobility_data/migrations/0005_alter_mobileunit_unit_id.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0004_auto_20211102_0806"),
     ]
diff --git a/mobility_data/migrations/0006_auto_20211112_1312.py b/mobility_data/migrations/0006_auto_20211112_1312.py
index 74906ae73..a515a53b4 100644
--- a/mobility_data/migrations/0006_auto_20211112_1312.py
+++ b/mobility_data/migrations/0006_auto_20211112_1312.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0005_alter_mobileunit_unit_id"),
     ]
diff --git a/mobility_data/migrations/0007_auto_20211112_1320.py b/mobility_data/migrations/0007_auto_20211112_1320.py
index dad0db099..c85b58c35 100644
--- a/mobility_data/migrations/0007_auto_20211112_1320.py
+++ b/mobility_data/migrations/0007_auto_20211112_1320.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0006_auto_20211112_1312"),
     ]
diff --git a/mobility_data/migrations/0008_auto_20211118_1256.py b/mobility_data/migrations/0008_auto_20211118_1256.py
index 00b5ce879..bb2ef763b 100644
--- a/mobility_data/migrations/0008_auto_20211118_1256.py
+++ b/mobility_data/migrations/0008_auto_20211118_1256.py
@@ -5,7 +5,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0007_auto_20211112_1320"),
     ]
diff --git a/mobility_data/migrations/0009_alter_contenttype_type_name.py b/mobility_data/migrations/0009_alter_contenttype_type_name.py
index 2f10396f2..adebf9344 100644
--- a/mobility_data/migrations/0009_alter_contenttype_type_name.py
+++ b/mobility_data/migrations/0009_alter_contenttype_type_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0008_auto_20211118_1256"),
     ]
diff --git a/mobility_data/migrations/0010_alter_mobileunit_unit_id.py b/mobility_data/migrations/0010_alter_mobileunit_unit_id.py
index 32ed7e492..b496cf815 100644
--- a/mobility_data/migrations/0010_alter_mobileunit_unit_id.py
+++ b/mobility_data/migrations/0010_alter_mobileunit_unit_id.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0009_alter_contenttype_type_name"),
     ]
diff --git a/mobility_data/migrations/0011_alter_contenttype_type_name.py b/mobility_data/migrations/0011_alter_contenttype_type_name.py
index 4ececacd5..d03b6de37 100644
--- a/mobility_data/migrations/0011_alter_contenttype_type_name.py
+++ b/mobility_data/migrations/0011_alter_contenttype_type_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0010_alter_mobileunit_unit_id"),
     ]
diff --git a/mobility_data/migrations/0012_add_table_datasource.py b/mobility_data/migrations/0012_add_table_datasource.py
index e2009f3f4..286c70f85 100644
--- a/mobility_data/migrations/0012_add_table_datasource.py
+++ b/mobility_data/migrations/0012_add_table_datasource.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0011_alter_contenttype_type_name"),
     ]
diff --git a/mobility_data/migrations/0013_add_contenttype_bike_service_station.py b/mobility_data/migrations/0013_add_contenttype_bike_service_station.py
index d780db5eb..111576112 100644
--- a/mobility_data/migrations/0013_add_contenttype_bike_service_station.py
+++ b/mobility_data/migrations/0013_add_contenttype_bike_service_station.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0012_add_table_datasource"),
     ]
diff --git a/mobility_data/migrations/0014_add_contenttype_share_car_parking_place.py b/mobility_data/migrations/0014_add_contenttype_share_car_parking_place.py
index 6f777b3a9..28dc128bc 100644
--- a/mobility_data/migrations/0014_add_contenttype_share_car_parking_place.py
+++ b/mobility_data/migrations/0014_add_contenttype_share_car_parking_place.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0013_add_contenttype_bike_service_station"),
     ]
diff --git a/mobility_data/migrations/0015_add_contenttype_brush_salted_and_sanded_bicycle_network.py b/mobility_data/migrations/0015_add_contenttype_brush_salted_and_sanded_bicycle_network.py
index 7e825917a..7bbaf1fb2 100644
--- a/mobility_data/migrations/0015_add_contenttype_brush_salted_and_sanded_bicycle_network.py
+++ b/mobility_data/migrations/0015_add_contenttype_brush_salted_and_sanded_bicycle_network.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0014_add_contenttype_share_car_parking_place"),
     ]
diff --git a/mobility_data/migrations/0016_add_content_type_marina_and_guest_marina_and_boat_parking.py b/mobility_data/migrations/0016_add_content_type_marina_and_guest_marina_and_boat_parking.py
index 4db804176..b6a88ee43 100644
--- a/mobility_data/migrations/0016_add_content_type_marina_and_guest_marina_and_boat_parking.py
+++ b/mobility_data/migrations/0016_add_content_type_marina_and_guest_marina_and_boat_parking.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         (
             "mobility_data",
diff --git a/mobility_data/migrations/0017_add_content_type_no_staff_parking.py b/mobility_data/migrations/0017_add_content_type_no_staff_parking.py
index a8d9a34d8..91a677ca7 100644
--- a/mobility_data/migrations/0017_add_content_type_no_staff_parking.py
+++ b/mobility_data/migrations/0017_add_content_type_no_staff_parking.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         (
             "mobility_data",
diff --git a/mobility_data/migrations/0018_add_address_zip_and_municipality_to_mobileunit.py b/mobility_data/migrations/0018_add_address_zip_and_municipality_to_mobileunit.py
index b0eaba8e2..731947c25 100644
--- a/mobility_data/migrations/0018_add_address_zip_and_municipality_to_mobileunit.py
+++ b/mobility_data/migrations/0018_add_address_zip_and_municipality_to_mobileunit.py
@@ -5,7 +5,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("munigeo", "0013_add_naturalsort_function"),
         ("mobility_data", "0017_add_content_type_no_staff_parking"),
diff --git a/mobility_data/migrations/0019_add_content_type_berth.py b/mobility_data/migrations/0019_add_content_type_berth.py
index a7bb004db..f2ae5ba0c 100644
--- a/mobility_data/migrations/0019_add_content_type_berth.py
+++ b/mobility_data/migrations/0019_add_content_type_berth.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0018_add_address_zip_and_municipality_to_mobileunit"),
     ]
diff --git a/mobility_data/migrations/0020_add_content_type_disabled_parking.py b/mobility_data/migrations/0020_add_content_type_disabled_parking.py
index c9420a06b..6d14ce912 100644
--- a/mobility_data/migrations/0020_add_content_type_disabled_parking.py
+++ b/mobility_data/migrations/0020_add_content_type_disabled_parking.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0019_add_content_type_berth"),
     ]
diff --git a/mobility_data/migrations/0021_add_content_type_loading_unloading_place.py b/mobility_data/migrations/0021_add_content_type_loading_unloading_place.py
index fcc5f8966..d53861082 100644
--- a/mobility_data/migrations/0021_add_content_type_loading_unloading_place.py
+++ b/mobility_data/migrations/0021_add_content_type_loading_unloading_place.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0020_add_content_type_disabled_parking"),
     ]
diff --git a/mobility_data/migrations/0022_add_content_type_marina_southwest_finland_and_slipway.py b/mobility_data/migrations/0022_add_content_type_marina_southwest_finland_and_slipway.py
index 0f04c423f..35d377c75 100644
--- a/mobility_data/migrations/0022_add_content_type_marina_southwest_finland_and_slipway.py
+++ b/mobility_data/migrations/0022_add_content_type_marina_southwest_finland_and_slipway.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0021_add_content_type_loading_unloading_place"),
     ]
diff --git a/mobility_data/migrations/0023_add_content_type_recreational_route.py b/mobility_data/migrations/0023_add_content_type_recreational_route.py
index fffa9a2e3..a9aefde55 100644
--- a/mobility_data/migrations/0023_add_content_type_recreational_route.py
+++ b/mobility_data/migrations/0023_add_content_type_recreational_route.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0022_add_content_type_marina_southwest_finland_and_slipway"),
     ]
diff --git a/mobility_data/migrations/0024_add_content_type_ferry_route_and_fishing_spot.py b/mobility_data/migrations/0024_add_content_type_ferry_route_and_fishing_spot.py
index 1be872497..abbd4078c 100644
--- a/mobility_data/migrations/0024_add_content_type_ferry_route_and_fishing_spot.py
+++ b/mobility_data/migrations/0024_add_content_type_ferry_route_and_fishing_spot.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0023_add_content_type_recreational_route"),
     ]
diff --git a/mobility_data/migrations/0025_add_content_type_paavonpolku.py b/mobility_data/migrations/0025_add_content_type_paavonpolku.py
index e59deb461..c6d5c1632 100644
--- a/mobility_data/migrations/0025_add_content_type_paavonpolku.py
+++ b/mobility_data/migrations/0025_add_content_type_paavonpolku.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0024_add_content_type_ferry_route_and_fishing_spot"),
     ]
diff --git a/mobility_data/migrations/0026_add_content_type_fitness_trail.py b/mobility_data/migrations/0026_add_content_type_fitness_trail.py
index 6b363e46d..3864afa9a 100644
--- a/mobility_data/migrations/0026_add_content_type_fitness_trail.py
+++ b/mobility_data/migrations/0026_add_content_type_fitness_trail.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0025_add_content_type_paavonpolku"),
     ]
diff --git a/mobility_data/migrations/0027_add_content_type_nature_trail.py b/mobility_data/migrations/0027_add_content_type_nature_trail.py
index eb591c0b6..85b0597e3 100644
--- a/mobility_data/migrations/0027_add_content_type_nature_trail.py
+++ b/mobility_data/migrations/0027_add_content_type_nature_trail.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0026_add_content_type_fitness_trail"),
     ]
diff --git a/mobility_data/migrations/0028_add_content_type_hiking_trail.py b/mobility_data/migrations/0028_add_content_type_hiking_trail.py
index 0be27c5db..fcbd9f6f1 100644
--- a/mobility_data/migrations/0028_add_content_type_hiking_trail.py
+++ b/mobility_data/migrations/0028_add_content_type_hiking_trail.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0027_add_content_type_nature_trail"),
     ]
diff --git a/mobility_data/migrations/0029_add_content_type_paddling_trail.py b/mobility_data/migrations/0029_add_content_type_paddling_trail.py
index 23bc0b1a0..230c041d1 100644
--- a/mobility_data/migrations/0029_add_content_type_paddling_trail.py
+++ b/mobility_data/migrations/0029_add_content_type_paddling_trail.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0028_add_content_type_hiking_trail"),
     ]
diff --git a/mobility_data/migrations/0030_add_unique_constraint_to_content_and_group_type_name.py b/mobility_data/migrations/0030_add_unique_constraint_to_content_and_group_type_name.py
index 5d772991d..200332733 100644
--- a/mobility_data/migrations/0030_add_unique_constraint_to_content_and_group_type_name.py
+++ b/mobility_data/migrations/0030_add_unique_constraint_to_content_and_group_type_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0029_add_content_type_paddling_trail"),
     ]
diff --git a/mobility_data/migrations/0031_increase_datasource_type_name_size.py b/mobility_data/migrations/0031_increase_datasource_type_name_size.py
index b34c88518..23dd3c673 100644
--- a/mobility_data/migrations/0031_increase_datasource_type_name_size.py
+++ b/mobility_data/migrations/0031_increase_datasource_type_name_size.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0030_add_unique_constraint_to_content_and_group_type_name"),
     ]
diff --git a/mobility_data/migrations/0032_remove_choices_from_type_name.py b/mobility_data/migrations/0032_remove_choices_from_type_name.py
index 37bbf6fa2..3cf5be960 100644
--- a/mobility_data/migrations/0032_remove_choices_from_type_name.py
+++ b/mobility_data/migrations/0032_remove_choices_from_type_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0031_increase_datasource_type_name_size"),
     ]
diff --git a/mobility_data/migrations/0033_remove_contenttype_type_name.py b/mobility_data/migrations/0033_remove_contenttype_type_name.py
index abf0125e1..63fd075d5 100644
--- a/mobility_data/migrations/0033_remove_contenttype_type_name.py
+++ b/mobility_data/migrations/0033_remove_contenttype_type_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0032_remove_choices_from_type_name"),
     ]
diff --git a/mobility_data/migrations/0034_remove_grouptype_type_name.py b/mobility_data/migrations/0034_remove_grouptype_type_name.py
index f56790277..2ae108676 100644
--- a/mobility_data/migrations/0034_remove_grouptype_type_name.py
+++ b/mobility_data/migrations/0034_remove_grouptype_type_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0033_remove_contenttype_type_name"),
     ]
diff --git a/mobility_data/migrations/0035_add_many_to_many_field_content_types_to_mobile_unit.py b/mobility_data/migrations/0035_add_many_to_many_field_content_types_to_mobile_unit.py
index 029d6b23f..7e4acbace 100644
--- a/mobility_data/migrations/0035_add_many_to_many_field_content_types_to_mobile_unit.py
+++ b/mobility_data/migrations/0035_add_many_to_many_field_content_types_to_mobile_unit.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0034_remove_grouptype_type_name"),
     ]
diff --git a/mobility_data/migrations/0036_populate_mobile_type_content_types.py b/mobility_data/migrations/0036_populate_mobile_type_content_types.py
index cafd7c797..35f488c9f 100644
--- a/mobility_data/migrations/0036_populate_mobile_type_content_types.py
+++ b/mobility_data/migrations/0036_populate_mobile_type_content_types.py
@@ -11,7 +11,6 @@ def make_many_to_many_content_types(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0035_add_many_to_many_field_content_types_to_mobile_unit"),
     ]
diff --git a/mobility_data/migrations/0037_remove_mobileunit_content_type.py b/mobility_data/migrations/0037_remove_mobileunit_content_type.py
index 8d0626c0a..7288e80fd 100644
--- a/mobility_data/migrations/0037_remove_mobileunit_content_type.py
+++ b/mobility_data/migrations/0037_remove_mobileunit_content_type.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0036_populate_mobile_type_content_types"),
     ]
diff --git a/mobility_data/migrations/0038_alter_contenttype_and_grouptype_ordering_to_field_name.py b/mobility_data/migrations/0038_alter_contenttype_and_grouptype_ordering_to_field_name.py
index 2d450fe7f..c1c951134 100644
--- a/mobility_data/migrations/0038_alter_contenttype_and_grouptype_ordering_to_field_name.py
+++ b/mobility_data/migrations/0038_alter_contenttype_and_grouptype_ordering_to_field_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0037_remove_mobileunit_content_type"),
     ]
diff --git a/mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py b/mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py
index aa0b6a828..5e0d35302 100644
--- a/mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py
+++ b/mobility_data/migrations/0039_contentype_and_grouptype_rename_name_to_type_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         (
             "mobility_data",
diff --git a/mobility_data/migrations/0040_contenttype_name_grouptype_name.py b/mobility_data/migrations/0040_contenttype_name_grouptype_name.py
index 6303c45d3..ed30f8320 100644
--- a/mobility_data/migrations/0040_contenttype_name_grouptype_name.py
+++ b/mobility_data/migrations/0040_contenttype_name_grouptype_name.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0039_contentype_and_grouptype_rename_name_to_type_name"),
     ]
diff --git a/mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py b/mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py
index c6d5e252d..6cd4f97df 100644
--- a/mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py
+++ b/mobility_data/migrations/0041_contenttype_grouptype_make_name_and_description_multilingual.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("mobility_data", "0040_contenttype_name_grouptype_name"),
     ]
diff --git a/mobility_data/models/content_type.py b/mobility_data/models/content_type.py
index 2acf722f9..a2f09b140 100644
--- a/mobility_data/models/content_type.py
+++ b/mobility_data/models/content_type.py
@@ -22,10 +22,8 @@ def __str__(self):
 
 
 class ContentType(BaseType):
-
     pass
 
 
 class GroupType(BaseType):
-
     pass
diff --git a/mobility_data/tests/test_import_gas_filling_stations.py b/mobility_data/tests/test_import_gas_filling_stations.py
index 11dd72c4f..5bd913c35 100644
--- a/mobility_data/tests/test_import_gas_filling_stations.py
+++ b/mobility_data/tests/test_import_gas_filling_stations.py
@@ -11,7 +11,6 @@
 
 @pytest.mark.django_db
 def test_importer(municipalities):
-
     from mobility_data.importers.gas_filling_station import (
         CONTENT_TYPE_NAME,
         get_filtered_gas_filling_station_objects,

From f60bda2f120d33dae49c5767f3816826e0f6f068 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 15 May 2023 10:08:49 +0300
Subject: [PATCH 30/81] Remove obsolete test importer

---
 .../management/commands/import_test_ali.py    | 57 -------------------
 1 file changed, 57 deletions(-)
 delete mode 100644 mobility_data/management/commands/import_test_ali.py

diff --git a/mobility_data/management/commands/import_test_ali.py b/mobility_data/management/commands/import_test_ali.py
deleted file mode 100644
index d8c9a33cf..000000000
--- a/mobility_data/management/commands/import_test_ali.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import xml.etree.ElementTree as ET
-
-import requests
-import xmltodict
-from django.contrib.gis.gdal import DataSource
-from django.contrib.gis.geos import GEOSGeometry, LineString, MultiPolygon, Polygon
-from django.core.management import BaseCommand
-
-from mobility_data.importers.under_and_overpasses import get_over_and_underpass_objects
-
-
-class Command(BaseCommand):
-    def handle(self, *args, **options):
-        response = requests.get(URL)
-        content_string = response.content.decode("utf-8")
-        # xml_string = ET.ElementTree(ET.fromstring(content_string))
-        json_data = xmltodict.parse(response.content)
-        num_ali = 0
-        num_yli = 0
-        num_oth = 0
-        for feature in json_data["wfs:FeatureCollection"]["gml:featureMembers"][
-            "digiroad:dr_tielinkki_toim_lk"
-        ]:
-            if (
-                feature.get("digiroad:KUNTAKOODI", None) == "853"
-                and feature.get("digiroad:TOIMINN_LK", None) == "8"
-            ):
-                silta_alik = int(feature.get("digiroad:SILTA_ALIK", None))
-                created = False
-                if silta_alik == -1:
-                    num_ali += 1
-                    createad = True
-                elif silta_alik == 1:
-                    num_yli += 1
-                    created = True
-                else:
-                    num_oth += 1
-                if created:
-                    coord_str = feature["digiroad:SHAPE"]["gml:LineString"][
-                        "gml:posList"
-                    ]
-                    # Split the string into a list of coordinate pairs
-                    coords = ()
-                    for x, y in pairwise(coord_str.split(" ")):
-                        coords += ((float(x), float(y)),)
-                    LineString(coords, srid=426)
-                    breakpoint()
-        print(
-            "yli: " + str(num_yli) + " ali " + str(num_ali) + " num_oth " + str(num_oth)
-        )
-        breakpoint()
-        # breakpoint()
-        # ds = DataSource(URL)
-        # breakpoint()
-        # layer = ds[0]
-        # for feature in layer:
-        #     breakpoint()

From 953b0d7b90f94fba14e6fd1b444a74f08ab2e430 Mon Sep 17 00:00:00 2001
From: Juuso Jokiniemi <68938778+juuso-j@users.noreply.github.com>
Date: Mon, 15 May 2023 10:38:32 +0300
Subject: [PATCH 31/81] Update run-tests.yml

---
 .github/workflows/run-tests.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 5c8c03bf1..c72649628 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -16,10 +16,7 @@ jobs:
       DATABASE_URL: postgis://postgres:postgres@localhost/smbackend
       ADDITIONAL_INSTALLED_APPS: smbackend_turku,ptv
       PTV_ID_OFFSET: 10000000
-      BIKE_SERVICE_STATIONS_IDS: service_node=500000,service=500000,units_offset=500000
-      GAS_FILLING_STATIONS_IDS: service_node=200000,service=200000,units_offset=200000
-      CHARGING_STATIONS_IDS: service_node=300000,service=300000,units_offset=300000
-      BICYCLE_STANDS_IDS: service_node=400000,service=400000,units_offset=400000
+      LAM_COUNTER_API_BASE_URL: https://tie.digitraffic.fi/api/tms/v1/history
 
     steps:
     - uses: actions/checkout@v2

From 8707cc3f3c3088aa2b996251e1a391e3c43604ed Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 16 May 2023 16:29:49 +0300
Subject: [PATCH 32/81] Add Telraam counter

---
 .../commands/import_counter_data.py           | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 16ab86263..6b0bb938f 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -20,6 +20,8 @@
     COUNTERS,
     ECO_COUNTER,
     LAM_COUNTER,
+    TELRAAM_COUNTER,
+    TELRAAM_START_MONTH,
     TRAFFIC_COUNTER,
     TRAFFIC_COUNTER_START_YEAR,
 )
@@ -41,6 +43,7 @@
     gen_eco_counter_test_csv,
     get_eco_counter_csv,
     get_lam_counter_csv,
+    get_telraam_counter_csv,
     get_test_dataframe,
     get_traffic_counter_csv,
     save_stations,
@@ -64,8 +67,10 @@
 
 class Command(BaseCommand):
     help = "Imports traffic counter data in the Turku region."
-    COUNTERS = [ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER]
-    COUNTER_CHOICES_STR = f"{ECO_COUNTER}, {TRAFFIC_COUNTER} and {LAM_COUNTER}"
+    COUNTERS = [ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER, TELRAAM_COUNTER]
+    COUNTER_CHOICES_STR = (
+        f"{ECO_COUNTER}, {TRAFFIC_COUNTER}, {TELRAAM_COUNTER} and {LAM_COUNTER}"
+    )
     TIMEZONE = pytz.timezone("Europe/Helsinki")
     """
     Movement types:
@@ -93,7 +98,8 @@ class Command(BaseCommand):
     type_dirs_lower = [TD.lower() for TD in TYPE_DIRS]
 
     def delete_tables(
-        self, csv_data_sources=[ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER]
+        self,
+        csv_data_sources=[ECO_COUNTER, TRAFFIC_COUNTER, LAM_COUNTER, TELRAAM_COUNTER],
     ):
         for csv_data_source in csv_data_sources:
             for station in Station.objects.filter(csv_data_source=csv_data_source):
@@ -438,14 +444,20 @@ def handle(self, *args, **options):
                         month=import_state.current_month_number,
                     )
                 else:
-                    start_time = f"{COUNTER_START_YEARS[counter]}-01-01"
+                    start_month = (
+                        TELRAAM_START_MONTH if counter == TELRAAM_COUNTER else "01"
+                    )
+                    start_time = f"{COUNTER_START_YEARS[counter]}-{start_month}-01"
 
                 start_time = dateutil.parser.parse(start_time)
                 start_time = self.TIMEZONE.localize(start_time)
                 # The timeformat for the input data is : 2020-03-01T00:00
                 # Convert starting time to input datas timeformat
                 start_time_string = start_time.strftime("%Y-%m-%dT%H:%M")
+                # start_index = None
                 match counter:
+                    case COUNTERS.TELRAAM_COUNTER:
+                        csv_data = get_telraam_counter_csv()
                     case COUNTERS.LAM_COUNTER:
                         csv_data = get_lam_counter_csv(start_time.date())
                     case COUNTERS.ECO_COUNTER:
@@ -456,7 +468,6 @@ def handle(self, *args, **options):
                         else:
                             start_year = TRAFFIC_COUNTER_START_YEAR
                         csv_data = get_traffic_counter_csv(start_year=start_year)
-
                 start_index = csv_data.index[
                     csv_data[TIMESTAMP_COL_NAME] == start_time_string
                 ].values[0]

From 90acc5fddde2a29fa8d6eca8308043c8b8d2b31e Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 16 May 2023 16:30:12 +0300
Subject: [PATCH 33/81] Add telraam data to csv importer

---
 .../commands/import_telraam_to_csv.py         | 185 ++++++++++++++++++
 1 file changed, 185 insertions(+)
 create mode 100644 eco_counter/management/commands/import_telraam_to_csv.py

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
new file mode 100644
index 000000000..fef44063c
--- /dev/null
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -0,0 +1,185 @@
+import json
+import logging
+from datetime import date, datetime, timedelta
+
+import pandas as pd
+import requests
+from django.conf import settings
+from django.core.management import BaseCommand
+
+from eco_counter.constants import TELRAAM_CSV_FILE_NAME
+from eco_counter.management.commands.utils import get_active_camera
+from mobility_data.importers.utils import get_root_dir
+
+TELRAAM_API_BASE_URL = "https://telraam-api.net"
+ALIVE_URL = f"{TELRAAM_API_BASE_URL}/v1"
+# Maximum 3 months at a time
+TRAFFIC_URL = f"{TELRAAM_API_BASE_URL}/v1/reports/traffic"
+AVAILABLE_CAMERAS_URL = f"{TELRAAM_API_BASE_URL}/v1/cameras"
+INDEX_COLUMN_NAME = "startTime"
+LEVEL = "instances"  # instance per individual can
+FORMAT = "per-hour"
+TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+DATA_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
+
+token = settings.TELRAAM_TOKEN
+assert token
+csv_file = f"{get_root_dir()}/media/{TELRAAM_CSV_FILE_NAME}"
+logger = logging.getLogger("eco_counter")
+
+HEAVY = "heavy"
+VEHICLE_TYPES = {
+    "pedestrian": "J",
+    "bike": "P",
+    "car": "A",
+    HEAVY: "A",  # Is added to car column
+}
+LEFT = "lft"
+RIGHT = "rgt"
+DIRECTIONS = [LEFT, RIGHT]
+
+
+def get_mappings(station_name, direction=True):
+    """
+    return mappings:
+    e.g.,
+    "pedestrian_lgt": "station_name KP"
+    """
+    dir1, dir2 = "K", "P"
+    if not direction:
+        dir1, dir2 = dir2, dir1
+    dirs = {
+        LEFT: dir1,
+        RIGHT: dir2,
+    }
+    column_mappings = {}
+    for veh in VEHICLE_TYPES.items():
+        for dir in DIRECTIONS:
+            key = f"{veh[0]}_{dir}"
+            value = f"{veh[1]}{dirs[dir]}"
+            column_mappings[key] = value
+    mappings = {}
+    for field in column_mappings.keys():
+        mappings[field] = f"{station_name} {column_mappings[field]}"
+
+    return mappings
+
+
+def get_hourly_data(from_date, end_date, camera_id):
+    headers = {
+        "X-Api-Key": token,
+        "Content-Type": "application/json",
+    }
+    data = {
+        "level": LEVEL,  # segments
+        "format": FORMAT,
+        "id": camera_id,
+        "time_start": from_date,
+        "time_end": end_date,
+    }
+    response = requests.post(TRAFFIC_URL, headers=headers, data=json.dumps(data))
+
+    from_date = datetime.strptime(from_date, TIME_FORMAT)
+    end_date = datetime.strptime(end_date, TIME_FORMAT)
+    delta = end_date - from_date
+    delta_hours = int(round(delta.total_seconds() / 3600))
+    logger.info(f"delta hours {delta_hours}")
+    report = response.json().get("report", [])
+    if not report:
+        report = [{} for a in range(delta_hours)]
+    else:
+        print("Report has data")
+    if len(report) != delta_hours:
+        report = [{} for a in range(delta_hours)]
+        delta_hours = len(report)
+    res = []
+    start_date = from_date
+    for i in range(delta_hours):
+        d = {}
+        d["date"] = datetime.strftime(start_date, TIME_FORMAT)
+        item = report[i]
+        for veh in VEHICLE_TYPES.keys():
+            for dir in DIRECTIONS:
+                key = f"{veh}_{dir}"
+                val = int(round(item.get(key, 0)))
+                d[key] = val
+        res.append(d)
+        start_date += timedelta(hours=1)
+    return res, delta_hours
+
+
+def get_active_cameras():
+    headers = {
+        "X-Api-Key": token,
+    }
+    active_cameras = []
+    response = requests.get(AVAILABLE_CAMERAS_URL, headers=headers)
+    cameras = response.json()["cameras"]
+    for camera in cameras:
+        if camera["status"] == "active":
+            active_cameras.append(camera)
+    return active_cameras
+
+
+def save_dataframe():
+    test_num_days = 6
+    test_num_cams = 2
+    # cam_off
+    start_date = date(2023, 5, 1)
+    end_date = date(2023, 5, 1 + test_num_days).strftime(TIME_FORMAT)
+    from_date = start_date.strftime(TIME_FORMAT)
+    try:
+        df = pd.read_csv(csv_file, index_col=INDEX_COLUMN_NAME)
+        from_date = df.iloc[-1].name
+        from_date = datetime.strptime(from_date, TIME_FORMAT) + timedelta(hours=1)
+        from_date = datetime.strftime(from_date, TIME_FORMAT)
+        logger.info(f"Read csv data to {from_date}")
+    except Exception:
+        logger.info("Creating new empty Pandas DataFrame")
+        df = pd.DataFrame()
+    # TODO, when Turku cameras online, add them
+    cameras = []
+    for i in range(2, 1 + test_num_cams):
+        cameras.append(get_active_camera(offset=i * 3))
+    df_created = pd.DataFrame()
+    reports = []
+    for camera in cameras:
+        report, delta_hours = get_hourly_data(
+            from_date, end_date, camera["instance_id"]
+        )
+        reports.append({"camera": camera, "report": report})
+
+    columns = {}
+    columns[INDEX_COLUMN_NAME] = []
+    for hour in range(delta_hours):
+        columns[INDEX_COLUMN_NAME].append(reports[0]["report"][hour]["date"])
+        for report in reports:
+            mappings = get_mappings(report["camera"]["mac"])
+            for mapping in mappings.items():
+                # key is the name of the column, e.g., name_ak
+                key = mapping[1]
+                value_key = mapping[0]
+                values_list = columns.get(key, [])
+                if HEAVY in value_key:
+                    # add heavy values to car column, as the mapping is same.
+                    values_list[-1] += report["report"][hour][value_key]
+                else:
+                    values_list.append(report["report"][hour][value_key])
+                columns[key] = values_list
+    df_created = pd.DataFrame(data=columns, index=columns["startTime"])
+
+    df_created = df_created.drop(columns=[INDEX_COLUMN_NAME], axis=1)
+    if df.empty:
+        df = df_created
+    else:
+        df = pd.concat([df, df_created], axis=0)
+    df.index.rename(INDEX_COLUMN_NAME, inplace=True)
+    df = df.fillna(0)
+    df = df.astype(int)
+    df.to_csv(csv_file)
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        logger.info("Importing Telraam data...")
+        save_dataframe()

From 17fb320e743346351d78771c9d013fb754d1ff1b Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 16 May 2023 16:30:58 +0300
Subject: [PATCH 34/81] Formatting

---
 eco_counter/migrations/0001_initial.py        |  1 -
 .../migrations/0002_auto_20211013_1018.py     |  1 -
 ...mportstate_current_year_number_and_more.py |  1 -
 ...e_en_name_fi_station_name_sv_to_station.py |  1 -
 ..._data_source_to_importstate_and_station.py |  1 -
 ...csv_data_source_and_current_year_number.py |  1 -
 .../0007_add_fields_for_bus_data.py           |  1 -
 .../0008_remove_importstate_rows_imported.py  |  1 -
 ..._data_source_to_station_and_importstate.py |  1 -
 ..._alter_max_length_of_station_name_to_64.py |  1 -
 .../migrations/0011_add_lam_id_to_station.py  |  1 -
 ...012_set_static_default_year_year_number.py |  1 -
 ...ate_current_month_and_year_to_nullables.py |  1 -
 ...emove_station_lam_id_station_station_id.py | 21 +++++++++++++++++++
 14 files changed, 21 insertions(+), 13 deletions(-)
 create mode 100644 eco_counter/migrations/0014_remove_station_lam_id_station_station_id.py

diff --git a/eco_counter/migrations/0001_initial.py b/eco_counter/migrations/0001_initial.py
index 03c92b877..2f05015ef 100644
--- a/eco_counter/migrations/0001_initial.py
+++ b/eco_counter/migrations/0001_initial.py
@@ -9,7 +9,6 @@
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = []
diff --git a/eco_counter/migrations/0002_auto_20211013_1018.py b/eco_counter/migrations/0002_auto_20211013_1018.py
index e8be909c9..3c1a2d709 100644
--- a/eco_counter/migrations/0002_auto_20211013_1018.py
+++ b/eco_counter/migrations/0002_auto_20211013_1018.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0001_initial"),
     ]
diff --git a/eco_counter/migrations/0003_alter_importstate_current_year_number_and_more.py b/eco_counter/migrations/0003_alter_importstate_current_year_number_and_more.py
index 5b76531de..8e1c98c0d 100644
--- a/eco_counter/migrations/0003_alter_importstate_current_year_number_and_more.py
+++ b/eco_counter/migrations/0003_alter_importstate_current_year_number_and_more.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0002_auto_20211013_1018"),
     ]
diff --git a/eco_counter/migrations/0004_add_name_en_name_fi_station_name_sv_to_station.py b/eco_counter/migrations/0004_add_name_en_name_fi_station_name_sv_to_station.py
index 89468829d..48b5e67cf 100644
--- a/eco_counter/migrations/0004_add_name_en_name_fi_station_name_sv_to_station.py
+++ b/eco_counter/migrations/0004_add_name_en_name_fi_station_name_sv_to_station.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0003_alter_importstate_current_year_number_and_more"),
     ]
diff --git a/eco_counter/migrations/0005_add_csv_data_source_to_importstate_and_station.py b/eco_counter/migrations/0005_add_csv_data_source_to_importstate_and_station.py
index 34048ec00..223b5e567 100644
--- a/eco_counter/migrations/0005_add_csv_data_source_to_importstate_and_station.py
+++ b/eco_counter/migrations/0005_add_csv_data_source_to_importstate_and_station.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0004_add_name_en_name_fi_station_name_sv_to_station"),
     ]
diff --git a/eco_counter/migrations/0006_alter_importstate_csv_data_source_and_current_year_number.py b/eco_counter/migrations/0006_alter_importstate_csv_data_source_and_current_year_number.py
index f4d38b4c8..2c409a5f5 100644
--- a/eco_counter/migrations/0006_alter_importstate_csv_data_source_and_current_year_number.py
+++ b/eco_counter/migrations/0006_alter_importstate_csv_data_source_and_current_year_number.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0005_add_csv_data_source_to_importstate_and_station"),
     ]
diff --git a/eco_counter/migrations/0007_add_fields_for_bus_data.py b/eco_counter/migrations/0007_add_fields_for_bus_data.py
index 0d78f427a..3ea588818 100644
--- a/eco_counter/migrations/0007_add_fields_for_bus_data.py
+++ b/eco_counter/migrations/0007_add_fields_for_bus_data.py
@@ -5,7 +5,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         (
             "eco_counter",
diff --git a/eco_counter/migrations/0008_remove_importstate_rows_imported.py b/eco_counter/migrations/0008_remove_importstate_rows_imported.py
index bbdb102bf..fa429c761 100644
--- a/eco_counter/migrations/0008_remove_importstate_rows_imported.py
+++ b/eco_counter/migrations/0008_remove_importstate_rows_imported.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0007_add_fields_for_bus_data"),
     ]
diff --git a/eco_counter/migrations/0009_add_lam_counter_csv_data_source_to_station_and_importstate.py b/eco_counter/migrations/0009_add_lam_counter_csv_data_source_to_station_and_importstate.py
index d8e4588ee..a326bf056 100644
--- a/eco_counter/migrations/0009_add_lam_counter_csv_data_source_to_station_and_importstate.py
+++ b/eco_counter/migrations/0009_add_lam_counter_csv_data_source_to_station_and_importstate.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0008_remove_importstate_rows_imported"),
     ]
diff --git a/eco_counter/migrations/0010_alter_max_length_of_station_name_to_64.py b/eco_counter/migrations/0010_alter_max_length_of_station_name_to_64.py
index 8268724e0..a6828ee2e 100644
--- a/eco_counter/migrations/0010_alter_max_length_of_station_name_to_64.py
+++ b/eco_counter/migrations/0010_alter_max_length_of_station_name_to_64.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         (
             "eco_counter",
diff --git a/eco_counter/migrations/0011_add_lam_id_to_station.py b/eco_counter/migrations/0011_add_lam_id_to_station.py
index bb89e01ce..93366d3d1 100644
--- a/eco_counter/migrations/0011_add_lam_id_to_station.py
+++ b/eco_counter/migrations/0011_add_lam_id_to_station.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0010_alter_max_length_of_station_name_to_64"),
     ]
diff --git a/eco_counter/migrations/0012_set_static_default_year_year_number.py b/eco_counter/migrations/0012_set_static_default_year_year_number.py
index 75e1b30c4..59d1676e9 100644
--- a/eco_counter/migrations/0012_set_static_default_year_year_number.py
+++ b/eco_counter/migrations/0012_set_static_default_year_year_number.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0011_add_lam_id_to_station"),
     ]
diff --git a/eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py b/eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py
index fc0118d47..763f680c0 100644
--- a/eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py
+++ b/eco_counter/migrations/0013_alter_importstate_current_month_and_year_to_nullables.py
@@ -5,7 +5,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("eco_counter", "0012_set_static_default_year_year_number"),
     ]
diff --git a/eco_counter/migrations/0014_remove_station_lam_id_station_station_id.py b/eco_counter/migrations/0014_remove_station_lam_id_station_station_id.py
new file mode 100644
index 000000000..ccd432beb
--- /dev/null
+++ b/eco_counter/migrations/0014_remove_station_lam_id_station_station_id.py
@@ -0,0 +1,21 @@
+# Generated by Django 4.2 on 2023-05-15 12:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("eco_counter", "0013_alter_importstate_current_month_and_year_to_nullables"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="station",
+            name="lam_id",
+        ),
+        migrations.AddField(
+            model_name="station",
+            name="station_id",
+            field=models.CharField(max_length=16, null=True),
+        ),
+    ]

From 057f5ec5815922faffd9ccce06764554ccc11b1e Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 16 May 2023 16:31:27 +0300
Subject: [PATCH 35/81] Formatting

---
 eco_counter/api/serializers.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/eco_counter/api/serializers.py b/eco_counter/api/serializers.py
index e0cbc80c8..0a1adca26 100644
--- a/eco_counter/api/serializers.py
+++ b/eco_counter/api/serializers.py
@@ -100,7 +100,6 @@ class Meta:
 
 
 class DaySerializer(serializers.ModelSerializer):
-
     station_name = serializers.PrimaryKeyRelatedField(
         many=False, source="station.name", read_only=True
     )
@@ -207,7 +206,6 @@ class Meta:
 
 
 class HourDataSerializer(serializers.ModelSerializer):
-
     day_info = DayInfoSerializer(source="day")
 
     class Meta:
@@ -229,7 +227,6 @@ class Meta:
 
 
 class DayDataSerializer(serializers.ModelSerializer):
-
     day_info = DayInfoSerializer(source="day")
 
     class Meta:
@@ -254,7 +251,6 @@ class Meta:
 
 
 class MonthDataSerializer(serializers.ModelSerializer):
-
     month_info = MonthInfoSerializer(source="month")
 
     class Meta:
@@ -267,7 +263,6 @@ class Meta:
 
 
 class YearDataSerializer(serializers.ModelSerializer):
-
     year_info = YearInfoSerializer(source="year")
 
     class Meta:

From 167052653f7c5632d84f61cdcba845ef7497e2db Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 16 May 2023 16:32:44 +0300
Subject: [PATCH 36/81] Add Telraam utility functions and class

---
 eco_counter/management/commands/utils.py | 68 +++++++++++++++++++++---
 1 file changed, 62 insertions(+), 6 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index c8f2d41a0..428b94358 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -17,6 +17,9 @@
     LAM_STATION_MUNICIPALITIES,
     LAM_STATIONS_API_FETCH_URL,
     LAM_STATIONS_DIRECTION_MAPPINGS,
+    TELRAAM_COUNTER,
+    TELRAAM_COUNTER_API_BASE_URL,
+    TELRAAM_CSV_FILE_NAME,
     TIMESTAMP_COL_NAME,
     TRAFFIC_COUNTER,
     TRAFFIC_COUNTER_CSV_URLS,
@@ -33,7 +36,7 @@ class LAMStation:
     def __init__(self, feature):
         if feature["municipality"].as_string() not in LAM_STATION_MUNICIPALITIES:
             self.active = False
-        self.lam_id = feature["tmsNumber"].as_int()
+        self.station_id = feature["tmsNumber"].as_int()
         names = json.loads(feature["names"].as_string())
         self.name = names["fi"]
         self.name_sv = names["sv"]
@@ -64,6 +67,15 @@ def __init__(self, feature):
         self.geom = geom
 
 
+class TelraamCounterStation:
+    def __init__(self, feature):
+        self.name = feature["mac"]
+        self.name_sv = feature["mac"]
+        self.name_en = feature["mac"]
+        self.geom = GEOSGeometry("POINT EMPTY")
+        self.station_id = feature["mac"]
+
+
 class ObservationStation(LAMStation, EcoCounterStation, TrafficCounterStation):
     def __init__(self, csv_data_source, feature):
         self.csv_data_source = csv_data_source
@@ -71,8 +83,10 @@ def __init__(self, csv_data_source, feature):
         self.name_sv = None
         self.name_en = None
         self.geom = None
-        self.lam_id = None
+        self.station_id = None
         match csv_data_source:
+            case COUNTERS.TELRAAM_COUNTER:
+                TelraamCounterStation.__init__(self, feature)
             case COUNTERS.LAM_COUNTER:
                 LAMStation.__init__(self, feature)
             case COUNTERS.ECO_COUNTER:
@@ -200,7 +214,6 @@ def get_lam_counter_csv(start_date):
     5. Shift the columns with the calculated shift_index, this must be done if there is no data
     for the station from the start_date. This ensures the data matches the timestamps.
     """
-
     drop_columns = [
         "pistetunnus",
         "sijainti",
@@ -223,7 +236,9 @@ def get_lam_counter_csv(start_date):
     for station in Station.objects.filter(csv_data_source=LAM_COUNTER):
         # In the source data the directions are 1 and 2.
         for direction in range(1, 3):
-            df = get_lam_station_dataframe(station.lam_id, direction, start_date, today)
+            df = get_lam_station_dataframe(
+                station.station_id, direction, start_date, today
+            )
             # Read the direction
             direction_name = df["suuntaselite"].iloc[0]
             # From the mappings determine the 'keskustaan päin' or 'poispäin keskustasta' direction.
@@ -317,10 +332,51 @@ def get_eco_counter_stations():
     return stations
 
 
+def get_active_camera(offset=0):
+    """
+    Function that gets pseudo random camera that is actvie for testing data.
+    NOTE, this function will be obsolete when Turku gets its cameras online.
+    """
+    url = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/cameras"
+
+    headers = {
+        "X-Api-Key": settings.TELRAAM_TOKEN,
+    }
+    response = requests.get(url, headers=headers)
+    cameras = response.json()["cameras"]
+    i = 0
+    for camera in cameras:
+        # time_end: null for active instances
+        if camera["status"] == "active" and not camera["time_end"]:
+            if i == offset:
+                return camera
+            i += 1
+    return None
+
+
+def get_telraam_counter_stations():
+    stations = []
+    cameras = []
+    for i in range(2, 5):
+        cameras.append(get_active_camera(offset=i * 3))
+    for feature in cameras:
+        stations.append(ObservationStation(TELRAAM_COUNTER, feature))
+    return stations
+
+
+def get_telraam_counter_csv():
+    csv_file = f"{get_root_dir()}/media/{TELRAAM_CSV_FILE_NAME}"
+    df = pd.read_csv(csv_file)
+    df[TIMESTAMP_COL_NAME] = pd.to_datetime(df[TIMESTAMP_COL_NAME])
+    return df
+
+
 def save_stations(csv_data_source):
     stations = []
     num_created = 0
     match csv_data_source:
+        case COUNTERS.TELRAAM_COUNTER:
+            stations = get_telraam_counter_stations()
         case COUNTERS.LAM_COUNTER:
             stations = get_lam_counter_stations()
         case COUNTERS.ECO_COUNTER:
@@ -338,13 +394,13 @@ def save_stations(csv_data_source):
             name_sv=station.name_sv,
             name_en=station.name_en,
             geom=station.geom,
-            lam_id=station.lam_id,
+            station_id=station.station_id,
+            csv_data_source=csv_data_source,
         )
         if obj.id in object_ids:
             object_ids.remove(obj.id)
         if created:
             num_created += 1
-
     Station.objects.filter(id__in=object_ids).delete()
     logger.info(
         f"Deleted {len(object_ids)} obsolete Stations for counter {csv_data_source}"

From c7a988b5f9409fe99e9d8253ce9bef3e942593b6 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 17 May 2023 14:38:26 +0300
Subject: [PATCH 37/81] Add telraam constants

---
 eco_counter/constants.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 4b3203ec8..4e8df1699 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -9,17 +9,19 @@
 TRAFFIC_COUNTER_END_YEAR = 2022
 ECO_COUNTER_START_YEAR = 2020
 LAM_COUNTER_START_YEAR = 2010
+TELRAAM_COUNTER_START_YEAR = 2023
 
 
 TRAFFIC_COUNTER = "TC"
 ECO_COUNTER = "EC"
 LAM_COUNTER = "LC"
+TELRAAM_COUNTER = "TR"
 
 COUNTERS = types.SimpleNamespace()
 COUNTERS.TRAFFIC_COUNTER = TRAFFIC_COUNTER
 COUNTERS.ECO_COUNTER = ECO_COUNTER
 COUNTERS.LAM_COUNTER = LAM_COUNTER
-
+COUNTERS.TELRAAM_COUNTER = TELRAAM_COUNTER
 
 CSV_DATA_SOURCES = (
     (TRAFFIC_COUNTER, "TrafficCounter"),
@@ -30,6 +32,7 @@
     ECO_COUNTER: ECO_COUNTER_START_YEAR,
     TRAFFIC_COUNTER: TRAFFIC_COUNTER_START_YEAR,
     LAM_COUNTER: LAM_COUNTER_START_YEAR,
+    TELRAAM_COUNTER: TELRAAM_COUNTER_START_YEAR,
 }
 
 TIMESTAMP_COL_NAME = "startTime"
@@ -68,3 +71,9 @@
         for k in keys
     ]
 )
+TELRAAM_COUNTER_API_BASE_URL = "https://telraam-api.net"
+# The start month of the start year as telraam data is not available
+# from the beginning of the start tear
+TELRAAM_COUNTER_START_MONTH = 5
+TELRAAM_API_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+TELRAAM_CSV_FILE_NAME = "telraam_data.csv"

From 70448270a3a2283d693a05bf5fadcf0cf0f13be3 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 17 May 2023 14:39:13 +0300
Subject: [PATCH 38/81] Add task import_telraam_to_csv

---
 eco_counter/tasks.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/eco_counter/tasks.py b/eco_counter/tasks.py
index 9b7fdb714..fbbc6fbde 100644
--- a/eco_counter/tasks.py
+++ b/eco_counter/tasks.py
@@ -11,3 +11,8 @@ def import_counter_data(args, name="import_counter_data"):
 @shared_task_email
 def initial_import_counter_data(args, name="initial_import_counter_data"):
     management.call_command("import_counter_data", "--init", args)
+
+
+@shared_task_email
+def import_telraam_to_csv(args, name="import_telraam_to_csv"):
+    management.call_command("import_telraam_to_csv", args)

From 4ff0a2d5bb23ee2efd83442e257e2f1fd68413aa Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 17 May 2023 14:39:55 +0300
Subject: [PATCH 39/81] Add TELRAAM_TOKEN

---
 smbackend/settings.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/smbackend/settings.py b/smbackend/settings.py
index 0e786c728..91306fdb0 100644
--- a/smbackend/settings.py
+++ b/smbackend/settings.py
@@ -62,6 +62,7 @@
     EMAIL_HOST_USER=(str, None),
     EMAIL_PORT=(int, None),
     EMAIL_USE_TLS=(bool, None),
+    TELRAAM_TOKEN=(str, None),
 )
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -313,7 +314,7 @@ def gettext(s):
 def preprocessing_filter_spec(endpoints):
     filtered = []
     for endpoint in DOC_ENDPOINTS:
-        for (path, path_regex, method, callback) in endpoints:
+        for path, path_regex, method, callback in endpoints:
             if path.startswith(endpoint):
                 filtered.append((path, path_regex, method, callback))
     return filtered
@@ -433,3 +434,4 @@ def preprocessing_filter_spec(endpoints):
 YIT_CONTRACTS_URL = env("YIT_CONTRACTS_URL")
 YIT_TOKEN_URL = env("YIT_TOKEN_URL")
 KUNTEC_KEY = env("KUNTEC_KEY")
+TELRAAM_TOKEN = env("TELRAAM_TOKEN")

From 1ff58205cc84daac23f04b8ea4d4873d466ee956 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 17 May 2023 14:40:26 +0300
Subject: [PATCH 40/81] Add functions to get telraam cameras

---
 eco_counter/management/commands/utils.py | 25 +++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 428b94358..f05246047 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -332,18 +332,22 @@ def get_eco_counter_stations():
     return stations
 
 
-def get_active_camera(offset=0):
-    """
-    Function that gets pseudo random camera that is actvie for testing data.
-    NOTE, this function will be obsolete when Turku gets its cameras online.
-    """
+def fetch_telraam_cameras():
     url = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/cameras"
 
     headers = {
         "X-Api-Key": settings.TELRAAM_TOKEN,
     }
     response = requests.get(url, headers=headers)
-    cameras = response.json()["cameras"]
+    return response.json().get("cameras", None)
+
+
+def get_active_telraam_camera(offset):
+    """
+    Function that gets pseudo random camera that is actvie for testing data.
+    NOTE, this function will be obsolete when Turku gets its cameras online.
+    """
+    cameras = fetch_telraam_cameras()
     i = 0
     for camera in cameras:
         # time_end: null for active instances
@@ -354,11 +358,14 @@ def get_active_camera(offset=0):
     return None
 
 
+def get_telraam_cameras():
+    # TODO, add Turku cameras when they are online
+    return [get_active_telraam_camera(3 + i * 3) for i in range(3)]
+
+
 def get_telraam_counter_stations():
     stations = []
-    cameras = []
-    for i in range(2, 5):
-        cameras.append(get_active_camera(offset=i * 3))
+    cameras = get_telraam_cameras()
     for feature in cameras:
         stations.append(ObservationStation(TELRAAM_COUNTER, feature))
     return stations

From 704b3c9ddbbacf58ed9495508cccff4c028e88e3 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 17 May 2023 14:41:06 +0300
Subject: [PATCH 41/81] Add start month for Telraam counter

---
 eco_counter/management/commands/import_counter_data.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 6b0bb938f..4cbcbfb94 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -21,7 +21,7 @@
     ECO_COUNTER,
     LAM_COUNTER,
     TELRAAM_COUNTER,
-    TELRAAM_START_MONTH,
+    TELRAAM_COUNTER_START_MONTH,
     TRAFFIC_COUNTER,
     TRAFFIC_COUNTER_START_YEAR,
 )
@@ -445,7 +445,9 @@ def handle(self, *args, **options):
                     )
                 else:
                     start_month = (
-                        TELRAAM_START_MONTH if counter == TELRAAM_COUNTER else "01"
+                        TELRAAM_COUNTER_START_MONTH
+                        if counter == TELRAAM_COUNTER
+                        else "01"
                     )
                     start_time = f"{COUNTER_START_YEARS[counter]}-{start_month}-01"
 

From 3f1d79d983d9f98d6b5ad7675d9c2da2bccd6ebb Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 17 May 2023 14:43:33 +0300
Subject: [PATCH 42/81] Improve/refactor importing of data to csv

---
 .../commands/import_telraam_to_csv.py         | 99 ++++++++++---------
 1 file changed, 53 insertions(+), 46 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index fef44063c..8bf794d8f 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -7,8 +7,12 @@
 from django.conf import settings
 from django.core.management import BaseCommand
 
-from eco_counter.constants import TELRAAM_CSV_FILE_NAME
-from eco_counter.management.commands.utils import get_active_camera
+from eco_counter.constants import (
+    TELRAAM_COUNTER_START_MONTH,
+    TELRAAM_COUNTER_START_YEAR,
+    TELRAAM_CSV_FILE_NAME,
+)
+from eco_counter.management.commands.utils import get_telraam_cameras
 from mobility_data.importers.utils import get_root_dir
 
 TELRAAM_API_BASE_URL = "https://telraam-api.net"
@@ -22,8 +26,8 @@
 TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 DATA_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
 
-token = settings.TELRAAM_TOKEN
-assert token
+TOKEN = settings.TELRAAM_TOKEN
+assert TOKEN
 csv_file = f"{get_root_dir()}/media/{TELRAAM_CSV_FILE_NAME}"
 logger = logging.getLogger("eco_counter")
 
@@ -61,13 +65,12 @@ def get_mappings(station_name, direction=True):
     mappings = {}
     for field in column_mappings.keys():
         mappings[field] = f"{station_name} {column_mappings[field]}"
-
     return mappings
 
 
-def get_hourly_data(from_date, end_date, camera_id):
+def fetch_traffic_report(from_date: str, end_date: str, camera_id: str):
     headers = {
-        "X-Api-Key": token,
+        "X-Api-Key": TOKEN,
         "Content-Type": "application/json",
     }
     data = {
@@ -78,26 +81,30 @@ def get_hourly_data(from_date, end_date, camera_id):
         "time_end": end_date,
     }
     response = requests.post(TRAFFIC_URL, headers=headers, data=json.dumps(data))
+    return response.json().get("report", [])
 
-    from_date = datetime.strptime(from_date, TIME_FORMAT)
-    end_date = datetime.strptime(end_date, TIME_FORMAT)
+
+def get_delta_hours(from_date: datetime, end_date: datetime) -> datetime:
     delta = end_date - from_date
     delta_hours = int(round(delta.total_seconds() / 3600))
-    logger.info(f"delta hours {delta_hours}")
-    report = response.json().get("report", [])
+    return delta_hours
+
+
+def get_hourly_data(from_date, end_date, camera_id):
+    report = fetch_traffic_report(from_date, end_date, camera_id)
+    from_date = datetime.strptime(from_date, TIME_FORMAT)
+    end_date = datetime.strptime(end_date, TIME_FORMAT)
+    delta_hours = get_delta_hours(from_date, end_date)
+    logger.info(f"Trying to import {delta_hours} hours for camera {camera_id}.")
     if not report:
         report = [{} for a in range(delta_hours)]
-    else:
-        print("Report has data")
-    if len(report) != delta_hours:
-        report = [{} for a in range(delta_hours)]
-        delta_hours = len(report)
+
+    delta_hours = len(report)
     res = []
     start_date = from_date
-    for i in range(delta_hours):
+    for item in report:
         d = {}
         d["date"] = datetime.strftime(start_date, TIME_FORMAT)
-        item = report[i]
         for veh in VEHICLE_TYPES.keys():
             for dir in DIRECTIONS:
                 key = f"{veh}_{dir}"
@@ -108,52 +115,50 @@ def get_hourly_data(from_date, end_date, camera_id):
     return res, delta_hours
 
 
-def get_active_cameras():
-    headers = {
-        "X-Api-Key": token,
-    }
-    active_cameras = []
-    response = requests.get(AVAILABLE_CAMERAS_URL, headers=headers)
-    cameras = response.json()["cameras"]
-    for camera in cameras:
-        if camera["status"] == "active":
-            active_cameras.append(camera)
-    return active_cameras
-
-
 def save_dataframe():
-    test_num_days = 6
-    test_num_cams = 2
-    # cam_off
-    start_date = date(2023, 5, 1)
-    end_date = date(2023, 5, 1 + test_num_days).strftime(TIME_FORMAT)
-    from_date = start_date.strftime(TIME_FORMAT)
     try:
         df = pd.read_csv(csv_file, index_col=INDEX_COLUMN_NAME)
         from_date = df.iloc[-1].name
+        logger.info(f"Found Telraam data until {from_date} in csv file.")
         from_date = datetime.strptime(from_date, TIME_FORMAT) + timedelta(hours=1)
         from_date = datetime.strftime(from_date, TIME_FORMAT)
-        logger.info(f"Read csv data to {from_date}")
     except Exception:
         logger.info("Creating new empty Pandas DataFrame")
         df = pd.DataFrame()
-    # TODO, when Turku cameras online, add them
-    cameras = []
-    for i in range(2, 1 + test_num_cams):
-        cameras.append(get_active_camera(offset=i * 3))
+        from_date = date(TELRAAM_COUNTER_START_YEAR, TELRAAM_COUNTER_START_MONTH, 1)
+        from_date = datetime.strftime(from_date, TIME_FORMAT)
+
+    end_date = datetime.now().strftime(TIME_FORMAT)
+    logger.info(f"Fetching Telraam data from {from_date} to {end_date}")
+
     df_created = pd.DataFrame()
     reports = []
+    min_delta_hours = 1_000_000
+    cameras = get_telraam_cameras()
     for camera in cameras:
         report, delta_hours = get_hourly_data(
             from_date, end_date, camera["instance_id"]
         )
+        if report:
+            logger.info(
+                f"Camera {camera['instance_id']} imported to {report[-1]['date']}"
+            )
+        else:
+            f"Imported empty report for camera {camera['instance_id']}"
+
+        # NOTE, reports length can vary as some have less data.
+        if delta_hours <= min_delta_hours:
+            min_delta_hours = delta_hours
         reports.append({"camera": camera, "report": report})
 
     columns = {}
     columns[INDEX_COLUMN_NAME] = []
-    for hour in range(delta_hours):
-        columns[INDEX_COLUMN_NAME].append(reports[0]["report"][hour]["date"])
-        for report in reports:
+    # NOTE, rows are only populated to that datetime where all cameras has data.
+    delta_hours = min_delta_hours
+    for i, report in enumerate(reports):
+        for hour in range(delta_hours):
+            if i == 0:
+                columns[INDEX_COLUMN_NAME].append(reports[0]["report"][hour]["date"])
             mappings = get_mappings(report["camera"]["mac"])
             for mapping in mappings.items():
                 # key is the name of the column, e.g., name_ak
@@ -167,7 +172,6 @@ def save_dataframe():
                     values_list.append(report["report"][hour][value_key])
                 columns[key] = values_list
     df_created = pd.DataFrame(data=columns, index=columns["startTime"])
-
     df_created = df_created.drop(columns=[INDEX_COLUMN_NAME], axis=1)
     if df.empty:
         df = df_created
@@ -177,6 +181,9 @@ def save_dataframe():
     df = df.fillna(0)
     df = df.astype(int)
     df.to_csv(csv_file)
+    logger.info(
+        f"Telraam data imported until {df.index[-1]} and DataFrame saved to {csv_file}"
+    )
 
 
 class Command(BaseCommand):

From 8d89253c0ee4181404822b70c7d287e86298b913 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 23 May 2023 12:12:13 +0300
Subject: [PATCH 43/81] Add Telraam constanst and INDEX_COLUMN_NAME

---
 eco_counter/constants.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 4e8df1699..fb4a1dca3 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -2,6 +2,10 @@
 
 from django.conf import settings
 
+from mobility_data.importers.utils import get_root_dir
+
+INDEX_COLUMN_NAME = "startTime"
+
 TRAFFIC_COUNTER_START_YEAR = 2015
 # Manually define the end year, as the source data comes from the page
 # defined in env variable TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL.
@@ -16,6 +20,7 @@
 ECO_COUNTER = "EC"
 LAM_COUNTER = "LC"
 TELRAAM_COUNTER = "TR"
+TELRAAM_CSV = "TV"
 
 COUNTERS = types.SimpleNamespace()
 COUNTERS.TRAFFIC_COUNTER = TRAFFIC_COUNTER
@@ -27,6 +32,8 @@
     (TRAFFIC_COUNTER, "TrafficCounter"),
     (ECO_COUNTER, "EcoCounter"),
     (LAM_COUNTER, "LamCounter"),
+    (TELRAAM_COUNTER, "TelraamCounter"),
+    (TELRAAM_CSV, "TelraamCSV"),
 )
 COUNTER_START_YEARS = {
     ECO_COUNTER: ECO_COUNTER_START_YEAR,
@@ -35,7 +42,6 @@
     TELRAAM_COUNTER: TELRAAM_COUNTER_START_YEAR,
 }
 
-TIMESTAMP_COL_NAME = "startTime"
 TRAFFIC_COUNTER_METADATA_GEOJSON = "traffic_counter_metadata.geojson"
 # LAM stations located in the municipalities list are included.
 LAM_STATION_MUNICIPALITIES = ["Turku", "Raisio", "Kaarina", "Lieto"]
@@ -72,8 +78,14 @@
     ]
 )
 TELRAAM_COUNTER_API_BASE_URL = "https://telraam-api.net"
+# Maximum 3 months at a time
+TELRAAM_COUNTER_TRAFFIC_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/reports/traffic"
+TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/cameras"
 # The start month of the start year as telraam data is not available
 # from the beginning of the start tear
 TELRAAM_COUNTER_START_MONTH = 5
-TELRAAM_API_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
-TELRAAM_CSV_FILE_NAME = "telraam_data.csv"
+TELRAAM_COUNTER_API_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+TELRAAM_COUNTER_CSV_FILE_PATH = f"{get_root_dir()}/media/telraam_data/"
+TELRAAM_COUNTER_CSV_FILE = (
+    TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{id}_{day}_{month}_{year}.csv"
+)

From bbe16889e17d0425bb70bb2382ed0b969790e5b8 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 23 May 2023 12:14:45 +0300
Subject: [PATCH 44/81] Generate CSV file for every day for every camera

---
 .../commands/import_telraam_to_csv.py         | 192 ++++++++++--------
 1 file changed, 105 insertions(+), 87 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index 8bf794d8f..cf8bb52c8 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 from datetime import date, datetime, timedelta
 
 import pandas as pd
@@ -8,27 +9,23 @@
 from django.core.management import BaseCommand
 
 from eco_counter.constants import (
+    INDEX_COLUMN_NAME,
+    TELRAAM_COUNTER_API_TIME_FORMAT,
+    TELRAAM_COUNTER_CSV_FILE,
+    TELRAAM_COUNTER_CSV_FILE_PATH,
     TELRAAM_COUNTER_START_MONTH,
     TELRAAM_COUNTER_START_YEAR,
-    TELRAAM_CSV_FILE_NAME,
+    TELRAAM_COUNTER_TRAFFIC_URL,
+    TELRAAM_CSV,
 )
 from eco_counter.management.commands.utils import get_telraam_cameras
-from mobility_data.importers.utils import get_root_dir
-
-TELRAAM_API_BASE_URL = "https://telraam-api.net"
-ALIVE_URL = f"{TELRAAM_API_BASE_URL}/v1"
-# Maximum 3 months at a time
-TRAFFIC_URL = f"{TELRAAM_API_BASE_URL}/v1/reports/traffic"
-AVAILABLE_CAMERAS_URL = f"{TELRAAM_API_BASE_URL}/v1/cameras"
-INDEX_COLUMN_NAME = "startTime"
+from eco_counter.models import ImportState
+
 LEVEL = "instances"  # instance per individual can
 FORMAT = "per-hour"
-TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
-DATA_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
 
 TOKEN = settings.TELRAAM_TOKEN
 assert TOKEN
-csv_file = f"{get_root_dir()}/media/{TELRAAM_CSV_FILE_NAME}"
 logger = logging.getLogger("eco_counter")
 
 HEAVY = "heavy"
@@ -80,8 +77,12 @@ def fetch_traffic_report(from_date: str, end_date: str, camera_id: str):
         "time_start": from_date,
         "time_end": end_date,
     }
-    response = requests.post(TRAFFIC_URL, headers=headers, data=json.dumps(data))
-    return response.json().get("report", [])
+    response = requests.post(
+        TELRAAM_COUNTER_TRAFFIC_URL, headers=headers, data=json.dumps(data)
+    )
+    report = response.json().get("report", [])
+    print(len(report))
+    return report
 
 
 def get_delta_hours(from_date: datetime, end_date: datetime) -> datetime:
@@ -90,21 +91,39 @@ def get_delta_hours(from_date: datetime, end_date: datetime) -> datetime:
     return delta_hours
 
 
-def get_hourly_data(from_date, end_date, camera_id):
-    report = fetch_traffic_report(from_date, end_date, camera_id)
-    from_date = datetime.strptime(from_date, TIME_FORMAT)
-    end_date = datetime.strptime(end_date, TIME_FORMAT)
-    delta_hours = get_delta_hours(from_date, end_date)
+def get_day_data(
+    day_date: date, camera_id: str, check_delta_hours: bool = True
+) -> list:
+    from_datetime = datetime(day_date.year, day_date.month, day_date.day, 0, 0, 0)
+    from_datetime_str = from_datetime.strftime(TELRAAM_COUNTER_API_TIME_FORMAT)
+    end_datetime = (
+        datetime(day_date.year, day_date.month, day_date.day)
+        + timedelta(hours=23)
+        + timedelta(minutes=59)
+    )
+    end_datetime_str = end_datetime.strftime(TELRAAM_COUNTER_API_TIME_FORMAT)
+    report = fetch_traffic_report(from_datetime_str, end_datetime_str, camera_id)
+
+    delta_hours = get_delta_hours(from_datetime, end_datetime)
     logger.info(f"Trying to import {delta_hours} hours for camera {camera_id}.")
     if not report:
+        logger.warning("No report found, populating with empty dicts")
         report = [{} for a in range(delta_hours)]
-
+    else:
+        logger.info(f"Imorted report with {len(report)} elements")
     delta_hours = len(report)
+    if check_delta_hours and delta_hours != 24:
+        dif = 24 - delta_hours
+        logger.warning(
+            f"Fetched report with delta_hours not equal to 24, appending missing {dif} elements empty dicts"
+        )
+        report += [{} for a in range(dif)]
+
     res = []
-    start_date = from_date
+    start_date = from_datetime
     for item in report:
         d = {}
-        d["date"] = datetime.strftime(start_date, TIME_FORMAT)
+        d["date"] = datetime.strftime(start_date, TELRAAM_COUNTER_API_TIME_FORMAT)
         for veh in VEHICLE_TYPES.keys():
             for dir in DIRECTIONS:
                 key = f"{veh}_{dir}"
@@ -116,74 +135,73 @@ def get_hourly_data(from_date, end_date, camera_id):
 
 
 def save_dataframe():
-    try:
-        df = pd.read_csv(csv_file, index_col=INDEX_COLUMN_NAME)
-        from_date = df.iloc[-1].name
-        logger.info(f"Found Telraam data until {from_date} in csv file.")
-        from_date = datetime.strptime(from_date, TIME_FORMAT) + timedelta(hours=1)
-        from_date = datetime.strftime(from_date, TIME_FORMAT)
-    except Exception:
-        logger.info("Creating new empty Pandas DataFrame")
-        df = pd.DataFrame()
-        from_date = date(TELRAAM_COUNTER_START_YEAR, TELRAAM_COUNTER_START_MONTH, 1)
-        from_date = datetime.strftime(from_date, TIME_FORMAT)
-
-    end_date = datetime.now().strftime(TIME_FORMAT)
-    logger.info(f"Fetching Telraam data from {from_date} to {end_date}")
-
-    df_created = pd.DataFrame()
-    reports = []
-    min_delta_hours = 1_000_000
-    cameras = get_telraam_cameras()
-    for camera in cameras:
-        report, delta_hours = get_hourly_data(
-            from_date, end_date, camera["instance_id"]
+    if not os.path.exists(TELRAAM_COUNTER_CSV_FILE_PATH):
+        os.makedirs(TELRAAM_COUNTER_CSV_FILE_PATH)
+        ImportState.objects.filter(csv_data_source=TELRAAM_CSV).delete()
+        import_state = ImportState.objects.create(
+            csv_data_source=TELRAAM_CSV,
+            current_year_number=TELRAAM_COUNTER_START_YEAR,
+            current_month_number=TELRAAM_COUNTER_START_MONTH,
+            current_day_number=1,
         )
-        if report:
-            logger.info(
-                f"Camera {camera['instance_id']} imported to {report[-1]['date']}"
-            )
-        else:
-            f"Imported empty report for camera {camera['instance_id']}"
-
-        # NOTE, reports length can vary as some have less data.
-        if delta_hours <= min_delta_hours:
-            min_delta_hours = delta_hours
-        reports.append({"camera": camera, "report": report})
-
-    columns = {}
-    columns[INDEX_COLUMN_NAME] = []
-    # NOTE, rows are only populated to that datetime where all cameras has data.
-    delta_hours = min_delta_hours
-    for i, report in enumerate(reports):
-        for hour in range(delta_hours):
-            if i == 0:
-                columns[INDEX_COLUMN_NAME].append(reports[0]["report"][hour]["date"])
-            mappings = get_mappings(report["camera"]["mac"])
-            for mapping in mappings.items():
-                # key is the name of the column, e.g., name_ak
-                key = mapping[1]
-                value_key = mapping[0]
-                values_list = columns.get(key, [])
-                if HEAVY in value_key:
-                    # add heavy values to car column, as the mapping is same.
-                    values_list[-1] += report["report"][hour][value_key]
-                else:
-                    values_list.append(report["report"][hour][value_key])
-                columns[key] = values_list
-    df_created = pd.DataFrame(data=columns, index=columns["startTime"])
-    df_created = df_created.drop(columns=[INDEX_COLUMN_NAME], axis=1)
-    if df.empty:
-        df = df_created
     else:
-        df = pd.concat([df, df_created], axis=0)
-    df.index.rename(INDEX_COLUMN_NAME, inplace=True)
-    df = df.fillna(0)
-    df = df.astype(int)
-    df.to_csv(csv_file)
-    logger.info(
-        f"Telraam data imported until {df.index[-1]} and DataFrame saved to {csv_file}"
+        import_state = ImportState.objects.filter(csv_data_source=TELRAAM_CSV).first()
+
+    from_date = date(
+        import_state.current_year_number,
+        import_state.current_month_number,
+        import_state.current_day_number,
     )
+    date_today = date.today()
+    logger.info(f"Fetching Telraam data from {str(from_date)} to {str(date_today)}")
+
+    cameras = get_telraam_cameras()
+    for camera in cameras:
+        start_date = from_date
+        while start_date <= date_today:
+            report, delta_hours = get_day_data(start_date, camera["instance_id"])
+            mappings = get_mappings(camera["mac"])
+            columns = {}
+            columns[INDEX_COLUMN_NAME] = []
+            for hour in range(delta_hours):
+                columns[INDEX_COLUMN_NAME].append(report[hour]["date"])
+                for mapping in mappings.items():
+                    # key is the name of the column, e.g., name_ak
+                    key = mapping[1]
+                    value_key = mapping[0]
+                    values_list = columns.get(key, [])
+                    if HEAVY in value_key:
+                        # add heavy values to car column, as the mapping is same.
+                        values_list[-1] += report[hour][value_key]
+                    else:
+                        values_list.append(report[hour][value_key])
+                    columns[key] = values_list
+            df = pd.DataFrame(data=columns, index=columns[INDEX_COLUMN_NAME])
+            df = df.drop(columns=[INDEX_COLUMN_NAME], axis=1)
+            df.index.rename(INDEX_COLUMN_NAME, inplace=True)
+            df = df.fillna(0)
+            df = df.astype(int)
+
+            csv_file = TELRAAM_COUNTER_CSV_FILE.format(
+                id=camera["mac"],
+                day=start_date.day,
+                month=start_date.month,
+                year=start_date.year,
+            )
+            if start_date == date_today:
+                # Remove latest csv, as it might not be populated until the end of day
+                if os.path.exists(csv_file):
+                    os.remove(csv_file)
+            if not os.path.exists(csv_file):
+                df.to_csv(csv_file)
+            start_date += timedelta(days=1)
+
+    start_date -= timedelta(days=1)
+    import_state.current_year_number = start_date.year
+    import_state.current_month_number = start_date.month
+    import_state.current_day_number = start_date.day
+    import_state.save()
+    logger.info(f"Telraam data imported until {str(start_date)}")
 
 
 class Command(BaseCommand):

From 7588bca537e23b9eb1eebb04e3a6023d918cf502 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Tue, 23 May 2023 12:16:15 +0300
Subject: [PATCH 45/81] Create Telraam dataframe from multiple CSV files

---
 eco_counter/management/commands/utils.py | 61 +++++++++++++++++++-----
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index f05246047..827935fa0 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -13,19 +13,23 @@
 from eco_counter.constants import (
     COUNTERS,
     ECO_COUNTER,
+    INDEX_COLUMN_NAME,
     LAM_COUNTER,
     LAM_STATION_MUNICIPALITIES,
     LAM_STATIONS_API_FETCH_URL,
     LAM_STATIONS_DIRECTION_MAPPINGS,
     TELRAAM_COUNTER,
     TELRAAM_COUNTER_API_BASE_URL,
-    TELRAAM_CSV_FILE_NAME,
-    TIMESTAMP_COL_NAME,
+    TELRAAM_COUNTER_API_TIME_FORMAT,
+    TELRAAM_COUNTER_CSV_FILE,
+    TELRAAM_COUNTER_START_MONTH,
+    TELRAAM_COUNTER_START_YEAR,
+    TELRAAM_CSV,
     TRAFFIC_COUNTER,
     TRAFFIC_COUNTER_CSV_URLS,
     TRAFFIC_COUNTER_METADATA_GEOJSON,
 )
-from eco_counter.models import Station
+from eco_counter.models import ImportState, Station
 from eco_counter.tests.test_import_counter_data import TEST_COLUMN_NAMES
 from mobility_data.importers.utils import get_root_dir
 
@@ -174,7 +178,7 @@ def get_traffic_counter_csv(start_year=2015):
     logger.info(df.info(verbose=False))
     logger.info(f"{ids_not_found} IDs not found in metadata.")
     # Move column 'startTime to first (0) position.
-    df.insert(0, TIMESTAMP_COL_NAME, df.pop(TIMESTAMP_COL_NAME))
+    df.insert(0, INDEX_COLUMN_NAME, df.pop(INDEX_COLUMN_NAME))
     # df.to_csv("tc_out.csv")
     return df
 
@@ -232,7 +236,7 @@ def get_lam_counter_csv(start_date):
     num_15min_freq = dif_time.total_seconds() / 60 / 15
     time_stamps = pd.date_range(start_time, freq="15T", periods=num_15min_freq)
     data_frame = pd.DataFrame()
-    data_frame[TIMESTAMP_COL_NAME] = time_stamps
+    data_frame[INDEX_COLUMN_NAME] = time_stamps
     for station in Station.objects.filter(csv_data_source=LAM_COUNTER):
         # In the source data the directions are 1 and 2.
         for direction in range(1, 3):
@@ -253,7 +257,7 @@ def get_lam_counter_csv(start_date):
             # Calculate shift index, i.e., if data starts from different position that the start_date.
             # then shift the rows to the correct position using the calculated shift_index.
             shift_index = data_frame.index[
-                getattr(data_frame, TIMESTAMP_COL_NAME) == str(start_time)
+                getattr(data_frame, INDEX_COLUMN_NAME) == str(start_time)
             ][0]
             column_name = f"{station.name} A{direction_value}"
             # Drop all unnecessary columns.
@@ -360,7 +364,7 @@ def get_active_telraam_camera(offset):
 
 def get_telraam_cameras():
     # TODO, add Turku cameras when they are online
-    return [get_active_telraam_camera(3 + i * 3) for i in range(3)]
+    return [get_active_telraam_camera(2 + i * 3) for i in range(3)]
 
 
 def get_telraam_counter_stations():
@@ -371,10 +375,45 @@ def get_telraam_counter_stations():
     return stations
 
 
-def get_telraam_counter_csv():
-    csv_file = f"{get_root_dir()}/media/{TELRAAM_CSV_FILE_NAME}"
-    df = pd.read_csv(csv_file)
-    df[TIMESTAMP_COL_NAME] = pd.to_datetime(df[TIMESTAMP_COL_NAME])
+def get_telraam_counter_csv():    
+    df = pd.DataFrame()
+    from_date = date(TELRAAM_COUNTER_START_YEAR, TELRAAM_COUNTER_START_MONTH, 1)
+    try:
+        import_state = ImportState.objects.get(csv_data_source=TELRAAM_CSV)
+    except ImportState.DoesNotExist:
+        return None
+    end_date = date(
+        import_state.current_year_number,
+        import_state.current_month_number,
+        import_state.current_day_number,
+    )
+    for camera in get_telraam_cameras():
+        df_cam = pd.DataFrame()
+        start_date = from_date
+
+        while start_date <= end_date:
+            csv_file = TELRAAM_COUNTER_CSV_FILE.format(
+                id=camera["mac"],
+                day=start_date.day,
+                month=start_date.month,
+                year=start_date.year,
+            )
+            try:
+                df_tmp = pd.read_csv(csv_file, index_col=False)
+            except FileNotFoundError:
+                logger.warning(f"File {csv_file} not found, skipping camera {camera}")
+                break
+            df_cam = pd.concat([df_cam, df_tmp])
+            start_date += timedelta(days=1)
+
+        if df.empty:
+            df = df_cam
+        else:
+            df = pd.merge(df, df_cam, on=INDEX_COLUMN_NAME)
+
+    df[INDEX_COLUMN_NAME] = pd.to_datetime(
+        df[INDEX_COLUMN_NAME], format=TELRAAM_COUNTER_API_TIME_FORMAT
+    )
     return df
 
 

From 0a688b4175b985727e7119a0e91ec291d2b32aa6 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 12:46:18 +0300
Subject: [PATCH 46/81] Fix logger output, remove print

---
 eco_counter/management/commands/import_telraam_to_csv.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index cf8bb52c8..dbb16b27b 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -80,9 +80,7 @@ def fetch_traffic_report(from_date: str, end_date: str, camera_id: str):
     response = requests.post(
         TELRAAM_COUNTER_TRAFFIC_URL, headers=headers, data=json.dumps(data)
     )
-    report = response.json().get("report", [])
-    print(len(report))
-    return report
+    return response.json().get("report", [])
 
 
 def get_delta_hours(from_date: datetime, end_date: datetime) -> datetime:
@@ -201,10 +199,11 @@ def save_dataframe():
     import_state.current_month_number = start_date.month
     import_state.current_day_number = start_date.day
     import_state.save()
-    logger.info(f"Telraam data imported until {str(start_date)}")
+    return start_date
 
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
         logger.info("Importing Telraam data...")
-        save_dataframe()
+        until_date = save_dataframe()
+        logger.info(f"Telraam data imported until {str(until_date)}")

From f0eaf6192c7f22f113950974536421d3795661f9 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 12:47:12 +0300
Subject: [PATCH 47/81] Add currrent_day_number and geometry. Rename geom to
 location

---
 eco_counter/models.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/eco_counter/models.py b/eco_counter/models.py
index 09d1308be..a96b5d74c 100644
--- a/eco_counter/models.py
+++ b/eco_counter/models.py
@@ -13,6 +13,9 @@ class ImportState(models.Model):
         validators=[MinValueValidator(1), MaxValueValidator(12)],
         null=True,  # , default=1
     )
+    current_day_number = models.PositiveSmallIntegerField(
+        null=True, blank=True, validators=[MinValueValidator(1), MaxValueValidator(31)]
+    )
     csv_data_source = models.CharField(
         max_length=2,
         choices=CSV_DATA_SOURCES,
@@ -21,17 +24,17 @@ class ImportState(models.Model):
 
 
 class Station(models.Model):
-
     name = models.CharField(max_length=64)
-    geom = models.PointField(srid=settings.DEFAULT_SRID)
+    location = models.PointField(srid=settings.DEFAULT_SRID)
+    geometry = models.GeometryField(srid=settings.DEFAULT_SRID, null=True)
     csv_data_source = models.CharField(
         max_length=2,
         choices=CSV_DATA_SOURCES,
         default=ECO_COUNTER,
     )
-    # For lam stations store the LAM station ID, this is
-    # required when fetching data from the API using the ID.
-    lam_id = models.PositiveSmallIntegerField(null=True)
+    # Optioal id of the station, used when fetching LAM
+    # and TELRAAM station data
+    station_id = models.CharField(max_length=16, null=True)
 
     def __str__(self):
         return "%s %s" % (self.name, self.geom)

From 38f7db6f6dba2552c78ee99ba4e160ea8dd6d181 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:08:04 +0300
Subject: [PATCH 48/81] Add Telraaam segments url, change end year of traffic
 counter

---
 eco_counter/constants.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index fb4a1dca3..79f9d0247 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -10,7 +10,7 @@
 # Manually define the end year, as the source data comes from the page
 # defined in env variable TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL.
 # Change end year when data for the next year is available.
-TRAFFIC_COUNTER_END_YEAR = 2022
+TRAFFIC_COUNTER_END_YEAR = 2023
 ECO_COUNTER_START_YEAR = 2020
 LAM_COUNTER_START_YEAR = 2010
 TELRAAM_COUNTER_START_YEAR = 2023
@@ -81,6 +81,9 @@
 # Maximum 3 months at a time
 TELRAAM_COUNTER_TRAFFIC_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/reports/traffic"
 TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/cameras"
+TELRAAM_COUNTER_CAMERA_SEGMENTS_URL = (
+    TELRAAM_COUNTER_API_BASE_URL + "/v1/segments/id/{id}"
+)
 # The start month of the start year as telraam data is not available
 # from the beginning of the start tear
 TELRAAM_COUNTER_START_MONTH = 5

From 0a92530160fc21596dd09982cf3b8ab4b630a332 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:09:00 +0300
Subject: [PATCH 49/81] Change geom to location

---
 eco_counter/tests/conftest.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/eco_counter/tests/conftest.py b/eco_counter/tests/conftest.py
index bbbfbb9ec..e5af61b79 100644
--- a/eco_counter/tests/conftest.py
+++ b/eco_counter/tests/conftest.py
@@ -41,21 +41,21 @@ def stations():
     stations.append(
         Station.objects.create(
             name=TEST_EC_STATION_NAME,
-            geom="POINT(60.4487578455581 22.269454227550053)",
+            location="POINT(60.4487578455581 22.269454227550053)",
             csv_data_source=ECO_COUNTER,
         )
     )
     stations.append(
         Station.objects.create(
             name=TEST_TC_STATION_NAME,
-            geom="POINT(60.4487578455581 22.269454227550053)",
+            location="POINT(60.4487578455581 22.269454227550053)",
             csv_data_source=TRAFFIC_COUNTER,
         )
     )
     stations.append(
         Station.objects.create(
             name=TEST_LC_STATION_NAME,
-            geom="POINT(60.4487578455581 22.269454227550053)",
+            location="POINT(60.4487578455581 22.269454227550053)",
             csv_data_source=LAM_COUNTER,
         )
     )

From aa821b03eabbdbf437d8c6b1ac17c3389b1cc13f Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:16:18 +0300
Subject: [PATCH 50/81] Serialize geometry and location, remove geom

---
 eco_counter/api/serializers.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/eco_counter/api/serializers.py b/eco_counter/api/serializers.py
index 0a1adca26..5fc7e3475 100644
--- a/eco_counter/api/serializers.py
+++ b/eco_counter/api/serializers.py
@@ -31,6 +31,7 @@ class StationSerializer(serializers.ModelSerializer):
     y = serializers.SerializerMethodField()
     lon = serializers.SerializerMethodField()
     lat = serializers.SerializerMethodField()
+    # geom = serializers.SerializerMethodField()
     sensor_types = serializers.SerializerMethodField()
 
     class Meta:
@@ -43,7 +44,9 @@ class Meta:
             "name_sv",
             "name_en",
             "csv_data_source",
-            "geom",
+            "location",
+            "geometry",
+            # "geom",
             "x",
             "y",
             "lon",
@@ -51,19 +54,25 @@ class Meta:
             "sensor_types",
         ]
 
+    # Field geom renamed to location, but the front end stil uses geom
+    # Serialize the geom to keep the functionality. TODO, remove when
+    # front end is updated
+    def get_geom(self, obj):
+        return obj.location
+
     def get_y(self, obj):
-        return obj.geom.y
+        return obj.location.y
 
     def get_lat(self, obj):
-        obj.geom.transform(4326)
-        return obj.geom.y
+        obj.location.transform(4326)
+        return obj.location.y
 
     def get_x(self, obj):
-        return obj.geom.x
+        return obj.location.x
 
     def get_lon(self, obj):
-        obj.geom.transform(4326)
-        return obj.geom.x
+        obj.location.transform(4326)
+        return obj.location.x
 
     def get_sensor_types(self, obj):
         # Return the sensor types(car, bike etc) that has a total year value >0.

From 29041bc963a292aa0f9c40f562f2306491f70a17 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:17:40 +0300
Subject: [PATCH 51/81] Add Telraam related info

---
 eco_counter/specification.swagger2.0.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/eco_counter/specification.swagger2.0.yaml b/eco_counter/specification.swagger2.0.yaml
index 767f05d8a..9505177f7 100755
--- a/eco_counter/specification.swagger2.0.yaml
+++ b/eco_counter/specification.swagger2.0.yaml
@@ -29,8 +29,11 @@ definitions:
         type: integer
       name:
         type: string
-      geom:
+      location:
         type: string
+      geometry:
+        type: string
+        description: Additional geometry, e.g., Telraam counters have a MultiLineString for its segment.
       lat:
         type: number
       lon:
@@ -245,7 +248,7 @@ paths:
       summary: "Returns a list of stations."
       parameters:
       - in: query
-        description: "The type of the counter EC(Eco Counter), TC(Traffic Counter), LC(LAM Counter)"
+        description: "The type of the counter EC(Eco Counter), TC(Traffic Counter), LC(LAM Counter), TR(Telraam Counter)"
         name: counter_type
         type: string
       responses:

From 9e4f2015b77454cd2f5fe47f154014107e42559e Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:21:36 +0300
Subject: [PATCH 52/81] Add INDEX_COLUMN_NAME

---
 eco_counter/management/commands/import_counter_data.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index 4cbcbfb94..ac7503e8d 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -19,6 +19,7 @@
     COUNTER_START_YEARS,
     COUNTERS,
     ECO_COUNTER,
+    INDEX_COLUMN_NAME,
     LAM_COUNTER,
     TELRAAM_COUNTER,
     TELRAAM_COUNTER_START_MONTH,
@@ -47,7 +48,6 @@
     get_test_dataframe,
     get_traffic_counter_csv,
     save_stations,
-    TIMESTAMP_COL_NAME,
 )
 
 logger = logging.getLogger("eco_counter")
@@ -471,7 +471,7 @@ def handle(self, *args, **options):
                             start_year = TRAFFIC_COUNTER_START_YEAR
                         csv_data = get_traffic_counter_csv(start_year=start_year)
                 start_index = csv_data.index[
-                    csv_data[TIMESTAMP_COL_NAME] == start_time_string
+                    csv_data[INDEX_COLUMN_NAME] == start_time_string
                 ].values[0]
                 # As LAM data is fetched with a timespan, no index data is available, instead
                 # show time.

From f7a10d9d57a190bc2c1da70f269853d94a8ae895 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:28:57 +0300
Subject: [PATCH 53/81] Import total values of bike, pedestrian, car

---
 .../commands/import_telraam_to_csv.py         | 42 ++++++++++++-------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index dbb16b27b..f199f2fb9 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -1,3 +1,9 @@
+"""
+Imports hourly Telraam data for given cameras.
+Saves a CSV file for every camera and every day to PROJECT_ROOT/media/telraam_data/
+
+"""
+
 import json
 import logging
 import os
@@ -21,9 +27,6 @@
 from eco_counter.management.commands.utils import get_telraam_cameras
 from eco_counter.models import ImportState
 
-LEVEL = "instances"  # instance per individual can
-FORMAT = "per-hour"
-
 TOKEN = settings.TELRAAM_TOKEN
 assert TOKEN
 logger = logging.getLogger("eco_counter")
@@ -37,28 +40,32 @@
 }
 LEFT = "lft"
 RIGHT = "rgt"
-DIRECTIONS = [LEFT, RIGHT]
+TOTAL = ""  # Total fields have no postfix in names
+DIRECTIONS = [LEFT, RIGHT, TOTAL]
 
 
-def get_mappings(station_name, direction=True):
+def get_mappings(station_name: str, direction: bool = True) -> dict:
     """
+    If direction is true, LEFT (lft) will be K (Keskustaan päin)
     return mappings:
     e.g.,
     "pedestrian_lgt": "station_name KP"
     """
-    dir1, dir2 = "K", "P"
+    dir1, dir2, dir_tot = "K", "P", "T"
     if not direction:
         dir1, dir2 = dir2, dir1
-    dirs = {
-        LEFT: dir1,
-        RIGHT: dir2,
-    }
+    dirs = {LEFT: dir1, RIGHT: dir2, TOTAL: dir_tot}
     column_mappings = {}
     for veh in VEHICLE_TYPES.items():
         for dir in DIRECTIONS:
-            key = f"{veh[0]}_{dir}"
+            if dir == TOTAL:
+                key = f"{veh[0]}{dir}"
+            else:
+                key = f"{veh[0]}_{dir}"
+
             value = f"{veh[1]}{dirs[dir]}"
             column_mappings[key] = value
+
     mappings = {}
     for field in column_mappings.keys():
         mappings[field] = f"{station_name} {column_mappings[field]}"
@@ -70,9 +77,10 @@ def fetch_traffic_report(from_date: str, end_date: str, camera_id: str):
         "X-Api-Key": TOKEN,
         "Content-Type": "application/json",
     }
+
     data = {
-        "level": LEVEL,  # segments
-        "format": FORMAT,
+        "level": "instances",  # Statistics for individual cameras
+        "format": "per-hour",
         "id": camera_id,
         "time_start": from_date,
         "time_end": end_date,
@@ -124,7 +132,11 @@ def get_day_data(
         d["date"] = datetime.strftime(start_date, TELRAAM_COUNTER_API_TIME_FORMAT)
         for veh in VEHICLE_TYPES.keys():
             for dir in DIRECTIONS:
-                key = f"{veh}_{dir}"
+                if dir == TOTAL:
+                    key = f"{veh}{dir}"
+                else:
+                    key = f"{veh}_{dir}"
+
                 val = int(round(item.get(key, 0)))
                 d[key] = val
         res.append(d)
@@ -132,7 +144,7 @@ def get_day_data(
     return res, delta_hours
 
 
-def save_dataframe():
+def save_dataframe() -> datetime:
     if not os.path.exists(TELRAAM_COUNTER_CSV_FILE_PATH):
         os.makedirs(TELRAAM_COUNTER_CSV_FILE_PATH)
         ImportState.objects.filter(csv_data_source=TELRAAM_CSV).delete()

From 974945fc999cebe49dce1da8a65c3558d6f98df4 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:31:14 +0300
Subject: [PATCH 54/81] Fix EC, TC stations bug, change geom to location, set
 Telraam stations geometry

---
 eco_counter/management/commands/utils.py | 67 +++++++++++++++++-------
 1 file changed, 49 insertions(+), 18 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 827935fa0..f627a1142 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -8,7 +8,7 @@
 import requests
 from django.conf import settings
 from django.contrib.gis.gdal import DataSource
-from django.contrib.gis.geos import GEOSGeometry, Point
+from django.contrib.gis.geos import GEOSGeometry, LineString, MultiLineString, Point
 
 from eco_counter.constants import (
     COUNTERS,
@@ -19,8 +19,9 @@
     LAM_STATIONS_API_FETCH_URL,
     LAM_STATIONS_DIRECTION_MAPPINGS,
     TELRAAM_COUNTER,
-    TELRAAM_COUNTER_API_BASE_URL,
     TELRAAM_COUNTER_API_TIME_FORMAT,
+    TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL,
+    TELRAAM_COUNTER_CAMERA_SEGMENTS_URL,
     TELRAAM_COUNTER_CSV_FILE,
     TELRAAM_COUNTER_START_MONTH,
     TELRAAM_COUNTER_START_YEAR,
@@ -48,8 +49,8 @@ def __init__(self, feature):
         # The source data has a obsolete Z dimension with value 0, remove it.
         geom = feature.geom.clone()
         geom.coord_dim = 2
-        self.geom = GEOSGeometry(geom.wkt, srid=4326)
-        self.geom.transform(settings.DEFAULT_SRID)
+        self.location = GEOSGeometry(geom.wkt, srid=4326)
+        self.location.transform(settings.DEFAULT_SRID)
 
 
 class EcoCounterStation:
@@ -57,8 +58,8 @@ def __init__(self, feature):
         self.name = feature["properties"]["Nimi"]
         lon = feature["geometry"]["coordinates"][0]
         lat = feature["geometry"]["coordinates"][1]
-        self.geom = Point(lon, lat, srid=4326)
-        self.geom.transform(settings.DEFAULT_SRID)
+        self.location = Point(lon, lat, srid=4326)
+        self.location.transform(settings.DEFAULT_SRID)
 
 
 class TrafficCounterStation:
@@ -68,25 +69,56 @@ def __init__(self, feature):
         self.name_en = feature["Osoite_en"].as_string()
         geom = GEOSGeometry(feature.geom.wkt, srid=feature.geom.srid)
         geom.transform(settings.DEFAULT_SRID)
-        self.geom = geom
+        self.location = geom
 
 
 class TelraamCounterStation:
+    # The Telraam API return the coordinates in EPSGS 31370
+    EPSG = 31370
+
+    def get_location_and_geometry(self, id):
+        url = TELRAAM_COUNTER_CAMERA_SEGMENTS_URL.format(id=id)
+        headers = {
+            "X-Api-Key": settings.TELRAAM_TOKEN,
+        }
+        response = requests.get(url, headers=headers)
+        assert (
+            response.status_code == 200
+        ), "Could not fetch segment for camera {id}".format(id)
+        json_data = response.json()
+        coords = json_data["features"][0]["geometry"]["coordinates"]
+        lss = []
+        for coord in coords:
+            ls = LineString(coord, srid=self.EPSG)
+            lss.append(ls)
+        geometry = MultiLineString(lss, srid=self.EPSG)
+        geometry.transform(settings.DEFAULT_SRID)
+        mid_line = round(len(coords) / 2)
+        mid_point = round(len(coords[mid_line]) / 2)
+        location = Point(coords[mid_line][mid_point], srid=self.EPSG)
+        location.transform(settings.DEFAULT_SRID)
+        return location, geometry
+
     def __init__(self, feature):
         self.name = feature["mac"]
         self.name_sv = feature["mac"]
         self.name_en = feature["mac"]
-        self.geom = GEOSGeometry("POINT EMPTY")
+        self.location, self.geometry = self.get_location_and_geometry(
+            feature["segment_id"]
+        )
         self.station_id = feature["mac"]
 
 
-class ObservationStation(LAMStation, EcoCounterStation, TrafficCounterStation):
+class ObservationStation(
+    LAMStation, EcoCounterStation, TrafficCounterStation, TelraamCounterStation
+):
     def __init__(self, csv_data_source, feature):
         self.csv_data_source = csv_data_source
         self.name = None
         self.name_sv = None
         self.name_en = None
-        self.geom = None
+        self.location = None
+        self.geometry = None
         self.station_id = None
         match csv_data_source:
             case COUNTERS.TELRAAM_COUNTER:
@@ -337,12 +369,10 @@ def get_eco_counter_stations():
 
 
 def fetch_telraam_cameras():
-    url = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/cameras"
-
     headers = {
         "X-Api-Key": settings.TELRAAM_TOKEN,
     }
-    response = requests.get(url, headers=headers)
+    response = requests.get(TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL, headers=headers)
     return response.json().get("cameras", None)
 
 
@@ -364,7 +394,7 @@ def get_active_telraam_camera(offset):
 
 def get_telraam_cameras():
     # TODO, add Turku cameras when they are online
-    return [get_active_telraam_camera(2 + i * 3) for i in range(3)]
+    return [get_active_telraam_camera(13 + i * 3) for i in range(3)]
 
 
 def get_telraam_counter_stations():
@@ -375,7 +405,7 @@ def get_telraam_counter_stations():
     return stations
 
 
-def get_telraam_counter_csv():    
+def get_telraam_counter_csv():
     df = pd.DataFrame()
     from_date = date(TELRAAM_COUNTER_START_YEAR, TELRAAM_COUNTER_START_MONTH, 1)
     try:
@@ -426,9 +456,9 @@ def save_stations(csv_data_source):
         case COUNTERS.LAM_COUNTER:
             stations = get_lam_counter_stations()
         case COUNTERS.ECO_COUNTER:
-            station = get_eco_counter_stations()
+            stations = get_eco_counter_stations()
         case COUNTERS.TRAFFIC_COUNTER:
-            station = get_traffic_counter_stations()
+            stations = get_traffic_counter_stations()
     object_ids = list(
         Station.objects.filter(csv_data_source=csv_data_source).values_list(
             "id", flat=True
@@ -439,7 +469,8 @@ def save_stations(csv_data_source):
             name=station.name,
             name_sv=station.name_sv,
             name_en=station.name_en,
-            geom=station.geom,
+            location=station.location,
+            geometry=station.geometry,
             station_id=station.station_id,
             csv_data_source=csv_data_source,
         )

From f723b1f562e25e5ae1a57fc25931e5fe11d55463 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Wed, 24 May 2023 14:33:01 +0300
Subject: [PATCH 55/81] Formatting

---
 eco_counter/api/views.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/eco_counter/api/views.py b/eco_counter/api/views.py
index 279e9f5af..04d3ea3b1 100644
--- a/eco_counter/api/views.py
+++ b/eco_counter/api/views.py
@@ -69,7 +69,6 @@ def list(self, request):
 
 
 class HourDataViewSet(viewsets.ReadOnlyModelViewSet):
-
     queryset = HourData.objects.all()
     serializer_class = HourDataSerializer
 
@@ -80,7 +79,6 @@ def get_hour_data(self, request):
 
 
 class DayDataViewSet(viewsets.ReadOnlyModelViewSet):
-
     queryset = DayData.objects.all()
     serializer_class = DayDataSerializer
 

From e11604b1c9bb9bd39ad0329345af49c3b6716e8f Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 09:14:56 +0300
Subject: [PATCH 56/81] Add info about Telraam

---
 eco_counter/README.md | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/eco_counter/README.md b/eco_counter/README.md
index a89da2dea..32bdcad68 100644
--- a/eco_counter/README.md
+++ b/eco_counter/README.md
@@ -2,8 +2,8 @@
 
 Imports/Processes data from:
 https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq 
-Imports both "Liikennelasketa-Ilmaisintiedot 15 min aikaväleillä"(Traffic Counter) and "Eco-Counter" (Eco Counter) datas. Imports/processes "LAM-Counter" (LAM Counter) data from https://www.digitraffic.fi/tieliikenne/lam/
-
+Imports both "Liikennelasketa-Ilmaisintiedot 15 min aikaväleillä"(Traffic Counter) and "Eco-Counter" (Eco Counter) datas. Imports/processes "LAM-Counter" (LAM Counter) data from https://www.digitraffic.fi/tieliikenne/lam/ and
+Telraam data from https://telraam-api.net/.
 
 ## Installation:
 Add following lines to the .env:
@@ -16,6 +16,8 @@ Note, The urls can change. Up-to-date urls can be found at:
 https://www.avoindata.fi/data/fi/dataset/turun-seudun-liikennemaaria
 and
 https://www.digitraffic.fi/tieliikenne/lam/
+Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py) https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api
+TELRAAM_TOKEN=
 
 ## Importing
 
@@ -23,14 +25,19 @@ https://www.digitraffic.fi/tieliikenne/lam/
 The initial import, this must be done before starting with the continous incremental imports:
 ./manage.py import_counter_data --init COUNTERS
 e.g. ./manage.py import_counter_data --init EC TC
-The counters are EC(Eco Counter), TC(Traffic Counter) and LC(Lam Counter).
+The counters are EC(Eco Counter), TC(Traffic Counter), LC(Lam Counter) and TR(Telraam Counter).
 
 ### Continous Import
 For continous (hourly) imports run:
 ./manage.py import_counter_data --counters COUNTERS
 e.g. ./manage.py import_counter_data --counters EC TC
-Counter names are: EC (Eco Counter), TC (Traffic Counter) and LC (Lam Counter).
-Note, Traffic Counter data is updated once a week.
+Counter names are: EC (Eco Counter), TC (Traffic Counter), LC (Lam Counter) and TR (Telraam Counter).
+Note, Traffic Counter data is updated once a week and Lam Counter data once a day.
+
+### Importing Telraam raw data
+In order to import Telraam data into the database the raw data has to be imported. The raw data is imported with the _import_telraam_to_csv_ management command.
+The imported should be set to be run once a hour (see: https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#telraam-to-csv-eco_countertasksimport_telraam_to_csv )
+Telraam raw data is imported to PROJECT_ROOT/media/telraam_data/. 
 
 ## Troubleshooting
 For reasons unknown, the amount of sensors can sometimes change in the source csv file, e.g. the amount of columns changes. If this happens, run the initial import: ./manage.py import_counter_data --init and after that it is safe to run the importer as normal.

From 21682c5296f24e82622d8034645934ec5af20bc7 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 09:32:11 +0300
Subject: [PATCH 57/81] Add camera that is located in Turku

---
 eco_counter/constants.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 79f9d0247..38be3c890 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -81,6 +81,8 @@
 # Maximum 3 months at a time
 TELRAAM_COUNTER_TRAFFIC_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/reports/traffic"
 TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/cameras"
+TELRAAM_COUNTER_CAMERAS_URL = TELRAAM_COUNTER_API_BASE_URL + "/v1/cameras/{mac_id}"
+
 TELRAAM_COUNTER_CAMERA_SEGMENTS_URL = (
     TELRAAM_COUNTER_API_BASE_URL + "/v1/segments/id/{id}"
 )
@@ -92,3 +94,7 @@
 TELRAAM_COUNTER_CSV_FILE = (
     TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{id}_{day}_{month}_{year}.csv"
 )
+TELRAAM_COUNTER_CAMERAS = {
+    # Mac id: Direction flag
+    350457790598039: False,  # Kristiinanankatu, Joelle katsottaessa vasemmalle
+}

From 964c449548c085061712d4c9ab33b11066cb91d4 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 09:37:37 +0300
Subject: [PATCH 58/81] Add date to get_telraam_counter_csv function call

---
 eco_counter/management/commands/import_counter_data.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eco_counter/management/commands/import_counter_data.py b/eco_counter/management/commands/import_counter_data.py
index ac7503e8d..0a607e682 100644
--- a/eco_counter/management/commands/import_counter_data.py
+++ b/eco_counter/management/commands/import_counter_data.py
@@ -459,7 +459,7 @@ def handle(self, *args, **options):
                 # start_index = None
                 match counter:
                     case COUNTERS.TELRAAM_COUNTER:
-                        csv_data = get_telraam_counter_csv()
+                        csv_data = get_telraam_counter_csv(start_time.date())
                     case COUNTERS.LAM_COUNTER:
                         csv_data = get_lam_counter_csv(start_time.date())
                     case COUNTERS.ECO_COUNTER:

From 42592c41ca5d279f7480623e511dca3b9f25c6e5 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 09:39:30 +0300
Subject: [PATCH 59/81] Add direction and utc_offset

---
 .../commands/import_telraam_to_csv.py         | 36 +++++++++++++------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index f199f2fb9..fca8fde5a 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -10,6 +10,7 @@
 from datetime import date, datetime, timedelta
 
 import pandas as pd
+import pytz
 import requests
 from django.conf import settings
 from django.core.management import BaseCommand
@@ -17,6 +18,7 @@
 from eco_counter.constants import (
     INDEX_COLUMN_NAME,
     TELRAAM_COUNTER_API_TIME_FORMAT,
+    TELRAAM_COUNTER_CAMERAS,
     TELRAAM_COUNTER_CSV_FILE,
     TELRAAM_COUNTER_CSV_FILE_PATH,
     TELRAAM_COUNTER_START_MONTH,
@@ -98,18 +100,20 @@ def get_delta_hours(from_date: datetime, end_date: datetime) -> datetime:
 
 
 def get_day_data(
-    day_date: date, camera_id: str, check_delta_hours: bool = True
-) -> list:
-    from_datetime = datetime(day_date.year, day_date.month, day_date.day, 0, 0, 0)
+    day_date: date, camera_id: str, utf_offset: datetime, check_delta_hours: bool = True
+) -> tuple[list, int]:
+    from_datetime = (
+        datetime(day_date.year, day_date.month, day_date.day, 0, 0, 0) - utf_offset
+    )
     from_datetime_str = from_datetime.strftime(TELRAAM_COUNTER_API_TIME_FORMAT)
     end_datetime = (
         datetime(day_date.year, day_date.month, day_date.day)
         + timedelta(hours=23)
         + timedelta(minutes=59)
-    )
+    ) - utf_offset
+
     end_datetime_str = end_datetime.strftime(TELRAAM_COUNTER_API_TIME_FORMAT)
     report = fetch_traffic_report(from_datetime_str, end_datetime_str, camera_id)
-
     delta_hours = get_delta_hours(from_datetime, end_datetime)
     logger.info(f"Trying to import {delta_hours} hours for camera {camera_id}.")
     if not report:
@@ -136,7 +140,6 @@ def get_day_data(
                     key = f"{veh}{dir}"
                 else:
                     key = f"{veh}_{dir}"
-
                 val = int(round(item.get(key, 0)))
                 d[key] = val
         res.append(d)
@@ -163,18 +166,31 @@ def save_dataframe() -> datetime:
         import_state.current_day_number,
     )
     date_today = date.today()
+    # Source data date time is in UTC. Calculate a utf_offset
+    utc_offset = pytz.timezone("Europe/Helsinki").utcoffset(datetime.now())
     logger.info(f"Fetching Telraam data from {str(from_date)} to {str(date_today)}")
-
     cameras = get_telraam_cameras()
     for camera in cameras:
         start_date = from_date
         while start_date <= date_today:
-            report, delta_hours = get_day_data(start_date, camera["instance_id"])
-            mappings = get_mappings(camera["mac"])
+            report, delta_hours = get_day_data(
+                start_date, camera["instance_id"], utc_offset
+            )
+            mappings = get_mappings(
+                camera["mac"], direction=TELRAAM_COUNTER_CAMERAS[camera["mac"]]
+            )
             columns = {}
             columns[INDEX_COLUMN_NAME] = []
             for hour in range(delta_hours):
-                columns[INDEX_COLUMN_NAME].append(report[hour]["date"])
+                col_date = (
+                    datetime.strptime(
+                        report[hour]["date"], TELRAAM_COUNTER_API_TIME_FORMAT
+                    )
+                    + utc_offset
+                )
+                col_date_str = col_date.strftime(TELRAAM_COUNTER_API_TIME_FORMAT)
+                columns[INDEX_COLUMN_NAME].append(col_date_str)
+
                 for mapping in mappings.items():
                     # key is the name of the column, e.g., name_ak
                     key = mapping[1]

From aacc79c38a5f442d75a6289a69a241688c2b6211 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 09:44:18 +0300
Subject: [PATCH 60/81] Remove geom

---
 eco_counter/api/serializers.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/eco_counter/api/serializers.py b/eco_counter/api/serializers.py
index 5fc7e3475..7b9f81713 100644
--- a/eco_counter/api/serializers.py
+++ b/eco_counter/api/serializers.py
@@ -31,12 +31,10 @@ class StationSerializer(serializers.ModelSerializer):
     y = serializers.SerializerMethodField()
     lon = serializers.SerializerMethodField()
     lat = serializers.SerializerMethodField()
-    # geom = serializers.SerializerMethodField()
     sensor_types = serializers.SerializerMethodField()
 
     class Meta:
         model = Station
-
         fields = [
             "id",
             "name",
@@ -46,7 +44,6 @@ class Meta:
             "csv_data_source",
             "location",
             "geometry",
-            # "geom",
             "x",
             "y",
             "lon",
@@ -54,12 +51,6 @@ class Meta:
             "sensor_types",
         ]
 
-    # Field geom renamed to location, but the front end stil uses geom
-    # Serialize the geom to keep the functionality. TODO, remove when
-    # front end is updated
-    def get_geom(self, obj):
-        return obj.location
-
     def get_y(self, obj):
         return obj.location.y
 

From c47628a4655aba5098a85238cf3bcdb1ea6edc88 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 09:45:04 +0300
Subject: [PATCH 61/81] Fix Telraam geometry bug, get Turku cameras

---
 eco_counter/management/commands/utils.py | 39 ++++++++++++++++--------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index f627a1142..19176d5f2 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -22,9 +22,9 @@
     TELRAAM_COUNTER_API_TIME_FORMAT,
     TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL,
     TELRAAM_COUNTER_CAMERA_SEGMENTS_URL,
+    TELRAAM_COUNTER_CAMERAS,
+    TELRAAM_COUNTER_CAMERAS_URL,
     TELRAAM_COUNTER_CSV_FILE,
-    TELRAAM_COUNTER_START_MONTH,
-    TELRAAM_COUNTER_START_YEAR,
     TELRAAM_CSV,
     TRAFFIC_COUNTER,
     TRAFFIC_COUNTER_CSV_URLS,
@@ -74,7 +74,8 @@ def __init__(self, feature):
 
 class TelraamCounterStation:
     # The Telraam API return the coordinates in EPSGS 31370
-    EPSG = 31370
+    SOURCE_SRID = 4326
+    TARGET_SRID = settings.DEFAULT_SRID
 
     def get_location_and_geometry(self, id):
         url = TELRAAM_COUNTER_CAMERA_SEGMENTS_URL.format(id=id)
@@ -84,19 +85,19 @@ def get_location_and_geometry(self, id):
         response = requests.get(url, headers=headers)
         assert (
             response.status_code == 200
-        ), "Could not fetch segment for camera {id}".format(id)
+        ), "Could not fetch segment for camera {id}".format(id=id)
         json_data = response.json()
         coords = json_data["features"][0]["geometry"]["coordinates"]
         lss = []
         for coord in coords:
-            ls = LineString(coord, srid=self.EPSG)
+            ls = LineString(coord, srid=self.SOURCE_SRID)
             lss.append(ls)
-        geometry = MultiLineString(lss, srid=self.EPSG)
-        geometry.transform(settings.DEFAULT_SRID)
+        geometry = MultiLineString(lss, srid=self.SOURCE_SRID)
+        geometry.transform(self.TARGET_SRID)
         mid_line = round(len(coords) / 2)
         mid_point = round(len(coords[mid_line]) / 2)
-        location = Point(coords[mid_line][mid_point], srid=self.EPSG)
-        location.transform(settings.DEFAULT_SRID)
+        location = Point(coords[mid_line][mid_point], srid=self.SOURCE_SRID)
+        location.transform(self.TARGET_SRID)
         return location, geometry
 
     def __init__(self, feature):
@@ -369,6 +370,7 @@ def get_eco_counter_stations():
 
 
 def fetch_telraam_cameras():
+    # NOTE, obsolete after Turku cameras are added
     headers = {
         "X-Api-Key": settings.TELRAAM_TOKEN,
     }
@@ -376,6 +378,16 @@ def fetch_telraam_cameras():
     return response.json().get("cameras", None)
 
 
+def fetch_telraam_camera(mac_id):
+    headers = {
+        "X-Api-Key": settings.TELRAAM_TOKEN,
+    }
+    url = TELRAAM_COUNTER_CAMERAS_URL.format(mac_id=mac_id)
+    response = requests.get(url, headers=headers)
+    return response.json()["camera"][0]
+    return response.json().get("camera", None)
+
+
 def get_active_telraam_camera(offset):
     """
     Function that gets pseudo random camera that is actvie for testing data.
@@ -393,8 +405,10 @@ def get_active_telraam_camera(offset):
 
 
 def get_telraam_cameras():
-    # TODO, add Turku cameras when they are online
-    return [get_active_telraam_camera(13 + i * 3) for i in range(3)]
+    cameras = []
+    for camera in TELRAAM_COUNTER_CAMERAS.items():
+        cameras.append(fetch_telraam_camera(camera[0]))
+    return cameras
 
 
 def get_telraam_counter_stations():
@@ -405,9 +419,8 @@ def get_telraam_counter_stations():
     return stations
 
 
-def get_telraam_counter_csv():
+def get_telraam_counter_csv(from_date):
     df = pd.DataFrame()
-    from_date = date(TELRAAM_COUNTER_START_YEAR, TELRAAM_COUNTER_START_MONTH, 1)
     try:
         import_state = ImportState.objects.get(csv_data_source=TELRAAM_CSV)
     except ImportState.DoesNotExist:

From 37d930aca8f2ac4ab2e16e2613abdefa1f5f33ee Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 10:04:01 +0300
Subject: [PATCH 62/81] Add migration

---
 ...r_and_traffic_csv_choice_to_importstate.py | 42 +++++++++++++++++++
 ...0016_add_importstate_current_day_number.py | 28 +++++++++++++
 .../0017_rename_geom_station_location.py      | 17 ++++++++
 .../0018_add_geometry_to_station.py           | 20 +++++++++
 4 files changed, 107 insertions(+)
 create mode 100644 eco_counter/migrations/0015_add_telraam_counter_and_traffic_csv_choice_to_importstate.py
 create mode 100644 eco_counter/migrations/0016_add_importstate_current_day_number.py
 create mode 100644 eco_counter/migrations/0017_rename_geom_station_location.py
 create mode 100644 eco_counter/migrations/0018_add_geometry_to_station.py

diff --git a/eco_counter/migrations/0015_add_telraam_counter_and_traffic_csv_choice_to_importstate.py b/eco_counter/migrations/0015_add_telraam_counter_and_traffic_csv_choice_to_importstate.py
new file mode 100644
index 000000000..4e5200a15
--- /dev/null
+++ b/eco_counter/migrations/0015_add_telraam_counter_and_traffic_csv_choice_to_importstate.py
@@ -0,0 +1,42 @@
+# Generated by Django 4.2 on 2023-05-22 09:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("eco_counter", "0014_remove_station_lam_id_station_station_id"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="importstate",
+            name="csv_data_source",
+            field=models.CharField(
+                choices=[
+                    ("TC", "TrafficCounter"),
+                    ("EC", "EcoCounter"),
+                    ("LC", "LamCounter"),
+                    ("TR", "TelraamCounter"),
+                    ("TV", "TelraamCSV"),
+                ],
+                default="EC",
+                max_length=2,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="station",
+            name="csv_data_source",
+            field=models.CharField(
+                choices=[
+                    ("TC", "TrafficCounter"),
+                    ("EC", "EcoCounter"),
+                    ("LC", "LamCounter"),
+                    ("TR", "TelraamCounter"),
+                    ("TV", "TelraamCSV"),
+                ],
+                default="EC",
+                max_length=2,
+            ),
+        ),
+    ]
diff --git a/eco_counter/migrations/0016_add_importstate_current_day_number.py b/eco_counter/migrations/0016_add_importstate_current_day_number.py
new file mode 100644
index 000000000..a6e6256bf
--- /dev/null
+++ b/eco_counter/migrations/0016_add_importstate_current_day_number.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2 on 2023-05-22 09:39
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "eco_counter",
+            "0015_add_telraam_counter_and_traffic_csv_choice_to_importstate",
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importstate",
+            name="current_day_number",
+            field=models.PositiveSmallIntegerField(
+                blank=True,
+                null=True,
+                validators=[
+                    django.core.validators.MinValueValidator(1),
+                    django.core.validators.MaxValueValidator(31),
+                ],
+            ),
+        ),
+    ]
diff --git a/eco_counter/migrations/0017_rename_geom_station_location.py b/eco_counter/migrations/0017_rename_geom_station_location.py
new file mode 100644
index 000000000..54cc3f846
--- /dev/null
+++ b/eco_counter/migrations/0017_rename_geom_station_location.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.2 on 2023-05-24 06:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("eco_counter", "0016_add_importstate_current_day_number"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="station",
+            old_name="geom",
+            new_name="location",
+        ),
+    ]
diff --git a/eco_counter/migrations/0018_add_geometry_to_station.py b/eco_counter/migrations/0018_add_geometry_to_station.py
new file mode 100644
index 000000000..ad08d6fd6
--- /dev/null
+++ b/eco_counter/migrations/0018_add_geometry_to_station.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2 on 2023-05-24 06:55
+
+import django.contrib.gis.db.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("eco_counter", "0017_rename_geom_station_location"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="station",
+            name="geometry",
+            field=django.contrib.gis.db.models.fields.GeometryField(
+                null=True, srid=3067
+            ),
+        ),
+    ]

From f9c9a998126c44c8d350236e706c47367a8809ca Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 10:05:10 +0300
Subject: [PATCH 63/81] Add Telraam token

---
 config_dev.env.example | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/config_dev.env.example b/config_dev.env.example
index c7b6241cb..296a49ad6 100644
--- a/config_dev.env.example
+++ b/config_dev.env.example
@@ -180,4 +180,8 @@ YIT_ROUTES_URL=https://api.autori.io/api/dailymaintenance-a3/route/
 YIT_VEHICLES_URL=https://api.autori.io/api/dailymaintenance-a3/route/types/vehicle/
 YIT_CONTRACTS_URL=https://api.autori.io/api/dailymaintenance-a3/contracts/
 YIT_TOKEN_URL=https://login.microsoftonline.com/86792d09-0d81-4899-8d66-95dfc96c8014/oauth2/v2.0/token?Scope=api://7f45c30e-cc67-4a93-85f1-0149b44c1cdf/.default
-KUNTEC_KEY=
\ No newline at end of file
+# API key to the Kuntec API
+KUNTEC_KEY=
+# Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py)
+# https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api
+TELRAAM_TOKEN=
\ No newline at end of file

From 850284f14b66892252789ae302c416d638491d63 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Fri, 26 May 2023 12:29:54 +0300
Subject: [PATCH 64/81] Change Pandas version to >=2.0.1

---
 requirements.in  |  2 +-
 requirements.txt | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/requirements.in b/requirements.in
index 85f1db3f1..6cdaf4f58 100644
--- a/requirements.in
+++ b/requirements.in
@@ -29,7 +29,7 @@ pep8-naming
 jedi
 parso
 whitenoise
-pandas>1.4
+pandas>=2.0.0
 pykml
 shapely
 celery
diff --git a/requirements.txt b/requirements.txt
index c5816cae9..9cbc1857e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
 #
 #    pip-compile requirements.in
 #
@@ -138,7 +138,7 @@ numpy==1.23.0
     #   pandas
 packaging==21.0
     # via pytest
-pandas==1.4.3
+pandas==2.0.1
     # via -r requirements.in
 parso==0.8.2
     # via
@@ -233,11 +233,15 @@ toml==0.10.2
     #   pytest
     #   pytest-cov
 tomli==1.2.1
-    # via pep517
+    # via
+    #   black
+    #   pep517
 tqdm==4.62.3
     # via -r requirements.in
 tzdata==2022.1
-    # via django-celery-beat
+    # via
+    #   django-celery-beat
+    #   pandas
 uritemplate==4.1.1
     # via drf-spectacular
 url-normalize==1.4.3

From 96eabb3ca500f4c0c8ea1a66b835b4bb45e91507 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 29 May 2023 12:34:36 +0300
Subject: [PATCH 65/81] Add TELRAAM_HTTP adapter

---
 eco_counter/constants.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 38be3c890..14cfd473c 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -1,6 +1,9 @@
 import types
 
+import requests
 from django.conf import settings
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from mobility_data.importers.utils import get_root_dir
 
@@ -80,7 +83,6 @@
 TELRAAM_COUNTER_API_BASE_URL = "https://telraam-api.net"
 # Maximum 3 months at a time
 TELRAAM_COUNTER_TRAFFIC_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/reports/traffic"
-TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL = f"{TELRAAM_COUNTER_API_BASE_URL}/v1/cameras"
 TELRAAM_COUNTER_CAMERAS_URL = TELRAAM_COUNTER_API_BASE_URL + "/v1/cameras/{mac_id}"
 
 TELRAAM_COUNTER_CAMERA_SEGMENTS_URL = (
@@ -95,6 +97,17 @@
     TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{id}_{day}_{month}_{year}.csv"
 )
 TELRAAM_COUNTER_CAMERAS = {
-    # Mac id: Direction flag
+    # Mac id: Direction flag (True=rgt prefix will be keskustaan päin)
     350457790598039: False,  # Kristiinanankatu, Joelle katsottaessa vasemmalle
+    350457790600975: True,  # Kristiinanankatu, Joelle katsottaessa oikealle
 }
+retry_strategy = Retry(
+    total=10,
+    status_forcelist=[429],
+    method_whitelist=["GET", "POST"],
+    backoff_factor=30,  # 30, 60, 120 , 240, ..seconds
+)
+adapter = HTTPAdapter(max_retries=retry_strategy)
+TELRAAM_HTTP = requests.Session()
+TELRAAM_HTTP.mount("https://", adapter)
+TELRAAM_HTTP.mount("http://", adapter)

From 677318f164dc4fe75d24a0b677073ec73d47da4b Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 29 May 2023 12:36:07 +0300
Subject: [PATCH 66/81] Remove obsole functions, use TELRAAM_HTTP adapter

---
 eco_counter/management/commands/utils.py | 43 +++++++-----------------
 1 file changed, 13 insertions(+), 30 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 19176d5f2..0c05e650e 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -20,12 +20,12 @@
     LAM_STATIONS_DIRECTION_MAPPINGS,
     TELRAAM_COUNTER,
     TELRAAM_COUNTER_API_TIME_FORMAT,
-    TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL,
     TELRAAM_COUNTER_CAMERA_SEGMENTS_URL,
     TELRAAM_COUNTER_CAMERAS,
     TELRAAM_COUNTER_CAMERAS_URL,
     TELRAAM_COUNTER_CSV_FILE,
     TELRAAM_CSV,
+    TELRAAM_HTTP,
     TRAFFIC_COUNTER,
     TRAFFIC_COUNTER_CSV_URLS,
     TRAFFIC_COUNTER_METADATA_GEOJSON,
@@ -369,45 +369,28 @@ def get_eco_counter_stations():
     return stations
 
 
-def fetch_telraam_cameras():
-    # NOTE, obsolete after Turku cameras are added
-    headers = {
-        "X-Api-Key": settings.TELRAAM_TOKEN,
-    }
-    response = requests.get(TELRAAM_COUNTER_AVAILABLE_CAMERAS_URL, headers=headers)
-    return response.json().get("cameras", None)
-
-
 def fetch_telraam_camera(mac_id):
     headers = {
         "X-Api-Key": settings.TELRAAM_TOKEN,
     }
     url = TELRAAM_COUNTER_CAMERAS_URL.format(mac_id=mac_id)
-    response = requests.get(url, headers=headers)
-    return response.json()["camera"][0]
-    return response.json().get("camera", None)
-
-
-def get_active_telraam_camera(offset):
-    """
-    Function that gets pseudo random camera that is actvie for testing data.
-    NOTE, this function will be obsolete when Turku gets its cameras online.
-    """
-    cameras = fetch_telraam_cameras()
-    i = 0
-    for camera in cameras:
-        # time_end: null for active instances
-        if camera["status"] == "active" and not camera["time_end"]:
-            if i == offset:
-                return camera
-            i += 1
-    return None
+    response = TELRAAM_HTTP.get(url, headers=headers)
+    cameras = response.json().get("camera", None)
+    if cameras:
+        # Return first camera
+        return cameras[0]
+    else:
+        return None
 
 
 def get_telraam_cameras():
     cameras = []
     for camera in TELRAAM_COUNTER_CAMERAS.items():
-        cameras.append(fetch_telraam_camera(camera[0]))
+        fetched_camera = fetch_telraam_camera(camera[0])
+        if fetched_camera:
+            cameras.append(fetched_camera)
+        else:
+            logger.warning(f"Could not fetch camera {camera[0]}")
     return cameras
 
 

From 53b185ee9f3d7678085641421d42a7a6968b5dd4 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 29 May 2023 12:39:55 +0300
Subject: [PATCH 67/81] Use TELRAAM_HTTP adapter

---
 eco_counter/management/commands/import_telraam_to_csv.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index fca8fde5a..1fdac8f25 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -11,7 +11,6 @@
 
 import pandas as pd
 import pytz
-import requests
 from django.conf import settings
 from django.core.management import BaseCommand
 
@@ -25,6 +24,7 @@
     TELRAAM_COUNTER_START_YEAR,
     TELRAAM_COUNTER_TRAFFIC_URL,
     TELRAAM_CSV,
+    TELRAAM_HTTP,
 )
 from eco_counter.management.commands.utils import get_telraam_cameras
 from eco_counter.models import ImportState
@@ -87,7 +87,8 @@ def fetch_traffic_report(from_date: str, end_date: str, camera_id: str):
         "time_start": from_date,
         "time_end": end_date,
     }
-    response = requests.post(
+
+    response = TELRAAM_HTTP.post(
         TELRAAM_COUNTER_TRAFFIC_URL, headers=headers, data=json.dumps(data)
     )
     return response.json().get("report", [])
@@ -155,7 +156,7 @@ def save_dataframe() -> datetime:
             csv_data_source=TELRAAM_CSV,
             current_year_number=TELRAAM_COUNTER_START_YEAR,
             current_month_number=TELRAAM_COUNTER_START_MONTH,
-            current_day_number=1,
+            current_day_number=25,
         )
     else:
         import_state = ImportState.objects.filter(csv_data_source=TELRAAM_CSV).first()

From b38392837e25db07b54900cc5c0c6645979e16d7 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 29 May 2023 13:40:35 +0300
Subject: [PATCH 68/81] Handle importing incomplete days

---
 .../commands/import_telraam_to_csv.py         | 28 +++++++++++++------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index 1fdac8f25..f7e646be1 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -115,21 +115,31 @@ def get_day_data(
 
     end_datetime_str = end_datetime.strftime(TELRAAM_COUNTER_API_TIME_FORMAT)
     report = fetch_traffic_report(from_datetime_str, end_datetime_str, camera_id)
-    delta_hours = get_delta_hours(from_datetime, end_datetime)
-    logger.info(f"Trying to import {delta_hours} hours for camera {camera_id}.")
+    delta_hours = len(report)
     if not report:
-        logger.warning("No report found, populating with empty dicts")
+        logger.warning(
+            f"No report found for camera {camera_id}, populating with empty dicts"
+        )
         report = [{} for a in range(delta_hours)]
     else:
-        logger.info(f"Imorted report with {len(report)} elements")
-    delta_hours = len(report)
+        logger.info(
+            f"Imorted report with {len(report)} elements for camera {camera_id}"
+        )
     if check_delta_hours and delta_hours != 24:
         dif = 24 - delta_hours
-        logger.warning(
-            f"Fetched report with delta_hours not equal to 24, appending missing {dif} elements empty dicts"
-        )
-        report += [{} for a in range(dif)]
+        if day_date == date.today():
+            logger.warning(
+                f"Fetched report with delta_hours not equal to 24, appending missing {dif} empty dicts."
+            )
+            report += [{} for a in range(dif)]
 
+        else:
+            # Case when camera gets turned on in the middle of day.
+            logger.warning(
+                f"Fetched report with delta_hours not equal to 24, adding missing {dif} empty dicts to start of report."
+            )
+            report = [{} for a in range(dif)] + report
+    delta_hours = len(report)
     res = []
     start_date = from_datetime
     for item in report:

From ada8b0266b0db98d305f082651abd72cdd0cb64b Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 29 May 2023 14:01:52 +0300
Subject: [PATCH 69/81] Add comment

---
 eco_counter/constants.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 14cfd473c..1ac5c6a7e 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -101,6 +101,7 @@
     350457790598039: False,  # Kristiinanankatu, Joelle katsottaessa vasemmalle
     350457790600975: True,  # Kristiinanankatu, Joelle katsottaessa oikealle
 }
+# For 429 (too many request) TELRAAM need a retry strategy
 retry_strategy = Retry(
     total=10,
     status_forcelist=[429],

From fb33802d9c7d3e8b19ae3ae48cc240b4449e4418 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 29 May 2023 14:02:20 +0300
Subject: [PATCH 70/81] Get Telraam cameras with TELRAAM_HTTP adapter

---
 eco_counter/management/commands/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 0c05e650e..566886f26 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -82,7 +82,7 @@ def get_location_and_geometry(self, id):
         headers = {
             "X-Api-Key": settings.TELRAAM_TOKEN,
         }
-        response = requests.get(url, headers=headers)
+        response = TELRAAM_HTTP.get(url, headers=headers)
         assert (
             response.status_code == 200
         ), "Could not fetch segment for camera {id}".format(id=id)

From 8d95dc24a693a5ef8c56e20e4e697645b1ad972d Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 29 May 2023 14:03:55 +0300
Subject: [PATCH 71/81] Set telraam initial day_number to 1

---
 eco_counter/management/commands/import_telraam_to_csv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index f7e646be1..8accba659 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -166,7 +166,7 @@ def save_dataframe() -> datetime:
             csv_data_source=TELRAAM_CSV,
             current_year_number=TELRAAM_COUNTER_START_YEAR,
             current_month_number=TELRAAM_COUNTER_START_MONTH,
-            current_day_number=25,
+            current_day_number=1,
         )
     else:
         import_state = ImportState.objects.filter(csv_data_source=TELRAAM_CSV).first()

From 54b68817e42bdb71bd16e5faf8a20e43943d02c6 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 8 Jun 2023 10:42:44 +0300
Subject: [PATCH 72/81] Add MEDIA_ROOT to TELRAAM_COUNTER_CSV_FILE_PATH

---
 eco_counter/constants.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 1ac5c6a7e..f56c6f7f3 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -92,7 +92,7 @@
 # from the beginning of the start tear
 TELRAAM_COUNTER_START_MONTH = 5
 TELRAAM_COUNTER_API_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
-TELRAAM_COUNTER_CSV_FILE_PATH = f"{get_root_dir()}/media/telraam_data/"
+TELRAAM_COUNTER_CSV_FILE_PATH = f"{settings.MEDIA_ROOT}/telraam_data/"
 TELRAAM_COUNTER_CSV_FILE = (
     TELRAAM_COUNTER_CSV_FILE_PATH + "telraam_data_{id}_{day}_{month}_{year}.csv"
 )

From f7408bce13de6277f4f46a4a3eef4e857d8e6deb Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 8 Jun 2023 10:44:10 +0300
Subject: [PATCH 73/81] Add args to import_telraam_to_csv task

---
 eco_counter/tasks.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/eco_counter/tasks.py b/eco_counter/tasks.py
index fbbc6fbde..664a44500 100644
--- a/eco_counter/tasks.py
+++ b/eco_counter/tasks.py
@@ -14,5 +14,9 @@ def initial_import_counter_data(args, name="initial_import_counter_data"):
 
 
 @shared_task_email
-def import_telraam_to_csv(args, name="import_telraam_to_csv"):
-    management.call_command("import_telraam_to_csv", args)
+def import_telraam_to_csv(*args, name="import_telraam_to_csv"):
+    if args:
+        management.call_command("import_telraam_to_csv", args)
+
+    else:
+        management.call_command("import_telraam_to_csv")

From 682fa1ce17bb91612320b66cf56ce445822c52df Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 8 Jun 2023 10:51:08 +0300
Subject: [PATCH 74/81] Add --from-date param

---
 .../commands/import_telraam_to_csv.py         | 36 ++++++++++++++-----
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/eco_counter/management/commands/import_telraam_to_csv.py b/eco_counter/management/commands/import_telraam_to_csv.py
index 8accba659..6ef2e0f5b 100644
--- a/eco_counter/management/commands/import_telraam_to_csv.py
+++ b/eco_counter/management/commands/import_telraam_to_csv.py
@@ -158,7 +158,8 @@ def get_day_data(
     return res, delta_hours
 
 
-def save_dataframe() -> datetime:
+def save_dataframe(from_date: date = True) -> datetime:
+    can_overwrite_csv_file = True if from_date else False
     if not os.path.exists(TELRAAM_COUNTER_CSV_FILE_PATH):
         os.makedirs(TELRAAM_COUNTER_CSV_FILE_PATH)
         ImportState.objects.filter(csv_data_source=TELRAAM_CSV).delete()
@@ -170,12 +171,12 @@ def save_dataframe() -> datetime:
         )
     else:
         import_state = ImportState.objects.filter(csv_data_source=TELRAAM_CSV).first()
-
-    from_date = date(
-        import_state.current_year_number,
-        import_state.current_month_number,
-        import_state.current_day_number,
-    )
+    if not from_date:
+        from_date = date(
+            import_state.current_year_number,
+            import_state.current_month_number,
+            import_state.current_day_number,
+        )
     date_today = date.today()
     # Source data date time is in UTC. Calculate a utf_offset
     utc_offset = pytz.timezone("Europe/Helsinki").utcoffset(datetime.now())
@@ -229,7 +230,7 @@ def save_dataframe() -> datetime:
                 # Remove latest csv, as it might not be populated until the end of day
                 if os.path.exists(csv_file):
                     os.remove(csv_file)
-            if not os.path.exists(csv_file):
+            if not os.path.exists(csv_file) or can_overwrite_csv_file:
                 df.to_csv(csv_file)
             start_date += timedelta(days=1)
 
@@ -242,7 +243,24 @@ def save_dataframe() -> datetime:
 
 
 class Command(BaseCommand):
+    def add_arguments(self, parser):
+        help_msg = (
+            "The date from which the import begins in YYYY-MM-DD format. Note, the date cannot be more than "
+            + "three months in the past, which is the maximum length of history the Telraam API supports."
+        )
+        parser.add_argument("--from-date", type=str, help=help_msg)
+
     def handle(self, *args, **options):
         logger.info("Importing Telraam data...")
-        until_date = save_dataframe()
+        from_date_arg = options.get("from_date", None)
+        from_date = None
+        if from_date_arg:
+            try:
+                from_date = datetime.strptime(from_date_arg, "%Y-%m-%d").date()
+            except ValueError:
+                logger.error("Invalid date argument format. use YYYY-MM-DD.")
+                return
+
+        until_date = save_dataframe(from_date)
+
         logger.info(f"Telraam data imported until {str(until_date)}")

From aafbc0ad357cec46f7ea261ad5b1d2348e704e03 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 8 Jun 2023 10:51:39 +0300
Subject: [PATCH 75/81] Improve missing Telraam CSV file handling

---
 eco_counter/management/commands/utils.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 566886f26..02de99ec9 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -427,10 +427,13 @@ def get_telraam_counter_csv(from_date):
             try:
                 df_tmp = pd.read_csv(csv_file, index_col=False)
             except FileNotFoundError:
-                logger.warning(f"File {csv_file} not found, skipping camera {camera}")
-                break
-            df_cam = pd.concat([df_cam, df_tmp])
-            start_date += timedelta(days=1)
+                logger.warning(
+                    f"File {csv_file} not found, skipping day{str(start_date)} for camera {camera}"
+                )
+            else:
+                df_cam = pd.concat([df_cam, df_tmp])
+            finally:
+                start_date += timedelta(days=1)
 
         if df.empty:
             df = df_cam

From c191f6ef930b0224a198750df74f0032317725ba Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 8 Jun 2023 11:24:41 +0300
Subject: [PATCH 76/81] Remove unused import

---
 eco_counter/constants.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index f56c6f7f3..11ebf577b 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -5,8 +5,6 @@
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
-from mobility_data.importers.utils import get_root_dir
-
 INDEX_COLUMN_NAME = "startTime"
 
 TRAFFIC_COUNTER_START_YEAR = 2015

From eb75f1ce620cf0df12a4a9b7516b7d34ec43cb66 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 8 Jun 2023 12:40:20 +0300
Subject: [PATCH 77/81] Add management command to delete all counter data

---
 .../commands/delete_all_counter_data.py         | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 eco_counter/management/commands/delete_all_counter_data.py

diff --git a/eco_counter/management/commands/delete_all_counter_data.py b/eco_counter/management/commands/delete_all_counter_data.py
new file mode 100644
index 000000000..9633df83c
--- /dev/null
+++ b/eco_counter/management/commands/delete_all_counter_data.py
@@ -0,0 +1,17 @@
+import logging
+
+from django import db
+from django.core.management.base import BaseCommand
+
+from eco_counter.models import ImportState, Station
+
+logger = logging.getLogger("eco_counter")
+
+
+class Command(BaseCommand):
+    @db.transaction.atomic
+    def handle(self, *args, **options):
+        logger.info("Deleting all counter data...")
+        logger.info(f"{Station.objects.all().delete()}")
+        logger.info(f"{ImportState.objects.all().delete()}")
+        logger.info("Deleted all counter data.")

From e3e0ef86217b582a6e373fce16ec85fb7c37c87b Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Thu, 8 Jun 2023 12:40:44 +0300
Subject: [PATCH 78/81] Add task to delete all counter data

---
 eco_counter/tasks.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/eco_counter/tasks.py b/eco_counter/tasks.py
index fbbc6fbde..e15288349 100644
--- a/eco_counter/tasks.py
+++ b/eco_counter/tasks.py
@@ -16,3 +16,8 @@ def initial_import_counter_data(args, name="initial_import_counter_data"):
 @shared_task_email
 def import_telraam_to_csv(args, name="import_telraam_to_csv"):
     management.call_command("import_telraam_to_csv", args)
+
+
+@shared_task_email
+def delete_all_counter_data(name="delete_all_counter_data"):
+    management.call_command("delete_all_counter_data")

From 54964f094245ea3d13a50d42c084ec5cdc1c3064 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 12 Jun 2023 11:00:54 +0300
Subject: [PATCH 79/81] Fix Station __str__ function

---
 eco_counter/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eco_counter/models.py b/eco_counter/models.py
index a96b5d74c..a58565184 100644
--- a/eco_counter/models.py
+++ b/eco_counter/models.py
@@ -11,7 +11,7 @@ class ImportState(models.Model):
     current_year_number = models.PositiveSmallIntegerField(null=True)
     current_month_number = models.PositiveSmallIntegerField(
         validators=[MinValueValidator(1), MaxValueValidator(12)],
-        null=True,  # , default=1
+        null=True,
     )
     current_day_number = models.PositiveSmallIntegerField(
         null=True, blank=True, validators=[MinValueValidator(1), MaxValueValidator(31)]
@@ -37,7 +37,7 @@ class Station(models.Model):
     station_id = models.CharField(max_length=16, null=True)
 
     def __str__(self):
-        return "%s %s" % (self.name, self.geom)
+        return "%s %s" % (self.name, self.location)
 
     class Meta:
         ordering = ["id"]

From 6ad85161d1de7cafe52a20327814eeafdfa7e29e Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 12 Jun 2023 11:01:26 +0300
Subject: [PATCH 80/81] Fix importing of LAM station to reflect new API

---
 eco_counter/management/commands/utils.py | 29 ++++++++++++++----------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py
index 02de99ec9..695eefb81 100644
--- a/eco_counter/management/commands/utils.py
+++ b/eco_counter/management/commands/utils.py
@@ -1,5 +1,4 @@
 import io
-import json
 import logging
 from datetime import date, timedelta
 
@@ -15,7 +14,8 @@
     ECO_COUNTER,
     INDEX_COLUMN_NAME,
     LAM_COUNTER,
-    LAM_STATION_MUNICIPALITIES,
+    LAM_STATION_LOCATIONS,
+    LAM_STATION_USER_HEADER,
     LAM_STATIONS_API_FETCH_URL,
     LAM_STATIONS_DIRECTION_MAPPINGS,
     TELRAAM_COUNTER,
@@ -39,13 +39,8 @@
 
 class LAMStation:
     def __init__(self, feature):
-        if feature["municipality"].as_string() not in LAM_STATION_MUNICIPALITIES:
-            self.active = False
         self.station_id = feature["tmsNumber"].as_int()
-        names = json.loads(feature["names"].as_string())
-        self.name = names["fi"]
-        self.name_sv = names["sv"]
-        self.name_en = names["en"]
+        self.name = self.name_sv = self.name_en = feature["name"].as_string()
         # The source data has a obsolete Z dimension with value 0, remove it.
         geom = feature.geom.clone()
         geom.coord_dim = 2
@@ -217,7 +212,7 @@ def get_traffic_counter_csv(start_year=2015):
 
 
 def get_lam_dataframe(csv_url):
-    response = requests.get(csv_url)
+    response = requests.get(csv_url, headers=LAM_STATION_USER_HEADER)
     string_data = response.content
     csv_data = pd.read_csv(io.StringIO(string_data.decode("utf-8")), delimiter=";")
     return csv_data
@@ -276,7 +271,7 @@ def get_lam_counter_csv(start_date):
             df = get_lam_station_dataframe(
                 station.station_id, direction, start_date, today
             )
-            # Read the direction
+            # Read the direction, e.g., Vaasa
             direction_name = df["suuntaselite"].iloc[0]
             # From the mappings determine the 'keskustaan päin' or 'poispäin keskustasta' direction.
             try:
@@ -337,11 +332,20 @@ def get_lam_counter_csv(start_date):
     return data_frame
 
 
+def has_list_elements_in_string(elements, string):
+    for element in elements:
+        if element in string:
+            return True
+    return False
+
+
 def get_lam_counter_stations():
     stations = []
     data_layer = DataSource(settings.LAM_COUNTER_STATIONS_URL)[0]
     for feature in data_layer:
-        if feature["municipality"].as_string() in LAM_STATION_MUNICIPALITIES:
+        if has_list_elements_in_string(
+            LAM_STATION_LOCATIONS, feature["name"].as_string()
+        ):
             stations.append(ObservationStation(LAM_COUNTER, feature))
     return stations
 
@@ -377,7 +381,8 @@ def fetch_telraam_camera(mac_id):
     response = TELRAAM_HTTP.get(url, headers=headers)
     cameras = response.json().get("camera", None)
     if cameras:
-        # Return first camera
+        # Return first camera, as currently only one camera is
+        # returned in Turku by mac_id
         return cameras[0]
     else:
         return None

From a521d70fedfb575e031e841bb0a76b7b7cf5c104 Mon Sep 17 00:00:00 2001
From: juuso-j <juuso.jokiniemi@turku.fi>
Date: Mon, 12 Jun 2023 11:01:47 +0300
Subject: [PATCH 81/81] Fix LAM_STATIONS_DIRECTION_MAPPINGS, add
 LAM_STATION_LOCATIONS and _HEADER

---
 eco_counter/constants.py | 54 +++++++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/eco_counter/constants.py b/eco_counter/constants.py
index 11ebf577b..7744b4229 100644
--- a/eco_counter/constants.py
+++ b/eco_counter/constants.py
@@ -1,3 +1,4 @@
+import platform
 import types
 
 import requests
@@ -44,31 +45,54 @@
 }
 
 TRAFFIC_COUNTER_METADATA_GEOJSON = "traffic_counter_metadata.geojson"
-# LAM stations located in the municipalities list are included.
-LAM_STATION_MUNICIPALITIES = ["Turku", "Raisio", "Kaarina", "Lieto"]
-
 LAM_STATIONS_API_FETCH_URL = (
     settings.LAM_COUNTER_API_BASE_URL
     + "?api=liikennemaara&tyyppi=h&pvm={start_date}&loppu={end_date}"
     + "&lam_type=option1&piste={id}&luokka=kaikki&suunta={direction}&sisallytakaistat=0"
 )
-# Maps the direction of the traffic of station, (P)oispäin or (K)eskustaan päin)
+# LAM stations in the locations list are included.
+LAM_STATION_LOCATIONS = ["Turku", "Raisio", "Kaarina", "Lieto", "Hauninen", "Oriketo"]
+# Header that is added to the request that fetches the LAM data.
+LAM_STATION_USER_HEADER = {
+    "Digitraffic-User": f"{platform.uname()[1]}/Turun Palvelukartta"
+}
+# Mappings are derived by the 'suunta' and  the 'suuntaselite' columns in the source data.
+# (P)oispäin or (K)eskustaan päin)
 LAM_STATIONS_DIRECTION_MAPPINGS = {
-    "1_Piikkiö": "P",
-    "1_Naantali": "P",
-    "2_Naantali": "K",
-    "1_Turku": "K",
+    # vt8_Raisio
+    "1_Vaasa": "P",
     "2_Turku": "K",
+    # vt1_Kaarina_Kirismäki
+    "1_Turku": "K",
     "2_Helsinki": "P",
-    "1_Suikkila.": "K",
-    "2_Artukainen.": "P",
-    "1_Vaasa": "P",
-    "1_Kuusisto": "P",
-    "2_Kaarina": "K",
-    "1_Tampere": "P",
+    # vt10_Lieto
     "1_Hämeenlinna": "P",
+    # "2_Turku": "K", Duplicate
+    # vt1_Turku_Kupittaa
+    # "1_Turku" Duplicate
+    # "2_Helsinki" Duplicate
+    # vt1_Turku_Kurkela_länsi
+    # "1_Turku" Duplicate
+    # "2_Helsinki" Duplicate
+    # vt1_Kaarina_Kurkela_itä
+    # "1_Turku" Duplicate
+    # "2_Helsinki" Duplicate
+    # vt1_Kaarina
+    # "1_Turku" Duplicate
+    # "2_Helsinki" Duplicate
+    # vt1_Kaarina_Piikkiö
+    # "1_Turku" Duplicate
+    # "2_Helsinki" Duplicate
+    # yt1851_Turku_Härkämäki
+    "1_Suikkila": "K",
+    "2_Artukainen": "P",
+    # kt40_Hauninen
+    "1_Piikkiö": "K",
+    "2_Naantali": "P",
+    # kt40_Oriketo
+    # "1_Piikkiö": "K", duplicate
+    # "2_Naantali": "P", dupicate
 }
-
 keys = [k for k in range(TRAFFIC_COUNTER_START_YEAR, TRAFFIC_COUNTER_END_YEAR + 1)]
 # Create a dict where the years to be importer are keys and the value is the url of the csv data.
 # e.g. {2015, "https://data.turku.fi/2yxpk2imqi2mzxpa6e6knq/2015_laskenta_juha.csv"}