From 6ee6dfac3b59d0b2526daaa69642d2809a75b3c4 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 23 Nov 2023 17:44:24 +0100 Subject: [PATCH] added test for "100 valid days" counting algorithm; removed duplicate test_data --- comptages/comptages.py | 5 ++-- comptages/core/statistics.py | 26 ++++++++++++++++ comptages/core/utils.py | 23 --------------- comptages/test/test_statistics.py | 49 +++++++++++++++++++++++++++---- comptages/test/utils.py | 5 ---- 5 files changed, 72 insertions(+), 36 deletions(-) diff --git a/comptages/comptages.py b/comptages/comptages.py index 4f7ef139..30be5f67 100644 --- a/comptages/comptages.py +++ b/comptages/comptages.py @@ -20,7 +20,8 @@ from comptages.core.filter_dialog import FilterDialog from comptages.core.yearly_report_dialog import YearlyReportDialog from comptages.core.delete_dialog import DeleteDialog -from comptages.core.utils import push_info, push_error, count_valid_days +from comptages.core.utils import push_info, push_error +from comptages.core.statistics import get_valid_days from comptages.datamodel import models from comptages.core import importer, importer_task, report, report_task from comptages.chart.chart_dialog import ChartDock @@ -422,7 +423,7 @@ def do_yearly_report_action(self): ) # Do not proceed unless the number of processed days exceeds 100 days - valid_days = count_valid_days(section_id, year) + valid_days = get_valid_days(section_id, year) if valid_days < 100: push_error( f"This section ({section_id}) lacks valid days for this year ({year}). Found only {valid_days} out of 100." diff --git a/comptages/core/statistics.py b/comptages/core/statistics.py index b07b9977..3eb96098 100644 --- a/comptages/core/statistics.py +++ b/comptages/core/statistics.py @@ -485,3 +485,29 @@ def get_month_data(section: models.Section, start, end): df = pd.DataFrame.from_records(qs) return df + + +def get_valid_days(section_id: str, year: int) -> int: + """ + Count valid days across all counts for `section` and `year`, + where a day is deemed valid just in case there are at least 14 1-hour blocks + between 6pm and 4pm with at least 1 vehicle. + """ + section = models.Section.objects.get(id=section_id) + df = get_time_data_yearly(year, section) + df.reset_index() + + valid_days_in_year = [] + one_hour_block = set() + prev_day = None + for _, row in df.iterrows(): + if any(k not in row for k in ("date", "hour", "thm")): + continue + if row["date"] != prev_day: + if len(one_hour_block) >= 14: + valid_days_in_year.append(prev_day) + one_hour_block.clear() + if 6 <= row["hour"] <= 22 and row["thm"] > 0: + one_hour_block.add(row["hour"]) + prev_day = row["date"] + return len(valid_days_in_year) diff --git a/comptages/core/utils.py b/comptages/core/utils.py index 1fdb9af0..777d3dbb 100644 --- a/comptages/core/utils.py +++ b/comptages/core/utils.py @@ -1,7 +1,6 @@ import os from datetime import datetime -from comptages.core.statistics import get_time_data_yearly from qgis.core import Qgis from qgis.PyQt.uic import loadUiType @@ -10,10 +9,8 @@ from qgis.PyQt.QtSql import QSqlDatabase from qgis.utils import iface -from django.db.models.functions import Trunc from comptages.core.settings import Settings -from comptages.datamodel import models def get_ui_class(ui_file): @@ -69,23 +66,3 @@ def connect_to_db(): db.open() return db - - -def count_valid_days(section_id: str, year: int) -> int: - """Count valid days across all counts for `section` and `year`.""" - section = models.Section.objects.get(id=section_id) - df = get_time_data_yearly(year, section) - df.reset_index() - - valid_days_in_year = set() - valid_hours_in_day = set() - prev_day = None - for _, row in df.iterrows(): - if row["date"] != prev_day: - if len(valid_hours_in_day) >= 14: - valid_days_in_year.add(prev_day) - valid_hours_in_day.clear() - if 6 <= row["hour"] <= 22 and row["thm"] > 0: - valid_hours_in_day.add(row["hour"]) - prev_day = row["date"] - return len(valid_days_in_year) diff --git a/comptages/test/test_statistics.py b/comptages/test/test_statistics.py index 60a2bc4c..99a71e93 100644 --- a/comptages/test/test_statistics.py +++ b/comptages/test/test_statistics.py @@ -1,3 +1,5 @@ +from itertools import islice +from pathlib import Path import pytz from datetime import datetime, timedelta from django.test import TransactionTestCase @@ -247,12 +249,12 @@ def test_get_speed_data_empty(self): tz = pytz.timezone("Europe/Zurich") count = models.Count.objects.create( - start_service_date=tz.localize(datetime(2022, 4, 25)), - end_service_date=tz.localize(datetime(2022, 4, 27)), - start_process_date=tz.localize(datetime(2022, 4, 25)), - end_process_date=tz.localize(datetime(2022, 4, 27)), - start_put_date=tz.localize(datetime(2022, 4, 25)), - end_put_date=tz.localize(datetime(2022, 4, 27)), + start_service_date=tz.localize(datetime(2021, 2, 1)), + end_service_date=tz.localize(datetime(2021, 12, 1)), + start_process_date=tz.localize(datetime(2021, 2, 2)), + end_process_date=tz.localize(datetime(2021, 12, 1)), + start_put_date=tz.localize(datetime(2021, 1, 1)), + end_put_date=tz.localize(datetime(2021, 12, 31)), id_model=model, id_device=device, id_sensor_type=sensor_type, @@ -269,3 +271,38 @@ def test_get_speed_data_empty(self): self.assertTrue(statistics.get_speed_data(count, sections[0]).empty) self.assertFalse(statistics.get_speed_data(count, sections[2]).empty) + + def test_get_valid_days(self): + section_id = "00107695" + section = models.Section.objects.get(id=section_id) + lane = models.Lane.objects.filter(id_section=section_id)[0] + installation = models.Installation.objects.get(id=lane.id_installation.id) + + model = models.Model.objects.all()[0] + device = models.Device.objects.all()[0] + sensor_type = models.SensorType.objects.all()[0] + class_ = models.Class.objects.get(name="SWISS10") + tz = pytz.timezone("Europe/Zurich") + + count = models.Count.objects.create( + start_service_date=tz.localize(datetime(2021, 2, 1)), + end_service_date=tz.localize(datetime(2022, 12, 10)), + start_process_date=tz.localize(datetime(2021, 2, 10)), + end_process_date=tz.localize(datetime(2021, 12, 15)), + start_put_date=tz.localize(datetime(2021, 1, 1)), + end_put_date=tz.localize(datetime(2021, 1, 5)), + id_model=model, + id_device=device, + id_sensor_type=sensor_type, + id_class=class_, + id_installation=installation, + ) + + path_to_files = Path(utils.test_data_path("SWISS10_vbv_year")) + test_files = islice(path_to_files.iterdir(), 5) + + for file in test_files: + importer.import_file(utils.test_data_path(file), count) + + valid = statistics.get_valid_days(section.id, 2021) + assert valid == 2 diff --git a/comptages/test/utils.py b/comptages/test/utils.py index 0f94bf7b..acc0787d 100644 --- a/comptages/test/utils.py +++ b/comptages/test/utils.py @@ -1,10 +1,5 @@ import os -from datetime import datetime -from django.contrib.gis.geos import LineString - -from comptages.datamodel import models - def test_data_path(file_path): """Return the path of file in the directory with the test data."""