From c7a75f2729b83df1b99ad9bad2c0af74d82217d6 Mon Sep 17 00:00:00 2001 From: juuso-j Date: Fri, 1 Mar 2024 09:09:21 +0200 Subject: [PATCH 01/64] Add Kupittaa station --- mobility_data/data/Pyorienkorjauspisteet_2022.geojson | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mobility_data/data/Pyorienkorjauspisteet_2022.geojson b/mobility_data/data/Pyorienkorjauspisteet_2022.geojson index 1cd597a4b..8b802faec 100755 --- a/mobility_data/data/Pyorienkorjauspisteet_2022.geojson +++ b/mobility_data/data/Pyorienkorjauspisteet_2022.geojson @@ -14,6 +14,8 @@ { "type": "Feature", "properties": { "id": null, "Kohde": "Lillmälö", "Osoite": "Saaristotie 3107, 21600 Parainen / Skärgårdsvägen 3107, 21600 Pargas", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Katoksellinen pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation med gapskjul innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station with roof includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23450878.808870975, "y": 6680753.3015155382 }, "geometry": { "type": "Point", "coordinates": [ 23450878.808870974928141, 6680753.30151553824544 ] } }, { "type": "Feature", "properties": { "id": null, "Kohde": "Hanka", "Osoite": "Luotojentie 1092, 21150 Naantali", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23442968.857040703, "y": 6686362.781391114 }, "geometry": { "type": "Point", "coordinates": [ 23442968.857040703296661, 6686362.781391113996506 ] } }, { "type": "Feature", "properties": { "id": null, "Kohde": "Röölä", "Osoite": "Rööläntie 402, 21150 Naantali", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23442390.872016005, "y": 6693030.3863375364 }, "geometry": { "type": "Point", "coordinates": [ 23442390.872016005218029, 6693030.386337536387146 ] } }, -{ "type": "Feature", "properties": { "id": null, "Kohde": "Nauvo / Nagu", "Osoite": "Nauvon ranta 6, 21660 Parainen / Nagu Strand 6, 21660 Pargas", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23439620.678092718, "y": 6676188.3548369883 }, "geometry": { "type": "Point", "coordinates": [ 23439620.67809271812439, 6676188.354836988262832 ] } } +{ "type": "Feature", "properties": { "id": null, "Kohde": "Nauvo / Nagu", "Osoite": "Nauvon ranta 6, 21660 Parainen / Nagu Strand 6, 21660 Pargas", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23439620.678092718, "y": 6676188.3548369883 }, "geometry": { "type": "Point", "coordinates": [ 23439620.67809271812439, 6676188.354836988262832 ] } }, +{ "type": "Feature", "properties": { "id": null, "Kohde": "Kupittaa / Kuppis / Kupittaa", "Osoite": "Joukahaisenkatu 6, 20520 Turku / Joukahainengatan 6, 20520 Turku", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "1.3.2024", "Kuvaus": "Pyöränkorjauspiste sisältää pyöränpumpun ja kaksi monitoimityökalua. Pyöränpumppuun on valittavissa erilaisia suukappaleita, jotka sopivat tavallisiin venttiilityyppeihin. Monitoimityökalut sisältävät kiintoavaimen (8,9,10 ja 15mm), kuusiokoloavaimen (3,4,5,6 ja 8mm) ja ruuvimeisselin (0,8x4mm). Pyöränkorjauspisteessä on myös kaksi kannatinkoukkua eri korkeudella, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller cykelpump och två multiverktyg. I cykelpump det finns multi-munstycke som passar alla gängse ventiltyper. Multiverktyg innehåller blocknyckel (8,9,10 och 15mm), sexkantnyckel (3,4,5,6 och 8mm) och skruvmejsel (0,8x4mm). Cykelservicestation har också två pelaren i olika nivåer som cykeln kan hänga upp. \nBike service station includes bicycle pump and two multifunctional tools. There are different mouthpiece options which are compatible to usual valve types. Multifunctional tools include a wrench (8,9,10 and 15mm), a hex head wrench (3,4,5,6 and 8mm) and a screwdriver (0,8x4mm). Bike service station also include two hooks in different heights where the bicycles can be lifted during the service.", "Maastossa": "Ei", "Lisätieto": "Merkki Care4bikes", "x": 23461183.976663336, "y": 6704468.582275727 }, "geometry": { "type": "Point", "coordinates": [ 23461183.976663336, 6704468.582275727 ] } } + ] } From f96fef3ef0227c035c97c8e16aa9eb66e4988207 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:31:54 +0200 Subject: [PATCH 02/64] Add headers and is_xml --- iot/models.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/iot/models.py b/iot/models.py index d2b4c7b22..25d8c9615 100644 --- a/iot/models.py +++ b/iot/models.py @@ -7,10 +7,18 @@ class IoTDataSource(models.Model): source_name = models.CharField( - max_length=3, unique=True, verbose_name="Three letter long name for the source" + max_length=3, + unique=True, + verbose_name="Three letter long identifier for the source. " + "Set the identifier as an argument to the Celery task that fetches the data.", ) source_full_name = models.CharField(max_length=64, null=True) url = models.URLField() + headers = models.JSONField( + null=True, + blank=True, + verbose_name='request headers in JSON format, e.g., {"key1": "value1", "key2": "value2"}', + ) def __str__(self): return self.source_name @@ -26,7 +34,7 @@ def clean(self): response.json() except json.decoder.JSONDecodeError: raise ValidationError( - f"Could not parse the JSON data for the given url {self.url}" + f"Could not parse the JSON data from the given url {self.url}" ) From 32d4453d7c0b04b023635e352af003d9cbfcd371 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:46:51 +0200 Subject: [PATCH 03/64] Add is_xml and XML validation --- iot/models.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/iot/models.py b/iot/models.py index 25d8c9615..a60d5cd1f 100644 --- a/iot/models.py +++ b/iot/models.py @@ -1,6 +1,8 @@ import json +from xml.parsers.expat import ExpatError import requests +import xmltodict from django.core.exceptions import ValidationError from django.db import models @@ -13,6 +15,9 @@ class IoTDataSource(models.Model): "Set the identifier as an argument to the Celery task that fetches the data.", ) source_full_name = models.CharField(max_length=64, null=True) + is_xml = models.BooleanField( + default=False, verbose_name="If True, XML data will be converted to JSON." + ) url = models.URLField() headers = models.JSONField( null=True, @@ -26,16 +31,26 @@ def __str__(self): def clean(self): # Test if url exists try: - response = requests.get(self.url) + response = requests.get(self.url, headers=self.headers) except requests.exceptions.ConnectionError: raise ValidationError(f"The given url {self.url} does not exist.") - # Test if valid json - try: - response.json() - except json.decoder.JSONDecodeError: - raise ValidationError( - f"Could not parse the JSON data from the given url {self.url}" - ) + + # Test if XML data can be parsed into JSON + if self.is_xml: + try: + xmltodict.parse(response.text) + except ExpatError as err: + raise ExpatError( + f"Could not parse XML data from the give url {self.url}. {err}" + ) + else: + # Test if valid JSON + try: + response.json() + except json.decoder.JSONDecodeError as err: + raise ValidationError( + f"Could not parse the JSON data from the given url {self.url}. {err}" + ) class IoTData(models.Model): From 2fdecb83a93c0d0d524d89c4ae6a42f7dfb5cc28 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:51:35 +0200 Subject: [PATCH 04/64] Add is_xml and headers info --- iot/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iot/README.md b/iot/README.md index e7935c21d..c92a5df19 100644 --- a/iot/README.md +++ b/iot/README.md @@ -3,9 +3,12 @@ The purpose of the IoT app is to store temporarly data from various IoT-data sou The data is stored as it is in JSON to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data source or when a data source is added. The cache is populated if empty when serving data. ## Adding IoT-data source from the Admin -* Give a tree letter long source name, this name will be the name for the source. Used for example when requesting the data. +* Give a tree letter long identifier, this will be used to identify the data +to be imported in the Celery task and when requesting data in the API. * Add the full name of the source * Add the Url to the JSON data. +* Set is_xml to True if the data is in XML format, the data will be converted to JSON. +* Add the optional headers for the request. ## Setting periodic importing using Celery from the Admin * Create a periodic task, give a descrpitive name. From 55bcbc17a2f884d4309d3052d639b08eb76a83f8 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:52:11 +0200 Subject: [PATCH 05/64] Add support for XML data --- iot/management/commands/import_iot_data.py | 24 +++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/iot/management/commands/import_iot_data.py b/iot/management/commands/import_iot_data.py index c1d90866e..eae05d817 100644 --- a/iot/management/commands/import_iot_data.py +++ b/iot/management/commands/import_iot_data.py @@ -1,7 +1,9 @@ import json import logging +from xml.parsers.expat import ExpatError import requests +import xmltodict from django.core.cache import cache from django.core.management.base import BaseCommand @@ -13,17 +15,25 @@ def save_data_to_db(source): IoTData.objects.filter(data_source=source).delete() - try: - response = requests.get(source.url) + response = requests.get(source.url, headers=source.headers) except requests.exceptions.ConnectionError: logger.error(f"Could not fetch data from: {source.url}") return - try: - json_data = response.json() - except json.decoder.JSONDecodeError: - logger.error(f"Could not decode data to json from: {source.url}") - return + if source.is_xml: + try: + json_data = xmltodict.parse(response.text) + except ExpatError as err: + logger.error( + f"Could not parse XML data from the give url {source.url}. {err}" + ) + return + else: + try: + json_data = response.json() + except json.decoder.JSONDecodeError as err: + logger.error(f"Could not decode data to JSON from: {source.url}. {err}") + return IoTData.objects.create(data_source=source, data=json_data) From fb36f963a6ff7429e521beec6126af0bdd35088a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:52:40 +0200 Subject: [PATCH 06/64] Add migration --- iot/migrations/0003_iotdatasource_headers.py | 20 ++++++++++ ...ml_alter_iotdatasource_headers_and_more.py | 39 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 iot/migrations/0003_iotdatasource_headers.py create mode 100644 iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py diff --git a/iot/migrations/0003_iotdatasource_headers.py b/iot/migrations/0003_iotdatasource_headers.py new file mode 100644 index 000000000..d70dd8d84 --- /dev/null +++ b/iot/migrations/0003_iotdatasource_headers.py @@ -0,0 +1,20 @@ +# Generated by Django 4.1.13 on 2024-03-13 06:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iot", "0002_add_ordering"), + ] + + operations = [ + migrations.AddField( + model_name="iotdatasource", + name="headers", + field=models.JSONField( + blank=True, null=True, verbose_name="request headers" + ), + ), + ] diff --git a/iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py b/iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py new file mode 100644 index 000000000..d8bdb9ddb --- /dev/null +++ b/iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 4.1.13 on 2024-03-13 08:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iot", "0003_iotdatasource_headers"), + ] + + operations = [ + migrations.AddField( + model_name="iotdatasource", + name="is_xml", + field=models.BooleanField( + default=False, + verbose_name="If True, XML data will be converted to JSON.", + ), + ), + migrations.AlterField( + model_name="iotdatasource", + name="headers", + field=models.JSONField( + blank=True, + null=True, + verbose_name='request headers in JSON format, e.g., {"key1": "value1", "key2": "value2"}', + ), + ), + migrations.AlterField( + model_name="iotdatasource", + name="source_name", + field=models.CharField( + max_length=3, + unique=True, + verbose_name="Three letter long identifier for the source. Set the identifier as an argument to the Celery task that fetches the data.", + ), + ), + ] From 94e4703afe2243d530e6c7720fb0f7a38bd98a48 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:37:41 +0200 Subject: [PATCH 07/64] Change name to identifier --- iot/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iot/README.md b/iot/README.md index c92a5df19..13955b3b3 100644 --- a/iot/README.md +++ b/iot/README.md @@ -15,7 +15,7 @@ to be imported in the Celery task and when requesting data in the API. * Select *iot.tasks.import_iot_data* as the Task (registered) * Choose the *Interval Schedule* * Set the Start DateTime -* Add the source name as *Positional Arguments*, e.g. ["R24"] would import the source_name R24. +* Add the identifier as *Positional Arguments*, e.g. ["R24"] would import the identifier R24. ## Manual import To manually import source: From 8c8b098476153604c759b9419427da967cac8ff2 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:54:46 +0200 Subject: [PATCH 08/64] Fix manual import info --- iot/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iot/README.md b/iot/README.md index 13955b3b3..49722766c 100644 --- a/iot/README.md +++ b/iot/README.md @@ -19,7 +19,7 @@ to be imported in the Celery task and when requesting data in the API. ## Manual import To manually import source: -`./manage.py import_iot_data source_name` +`./manage.py import_iot_data identifier` Or by running the perioc task from the admin. ## Retriving data From a6e02a6960fc2fed36904b1dc4bc0b3e9f595b8e Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:27:37 +0200 Subject: [PATCH 09/64] Fix --- iot/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iot/README.md b/iot/README.md index 49722766c..521393f68 100644 --- a/iot/README.md +++ b/iot/README.md @@ -1,6 +1,6 @@ ## About The purpose of the IoT app is to store temporarly data from various IoT-data sources, that do not allow frequent fetching of their data. -The data is stored as it is in JSON to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data source or when a data source is added. The cache is populated if empty when serving data. +The data is stored as it in JSON format to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data source or when a data source is added. The cache is populated if empty when serving data. ## Adding IoT-data source from the Admin * Give a tree letter long identifier, this will be used to identify the data From d39b56a4e6f5bc1f4a2c3c082bebafb88bda0a8e Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:43:04 +0200 Subject: [PATCH 10/64] Remove useless source word --- iot/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iot/README.md b/iot/README.md index 521393f68..d40de5a32 100644 --- a/iot/README.md +++ b/iot/README.md @@ -1,6 +1,6 @@ ## About The purpose of the IoT app is to store temporarly data from various IoT-data sources, that do not allow frequent fetching of their data. -The data is stored as it in JSON format to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data source or when a data source is added. The cache is populated if empty when serving data. +The data is stored as it in JSON format to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data or when a data source is added. The cache is populated if empty when serving data. ## Adding IoT-data source from the Admin * Give a tree letter long identifier, this will be used to identify the data From 5d73648fb8c72e88d400654f44f6242de8331772 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 19 Mar 2024 08:28:46 +0200 Subject: [PATCH 11/64] Set TRAFFIC_COUNTER_END_YEAR to current year --- eco_counter/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eco_counter/constants.py b/eco_counter/constants.py index c41829e77..12c9b2f54 100644 --- a/eco_counter/constants.py +++ b/eco_counter/constants.py @@ -1,5 +1,6 @@ import platform import types +from datetime import datetime import requests from django.conf import settings @@ -12,7 +13,7 @@ # Manually define the end year, as the source data comes from the page # defined in env variable TRAFFIC_COUNTER_OBSERVATIONS_BASE_URL. # Change end year when data for the next year is available. -TRAFFIC_COUNTER_END_YEAR = 2023 +TRAFFIC_COUNTER_END_YEAR = datetime.today().year ECO_COUNTER_START_YEAR = 2020 LAM_COUNTER_START_YEAR = 2010 TELRAAM_COUNTER_START_YEAR = 2023 From 4b7b11e325f69526ca23720f77267719eb82f873 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 19 Mar 2024 08:29:10 +0200 Subject: [PATCH 12/64] Handle AssertionError --- eco_counter/management/commands/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eco_counter/management/commands/utils.py b/eco_counter/management/commands/utils.py index 5a9169129..e358db319 100644 --- a/eco_counter/management/commands/utils.py +++ b/eco_counter/management/commands/utils.py @@ -206,7 +206,10 @@ def get_traffic_counter_csv(start_year=2015): # data from years before the start year. if key <= start_year: continue - concat_df = get_dataframe(TRAFFIC_COUNTER_CSV_URLS[key]) + try: + concat_df = get_dataframe(TRAFFIC_COUNTER_CSV_URLS[key]) + except AssertionError: + continue # ignore_index=True, do not use the index values along the concatenation axis. # The resulting axis will be labeled 0, …, n - 1. df = pd.concat([df, concat_df], ignore_index=True) From b08e7750a3e2898aa42f9a7482d22b3de16a72f6 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 19 Mar 2024 08:54:00 +0200 Subject: [PATCH 13/64] Add Tapion Polku --- mobility_data/importers/culture_routes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mobility_data/importers/culture_routes.py b/mobility_data/importers/culture_routes.py index 2b8eee4d9..9427a3518 100644 --- a/mobility_data/importers/culture_routes.py +++ b/mobility_data/importers/culture_routes.py @@ -26,6 +26,11 @@ SOURCE_DATA_SRID = 4326 # Routes are from https://citynomadi.com/route/?keywords=turku URLS = { + "Tapion Polku": { + "fi": "https://www.citynomadi.com/api/route/5b6669fa989c1b8c2fc552b2b2afdbd1/kml?lang=fi", + "sv": "https://www.citynomadi.com/api/route/5b6669fa989c1b8c2fc552b2b2afdbd1/kml?lang=sv", + "en": "https://www.citynomadi.com/api/route/5b6669fa989c1b8c2fc552b2b2afdbd1/kml?lang=en", + }, "Sotiemme Turku": { "fi": "https://citynomadi.com/api/route/fb656ce4fc31868f4b90168ecc3fabdb/kml?lang=fi", "sv": "https://citynomadi.com/api/route/fb656ce4fc31868f4b90168ecc3fabdb/kml?lang=sv", From 6e3002539ea4883a765e44193c9d42bf91baf51d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:44:28 +0200 Subject: [PATCH 14/64] Add ExceptionalSitatuonsConfig to INSTALLED_APPS --- smbackend/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index 6e72fcc9f..13bfb2260 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -133,6 +133,7 @@ "iot.apps.IotConfig", "street_maintenance.apps.StreetMaintenanceConfig", "environment_data.apps.EnvironmentDataConfig", + "exceptional_situations.apps.ExceptionalSituationsConfig", ] if env("ADDITIONAL_INSTALLED_APPS"): From 884ed5662b647e6eabaf0601fd4d02efcf12d37d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:50:25 +0200 Subject: [PATCH 15/64] Add __init__.py --- exceptional_situations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 exceptional_situations/__init__.py diff --git a/exceptional_situations/__init__.py b/exceptional_situations/__init__.py new file mode 100644 index 000000000..e69de29bb From c474d3562fc3adfa6799760b5e5199da50829f75 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:50:54 +0200 Subject: [PATCH 16/64] Add AppConfig --- exceptional_situations/apps.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 exceptional_situations/apps.py diff --git a/exceptional_situations/apps.py b/exceptional_situations/apps.py new file mode 100644 index 000000000..c1addb764 --- /dev/null +++ b/exceptional_situations/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ExceptionalSituationsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "exceptional_situations" From 2530d9434208089f51e224e5081a476cc6897ecb Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:21:35 +0200 Subject: [PATCH 17/64] Add initial API --- exceptional_situations/api/serializers.py | 50 +++++++++++++++++++++++ exceptional_situations/api/urls.py | 24 +++++++++++ exceptional_situations/api/views.py | 34 +++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 exceptional_situations/api/serializers.py create mode 100644 exceptional_situations/api/urls.py create mode 100644 exceptional_situations/api/views.py diff --git a/exceptional_situations/api/serializers.py b/exceptional_situations/api/serializers.py new file mode 100644 index 000000000..4d79c5a1a --- /dev/null +++ b/exceptional_situations/api/serializers.py @@ -0,0 +1,50 @@ +from rest_framework import serializers + +from exceptional_situations.models import ( + Situation, + SituationAnnouncement, + SituationLocation, + SituationType, +) + + +class SituationSerializer(serializers.ModelSerializer): + class Meta: + model = Situation + fields = [ + "id", + "is_active", + "situation_id", + "release_time", + "situation_type", + "situation_type_str", + "situation_sub_type_str", + ] + + def to_representation(self, obj): + representation = super().to_representation(obj) + representation["locations"] = SituationLocationSerializer( + obj.locations, many=True + ).data + representation["announcements"] = SituationAnnouncementSerializer( + obj.announcements, many=True + ).data + return representation + + +class SituationAnnouncementSerializer(serializers.ModelSerializer): + class Meta: + model = SituationAnnouncement + fields = "__all__" + + +class SituationLocationSerializer(serializers.ModelSerializer): + class Meta: + model = SituationLocation + fields = "__all__" + + +class SituationTypeSerializer(serializers.ModelSerializer): + class Meta: + model = SituationType + fields = "__all__" diff --git a/exceptional_situations/api/urls.py b/exceptional_situations/api/urls.py new file mode 100644 index 000000000..eef3621a2 --- /dev/null +++ b/exceptional_situations/api/urls.py @@ -0,0 +1,24 @@ +from django.urls import include, path +from rest_framework import routers + +from exceptional_situations.api import views + +app_name = "exceptional_stituations" + + +router = routers.DefaultRouter() + +router.register("situation", views.SituationViewSet, basename="situation") +router.register("situation_type", views.SituationTypeViewSet, basename="situation_type") +router.register( + "situation_location", views.SituationLocationViewSet, basename="situation_location" +) +router.register( + "situation_announcement", + views.SituationAnnouncementViewSet, + basename="situation_announcement", +) + +urlpatterns = [ + path("api/v1/", include(router.urls), name="exceptional_stituations"), +] diff --git a/exceptional_situations/api/views.py b/exceptional_situations/api/views.py new file mode 100644 index 000000000..44f3bc76f --- /dev/null +++ b/exceptional_situations/api/views.py @@ -0,0 +1,34 @@ +from rest_framework import viewsets + +from exceptional_situations.api.serializers import ( + SituationAnnouncementSerializer, + SituationLocationSerializer, + SituationSerializer, + SituationTypeSerializer, +) +from exceptional_situations.models import ( + Situation, + SituationAnnouncement, + SituationLocation, + SituationType, +) + + +class SituationViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Situation.objects.all() + serializer_class = SituationSerializer + + +class SituationLocationViewSet(viewsets.ReadOnlyModelViewSet): + queryset = SituationLocation.objects.all() + serializer_class = SituationLocationSerializer + + +class SituationAnnouncementViewSet(viewsets.ReadOnlyModelViewSet): + queryset = SituationAnnouncement.objects.all() + serializer_class = SituationAnnouncementSerializer + + +class SituationTypeViewSet(viewsets.ReadOnlyModelViewSet): + queryset = SituationType.objects.all() + serializer_class = SituationTypeSerializer From d795379467d25283863e07b0ecef5f7d631cb1e9 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:22:04 +0200 Subject: [PATCH 18/64] Add __init__.py --- exceptional_situations/migrations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 exceptional_situations/migrations/__init__.py diff --git a/exceptional_situations/migrations/__init__.py b/exceptional_situations/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb From c804f205a35d9f26b6c3def3732865d0f5cdbb71 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:57:45 +0200 Subject: [PATCH 19/64] Add initial version of traffic situations importer --- .../commands/import_traffic_situations.py | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 exceptional_situations/management/commands/import_traffic_situations.py diff --git a/exceptional_situations/management/commands/import_traffic_situations.py b/exceptional_situations/management/commands/import_traffic_situations.py new file mode 100644 index 000000000..629a42432 --- /dev/null +++ b/exceptional_situations/management/commands/import_traffic_situations.py @@ -0,0 +1,140 @@ +import logging +from copy import deepcopy + +import requests +from dateutil import parser +from django.contrib.gis.geos import GEOSGeometry, Polygon +from django.core.management import BaseCommand + +from exceptional_situations.models import ( + PROJECTION_SRID, + Situation, + SituationAnnouncement, + SituationLocation, + SituationType, +) +from mobility_data.importers.constants import ( + SOUTHWEST_FINLAND_BOUNDARY, + SOUTHWEST_FINLAND_BOUNDARY_SRID, +) + +logger = logging.getLogger(__name__) + +ROAD_WORK_URL = ( + "https://tie.digitraffic.fi/api/traffic-message/v1/messages" + "?inactiveHours=0&includeAreaGeometry=true&situationType=ROAD_WORK" +) +TRAFFIC_ANNOUNCEMENT_URL = ( + "https://tie.digitraffic.fi/api/traffic-message/v1/messages" + "?inactiveHours=0&includeAreaGeometry=true&situationType=TRAFFIC_ANNOUNCEMENT" +) +URLS = [ROAD_WORK_URL, TRAFFIC_ANNOUNCEMENT_URL] +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +SOUTHWEST_FINLAND_POLYGON = Polygon( + SOUTHWEST_FINLAND_BOUNDARY, srid=SOUTHWEST_FINLAND_BOUNDARY_SRID +) + + +class Command(BaseCommand): + def get_geos_geometry(self, feature_data): + return GEOSGeometry(str(feature_data["geometry"]), srid=PROJECTION_SRID) + + def create_location(self, geometry, announcement_data): + location = None + details = announcement_data["locationDetails"].get("roadAddressLocation", None) + details.update(announcement_data.get("location", None)) + filter = { + "geometry": geometry, + "location": location, + "details": details, + } + situation_location = SituationLocation.objects.create(**filter) + return situation_location + + def create_announcement(self, announcement_data, situation_location): + title = announcement_data.get("title", "") + description = announcement_data["location"].get("description", "") + additional_info = {} + for road_work_phase in announcement_data.get("roadWorkPhases", []): + del road_work_phase["locationDetails"] + del road_work_phase["location"] + additional_info.update(road_work_phase) + + additional_info.update( + { + "additionalInformation": announcement_data.get( + "additionalInformation", None + ) + } + ) + additional_info.update({"sender": announcement_data.get("sender", None)}) + start_time = parser.parse( + announcement_data["timeAndDuration"].get("startTime", None) + ) + end_time = announcement_data["timeAndDuration"].get("endTime", None) + # Note, endTime can be None (unknown) + if end_time: + end_time = parser.parse(end_time) + filter = { + "location": situation_location, + "title": title, + "description": description, + "additional_info": additional_info, + "start_time": start_time, + "end_time": end_time, + } + situation_announcement = SituationAnnouncement.objects.create(**filter) + return situation_announcement + + def handle(self, *args, **options): + for url in URLS: + try: + response = requests.get(url) + assert response.status_code == 200 + except AssertionError: + continue + features = response.json()["features"] + + for feature_data in features: + geometry = self.get_geos_geometry(feature_data) + if not SOUTHWEST_FINLAND_POLYGON.intersects(geometry): + continue + + properties = feature_data.get("properties", None) + if not properties: + continue + situation_id = properties.get("situationId", None) + release_time = properties.get("releaseTime", None) + try: + release_time = parser.parse(release_time) + except parser.ParserError: + logger.error(f"Invalid release time {release_time}") + continue + + type_name = properties.get("situationType", None) + sub_type_name = properties.get("trafficAnnouncementType", None) + + situation_type, _ = SituationType.objects.get_or_create( + type_name=type_name, sub_type_name=sub_type_name + ) + + filter = { + "situation_id": situation_id, + "release_time": release_time, + "situation_type": situation_type, + } + situation, _ = Situation.objects.get_or_create(**filter) + + SituationLocation.objects.filter(situation=situation).delete() + SituationAnnouncement.objects.filter(situation=situation).delete() + situation.locations.clear() + situation.announcements.clear() + for announcement_data in properties.get("announcements", []): + situation_location = self.create_location( + geometry, announcement_data + ) + situation.locations.add(situation_location) + situation_announcement = self.create_announcement( + deepcopy(announcement_data), situation_location + ) + situation.announcements.add(situation_announcement) From d823748812b723a382167a24cf033960320aca08 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:58:17 +0200 Subject: [PATCH 20/64] Initial version of model --- exceptional_situations/models.py | 81 +++++++++++++++++++++++++++ exceptional_situations/translation.py | 13 +++++ 2 files changed, 94 insertions(+) create mode 100644 exceptional_situations/models.py create mode 100644 exceptional_situations/translation.py diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py new file mode 100644 index 000000000..4c7dcb841 --- /dev/null +++ b/exceptional_situations/models.py @@ -0,0 +1,81 @@ +from django.contrib.gis.db import models +from django.utils import timezone + +PROJECTION_SRID = 4326 + + +class SituationType(models.Model): + type_name = models.CharField(max_length=64) + sub_type_name = models.CharField(max_length=64, null=True, blank=True) + + +class SituationLocation(models.Model): + location = models.PointField(null=True, blank=True, srid=PROJECTION_SRID) + geometry = models.GeometryField(null=True, blank=True, srid=PROJECTION_SRID) + details = models.JSONField(null=True, blank=True) + + +class SituationAnnouncement(models.Model): + title = models.CharField(max_length=128) + description = models.TextField() + start_time = models.DateTimeField() + end_time = models.DateTimeField(null=True, blank=True) + additional_info = models.JSONField(null=True, blank=True) + location = models.OneToOneField(SituationLocation, on_delete=models.CASCADE) + + +class Situation(models.Model): + situation_id = models.CharField(max_length=64) + situation_type = models.ForeignKey(SituationType, on_delete=models.CASCADE) + release_time = models.DateTimeField() + locations = models.ManyToManyField(SituationLocation) + announcements = models.ManyToManyField(SituationAnnouncement) + + @property + def situation_type_str(self): + return self.situation_type.type_name + + @property + def situation_sub_type_str(self): + return self.situation_type.sub_type_name + + @property + def is_active(self): + # If one or more end_time is null(unknown?) the situation is active + if self.announcements.filter(end_time__isnull=True).exists(): + return True + + # If end_time is past for all announcements, retrun True, else False + return all( + { + not a.end_time < timezone.now() + for a in self.announcements.filter(end_time__isnull=False) + } + ) + + @property + def situation_start_time(self): + """ + Return start_time furthest in history + """ + start_time = None + for announcement in self.announcements.all(): + if not start_time: + start_time = announcement.start_time + if start_time < announcement.start_time: + start_time = announcement.start_time + return start_time + + @property + def situation_end_time(self): + """ + Return end_time furthest in future + """ + end_time = None + for announcement in self.announcements.filter(end_time__isnull=False): + if not end_time: + end_time = announcement.end_time + + if end_time > announcement.end_time: + end_time = announcement.end_time + return end_time diff --git a/exceptional_situations/translation.py b/exceptional_situations/translation.py new file mode 100644 index 000000000..08f98800d --- /dev/null +++ b/exceptional_situations/translation.py @@ -0,0 +1,13 @@ +from modeltranslation.translator import TranslationOptions, translator + +from exceptional_situations.models import SituationAnnouncement + + +class SituationAnnouncementTranslationOptions(TranslationOptions): + fields = ( + "title", + "description", + ) + + +translator.register(SituationAnnouncement, SituationAnnouncementTranslationOptions) From 88fdc99b2b1911bde45eb43314172b20e870d431 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:10:55 +0200 Subject: [PATCH 21/64] Add exceptional situations url --- smbackend/urls.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/smbackend/urls.py b/smbackend/urls.py index b04e54fab..94555aa30 100644 --- a/smbackend/urls.py +++ b/smbackend/urls.py @@ -10,6 +10,7 @@ import bicycle_network.api.urls import eco_counter.api.urls import environment_data.api.urls +import exceptional_situations.api.urls import mobility_data.api.urls import street_maintenance.api.urls from iot.api import IoTViewSet @@ -71,6 +72,11 @@ include(environment_data.api.urls), name="environmet_data", ), + re_path( + r"^exceptional_situations/", + include(exceptional_situations.api.urls), + name="exceptional_situations", + ), re_path( r"^street_maintenance/", include(street_maintenance.api.urls), From 9010634bac40f58a94d3e67d9cfe237efbe89f21 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:59:17 +0200 Subject: [PATCH 22/64] Add logging settings for Exceptional Situations APP --- smbackend/settings.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index 13bfb2260..6a9915f2c 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -79,6 +79,7 @@ BICYCLE_NETWORK_LOG_LEVEL=(str, "INFO"), STREET_MAINTENANCE_LOG_LEVEL=(str, "INFO"), ENVIRONMENT_DATA_LOG_LEVEL=(str, "INFO"), + EXCEPTIONAL_SITUATIONS_LOG_LEVEL=(str, "INFO"), ) @@ -103,6 +104,7 @@ BICYCLE_NETWORK_LOG_LEVEL = env("BICYCLE_NETWORK_LOG_LEVEL") STREET_MAINTENANCE_LOG_LEVEL = env("STREET_MAINTENANCE_LOG_LEVEL") ENVIRONMENT_DATA_LOG_LEVEL = env("ENVIRONMENT_DATA_LOG_LEVEL") +EXCEPTIONAL_SITUATIONS_LOG_LEVEL = env("EXCEPTIONAL_SITUATIONS_LOG_LEVEL") # Application definition INSTALLED_APPS = [ @@ -337,6 +339,10 @@ def gettext(s): "handlers": ["console"], "level": ENVIRONMENT_DATA_LOG_LEVEL, }, + "exceptional_situations": { + "handlers": ["console"], + "level": EXCEPTIONAL_SITUATIONS_LOG_LEVEL, + }, }, } logging.config.dictConfig(LOGGING) From fc1452323a5f5c23942e95f6a0709ba25060e300 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 08:24:23 +0200 Subject: [PATCH 23/64] Add ordering and __str__ function to Model --- exceptional_situations/models.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py index 4c7dcb841..b1891d901 100644 --- a/exceptional_situations/models.py +++ b/exceptional_situations/models.py @@ -8,12 +8,21 @@ class SituationType(models.Model): type_name = models.CharField(max_length=64) sub_type_name = models.CharField(max_length=64, null=True, blank=True) + class Meta: + ordering = ["id"] + + def __str__(self): + return "%s (%s)" % (self.type_name, self.id) + class SituationLocation(models.Model): location = models.PointField(null=True, blank=True, srid=PROJECTION_SRID) geometry = models.GeometryField(null=True, blank=True, srid=PROJECTION_SRID) details = models.JSONField(null=True, blank=True) + class Meta: + ordering = ["id"] + class SituationAnnouncement(models.Model): title = models.CharField(max_length=128) @@ -23,6 +32,12 @@ class SituationAnnouncement(models.Model): additional_info = models.JSONField(null=True, blank=True) location = models.OneToOneField(SituationLocation, on_delete=models.CASCADE) + class Meta: + ordering = ["start_time"] + + def __str__(self): + return "%s (%s)" % (self.title, self.id) + class Situation(models.Model): situation_id = models.CharField(max_length=64) @@ -31,6 +46,9 @@ class Situation(models.Model): locations = models.ManyToManyField(SituationLocation) announcements = models.ManyToManyField(SituationAnnouncement) + class Meta: + ordering = ["id"] + @property def situation_type_str(self): return self.situation_type.type_name @@ -54,9 +72,9 @@ def is_active(self): ) @property - def situation_start_time(self): + def start_time(self): """ - Return start_time furthest in history + Return the start_time that is furthest in history """ start_time = None for announcement in self.announcements.all(): @@ -67,9 +85,9 @@ def situation_start_time(self): return start_time @property - def situation_end_time(self): + def end_time(self): """ - Return end_time furthest in future + Return the end_time that is furthest in future """ end_time = None for announcement in self.announcements.filter(end_time__isnull=False): From 185e75276b7e0394e29652e8c357895fbbe24a5a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 08:28:05 +0200 Subject: [PATCH 24/64] Remove "situation_" prefix from start_time and end_time --- exceptional_situations/api/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/exceptional_situations/api/serializers.py b/exceptional_situations/api/serializers.py index 4d79c5a1a..8c81a6120 100644 --- a/exceptional_situations/api/serializers.py +++ b/exceptional_situations/api/serializers.py @@ -14,6 +14,8 @@ class Meta: fields = [ "id", "is_active", + "start_time", + "end_time", "situation_id", "release_time", "situation_type", From dfdaae24c5600e926220ae4c257cbc37c0a9e1e1 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 08:28:59 +0200 Subject: [PATCH 25/64] Add logger output and explanation comment --- .../management/commands/import_traffic_situations.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/exceptional_situations/management/commands/import_traffic_situations.py b/exceptional_situations/management/commands/import_traffic_situations.py index 629a42432..3132434cc 100644 --- a/exceptional_situations/management/commands/import_traffic_situations.py +++ b/exceptional_situations/management/commands/import_traffic_situations.py @@ -1,3 +1,7 @@ +""" +Imports road works and traffic announcements in Southwest Finland from digitraffic.fi. +""" + import logging from copy import deepcopy @@ -19,7 +23,6 @@ ) logger = logging.getLogger(__name__) - ROAD_WORK_URL = ( "https://tie.digitraffic.fi/api/traffic-message/v1/messages" "?inactiveHours=0&includeAreaGeometry=true&situationType=ROAD_WORK" @@ -87,6 +90,7 @@ def create_announcement(self, announcement_data, situation_location): return situation_announcement def handle(self, *args, **options): + num_imported = 0 for url in URLS: try: response = requests.get(url) @@ -138,3 +142,5 @@ def handle(self, *args, **options): deepcopy(announcement_data), situation_location ) situation.announcements.add(situation_announcement) + num_imported += 1 + logger.info(f"Imported/updated {num_imported} traffic situations.") From fe80936026076f4dd197a9bc3a9c8109075c1f23 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 08:30:03 +0200 Subject: [PATCH 26/64] Add filters SituationViewSet --- exceptional_situations/api/views.py | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/exceptional_situations/api/views.py b/exceptional_situations/api/views.py index 44f3bc76f..f14488ba1 100644 --- a/exceptional_situations/api/views.py +++ b/exceptional_situations/api/views.py @@ -1,3 +1,5 @@ +import django_filters +from django_filters.rest_framework import DjangoFilterBackend from rest_framework import viewsets from exceptional_situations.api.serializers import ( @@ -14,9 +16,60 @@ ) +class SituationFilter(django_filters.FilterSet): + is_active = django_filters.BooleanFilter(method="filter_is_active") + situation_type_str = django_filters.CharFilter(method="filter_situation_type_str") + start_time__gt = django_filters.DateTimeFilter(method="filter_start_time__gt") + start_time__lt = django_filters.DateTimeFilter(method="filter_start_time__lt") + end_time__gt = django_filters.DateTimeFilter(method="filter_end_time__gt") + end_time__lt = django_filters.DateTimeFilter(method="filter_end_time__lt") + + class Meta: + model = Situation + fields = { + "situation_type": ["exact"], + "situation_id": ["exact"], + "release_time": ["lt", "gt"], + } + + def filter_situation_type_str(self, queryset, fields, situation_type_str): + ids = [ + obj.id for obj in queryset if obj.situation_type_str == situation_type_str + ] + return queryset.filter(id__in=ids) + + def filter_is_active(self, queryset, fields, active): + ids = [obj.id for obj in queryset if obj.is_active == bool(active)] + return queryset.filter(id__in=ids) + + def filter_start_time__gt(self, queryset, fields, start_time): + ids = [obj.id for obj in queryset if obj.start_time > start_time] + return queryset.filter(id__in=ids) + + def filter_start_time__lt(self, queryset, fields, start_time): + ids = [obj.id for obj in queryset if obj.start_time < start_time] + return queryset.filter(id__in=ids) + + def filter_end_time__gt(self, queryset, fields, start_time): + ids = [obj.id for obj in queryset if obj.start_time > start_time] + return queryset.filter(id__in=ids) + + def filter_end_time__lt(self, queryset, fields, start_time): + ids = [obj.id for obj in queryset if obj.start_time < start_time] + return queryset.filter(id__in=ids) + + class SituationViewSet(viewsets.ReadOnlyModelViewSet): queryset = Situation.objects.all() serializer_class = SituationSerializer + filter_backends = [DjangoFilterBackend] + filterset_class = SituationFilter + + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.queryset) + page = self.paginate_queryset(queryset) + serializer = self.serializer_class(page, many=True) + return self.get_paginated_response(serializer.data) class SituationLocationViewSet(viewsets.ReadOnlyModelViewSet): From 15ff24f14068e7e61acbed6d19cb7b2f2b518996 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:03:22 +0200 Subject: [PATCH 27/64] Remove M2M relation location from Situation --- exceptional_situations/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py index b1891d901..ae12cbd7b 100644 --- a/exceptional_situations/models.py +++ b/exceptional_situations/models.py @@ -43,7 +43,6 @@ class Situation(models.Model): situation_id = models.CharField(max_length=64) situation_type = models.ForeignKey(SituationType, on_delete=models.CASCADE) release_time = models.DateTimeField() - locations = models.ManyToManyField(SituationLocation) announcements = models.ManyToManyField(SituationAnnouncement) class Meta: From 830c0ce8185805714210e23c254c956a9eb19bba Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:03:54 +0200 Subject: [PATCH 28/64] Serialize location in announcement --- exceptional_situations/api/serializers.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/exceptional_situations/api/serializers.py b/exceptional_situations/api/serializers.py index 8c81a6120..5af19f5bc 100644 --- a/exceptional_situations/api/serializers.py +++ b/exceptional_situations/api/serializers.py @@ -25,9 +25,6 @@ class Meta: def to_representation(self, obj): representation = super().to_representation(obj) - representation["locations"] = SituationLocationSerializer( - obj.locations, many=True - ).data representation["announcements"] = SituationAnnouncementSerializer( obj.announcements, many=True ).data @@ -39,6 +36,17 @@ class Meta: model = SituationAnnouncement fields = "__all__" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields.pop("location") + + def to_representation(self, obj): + representation = super().to_representation(obj) + representation["location"] = SituationLocationSerializer( + obj.location, many=False + ).data + return representation + class SituationLocationSerializer(serializers.ModelSerializer): class Meta: From e1759f4f1a4c14240ed5b888c4e7eb9a47eeafe6 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:04:21 +0200 Subject: [PATCH 29/64] Add management command that deletes inactive situations --- .../commands/delete_inactive_situations.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 exceptional_situations/management/commands/delete_inactive_situations.py diff --git a/exceptional_situations/management/commands/delete_inactive_situations.py b/exceptional_situations/management/commands/delete_inactive_situations.py new file mode 100644 index 000000000..629497b1d --- /dev/null +++ b/exceptional_situations/management/commands/delete_inactive_situations.py @@ -0,0 +1,18 @@ +import logging + +from django.core.management import BaseCommand + +from exceptional_situations.models import Situation, SituationAnnouncement + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + def handle(self, *args, **options): + num_deleted = 0 + for situation in Situation.objects.all(): + if situation.is_active is False: + SituationAnnouncement.objects.filter(situation=situation).delete() + situation.delete() + num_deleted += 1 + logger.info(f"Deleted {num_deleted} inactive situations.") From 580e50842024e370c7948991990899dc8bab5f32 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:04:59 +0200 Subject: [PATCH 30/64] Remove adding M2M relation locations --- .../management/commands/import_traffic_situations.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/exceptional_situations/management/commands/import_traffic_situations.py b/exceptional_situations/management/commands/import_traffic_situations.py index 3132434cc..501fa284a 100644 --- a/exceptional_situations/management/commands/import_traffic_situations.py +++ b/exceptional_situations/management/commands/import_traffic_situations.py @@ -129,15 +129,12 @@ def handle(self, *args, **options): } situation, _ = Situation.objects.get_or_create(**filter) - SituationLocation.objects.filter(situation=situation).delete() SituationAnnouncement.objects.filter(situation=situation).delete() - situation.locations.clear() situation.announcements.clear() for announcement_data in properties.get("announcements", []): situation_location = self.create_location( geometry, announcement_data ) - situation.locations.add(situation_location) situation_announcement = self.create_announcement( deepcopy(announcement_data), situation_location ) From 52b2d61ee568e298c2c3abf836b1a546362f029a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:06:50 +0200 Subject: [PATCH 31/64] Fix typos --- exceptional_situations/api/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exceptional_situations/api/urls.py b/exceptional_situations/api/urls.py index eef3621a2..13ae06a4e 100644 --- a/exceptional_situations/api/urls.py +++ b/exceptional_situations/api/urls.py @@ -3,7 +3,7 @@ from exceptional_situations.api import views -app_name = "exceptional_stituations" +app_name = "exceptional_situations" router = routers.DefaultRouter() @@ -20,5 +20,5 @@ ) urlpatterns = [ - path("api/v1/", include(router.urls), name="exceptional_stituations"), + path("api/v1/", include(router.urls), name="exceptional_situations"), ] From 225c84b7fe15e0cc91c0203805ca6eccabb36061 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:56:48 +0200 Subject: [PATCH 32/64] Fix Situation is_active @property --- exceptional_situations/models.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py index ae12cbd7b..373939125 100644 --- a/exceptional_situations/models.py +++ b/exceptional_situations/models.py @@ -26,11 +26,13 @@ class Meta: class SituationAnnouncement(models.Model): title = models.CharField(max_length=128) - description = models.TextField() + description = models.TextField(null=True, blank=True) start_time = models.DateTimeField() end_time = models.DateTimeField(null=True, blank=True) additional_info = models.JSONField(null=True, blank=True) - location = models.OneToOneField(SituationLocation, on_delete=models.CASCADE) + location = models.OneToOneField( + SituationLocation, on_delete=models.CASCADE, null=True, blank=True + ) class Meta: ordering = ["start_time"] @@ -58,14 +60,16 @@ def situation_sub_type_str(self): @property def is_active(self): + if not self.announcements.exists(): + return False # If one or more end_time is null(unknown?) the situation is active if self.announcements.filter(end_time__isnull=True).exists(): return True - # If end_time is past for all announcements, retrun True, else False - return all( + # If end_time is past for all announcements, return True, else False + return any( { - not a.end_time < timezone.now() + a.end_time > timezone.now() for a in self.announcements.filter(end_time__isnull=False) } ) From 6e89e0a8c2c7a292696c0208d947b5e3d8d794ad Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:14:32 +0200 Subject: [PATCH 33/64] Fix bugs in start_time and end_time --- exceptional_situations/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py index 373939125..668d2784a 100644 --- a/exceptional_situations/models.py +++ b/exceptional_situations/models.py @@ -44,7 +44,7 @@ def __str__(self): class Situation(models.Model): situation_id = models.CharField(max_length=64) situation_type = models.ForeignKey(SituationType, on_delete=models.CASCADE) - release_time = models.DateTimeField() + release_time = models.DateTimeField(null=True, blank=True) announcements = models.ManyToManyField(SituationAnnouncement) class Meta: @@ -83,7 +83,7 @@ def start_time(self): for announcement in self.announcements.all(): if not start_time: start_time = announcement.start_time - if start_time < announcement.start_time: + if announcement.start_time < start_time: start_time = announcement.start_time return start_time @@ -97,6 +97,6 @@ def end_time(self): if not end_time: end_time = announcement.end_time - if end_time > announcement.end_time: + if announcement.end_time > end_time: end_time = announcement.end_time return end_time From 43a764e6e77acde934c73ddf8142614d4c38b7d6 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:50:18 +0200 Subject: [PATCH 34/64] Check start times in is_active --- exceptional_situations/models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py index 668d2784a..a6f9b5a1c 100644 --- a/exceptional_situations/models.py +++ b/exceptional_situations/models.py @@ -62,6 +62,13 @@ def situation_sub_type_str(self): def is_active(self): if not self.announcements.exists(): return False + + start_times_in_future = all( + {a.start_time > timezone.now() for a in self.announcements.all()} + ) + # If all start times are in future, return False + if start_times_in_future: + return False # If one or more end_time is null(unknown?) the situation is active if self.announcements.filter(end_time__isnull=True).exists(): return True From 74cf546127c46615d5af916e0e324d247427dd4b Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:51:02 +0200 Subject: [PATCH 35/64] Add Situation model property tests --- exceptional_situations/tests/test_models.py | 83 +++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 exceptional_situations/tests/test_models.py diff --git a/exceptional_situations/tests/test_models.py b/exceptional_situations/tests/test_models.py new file mode 100644 index 000000000..b4028faa5 --- /dev/null +++ b/exceptional_situations/tests/test_models.py @@ -0,0 +1,83 @@ +from datetime import datetime, timedelta + +import pytest +from django.utils import timezone + +from exceptional_situations.models import ( + Situation, + SituationAnnouncement, + SituationLocation, + SituationType, +) + +NOW = timezone.now() + + +@pytest.mark.django_db +def test_situation_is_active(situation_types): + announcement_1 = SituationAnnouncement.objects.create(start_time=NOW, title="test1") + announcement_2 = SituationAnnouncement.objects.create(start_time=NOW, title="test2") + situation = Situation.objects.create( + release_time=NOW, situation_type=situation_types.first(), situation_id="TestID" + ) + assert situation.is_active is False + + situation.announcements.add(announcement_1) + situation.announcements.add(announcement_2) + assert situation.is_active is True + + announcement_1.start_time = NOW - timedelta(days=2) + announcement_1.end_time = NOW - timedelta(days=1) + announcement_1.save() + announcement_2.start_time = NOW - timedelta(hours=2) + announcement_2.end_time = NOW - timedelta(hours=1) + announcement_2.save() + assert situation.is_active is False + + announcement_2.start_time = NOW - timedelta(hours=2) + announcement_2.end_time = NOW + timedelta(hours=1) + announcement_2.save() + assert situation.is_active is True + # Test that returns False if all start times are in future + announcement_1.start_time = NOW + timedelta(days=2) + announcement_1.end_time = NOW + timedelta(days=3) + announcement_1.save() + announcement_2.start_time = NOW + timedelta(hours=2) + announcement_2.end_time = NOW + timedelta(hours=3) + announcement_2.save() + assert situation.is_active is False + + +@pytest.mark.django_db +def test_situation_start_time(situation_types): + announcement_1 = SituationAnnouncement.objects.create( + start_time=NOW, title="starts now" + ) + announcement_2 = SituationAnnouncement.objects.create( + start_time=NOW - timedelta(hours=1), title="started an hour ago" + ) + situation = Situation.objects.create( + release_time=NOW, situation_type=situation_types.first(), situation_id="TestID" + ) + situation.announcements.add(announcement_1) + situation.announcements.add(announcement_2) + assert situation.start_time == announcement_2.start_time + + +@pytest.mark.django_db +def test_situation_end_time(situation_types): + announcement_1 = SituationAnnouncement.objects.create( + start_time=NOW - timedelta(hours=1), + end_time=NOW + timedelta(hours=2), + title="ends after two hours", + ) + announcement_2 = SituationAnnouncement.objects.create( + start_time=NOW, end_time=NOW + timedelta(days=2), title="ends after two days" + ) + situation = Situation.objects.create( + release_time=NOW, situation_type=situation_types.first(), situation_id="TestID" + ) + situation.announcements.add(announcement_1) + situation.announcements.add(announcement_2) + assert situation.end_time == announcement_2.end_time + assert situation.start_time == announcement_1.start_time From 58636988a715a042b879f8da19775164b8699439 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:52:07 +0200 Subject: [PATCH 36/64] Strptime release_time and remove microseconds --- .../commands/import_traffic_situations.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/exceptional_situations/management/commands/import_traffic_situations.py b/exceptional_situations/management/commands/import_traffic_situations.py index 501fa284a..80e23c37b 100644 --- a/exceptional_situations/management/commands/import_traffic_situations.py +++ b/exceptional_situations/management/commands/import_traffic_situations.py @@ -4,11 +4,13 @@ import logging from copy import deepcopy +from datetime import datetime import requests from dateutil import parser from django.contrib.gis.geos import GEOSGeometry, Polygon from django.core.management import BaseCommand +from django.utils import timezone from exceptional_situations.models import ( PROJECTION_SRID, @@ -32,7 +34,7 @@ "?inactiveHours=0&includeAreaGeometry=true&situationType=TRAFFIC_ANNOUNCEMENT" ) URLS = [ROAD_WORK_URL, TRAFFIC_ANNOUNCEMENT_URL] -DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" SOUTHWEST_FINLAND_POLYGON = Polygon( SOUTHWEST_FINLAND_BOUNDARY, srid=SOUTHWEST_FINLAND_BOUNDARY_SRID ) @@ -109,11 +111,10 @@ def handle(self, *args, **options): continue situation_id = properties.get("situationId", None) release_time = properties.get("releaseTime", None) - try: - release_time = parser.parse(release_time) - except parser.ParserError: - logger.error(f"Invalid release time {release_time}") - continue + release_time = datetime.strptime(release_time, DATETIME_FORMAT).replace( + microsecond=0 + ) + release_time = timezone.make_aware(release_time, timezone.utc) type_name = properties.get("situationType", None) sub_type_name = properties.get("trafficAnnouncementType", None) @@ -124,10 +125,11 @@ def handle(self, *args, **options): filter = { "situation_id": situation_id, - "release_time": release_time, "situation_type": situation_type, } situation, _ = Situation.objects.get_or_create(**filter) + situation.release_time = release_time + situation.save() SituationAnnouncement.objects.filter(situation=situation).delete() situation.announcements.clear() From 572e1c36d7fc2e24ae2c41b6236958dba79df702 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:56:32 +0200 Subject: [PATCH 37/64] Test delete inactive situations --- .../tests/test_delete_inactive_situations.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 exceptional_situations/tests/test_delete_inactive_situations.py diff --git a/exceptional_situations/tests/test_delete_inactive_situations.py b/exceptional_situations/tests/test_delete_inactive_situations.py new file mode 100644 index 000000000..f017e1391 --- /dev/null +++ b/exceptional_situations/tests/test_delete_inactive_situations.py @@ -0,0 +1,13 @@ +import pytest +from django.core.management import call_command + +from exceptional_situations.models import Situation, SituationAnnouncement + + +@pytest.mark.django_db +def test_delete_inactive_situations(inactive_situations, inactive_announcements): + assert Situation.objects.count() == 1 + assert SituationAnnouncement.objects.count() == 1 + call_command("delete_inactive_situations") + assert Situation.objects.count() == 0 + assert SituationAnnouncement.objects.count() == 0 From d912e1a1a59adb7659807aedad17c15b0a62e229 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:57:41 +0200 Subject: [PATCH 38/64] Remove unused imports --- exceptional_situations/tests/test_models.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/exceptional_situations/tests/test_models.py b/exceptional_situations/tests/test_models.py index b4028faa5..db1c38fcd 100644 --- a/exceptional_situations/tests/test_models.py +++ b/exceptional_situations/tests/test_models.py @@ -1,14 +1,9 @@ -from datetime import datetime, timedelta +from datetime import timedelta import pytest from django.utils import timezone -from exceptional_situations.models import ( - Situation, - SituationAnnouncement, - SituationLocation, - SituationType, -) +from exceptional_situations.models import Situation, SituationAnnouncement NOW = timezone.now() From 7e560637bc5bf56b5fda0dfb24f1580598b3ebfd Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 09:43:41 +0200 Subject: [PATCH 39/64] Refactor --- exceptional_situations/api/serializers.py | 66 +++++++++++------------ 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/exceptional_situations/api/serializers.py b/exceptional_situations/api/serializers.py index 5af19f5bc..983cde86a 100644 --- a/exceptional_situations/api/serializers.py +++ b/exceptional_situations/api/serializers.py @@ -8,53 +8,51 @@ ) -class SituationSerializer(serializers.ModelSerializer): +class SituationLocationSerializer(serializers.ModelSerializer): class Meta: - model = Situation - fields = [ - "id", - "is_active", - "start_time", - "end_time", - "situation_id", - "release_time", - "situation_type", - "situation_type_str", - "situation_sub_type_str", - ] - - def to_representation(self, obj): - representation = super().to_representation(obj) - representation["announcements"] = SituationAnnouncementSerializer( - obj.announcements, many=True - ).data - return representation + model = SituationLocation + fields = ["id", "location", "geometry", "details"] class SituationAnnouncementSerializer(serializers.ModelSerializer): + location = SituationLocationSerializer() + class Meta: model = SituationAnnouncement - fields = "__all__" + fields = [ + "id", + "title", + "description", + "start_time", + "end_time", + "additional_info", + "location", + ] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields.pop("location") - def to_representation(self, obj): - representation = super().to_representation(obj) - representation["location"] = SituationLocationSerializer( - obj.location, many=False - ).data - return representation - -class SituationLocationSerializer(serializers.ModelSerializer): +class SituationTypeSerializer(serializers.ModelSerializer): class Meta: - model = SituationLocation + model = SituationType fields = "__all__" -class SituationTypeSerializer(serializers.ModelSerializer): +class SituationSerializer(serializers.ModelSerializer): + announcements = SituationAnnouncementSerializer(many=True, read_only=True) + class Meta: - model = SituationType - fields = "__all__" + model = Situation + fields = [ + "id", + "is_active", + "start_time", + "end_time", + "situation_id", + "release_time", + "situation_type", + "situation_type_str", + "situation_sub_type_str", + "announcements", + ] From 3a435c969bd07affb10f060dfb0e89351b80b489 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 09:43:55 +0200 Subject: [PATCH 40/64] Add type hints to properties for drf-spectacular --- exceptional_situations/models.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py index a6f9b5a1c..b1c590a4c 100644 --- a/exceptional_situations/models.py +++ b/exceptional_situations/models.py @@ -1,3 +1,5 @@ +from datetime import datetime + from django.contrib.gis.db import models from django.utils import timezone @@ -51,15 +53,15 @@ class Meta: ordering = ["id"] @property - def situation_type_str(self): + def situation_type_str(self) -> str: return self.situation_type.type_name @property - def situation_sub_type_str(self): + def situation_sub_type_str(self) -> str: return self.situation_type.sub_type_name @property - def is_active(self): + def is_active(self) -> bool: if not self.announcements.exists(): return False @@ -82,7 +84,7 @@ def is_active(self): ) @property - def start_time(self): + def start_time(self) -> datetime: """ Return the start_time that is furthest in history """ @@ -95,7 +97,7 @@ def start_time(self): return start_time @property - def end_time(self): + def end_time(self) -> datetime: """ Return the end_time that is furthest in future """ From 697bc52da6486490968792582f7cc3d77df76a32 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:01:03 +0200 Subject: [PATCH 41/64] Add API tests --- exceptional_situations/tests/test_api.py | 215 +++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 exceptional_situations/tests/test_api.py diff --git a/exceptional_situations/tests/test_api.py b/exceptional_situations/tests/test_api.py new file mode 100644 index 000000000..12c78b227 --- /dev/null +++ b/exceptional_situations/tests/test_api.py @@ -0,0 +1,215 @@ +from datetime import datetime, timedelta + +import pytest +from django.utils import timezone +from rest_framework.reverse import reverse + +SITUATION_LIST_URL = reverse("exceptional_situations:situation-list") +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S" + + +@pytest.mark.django_db +def test_situations_list(api_client, situations, inactive_situations): + response = api_client.get(SITUATION_LIST_URL) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == {"count", "next", "previous", "results"} + assert json_data["count"] == 3 + result_data = json_data["results"][0] + assert result_data.keys() == { + "id", + "is_active", + "start_time", + "end_time", + "situation_id", + "release_time", + "situation_type", + "situation_type_str", + "situation_sub_type_str", + "announcements", + } + assert len(result_data["announcements"]) == 1 + announcement = result_data["announcements"][0] + assert announcement.keys() == { + "id", + "title", + "description", + "start_time", + "end_time", + "additional_info", + "location", + } + location = announcement["location"] + assert location.keys() == {"id", "location", "geometry", "details"} + + +@pytest.mark.django_db +def test_situation_retrieve(api_client, situations): + response = api_client.get( + reverse( + "exceptional_situations:situation-detail", kwargs={"pk": situations[0].pk} + ) + ) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == { + "id", + "is_active", + "start_time", + "end_time", + "situation_id", + "release_time", + "situation_type", + "situation_type_str", + "situation_sub_type_str", + "announcements", + } + assert json_data["id"] == situations[0].pk + assert json_data["is_active"] is True + + +@pytest.mark.django_db +def test_situation_filter_by_start_time(api_client, situations): + start_time = timezone.now() + response = api_client.get( + SITUATION_LIST_URL + + f"?start_time__gt={datetime.strftime(start_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 1 + response = api_client.get( + SITUATION_LIST_URL + + f"?start_time__lt={datetime.strftime(start_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 1 + + start_time = timezone.now() - timedelta(days=2) + response = api_client.get( + SITUATION_LIST_URL + + f"?start_time__gt={datetime.strftime(start_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 2 + response = api_client.get( + SITUATION_LIST_URL + + f"?start_time__lt={datetime.strftime(start_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 0 + + +@pytest.mark.django_db +def test_situation_filter_by_end_time(api_client, situations): + end_time = timezone.now() + response = api_client.get( + SITUATION_LIST_URL + + f"?end_time__gt={datetime.strftime(end_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 1 + response = api_client.get( + SITUATION_LIST_URL + + f"?end_time__lt={datetime.strftime(end_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 1 + + end_time = timezone.now() - timedelta(days=2) + response = api_client.get( + SITUATION_LIST_URL + + f"?end_time__gt={datetime.strftime(end_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 2 + response = api_client.get( + SITUATION_LIST_URL + + f"?end_time__lt={datetime.strftime(end_time, DATETIME_FORMAT)}" + ) + assert response.json()["count"] == 0 + + +@pytest.mark.django_db +def test_situation_types_list(api_client, situation_types): + response = api_client.get(reverse("exceptional_situations:situation_type-list")) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == {"count", "next", "previous", "results"} + assert json_data["count"] == situation_types.count() + + +@pytest.mark.django_db +def test_situation_types_retrieve(api_client, situation_types): + response = api_client.get( + reverse( + "exceptional_situations:situation_type-detail", + kwargs={"pk": situation_types[0].pk}, + ) + ) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == {"id", "type_name", "sub_type_name"} + assert json_data["id"] == situation_types[0].pk + + +@pytest.mark.django_db +def test_announcement_list(api_client, announcements): + response = api_client.get( + reverse("exceptional_situations:situation_announcement-list") + ) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == {"count", "next", "previous", "results"} + assert json_data["count"] == announcements.count() + result_data = json_data["results"][0] + assert result_data.keys() == { + "id", + "title", + "description", + "start_time", + "end_time", + "additional_info", + "location", + } + location = result_data["location"] + assert location.keys() == {"id", "location", "geometry", "details"} + + +@pytest.mark.django_db +def test_announcement_retrieve(api_client, announcements): + response = api_client.get( + reverse( + "exceptional_situations:situation_announcement-detail", + kwargs={"pk": announcements[0].pk}, + ) + ) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == { + "id", + "title", + "description", + "start_time", + "end_time", + "additional_info", + "location", + } + assert json_data["id"] == announcements[0].pk + + +@pytest.mark.django_db +def test_location_list(api_client, locations): + response = api_client.get(reverse("exceptional_situations:situation_location-list")) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == {"count", "next", "previous", "results"} + assert json_data["count"] == locations.count() + result_data = json_data["results"][0] + assert result_data.keys() == {"id", "location", "geometry", "details"} + + +@pytest.mark.django_db +def test_location_retrieve(api_client, locations): + response = api_client.get( + reverse( + "exceptional_situations:situation_location-detail", + kwargs={"pk": locations[0].pk}, + ) + ) + assert response.status_code == 200 + json_data = response.json() + assert json_data.keys() == {"id", "location", "geometry", "details"} + assert json_data["id"] == locations[0].pk From ca79b5db1549123142a3bf73e50640abfc57007c Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:02:01 +0200 Subject: [PATCH 42/64] Add test fixtures --- exceptional_situations/tests/conftest.py | 114 +++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 exceptional_situations/tests/conftest.py diff --git a/exceptional_situations/tests/conftest.py b/exceptional_situations/tests/conftest.py new file mode 100644 index 000000000..10e5f2a6a --- /dev/null +++ b/exceptional_situations/tests/conftest.py @@ -0,0 +1,114 @@ +from datetime import datetime, timedelta + +import pytest +from django.contrib.gis.geos import GEOSGeometry +from rest_framework.test import APIClient + +from exceptional_situations.models import ( + Situation, + SituationAnnouncement, + SituationLocation, + SituationType, +) + +NOW = datetime.now() + + +@pytest.fixture +def api_client(): + return APIClient() + + +@pytest.mark.django_db +@pytest.fixture +def situation_types(): + SituationType.objects.create( + type_name="test type name", sub_type_name="test sub type name" + ) + return SituationType.objects.all() + + +@pytest.mark.django_db +@pytest.fixture +def locations(): + json_data = {"test_key": "test_value"} + SituationLocation.objects.create( + details=json_data, geometry=GEOSGeometry("POINT(0 0)") + ) + SituationLocation.objects.create( + details=json_data, geometry=GEOSGeometry("POINT(1 0)") + ) + SituationLocation.objects.create( + details=json_data, geometry=GEOSGeometry("POINT(0 1)") + ) + + return SituationLocation.objects.all() + + +@pytest.mark.django_db +@pytest.fixture +def announcements(locations): + json_data = {"test_key": "test_value"} + SituationAnnouncement.objects.create( + title="two hours", + description="two hours long situation", + additional_info=json_data, + location=locations[0], + start_time=NOW - timedelta(hours=1), + end_time=NOW + timedelta(hours=1), + ) + SituationAnnouncement.objects.create( + title="two days", + description="two days long situation", + additional_info=json_data, + location=locations[1], + start_time=NOW - timedelta(days=1), + end_time=NOW + timedelta(days=1), + ) + + return SituationAnnouncement.objects.all() + + +@pytest.mark.django_db +@pytest.fixture +def inactive_announcements(locations): + json_data = {"test_key": "test_value"} + SituationAnnouncement.objects.create( + title="in past", + description="inactive announcement", + additional_info=json_data, + location=locations[2], + start_time=NOW - timedelta(days=2), + end_time=NOW - timedelta(days=1), + ) + return SituationAnnouncement.objects.all() + + +@pytest.mark.django_db +@pytest.fixture +def inactive_situations(situation_types, inactive_announcements): + situation = Situation.objects.create( + release_time=NOW, + situation_id="inactive", + situation_type=situation_types.first(), + ) + situation.announcements.add(inactive_announcements.first()) + return Situation.objects.all() + + +@pytest.mark.django_db +@pytest.fixture +def situations(situation_types, announcements): + situation = Situation.objects.create( + release_time=NOW, + situation_id="TwoHoursLong", + situation_type=situation_types.first(), + ) + situation.announcements.add(announcements[0]) + situation = Situation.objects.create( + release_time=NOW - timedelta(days=1), + situation_id="TwoDaysLong", + situation_type=situation_types.first(), + ) + situation.announcements.add(announcements[1]) + return Situation.objects.all() From 4142e06b14a46b360074de069efffd43a4ad5019 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:37:54 +0200 Subject: [PATCH 43/64] Add Celery tasks --- exceptional_situations/tasks.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 exceptional_situations/tasks.py diff --git a/exceptional_situations/tasks.py b/exceptional_situations/tasks.py new file mode 100644 index 000000000..3dcc90208 --- /dev/null +++ b/exceptional_situations/tasks.py @@ -0,0 +1,13 @@ +from django.core import management + +from smbackend.utils import shared_task_email + + +@shared_task_email +def import_traffic_situations(name="import_traffic_situations"): + management.call_command("import_traffic_situations") + + +@shared_task_email +def delete_inactive_situations(name="delete_inactive_situations"): + management.call_command("delete_inactive_situations") From c81d4c630629201d1d2563d7d813cf37c393c885 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:39:39 +0200 Subject: [PATCH 44/64] Add basic readme --- exceptional_situations/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 exceptional_situations/README.md diff --git a/exceptional_situations/README.md b/exceptional_situations/README.md new file mode 100644 index 000000000..59d54b69f --- /dev/null +++ b/exceptional_situations/README.md @@ -0,0 +1,15 @@ +# Exceptional Situations APP +APP for importing, storing and serving exceptional situations + +## Importing data +### Traffic Announcements +Imports road works and traffic announcements in Southwest Finland from digitraffic.fi. +To import type: +`./manage.py import_traffic_situations` + +### Delete inactive situations +`./manage.py delete_inactive_situations` +Deletes also the related announcements and locations. + +## API Documentation +See online swagger documentation. From 5a08d1fe716b47bd86273237781d0b7c6a18ed19 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:20:24 +0200 Subject: [PATCH 45/64] Add admin --- exceptional_situations/admin.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 exceptional_situations/admin.py diff --git a/exceptional_situations/admin.py b/exceptional_situations/admin.py new file mode 100644 index 000000000..00d6027e0 --- /dev/null +++ b/exceptional_situations/admin.py @@ -0,0 +1,33 @@ +from django.contrib import admin + +from exceptional_situations.models import ( + Situation, + SituationAnnouncement, + SituationLocation, + SituationType, +) + + +class SituationAdmin(admin.ModelAdmin): + list_display = ("is_active", "start_time", "end_time") + + +class SituationTypeAdmin(admin.ModelAdmin): + list_display = ("type_name", "sub_type_name") + + +class SituationAnnouncementAdmin(admin.ModelAdmin): + list_display = ("title", "start_time", "end_time") + + +class SituationLocationAdmin(admin.ModelAdmin): + list_display = ("id", "title", "geometry") + + def title(self, obj): + return obj.announcement.title + + +admin.site.register(Situation, SituationAdmin) +admin.site.register(SituationType, SituationTypeAdmin) +admin.site.register(SituationAnnouncement, SituationAnnouncementAdmin) +admin.site.register(SituationLocation, SituationLocationAdmin) From b4f1b29a2f2287c1c20bd87d9826f38b1f556e24 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:20:51 +0200 Subject: [PATCH 46/64] Add related_name to location relation --- exceptional_situations/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/exceptional_situations/models.py b/exceptional_situations/models.py index b1c590a4c..7b138ec46 100644 --- a/exceptional_situations/models.py +++ b/exceptional_situations/models.py @@ -33,7 +33,11 @@ class SituationAnnouncement(models.Model): end_time = models.DateTimeField(null=True, blank=True) additional_info = models.JSONField(null=True, blank=True) location = models.OneToOneField( - SituationLocation, on_delete=models.CASCADE, null=True, blank=True + SituationLocation, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="announcement", ) class Meta: From bef73bdfbd17446928e85526af8a8271cdd48ff4 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:21:51 +0200 Subject: [PATCH 47/64] Add exceptional_situations to DOC_ENDPOINTS --- smbackend/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index 6a9915f2c..59bfb9b55 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -355,6 +355,8 @@ def gettext(s): "/environment_data/api/v1/stations/", "/environment_data/api/v1/parameters/", "/environment_data/api/v1/data/", + "/exceptional_situations/api/v1/situation/", + "/exceptional_situations/api/v1/situation_type/", ] From 31c4e8f5ab87205fa6256deee3ac83537953c3f7 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:22:22 +0200 Subject: [PATCH 48/64] Add initial migrations --- .../migrations/0001_initial.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 exceptional_situations/migrations/0001_initial.py diff --git a/exceptional_situations/migrations/0001_initial.py b/exceptional_situations/migrations/0001_initial.py new file mode 100644 index 000000000..7ae959154 --- /dev/null +++ b/exceptional_situations/migrations/0001_initial.py @@ -0,0 +1,137 @@ +# Generated by Django 4.1.13 on 2024-03-27 11:12 + +import django.contrib.gis.db.models.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="SituationLocation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "location", + django.contrib.gis.db.models.fields.PointField( + blank=True, null=True, srid=4326 + ), + ), + ( + "geometry", + django.contrib.gis.db.models.fields.GeometryField( + blank=True, null=True, srid=4326 + ), + ), + ("details", models.JSONField(blank=True, null=True)), + ], + options={ + "ordering": ["id"], + }, + ), + migrations.CreateModel( + name="SituationType", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("type_name", models.CharField(max_length=64)), + ( + "sub_type_name", + models.CharField(blank=True, max_length=64, null=True), + ), + ], + options={ + "ordering": ["id"], + }, + ), + migrations.CreateModel( + name="SituationAnnouncement", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("title", models.CharField(max_length=128)), + ("title_fi", models.CharField(max_length=128, null=True)), + ("title_sv", models.CharField(max_length=128, null=True)), + ("title_en", models.CharField(max_length=128, null=True)), + ("description", models.TextField(blank=True, null=True)), + ("description_fi", models.TextField(blank=True, null=True)), + ("description_sv", models.TextField(blank=True, null=True)), + ("description_en", models.TextField(blank=True, null=True)), + ("start_time", models.DateTimeField()), + ("end_time", models.DateTimeField(blank=True, null=True)), + ("additional_info", models.JSONField(blank=True, null=True)), + ( + "location", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="announcement", + to="exceptional_situations.situationlocation", + ), + ), + ], + options={ + "ordering": ["start_time"], + }, + ), + migrations.CreateModel( + name="Situation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("situation_id", models.CharField(max_length=64)), + ("release_time", models.DateTimeField(blank=True, null=True)), + ( + "announcements", + models.ManyToManyField( + to="exceptional_situations.situationannouncement" + ), + ), + ( + "situation_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="exceptional_situations.situationtype", + ), + ), + ], + options={ + "ordering": ["id"], + }, + ), + ] From 4dd9fb8745b921993711cf33cd7efda79cf44a3b Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:24:16 +0200 Subject: [PATCH 49/64] Replace datetime.now() with timezone aware timezone.now() --- exceptional_situations/tests/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/exceptional_situations/tests/conftest.py b/exceptional_situations/tests/conftest.py index 10e5f2a6a..a91ed5434 100644 --- a/exceptional_situations/tests/conftest.py +++ b/exceptional_situations/tests/conftest.py @@ -1,7 +1,8 @@ -from datetime import datetime, timedelta +from datetime import timedelta import pytest from django.contrib.gis.geos import GEOSGeometry +from django.utils import timezone from rest_framework.test import APIClient from exceptional_situations.models import ( @@ -11,7 +12,7 @@ SituationType, ) -NOW = datetime.now() +NOW = timezone.now() @pytest.fixture From 9fb8d61d8359fd273b2313579e541431ad548822 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:24:59 +0200 Subject: [PATCH 50/64] Add __init__.py --- exceptional_situations/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 exceptional_situations/tests/__init__.py diff --git a/exceptional_situations/tests/__init__.py b/exceptional_situations/tests/__init__.py new file mode 100644 index 000000000..e69de29bb From 8f36228b5e66826f474382bcb60165b44742ec4e Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 28 Mar 2024 11:09:20 +0200 Subject: [PATCH 51/64] Change base class of SitutaionLocationAdmin to OSMGeoAdmin --- exceptional_situations/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exceptional_situations/admin.py b/exceptional_situations/admin.py index 00d6027e0..46cb8a8bb 100644 --- a/exceptional_situations/admin.py +++ b/exceptional_situations/admin.py @@ -1,4 +1,4 @@ -from django.contrib import admin +from django.contrib.gis import admin from exceptional_situations.models import ( Situation, @@ -20,7 +20,7 @@ class SituationAnnouncementAdmin(admin.ModelAdmin): list_display = ("title", "start_time", "end_time") -class SituationLocationAdmin(admin.ModelAdmin): +class SituationLocationAdmin(admin.OSMGeoAdmin): list_display = ("id", "title", "geometry") def title(self, obj): From 1b35b293958063189686364fc7122df4622b1dc0 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 2 Apr 2024 13:26:16 +0300 Subject: [PATCH 52/64] Add content type StreetAreaInformation --- mobility_data/importers/data/content_types.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index 9584a2aed..e8fb85841 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -413,6 +413,11 @@ content_types: sv: Tillgänglighetsområde för skola och daghem en: School and kindergarten accessibility area + - content_type_name: StreetAreaInformation + name: + fi: KatuAlueTieto + sv: GatuOmrådeInformation + en: StreetAreaInformation # End of content types importer from opaskarta.turku.fi - content_type_name: Underpass From d676d67a198aef066d045d6b9610529ea3ebcedd Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 2 Apr 2024 13:27:22 +0300 Subject: [PATCH 53/64] Add configuration for StreetAreaInformation --- .../importers/data/wfs_importer_config.yml | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index a5a71403f..3cb19fcd3 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -1,4 +1,32 @@ features: + - content_type_name: StreetAreaInformation + wfs_layer: GIS:Katualueet + max_features: 100000 + fields: + name: + fi: Kadunnimi + extra_fields: + omistaja: + wfs_field: Omistaja + omistaja_koodi: + wfs_field: Omistaja_koodi + kunnossapitaja: + wfs_field: Kunnossapitaja + kunnossapitoluokka: + wfs_field: Kunnossapitoluokka + kunnossapitoluokka_koodi: + wfs_field: Kunnossapitoluokka_koodi + talvikunnossapito: + wfs_field: Talvikunnossapito + talvikunnossapito_koodi: + wfs_field: Talvikunnossapito_koodi + pintamateriaaliryhma: + wfs_field: Pintamateriaaliryhma + pintamateriaali: + wfs_field: Pintamateriaali + pintamateriaali_koodi: + wfs_field: Pintamateriaali_koodi + - content_type_name: PlayGround wfs_layer: GIS:Viheralueet max_features: 50000 @@ -335,18 +363,21 @@ features: - content_type_name: ScooterParkingArea wfs_layer: GIS:Sahkopotkulautaparkki + locates_in_turku: False - content_type_name: ScooterSpeedLimitArea wfs_layer: GIS:Sahkopotkulauta_nopeusrajoitus + locates_in_turku: False - content_type_name: ScooterNoParkingArea wfs_layer: GIS:Sahkopotkulauta_pysakointikielto + locates_in_turku: False + - content_type_name: PublicToilet wfs_layer: GIS:Varusteet max_features: 10000 # Default is False, if True include only if geometry locates in Turku. - locates_in_turku: True # Include feature if field 'Tyyppi' has value 'WC' include: Tyyppi: WC @@ -425,7 +456,6 @@ features: - content_type_name: PublicBench wfs_layer: GIS:Varusteet max_features: 10000 - locates_in_turku: True include: Tyyppi: Penkki extra_fields: From ec2d4eaf172b10411a5512f65411a5571532cde2 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 2 Apr 2024 13:32:02 +0300 Subject: [PATCH 54/64] Add task that imports street area information --- mobility_data/tasks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mobility_data/tasks.py b/mobility_data/tasks.py index 6539ba71c..e7f7b2678 100644 --- a/mobility_data/tasks.py +++ b/mobility_data/tasks.py @@ -163,6 +163,11 @@ def import_under_and_overpasses(name="import_under_and_overpasses"): management.call_command("import_under_and_overpasses") +@shared_task_email +def import_street_area_information(name="import_street_area_information"): + management.call_command("import_wfs", "StreetAreaInformation") + + @shared_task_email def delete_obsolete_data(name="delete_obsolete_data"): MobileUnit.objects.filter(content_types__isnull=True).delete() From b8921f0773e6f9dd4d7d6b028b33b6ef16748c19 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:46:02 +0300 Subject: [PATCH 55/64] Filter PublicBench and PublicToilet by location --- mobility_data/importers/data/wfs_importer_config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index 3cb19fcd3..e4fbcb1ab 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -377,6 +377,7 @@ features: - content_type_name: PublicToilet wfs_layer: GIS:Varusteet max_features: 10000 + locates_in_turku: True # Default is False, if True include only if geometry locates in Turku. # Include feature if field 'Tyyppi' has value 'WC' include: @@ -456,6 +457,7 @@ features: - content_type_name: PublicBench wfs_layer: GIS:Varusteet max_features: 10000 + locates_in_turku: True include: Tyyppi: Penkki extra_fields: From 00878771b6ee53ddc55171adf6665d3356c5e686 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:57:51 +0300 Subject: [PATCH 56/64] Change include to 'Varustelaji: Grillauspaikka' for BarbecuePlace --- mobility_data/importers/data/wfs_importer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index a5a71403f..f6133e9b2 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -38,7 +38,7 @@ features: wfs_layer: GIS:Varusteet max_features: 100000 include: - Tyyppi: Grillipaikka + Varustelaji: Grillauspaikka extra_fields: valmistaja: wfs_field: Valmistaja From a447fe5d71879652cc9687a6f3d7dcac5bb28c8d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 5 Apr 2024 08:09:04 +0300 Subject: [PATCH 57/64] Fix logger.warning output --- mobility_data/importers/wfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobility_data/importers/wfs.py b/mobility_data/importers/wfs.py index 137f36116..666a46f4a 100644 --- a/mobility_data/importers/wfs.py +++ b/mobility_data/importers/wfs.py @@ -151,10 +151,10 @@ def get_data_source(config, max_features): def import_wfs_feature(config, data_file=None): if "content_type_name" not in config: - logger.warning(f"Skipping feature {config}, 'content_type_name' is required.") + logger.warning(f"Discarding feature {config}, 'content_type_name' is required.") return False if "wfs_layer" not in config: - logger.warning(f"Skipping feature {config}, no wfs_layer defined.") + logger.warning(f"Dicarding feature {config}, no wfs_layer defined.") return False if "max_features" in config: max_features = config["max_features"] From 9452e1e572bd1937532a24178f98c442054c0e01 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 5 Apr 2024 08:09:37 +0300 Subject: [PATCH 58/64] Add import_outdoor_places task Import barbecue places and lean-tos --- mobility_data/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobility_data/tasks.py b/mobility_data/tasks.py index 6539ba71c..486707384 100644 --- a/mobility_data/tasks.py +++ b/mobility_data/tasks.py @@ -44,8 +44,8 @@ def import_accessories(name="import_accessories"): @shared_task_email -def import_barbecue_places(name="import_barbecue_places"): - management.call_command("import_wfs", ["BarbecuePlace"]) +def import_outdoor_places(name="import_outdoor_places"): + management.call_command("import_wfs", ["BarbecuePlace", "LeanTo"]) @shared_task_email From 0e0d205564bd1145f9bf0a0f24e7952ac64a7153 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 5 Apr 2024 08:10:55 +0300 Subject: [PATCH 59/64] Add LeanTo content type name --- mobility_data/importers/data/content_types.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mobility_data/importers/data/content_types.yml b/mobility_data/importers/data/content_types.yml index 9584a2aed..fa840547f 100644 --- a/mobility_data/importers/data/content_types.yml +++ b/mobility_data/importers/data/content_types.yml @@ -195,6 +195,12 @@ content_types: sv: Grillplats en: Barbecue place + - content_type_name: LeanTo + name: + fi: Laavu + sv: Vindskydd + en: LeanTo + - content_type_name: TicketMachineSign # Liikennemerkki: 990 Lippuautomaatti name: From 2dd2875751e4b22682caeb5a5fbaa6ffb6467871 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 5 Apr 2024 08:11:46 +0300 Subject: [PATCH 60/64] Add configuration for lean-tos --- mobility_data/importers/data/wfs_importer_config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index f6133e9b2..8fa6e7540 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -70,6 +70,12 @@ features: asennus: wfs_field: Asennus + - content_type_name: LeanTo + wfs_layer: GIS:Varusteet + max_features: 100000 + include: + Tyyppi: Laavu + - content_type_name: TicketMachineSign wfs_layer: GIS:Liikennemerkit include: From b1e5920fd43e9cd86a9acca536adbf6babb2caf8 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 5 Apr 2024 08:29:47 +0300 Subject: [PATCH 61/64] Fix logger output --- mobility_data/management/commands/import_wfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobility_data/management/commands/import_wfs.py b/mobility_data/management/commands/import_wfs.py index 68d033fdd..1d750da45 100644 --- a/mobility_data/management/commands/import_wfs.py +++ b/mobility_data/management/commands/import_wfs.py @@ -65,4 +65,4 @@ def handle(self, *args, **options): try: import_wfs_feature(feature, data_file) except Exception as e: - logger.warning(f"Skipping content_type {feature} : {e}") + logger.warning(f"Discarding content_type {feature} : {e}") From 5ad54a079b186a50acac5b86868e1fab6af38abe Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 5 Apr 2024 10:04:23 +0300 Subject: [PATCH 62/64] Add swedish addresses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes bug that front end shows street name number e.g., Rööläntie number --- mobility_data/data/Pyorienkorjauspisteet_2022.geojson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobility_data/data/Pyorienkorjauspisteet_2022.geojson b/mobility_data/data/Pyorienkorjauspisteet_2022.geojson index 8b802faec..bf166f41a 100755 --- a/mobility_data/data/Pyorienkorjauspisteet_2022.geojson +++ b/mobility_data/data/Pyorienkorjauspisteet_2022.geojson @@ -12,8 +12,8 @@ { "type": "Feature", "properties": { "id": null, "Kohde": "Pääkirjaston sisäpiha / Huvudbiblioteks gård / Main librarys inner court", "Osoite": "Läntinen rantakatu 1, 20100 Turku / Västra Strandgatan 1, 20100 Turku", "Varustelaji": "Polkupyörän pumppu / cykelpump / Bicycle pump", "Pvm": "6.5.2022", "Kuvaus": "Kasikayttoinen pyoranpumppu kahdella eri suukappaleella.\nCykelpump med två olika munstycken.\nBicycle pump with two different mouthpiece options.", "Maastossa": "Kyllä", "Lisätieto": "Epakunnossa", "x": 23459938.338352039, "y": 6704503.6369999349 }, "geometry": { "type": "Point", "coordinates": [ 23459938.338352039456367, 6704503.636999934911728 ] } }, { "type": "Feature", "properties": { "id": null, "Kohde": "Brankkiksenaukio / Brankisplan / Brankis plaza", "Osoite": "Kauppiaskatu 4, 21600 Parainen / Köpmansgatan 4, 21600 Pargas", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Katoksellinen pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation med gapskjul innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station with roof includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23461398.542189036, "y": 6687555.9403193807 }, "geometry": { "type": "Point", "coordinates": [ 23461398.542189035564661, 6687555.94031938072294 ] } }, { "type": "Feature", "properties": { "id": null, "Kohde": "Lillmälö", "Osoite": "Saaristotie 3107, 21600 Parainen / Skärgårdsvägen 3107, 21600 Pargas", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Katoksellinen pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation med gapskjul innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station with roof includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23450878.808870975, "y": 6680753.3015155382 }, "geometry": { "type": "Point", "coordinates": [ 23450878.808870974928141, 6680753.30151553824544 ] } }, -{ "type": "Feature", "properties": { "id": null, "Kohde": "Hanka", "Osoite": "Luotojentie 1092, 21150 Naantali", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23442968.857040703, "y": 6686362.781391114 }, "geometry": { "type": "Point", "coordinates": [ 23442968.857040703296661, 6686362.781391113996506 ] } }, -{ "type": "Feature", "properties": { "id": null, "Kohde": "Röölä", "Osoite": "Rööläntie 402, 21150 Naantali", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23442390.872016005, "y": 6693030.3863375364 }, "geometry": { "type": "Point", "coordinates": [ 23442390.872016005218029, 6693030.386337536387146 ] } }, +{ "type": "Feature", "properties": { "id": null, "Kohde": "Hanka", "Osoite": "Luotojentie 1092, 21150 Naantali / Luotojentie 1092, 21150 Naantali", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23442968.857040703, "y": 6686362.781391114 }, "geometry": { "type": "Point", "coordinates": [ 23442968.857040703296661, 6686362.781391113996506 ] } }, +{ "type": "Feature", "properties": { "id": null, "Kohde": "Röölä", "Osoite": "Rööläntie 402, 21150 Naantali / Rööläntie 402, 21150 Naantali", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23442390.872016005, "y": 6693030.3863375364 }, "geometry": { "type": "Point", "coordinates": [ 23442390.872016005218029, 6693030.386337536387146 ] } }, { "type": "Feature", "properties": { "id": null, "Kohde": "Nauvo / Nagu", "Osoite": "Nauvon ranta 6, 21660 Parainen / Nagu Strand 6, 21660 Pargas", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "16.05.2022", "Kuvaus": "Pyöränkorjauspiste sisältää ilmapumpun ja työkaluja. Ilmapumpussa on moniventtiilisuulake ja painemittari. Korjauspiste sisältää kolme ruuvimeisseliä, (ristipää- ura- ja TORX T25 -ruuvimeisselin), jakoavaimen, Gedore kärkipihdit, kaksi kiintoavainta (8x10mm ja 13x15mm), kuusiokoloavainsarjan (2-8mm) ja rengasraudat. Pyöränkorjauspisteessä on kannatinkoukut, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller en pump och verktygs. I pumpen finns multi-munstycke och en manometer. Servicestation innehåller tre skruvmejsel (krysspårmejsel, spårskruvmejsel och TORX T25 skruvmejsel), skiftnyckeln, Gedora spetstång, två blocknycklar (8x10mm och 13x15mm), sexkantnyckelsats (2-8mm) och däckjärn. Cykelservicestation har också två pelaren som cykeln kan hänga upp.\nBike service station includes air pump and tools. There are different mouthpiece options and a manometer. Service station tools include three different screwdrivers (a Philips screwdriver, a slotted screwdriver and a TORX T25 screwdriver), adjustable wrench, Gedora nippers, two wrench (8x10mm and 13x15mm), a series of hex head wrench (2-8mm) and tyre irons. Bike service station also include hooks where the bicycle can be lifted during the service. \n", "Maastossa": "Kyllä", "Lisätieto": "Merkki iBOMBO PRS SCANDIC", "x": 23439620.678092718, "y": 6676188.3548369883 }, "geometry": { "type": "Point", "coordinates": [ 23439620.67809271812439, 6676188.354836988262832 ] } }, { "type": "Feature", "properties": { "id": null, "Kohde": "Kupittaa / Kuppis / Kupittaa", "Osoite": "Joukahaisenkatu 6, 20520 Turku / Joukahainengatan 6, 20520 Turku", "Varustelaji": "Pyöränkorjauspiste / Cykelservicestation / Bike service station", "Pvm": "1.3.2024", "Kuvaus": "Pyöränkorjauspiste sisältää pyöränpumpun ja kaksi monitoimityökalua. Pyöränpumppuun on valittavissa erilaisia suukappaleita, jotka sopivat tavallisiin venttiilityyppeihin. Monitoimityökalut sisältävät kiintoavaimen (8,9,10 ja 15mm), kuusiokoloavaimen (3,4,5,6 ja 8mm) ja ruuvimeisselin (0,8x4mm). Pyöränkorjauspisteessä on myös kaksi kannatinkoukkua eri korkeudella, joihin pyörän voi nostaa huollon ajaksi.\nCykelservicestation innehåller cykelpump och två multiverktyg. I cykelpump det finns multi-munstycke som passar alla gängse ventiltyper. Multiverktyg innehåller blocknyckel (8,9,10 och 15mm), sexkantnyckel (3,4,5,6 och 8mm) och skruvmejsel (0,8x4mm). Cykelservicestation har också två pelaren i olika nivåer som cykeln kan hänga upp. \nBike service station includes bicycle pump and two multifunctional tools. There are different mouthpiece options which are compatible to usual valve types. Multifunctional tools include a wrench (8,9,10 and 15mm), a hex head wrench (3,4,5,6 and 8mm) and a screwdriver (0,8x4mm). Bike service station also include two hooks in different heights where the bicycles can be lifted during the service.", "Maastossa": "Ei", "Lisätieto": "Merkki Care4bikes", "x": 23461183.976663336, "y": 6704468.582275727 }, "geometry": { "type": "Point", "coordinates": [ 23461183.976663336, 6704468.582275727 ] } } From 013cf89bce16f2c0df4da12c4742e1a82d6236ae Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:42:34 +0300 Subject: [PATCH 63/64] Add kohde_ID to SchoolAndKindergartenAccessibilityArea --- mobility_data/importers/data/wfs_importer_config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mobility_data/importers/data/wfs_importer_config.yml b/mobility_data/importers/data/wfs_importer_config.yml index a5a71403f..bf3ae40a7 100644 --- a/mobility_data/importers/data/wfs_importer_config.yml +++ b/mobility_data/importers/data/wfs_importer_config.yml @@ -522,9 +522,12 @@ features: name: fi: Kohde extra_fields: - Minuutit: + kohde_ID: + wfs_field: Kohde_ID + wfs_type: int + minuutit: wfs_field: Minuutit wfs_type: int - Kulkumuoto: + kulkumuoto: wfs_field: Kulkumuoto From fbfc97c926fbf658e740ebc230a04a9a97a0ef5a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:00:00 +0300 Subject: [PATCH 64/64] csv_data_source only readonly field. add list_display fields --- eco_counter/admin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/eco_counter/admin.py b/eco_counter/admin.py index 4a8bd6c82..452689ba5 100644 --- a/eco_counter/admin.py +++ b/eco_counter/admin.py @@ -55,8 +55,16 @@ def get_date(self, obj): class ImportStateAdmin(admin.ModelAdmin): + list_display = ( + "id", + "csv_data_source", + "current_year_number", + "current_month_number", + "current_day_number", + ) + def get_readonly_fields(self, request, obj=None): - return [f.name for f in self.model._meta.fields] + return ["csv_data_source"] class StationAdmin(admin.ModelAdmin):