diff --git a/iot/README.md b/iot/README.md index e7935c21d..d40de5a32 100644 --- a/iot/README.md +++ b/iot/README.md @@ -1,22 +1,25 @@ ## About The purpose of the IoT app is to store temporarly data from various IoT-data sources, that do not allow frequent fetching of their data. -The data is stored as it is in JSON to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data source or when a data source is added. The cache is populated if empty when serving data. +The data is stored as it in JSON format to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data or when a data source is added. The cache is populated if empty when serving data. ## Adding IoT-data source from the Admin -* Give a tree letter long source name, this name will be the name for the source. Used for example when requesting the data. +* Give a tree letter long identifier, this will be used to identify the data +to be imported in the Celery task and when requesting data in the API. * Add the full name of the source * Add the Url to the JSON data. +* Set is_xml to True if the data is in XML format, the data will be converted to JSON. +* Add the optional headers for the request. ## Setting periodic importing using Celery from the Admin * Create a periodic task, give a descrpitive name. * Select *iot.tasks.import_iot_data* as the Task (registered) * Choose the *Interval Schedule* * Set the Start DateTime -* Add the source name as *Positional Arguments*, e.g. ["R24"] would import the source_name R24. +* Add the identifier as *Positional Arguments*, e.g. ["R24"] would import the identifier R24. ## Manual import To manually import source: -`./manage.py import_iot_data source_name` +`./manage.py import_iot_data identifier` Or by running the perioc task from the admin. ## Retriving data diff --git a/iot/management/commands/import_iot_data.py b/iot/management/commands/import_iot_data.py index c1d90866e..eae05d817 100644 --- a/iot/management/commands/import_iot_data.py +++ b/iot/management/commands/import_iot_data.py @@ -1,7 +1,9 @@ import json import logging +from xml.parsers.expat import ExpatError import requests +import xmltodict from django.core.cache import cache from django.core.management.base import BaseCommand @@ -13,17 +15,25 @@ def save_data_to_db(source): IoTData.objects.filter(data_source=source).delete() - try: - response = requests.get(source.url) + response = requests.get(source.url, headers=source.headers) except requests.exceptions.ConnectionError: logger.error(f"Could not fetch data from: {source.url}") return - try: - json_data = response.json() - except json.decoder.JSONDecodeError: - logger.error(f"Could not decode data to json from: {source.url}") - return + if source.is_xml: + try: + json_data = xmltodict.parse(response.text) + except ExpatError as err: + logger.error( + f"Could not parse XML data from the give url {source.url}. {err}" + ) + return + else: + try: + json_data = response.json() + except json.decoder.JSONDecodeError as err: + logger.error(f"Could not decode data to JSON from: {source.url}. {err}") + return IoTData.objects.create(data_source=source, data=json_data) diff --git a/iot/migrations/0003_iotdatasource_headers.py b/iot/migrations/0003_iotdatasource_headers.py new file mode 100644 index 000000000..d70dd8d84 --- /dev/null +++ b/iot/migrations/0003_iotdatasource_headers.py @@ -0,0 +1,20 @@ +# Generated by Django 4.1.13 on 2024-03-13 06:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iot", "0002_add_ordering"), + ] + + operations = [ + migrations.AddField( + model_name="iotdatasource", + name="headers", + field=models.JSONField( + blank=True, null=True, verbose_name="request headers" + ), + ), + ] diff --git a/iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py b/iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py new file mode 100644 index 000000000..d8bdb9ddb --- /dev/null +++ b/iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 4.1.13 on 2024-03-13 08:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iot", "0003_iotdatasource_headers"), + ] + + operations = [ + migrations.AddField( + model_name="iotdatasource", + name="is_xml", + field=models.BooleanField( + default=False, + verbose_name="If True, XML data will be converted to JSON.", + ), + ), + migrations.AlterField( + model_name="iotdatasource", + name="headers", + field=models.JSONField( + blank=True, + null=True, + verbose_name='request headers in JSON format, e.g., {"key1": "value1", "key2": "value2"}', + ), + ), + migrations.AlterField( + model_name="iotdatasource", + name="source_name", + field=models.CharField( + max_length=3, + unique=True, + verbose_name="Three letter long identifier for the source. Set the identifier as an argument to the Celery task that fetches the data.", + ), + ), + ] diff --git a/iot/models.py b/iot/models.py index d2b4c7b22..a60d5cd1f 100644 --- a/iot/models.py +++ b/iot/models.py @@ -1,16 +1,29 @@ import json +from xml.parsers.expat import ExpatError import requests +import xmltodict from django.core.exceptions import ValidationError from django.db import models class IoTDataSource(models.Model): source_name = models.CharField( - max_length=3, unique=True, verbose_name="Three letter long name for the source" + max_length=3, + unique=True, + verbose_name="Three letter long identifier for the source. " + "Set the identifier as an argument to the Celery task that fetches the data.", ) source_full_name = models.CharField(max_length=64, null=True) + is_xml = models.BooleanField( + default=False, verbose_name="If True, XML data will be converted to JSON." + ) url = models.URLField() + headers = models.JSONField( + null=True, + blank=True, + verbose_name='request headers in JSON format, e.g., {"key1": "value1", "key2": "value2"}', + ) def __str__(self): return self.source_name @@ -18,16 +31,26 @@ def __str__(self): def clean(self): # Test if url exists try: - response = requests.get(self.url) + response = requests.get(self.url, headers=self.headers) except requests.exceptions.ConnectionError: raise ValidationError(f"The given url {self.url} does not exist.") - # Test if valid json - try: - response.json() - except json.decoder.JSONDecodeError: - raise ValidationError( - f"Could not parse the JSON data for the given url {self.url}" - ) + + # Test if XML data can be parsed into JSON + if self.is_xml: + try: + xmltodict.parse(response.text) + except ExpatError as err: + raise ExpatError( + f"Could not parse XML data from the give url {self.url}. {err}" + ) + else: + # Test if valid JSON + try: + response.json() + except json.decoder.JSONDecodeError as err: + raise ValidationError( + f"Could not parse the JSON data from the given url {self.url}. {err}" + ) class IoTData(models.Model):