Merge pull request #335 from City-of-Turku/feature/iot-app-add-header…

…-and-xml-support

Feature/iot app add header and xml support

iot/README.md

@@ -1,22 +1,25 @@
 ## About
 The purpose of the IoT app is to store temporarly data from various IoT-data sources, that do not allow frequent fetching of their data. 
-The data is stored as it is in JSON to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data source or when a data source is added. The cache is populated if empty when serving data. 
+The data is stored as it in JSON format to a JSONField and served as JSON. The app uses caching to cache all its queries and serialized data. The Cache is cleared for the source when importing the data or when a data source is added. The cache is populated if empty when serving data. 
 
 ## Adding IoT-data source from the Admin
-* Give a tree letter long source name, this name will be the name for the source. Used for example when requesting the data.
+* Give a tree letter long identifier, this will be used to identify the data
+to be imported in the Celery task and when requesting data in the API.
 * Add the full name of the source 
 * Add the Url to the JSON data. 
+* Set is_xml to True if the data is in XML format, the data will be converted to JSON.
+* Add the optional headers for the request.
 
 ## Setting periodic importing using Celery from the Admin
 * Create a periodic task, give a descrpitive name.
 * Select *iot.tasks.import_iot_data* as the Task (registered) 
 * Choose the *Interval Schedule*
 * Set the Start DateTime
-* Add the source name as *Positional Arguments*, e.g. ["R24"] would import the source_name R24. 
+* Add the identifier as *Positional Arguments*, e.g. ["R24"] would import the identifier R24. 
 
 ## Manual import
 To manually import source:
-`./manage.py import_iot_data source_name`
+`./manage.py import_iot_data identifier`
 Or by running the perioc task from the admin. 
 
 ## Retriving data

iot/management/commands/import_iot_data.py

@@ -1,7 +1,9 @@
 import json
 import logging
+from xml.parsers.expat import ExpatError
 
 import requests
+import xmltodict
 from django.core.cache import cache
 from django.core.management.base import BaseCommand
 
@@ -13,17 +15,25 @@
 
 def save_data_to_db(source):
  IoTData.objects.filter(data_source=source).delete()
-
  try:
- response = requests.get(source.url)
+ response = requests.get(source.url, headers=source.headers)
  except requests.exceptions.ConnectionError:
  logger.error(f"Could not fetch data from: {source.url}")
  return
- try:
- json_data = response.json()
- except json.decoder.JSONDecodeError:
- logger.error(f"Could not decode data to json from: {source.url}")
- return
+ if source.is_xml:
+ try:
+ json_data = xmltodict.parse(response.text)
+ except ExpatError as err:
+ logger.error(
+ f"Could not parse XML data from the give url {source.url}. {err}"
+ )
+ return
+ else:
+ try:
+ json_data = response.json()
+ except json.decoder.JSONDecodeError as err:
+ logger.error(f"Could not decode data to JSON from: {source.url}. {err}")
+ return
 
  IoTData.objects.create(data_source=source, data=json_data)
 

iot/migrations/0003_iotdatasource_headers.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.13 on 2024-03-13 06:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("iot", "0002_add_ordering"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="iotdatasource",
+ name="headers",
+ field=models.JSONField(
+ blank=True, null=True, verbose_name="request headers"
+ ),
+ ),
+ ]

iot/migrations/0004_iotdatasource_is_xml_alter_iotdatasource_headers_and_more.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.1.13 on 2024-03-13 08:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("iot", "0003_iotdatasource_headers"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="iotdatasource",
+ name="is_xml",
+ field=models.BooleanField(
+ default=False,
+ verbose_name="If True, XML data will be converted to JSON.",
+ ),
+ ),
+ migrations.AlterField(
+ model_name="iotdatasource",
+ name="headers",
+ field=models.JSONField(
+ blank=True,
+ null=True,
+ verbose_name='request headers in JSON format, e.g., {"key1": "value1", "key2": "value2"}',
+ ),
+ ),
+ migrations.AlterField(
+ model_name="iotdatasource",
+ name="source_name",
+ field=models.CharField(
+ max_length=3,
+ unique=True,
+ verbose_name="Three letter long identifier for the source. Set the identifier as an argument to the Celery task that fetches the data.",
+ ),
+ ),
+ ]

iot/models.py

@@ -1,33 +1,56 @@
 import json
+from xml.parsers.expat import ExpatError
 
 import requests
+import xmltodict
 from django.core.exceptions import ValidationError
 from django.db import models
 
 
 class IoTDataSource(models.Model):
  source_name = models.CharField(
- max_length=3, unique=True, verbose_name="Three letter long name for the source"
+ max_length=3,
+ unique=True,
+ verbose_name="Three letter long identifier for the source. "
+ "Set the identifier as an argument to the Celery task that fetches the data.",
  )
  source_full_name = models.CharField(max_length=64, null=True)
+ is_xml = models.BooleanField(
+ default=False, verbose_name="If True, XML data will be converted to JSON."
+ )
  url = models.URLField()
+ headers = models.JSONField(
+ null=True,
+ blank=True,
+ verbose_name='request headers in JSON format, e.g., {"key1": "value1", "key2": "value2"}',
+ )
 
  def __str__(self):
  return self.source_name
 
  def clean(self):
  # Test if url exists
  try:
- response = requests.get(self.url)
+ response = requests.get(self.url, headers=self.headers)
  except requests.exceptions.ConnectionError:
  raise ValidationError(f"The given url {self.url} does not exist.")
- # Test if valid json
- try:
- response.json()
- except json.decoder.JSONDecodeError:
- raise ValidationError(
- f"Could not parse the JSON data for the given url {self.url}"
- )
+
+ # Test if XML data can be parsed into JSON
+ if self.is_xml:
+ try:
+ xmltodict.parse(response.text)
+ except ExpatError as err:
+ raise ExpatError(
+ f"Could not parse XML data from the give url {self.url}. {err}"
+ )
+ else:
+ # Test if valid JSON
+ try:
+ response.json()
+ except json.decoder.JSONDecodeError as err:
+ raise ValidationError(
+ f"Could not parse the JSON data from the given url {self.url}. {err}"
+ )
 
 
 class IoTData(models.Model):