Merge pull request #64 from DPIclimate/issue_62

Added axistech poller, changed logging in docker compose.

compose/.env

@@ -38,3 +38,5 @@ TZ=Australia/Sydney
 
 # Used by the Intersect DataBolt delivery service.
 DATABOLT_SHARED_DIR=/some/where/raw_data
+
+AXISTECH_TOKEN=

compose/docker-compose.yml

@@ -3,7 +3,11 @@ version: '3.1'
 services:
   db:
     image: postgres:14.2
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     volumes:
@@ -18,7 +22,11 @@ services:
   mq:
     hostname: "mq"
     image: rabbitmq:3.9-management
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     volumes:
@@ -30,7 +38,11 @@ services:
 
   restapi:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     depends_on:
@@ -39,27 +51,32 @@ services:
       - ../src/python:/home/broker/python
     working_dir: "/home/broker/python/restapi"
     entrypoint: [ "/home/broker/.local/bin/uvicorn", "--proxy-headers", "--host", "0.0.0.0", "--port", "5687", "RestAPI:app" ]
-    healthcheck:
-      test: curl --fail -I http://localhost:5687/openapi.json || exit 1
-      interval: 1m
-      start_period: 20s
 
   website:
     image: broker/mgmt-app
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     environment:
       - PYTHONPATH=/app:/iota
     depends_on:
-      restapi:
+      db:
         condition: "service_healthy"
     volumes:
       - ../src/www:/app
       - ../src/python:/iota
 
   ttn_webhook:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -74,7 +91,11 @@ services:
 
   ttn_processor:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -93,7 +114,11 @@ services:
 
   ttn_decoder:
     image: broker/ttn_decoder
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -106,7 +131,11 @@ services:
 
   ydoc:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -123,11 +152,14 @@ services:
 
   wombat:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
-    profiles:
-      - wombat
+    profiles: ["wombat", "frred"]
     depends_on:
       db:
         condition: "service_healthy"
@@ -140,7 +172,11 @@ services:
 
   lm:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     depends_on:
@@ -155,6 +191,10 @@ services:
 
   delivery:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -172,6 +212,10 @@ services:
 
   pollers:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -189,6 +233,10 @@ services:
 
   frred:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -204,3 +252,24 @@ services:
       - ${DATABOLT_SHARED_DIR}/nectar_raw_data:/raw_data
     working_dir: "/home/broker/python"
     entrypoint: [ "python", "-m", "delivery.FRRED" ]
+
+  axistech:
+    image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
+    env_file:
+      - .env
+    profiles:
+      - frred
+    depends_on:
+      db:
+        condition: "service_healthy"
+      mq:
+        condition: "service_healthy"
+    volumes:
+      - ../src/python:/home/broker/python
+    working_dir: "/home/broker/python"
+    entrypoint: [ "python", "-m", "pollers.axistech" ]

db/init.d/init_db.sql

@@ -90,7 +90,7 @@ create table if not exists physical_logical_map (
     primary key(physical_uid, logical_uid, start_time)
 );
 
-create table if not exists users(
+create table if not exists users (
     uid integer generated always as identity primary key,
     username text not null unique,
     salt text not null,
@@ -100,9 +100,14 @@ create table if not exists users(
     read_only boolean default True not null
 );
 
+create table if not exists version (
+    version integer not null
+);
+
 create index if not exists pd_src_id_idx on physical_devices using GIN (source_ids);
 
 insert into sources values ('ttn'), ('greenbrain'), ('wombat'), ('ydoc'), ('ict_eagleio');
+insert into version values (1);
 
 -- Enable the PostGIS extensions
 -- CREATE EXTENSION postgis;

images/restapi/requirements.txt

@@ -6,37 +6,41 @@ charset-normalizer==2.1.0
 click==8.1.3
 dnspython==2.2.1
 email-validator==1.2.1
+exceptiongroup==1.1.1
 fastapi==0.98.0
 h11==0.13.0
 httpcore==0.17.2
 httptools==0.4.0
 httpx==0.24.1
 idna==3.3
+iniconfig==2.0.0
 itsdangerous==2.1.2
 Jinja2==3.1.2
 MarkupSafe==2.1.1
+numpy==1.26.4
 orjson==3.7.7
+packaging==23.1
+pandas==2.2.1
 pika==1.3.0
+pluggy==1.2.0
 psycopg2-binary==2.9.3
 pydantic==1.9.1
+pytest==7.4.0
 python-dateutil==2.8.2
 python-dotenv==0.20.0
 python-multipart==0.0.5
+pytz==2024.1
 PyYAML==6.0
 requests==2.28.1
 six==1.16.0
 sniffio==1.2.0
 starlette==0.27.0
+tomli==2.0.1
 typing_extensions==4.3.0
+tzdata==2024.1
 ujson==5.4.0
 urllib3==1.26.10
 uvicorn==0.17.6
 uvloop==0.16.0
 watchgod==0.8.2
 websockets==10.3
-exceptiongroup==1.1.1
-iniconfig==2.0.0
-packaging==23.1
-pluggy==1.2.0
-pytest==7.4.0
-tomli==2.0.1

src/python/BrokerConstants.py

@@ -13,6 +13,7 @@
 WOMBAT = 'wombat'
 YDOC = 'ydoc'
 ICT_EAGLEIO = 'ict_eagleio'
+AXISTECH = 'axistech'
 
 CREATION_CORRELATION_ID_KEY = 'creation_correlation_id'
 SENSOR_GROUP_ID_KEY = 'sensor_group_id'

src/python/api/client/DAO.py

@@ -172,6 +172,32 @@ def get_all_physical_sources() -> List[PhysicalDevice]:
             free_conn(conn)
 
 
+def add_physical_source(name: str) -> None:
+    """
+    Add a new physical device source if it does not already exist.
+    """
+    conn = None
+    try:
+        with _get_connection() as conn, conn.cursor() as cursor:
+            conn.autocommit = False
+            cursor.execute('select source_name from sources where source_name = %s', (name, ))
+            if cursor.rowcount == 0:
+                cursor.execute("insert into sources values (%s)", (name, ))
+                cursor.execute('select source_name from sources where source_name = %s', (name, ))
+                if cursor.rowcount == 1:
+                    logging.info(f'Added physical device source {name}')
+                    conn.commit()
+                else:
+                    logging.error(f'Failed to add physical device source {name}')
+                    conn.rollback()
+    except Exception as err:
+        conn.rollback()
+        raise err if isinstance(err, DAOException) else DAOException('add_physical_source failed.', err)
+    finally:
+        if conn is not None:
+            free_conn(conn)
+
+
 """
 Physical device CRUD methods
 """
@@ -1269,4 +1295,4 @@ def token_enable(uname)-> None:
     finally:
         if conn is not None:
             free_conn(conn)
-
+

src/python/delivery/FRRED.py

@@ -147,7 +147,7 @@ def on_message(channel, method, properties, body):
             _channel.basic_ack(delivery_tag)
             return
 
-        if BrokerConstants.WOMBAT != pd.source_name:
+        if pd.source_name not in [BrokerConstants.WOMBAT, BrokerConstants.AXISTECH]:
             _channel.basic_ack(delivery_tag)
             return
 

test/python/test_dao.py

@@ -22,14 +22,32 @@ def setUp(self):
                     truncate physical_logical_map cascade;
                     truncate device_notes cascade;
                     truncate physical_timeseries cascade;
-                    truncate raw_messages cascade''')
+                    truncate raw_messages cascade;
+                    delete from sources where source_name = 'axistech';''')
         finally:
             dao.free_conn(conn)
 
     def test_get_all_physical_sources(self):
         sources = dao.get_all_physical_sources()
         self.assertEqual(sources, ['greenbrain', 'ict_eagleio', 'ttn', 'wombat', 'ydoc'])
 
+    def test_add_physical_source(self):
+        sources = dao.get_all_physical_sources()
+        self.assertFalse(BrokerConstants.AXISTECH in sources)
+        dao.add_physical_source(BrokerConstants.AXISTECH)
+        sources = dao.get_all_physical_sources()
+        self.assertTrue(BrokerConstants.AXISTECH in sources)
+
+        # Do it again to ensure it doesn't crash, and there is only one instance of the string.
+        dao.add_physical_source(BrokerConstants.AXISTECH)
+        sources = dao.get_all_physical_sources()
+        i = 0
+        for s in sources:
+            if s == BrokerConstants.AXISTECH:
+                i += 1
+
+        self.assertEqual(1, i)
+
     def now(self):
         return datetime.datetime.now(tz=datetime.timezone.utc)