diff --git a/compose/.env b/compose/.env index e8f14b3b..d2cf7f4f 100755 --- a/compose/.env +++ b/compose/.env @@ -38,3 +38,5 @@ TZ=Australia/Sydney # Used by the Intersect DataBolt delivery service. DATABOLT_SHARED_DIR=/some/where/raw_data + +AXISTECH_TOKEN= diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 00049743..8d7a338c 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -3,7 +3,11 @@ version: '3.1' services: db: image: postgres:14.2 - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env volumes: @@ -18,7 +22,11 @@ services: mq: hostname: "mq" image: rabbitmq:3.9-management - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env volumes: @@ -30,7 +38,11 @@ services: restapi: image: broker/python-base - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env depends_on: @@ -39,19 +51,20 @@ services: - ../src/python:/home/broker/python working_dir: "/home/broker/python/restapi" entrypoint: [ "/home/broker/.local/bin/uvicorn", "--proxy-headers", "--host", "0.0.0.0", "--port", "5687", "RestAPI:app" ] - healthcheck: - test: curl --fail -I http://localhost:5687/openapi.json || exit 1 - interval: 1m - start_period: 20s website: image: broker/mgmt-app + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env environment: - PYTHONPATH=/app:/iota depends_on: - restapi: + db: condition: "service_healthy" volumes: - ../src/www:/app @@ -59,7 +72,11 @@ services: ttn_webhook: image: broker/python-base - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env profiles: @@ -74,7 +91,11 @@ services: ttn_processor: image: broker/python-base - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env profiles: @@ -93,7 +114,11 @@ services: ttn_decoder: image: broker/ttn_decoder - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env profiles: @@ -106,7 +131,11 @@ services: ydoc: image: broker/python-base - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env profiles: @@ -123,11 +152,14 @@ services: wombat: image: broker/python-base - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env - profiles: - - wombat + profiles: ["wombat", "frred"] depends_on: db: condition: "service_healthy" @@ -140,7 +172,11 @@ services: lm: image: broker/python-base - restart: "no" + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped env_file: - .env depends_on: @@ -155,6 +191,10 @@ services: delivery: image: broker/python-base + logging: + driver: local + options: + max-file: "3" restart: unless-stopped env_file: - .env @@ -172,6 +212,10 @@ services: pollers: image: broker/python-base + logging: + driver: local + options: + max-file: "3" restart: unless-stopped env_file: - .env @@ -189,6 +233,10 @@ services: frred: image: broker/python-base + logging: + driver: local + options: + max-file: "3" restart: unless-stopped env_file: - .env @@ -204,3 +252,24 @@ services: - ${DATABOLT_SHARED_DIR}/nectar_raw_data:/raw_data working_dir: "/home/broker/python" entrypoint: [ "python", "-m", "delivery.FRRED" ] + + axistech: + image: broker/python-base + logging: + driver: local + options: + max-file: "3" + restart: unless-stopped + env_file: + - .env + profiles: + - frred + depends_on: + db: + condition: "service_healthy" + mq: + condition: "service_healthy" + volumes: + - ../src/python:/home/broker/python + working_dir: "/home/broker/python" + entrypoint: [ "python", "-m", "pollers.axistech" ] diff --git a/db/init.d/init_db.sql b/db/init.d/init_db.sql index a37652c9..ec3638d4 100755 --- a/db/init.d/init_db.sql +++ b/db/init.d/init_db.sql @@ -90,7 +90,7 @@ create table if not exists physical_logical_map ( primary key(physical_uid, logical_uid, start_time) ); -create table if not exists users( +create table if not exists users ( uid integer generated always as identity primary key, username text not null unique, salt text not null, @@ -100,9 +100,14 @@ create table if not exists users( read_only boolean default True not null ); +create table if not exists version ( + version integer not null +); + create index if not exists pd_src_id_idx on physical_devices using GIN (source_ids); insert into sources values ('ttn'), ('greenbrain'), ('wombat'), ('ydoc'), ('ict_eagleio'); +insert into version values (1); -- Enable the PostGIS extensions -- CREATE EXTENSION postgis; diff --git a/images/restapi/requirements.txt b/images/restapi/requirements.txt index c440f164..1c5ca2dc 100644 --- a/images/restapi/requirements.txt +++ b/images/restapi/requirements.txt @@ -6,37 +6,41 @@ charset-normalizer==2.1.0 click==8.1.3 dnspython==2.2.1 email-validator==1.2.1 +exceptiongroup==1.1.1 fastapi==0.98.0 h11==0.13.0 httpcore==0.17.2 httptools==0.4.0 httpx==0.24.1 idna==3.3 +iniconfig==2.0.0 itsdangerous==2.1.2 Jinja2==3.1.2 MarkupSafe==2.1.1 +numpy==1.26.4 orjson==3.7.7 +packaging==23.1 +pandas==2.2.1 pika==1.3.0 +pluggy==1.2.0 psycopg2-binary==2.9.3 pydantic==1.9.1 +pytest==7.4.0 python-dateutil==2.8.2 python-dotenv==0.20.0 python-multipart==0.0.5 +pytz==2024.1 PyYAML==6.0 requests==2.28.1 six==1.16.0 sniffio==1.2.0 starlette==0.27.0 +tomli==2.0.1 typing_extensions==4.3.0 +tzdata==2024.1 ujson==5.4.0 urllib3==1.26.10 uvicorn==0.17.6 uvloop==0.16.0 watchgod==0.8.2 websockets==10.3 -exceptiongroup==1.1.1 -iniconfig==2.0.0 -packaging==23.1 -pluggy==1.2.0 -pytest==7.4.0 -tomli==2.0.1 \ No newline at end of file diff --git a/src/python/BrokerConstants.py b/src/python/BrokerConstants.py index 62f2a0a4..ccf74c26 100644 --- a/src/python/BrokerConstants.py +++ b/src/python/BrokerConstants.py @@ -13,6 +13,7 @@ WOMBAT = 'wombat' YDOC = 'ydoc' ICT_EAGLEIO = 'ict_eagleio' +AXISTECH = 'axistech' CREATION_CORRELATION_ID_KEY = 'creation_correlation_id' SENSOR_GROUP_ID_KEY = 'sensor_group_id' diff --git a/src/python/api/client/DAO.py b/src/python/api/client/DAO.py index e3f1f284..b47456fb 100644 --- a/src/python/api/client/DAO.py +++ b/src/python/api/client/DAO.py @@ -172,6 +172,32 @@ def get_all_physical_sources() -> List[PhysicalDevice]: free_conn(conn) +def add_physical_source(name: str) -> None: + """ + Add a new physical device source if it does not already exist. + """ + conn = None + try: + with _get_connection() as conn, conn.cursor() as cursor: + conn.autocommit = False + cursor.execute('select source_name from sources where source_name = %s', (name, )) + if cursor.rowcount == 0: + cursor.execute("insert into sources values (%s)", (name, )) + cursor.execute('select source_name from sources where source_name = %s', (name, )) + if cursor.rowcount == 1: + logging.info(f'Added physical device source {name}') + conn.commit() + else: + logging.error(f'Failed to add physical device source {name}') + conn.rollback() + except Exception as err: + conn.rollback() + raise err if isinstance(err, DAOException) else DAOException('add_physical_source failed.', err) + finally: + if conn is not None: + free_conn(conn) + + """ Physical device CRUD methods """ @@ -1269,4 +1295,4 @@ def token_enable(uname)-> None: finally: if conn is not None: free_conn(conn) - \ No newline at end of file + diff --git a/src/python/delivery/FRRED.py b/src/python/delivery/FRRED.py index 40b168db..aed79c7b 100644 --- a/src/python/delivery/FRRED.py +++ b/src/python/delivery/FRRED.py @@ -147,7 +147,7 @@ def on_message(channel, method, properties, body): _channel.basic_ack(delivery_tag) return - if BrokerConstants.WOMBAT != pd.source_name: + if pd.source_name not in [BrokerConstants.WOMBAT, BrokerConstants.AXISTECH]: _channel.basic_ack(delivery_tag) return diff --git a/test/python/test_dao.py b/test/python/test_dao.py index 2844e496..33597f92 100644 --- a/test/python/test_dao.py +++ b/test/python/test_dao.py @@ -22,7 +22,8 @@ def setUp(self): truncate physical_logical_map cascade; truncate device_notes cascade; truncate physical_timeseries cascade; - truncate raw_messages cascade''') + truncate raw_messages cascade; + delete from sources where source_name = 'axistech';''') finally: dao.free_conn(conn) @@ -30,6 +31,23 @@ def test_get_all_physical_sources(self): sources = dao.get_all_physical_sources() self.assertEqual(sources, ['greenbrain', 'ict_eagleio', 'ttn', 'wombat', 'ydoc']) + def test_add_physical_source(self): + sources = dao.get_all_physical_sources() + self.assertFalse(BrokerConstants.AXISTECH in sources) + dao.add_physical_source(BrokerConstants.AXISTECH) + sources = dao.get_all_physical_sources() + self.assertTrue(BrokerConstants.AXISTECH in sources) + + # Do it again to ensure it doesn't crash, and there is only one instance of the string. + dao.add_physical_source(BrokerConstants.AXISTECH) + sources = dao.get_all_physical_sources() + i = 0 + for s in sources: + if s == BrokerConstants.AXISTECH: + i += 1 + + self.assertEqual(1, i) + def now(self): return datetime.datetime.now(tz=datetime.timezone.utc)