feat: adjust docker, CI, and refactor for new structure

.dockerignore

@@ -1,6 +1,6 @@
 *
-!/app.py
-!/app.py
+!/endpoint/
 !/endpoints/
 !/requirements.txt
 !/tests/
+!/pyproject.toml

.github/workflows/test-endpoint.yml

@@ -70,11 +70,10 @@ jobs:
         env:
           PYTEST_ADDOPTS: "--color=yes"
           AUTH_TOKEN: "abcd1234"
-          KAFKA_HOST: "localhost"
-          KAFKA_PORT: 9092
           KAFKA_BOOTSTRAP_SERVERS: "localhost:9092"
-          LOG_LEVEL: "DEBUG"
+          UVICORN_LOG_LEVEL: "debug"
+          ENDPOINT_CONFIG_URL: "tests/endpoint_config"
           DEBUG: 1
         run: |
           uvicorn app:app --host 0.0.0.0 --port 8000 --proxy-headers &&
-          pytest -v tests/test_api.py
+          pytest

.gitignore

@@ -161,4 +161,4 @@ cython_debug/
 .idea/
 
 # Version file generated by setuptools-scm
-/_version.py
+_version.py

.pre-commit-config.yaml

@@ -9,18 +9,3 @@ repos:
     rev: 'v0.1.8'
     hooks:
       - id: ruff
-  - repo: https://github.com/jazzband/pip-tools
-    rev: 7.3.0
-    hooks:
-      - id: pip-compile
-        name: pip-compile requirements.txt
-        args: [--strip-extras, --output-file=requirements.txt]
-        files: ^(pyproject\.toml|requirements\.txt)$
-      - id: pip-compile
-        name: pip-compile requirements-test.txt
-        args: [--extra=test, --strip-extras, --output-file=requirements-test.txt]
-        files: ^(pyproject\.toml|requirements-test\.txt)$
-      - id: pip-compile
-        name: pip-compile requirements-dev.txt
-        args: [--extra=dev, --strip-extras, --output-file=requirements-dev.txt]
-        files: ^(pyproject\.toml|requirements-dev\.txt)$

Dockerfile

@@ -19,14 +19,15 @@ RUN apk add --no-cache \
 COPY --chown=app:app requirements.txt .
 RUN pip install --no-cache-dir --no-compile --upgrade -r requirements.txt
 
-COPY --chown=app:app . .
+COPY --chown=app:app endpoint/ ./endpoint
+COPY --chown=app:app endpoints/ ./endpoints
 
 # Support Arbitrary User IDs
 RUN chgrp -R 0 /home/app && \
   chmod -R g+rwX /home/app
 
 USER app
 
-HEALTHCHECK CMD wget --no-verbose --tries=1 --spider localhost:8000/healthz || exit
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"]
+HEALTHCHECK CMD wget --no-verbose --tries=1 --spider localhost:8000/liveness || exit
+CMD ["uvicorn", "endpoint.endpoint:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"]
 EXPOSE 8000/tcp

README.md

@@ -1 +1,10 @@
 # Mittaridatapumppu endpoint
+
+```
+pip install pip-tools pre-commit
+. venv/bin/activate
+pre-commit install
+pip-sync requirements*.txt
+uvicorn endpoint.endpoint:app --host 0.0.0.0 --port 8080 --proxy-headers
+API_TOKEN=abcdef1234567890abcdef1234567890abcdef12 venv/bin/python tests/test_api2.py
+```

app.py → endpoint/endpoint.py

@@ -2,13 +2,13 @@
 import logging
 import os
 import pprint
+import json
 from contextlib import asynccontextmanager
 
 import httpx
 from fastapi import FastAPI
 from fastapi.requests import Request
 from fastapi.responses import JSONResponse, PlainTextResponse, Response
-from fvhiot.utils import init_script
 from fvhiot.utils.aiokafka import (get_aiokafka_producer_by_envs,
                                    on_send_error, on_send_success)
 from fvhiot.utils.data import data_pack
@@ -20,7 +20,6 @@
 
 app_producer = None
 app_endpoints = {}
-init_script()
 
 # TODO: for testing, add better defaults (or remove completely to make sure it is set in env)
 ENDPOINT_CONFIG_URL = os.getenv(
@@ -80,29 +79,35 @@ async def get_endpoints_from_device_registry(fail_on_error: bool) -> dict:
     Update endpoints from device registry. This is done on startup and when device registry is updated.
     """
     endpoints = {}
-    # Create request to ENDPOINTS_URL and get data using httpx
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.get(
-                ENDPOINT_CONFIG_URL, headers=device_registry_request_headers
-            )
-            if response.status_code == 200:
-                data = response.json()
-                logging.info(
-                    f"Got {len(data['endpoints'])} endpoints from device registry {ENDPOINT_CONFIG_URL}"
+    data = {}
+    if ENDPOINT_CONFIG_URL.startswith("http"):
+        # Create request to ENDPOINTS_URL and get data using httpx
+        async with httpx.AsyncClient() as client:
+            try:
+                response = await client.get(
+                    ENDPOINT_CONFIG_URL, headers=device_registry_request_headers
                 )
-            else:
+                if response.status_code == 200:
+                    data = response.json()
+                    logging.info(
+                        f"Got {len(data['endpoints'])} endpoints from device registry {ENDPOINT_CONFIG_URL}"
+                    )
+                else:
+                    logging.error(
+                        f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}"
+                    )
+                    return endpoints
+            except Exception as e:
                 logging.error(
-                    f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}"
+                    f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}: {e}"
                 )
-                return endpoints
-        except Exception as e:
-            logging.error(
-                f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}: {e}"
-            )
-            if fail_on_error:
-                raise e
+                if fail_on_error:
+                    raise e
+    else:
+        with open(ENDPOINT_CONFIG_URL, "r") as file:
+            return json.loads(file.read())
     for endpoint in data["endpoints"]:
+        logging.debug(f"{endpoint}")
         # Import requesthandler module. It must exist in python path.
         try:
             request_handler_module = importlib.import_module(
@@ -177,15 +182,16 @@ async def api_v2(request: Request, endpoint: dict) -> Response:
         "request_handler"
     ].process_request(request_data, endpoint)
     response_message = str(response_message)
-    print("REMOVE ME", auth_ok, device_id, topic_name, response_message, status_code)
+    print("REMOVE ME", auth_ok, device_id,
+          topic_name, response_message, status_code)
     # add extracted device id to request data before pushing to kafka raw data topic
     request_data["device_id"] = device_id
     # We assume device data is valid here
     logging.debug(pprint.pformat(request_data))
     if auth_ok and topic_name:
         if app_producer:
             logging.info(f'Sending path "{path}" data to {topic_name}')
-            packed_data = data_pack(request_data)
+            packed_data = data_pack(request_data) or {}
             logging.debug(packed_data[:1000])
             try:
                 res = await app_producer.send_and_wait(topic_name, value=packed_data)

endpoints/__init__.py

@@ -1,5 +1,4 @@
 import abc
-
 import ipaddress
 import logging
 import os
@@ -57,9 +56,6 @@ class AsyncRequestHandler(abc.ABC):
     """
 
     @abc.abstractmethod
-    def __init__(self):
-        pass
-
     async def validate(
         self, request_data: dict, endpoint_data: dict
     ) -> Tuple[bool, Union[str, None], Union[int, None]]:

pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
-version_file = "_version.py"
+version_file = "endpoint/_version.py"
 
 [tool.ruff]
 line-length = 120
@@ -15,16 +15,16 @@ select = ["E", "F", "B", "Q"]
 name = "mittaridatapumppu-endpoint"
 description = "A FastAPI app that receives sensor data in POST requests and produces them to Kafka."
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dynamic = ["version"]
 dependencies = [
-  "fastapi",
-  "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl",
-  "httpx",
-  "kafka-python",
-  "python-multipart",
-  "sentry-asgi",
-  "uvicorn",
+  "fastapi ~= 0.105",
+  "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.2/FVHIoT-1.0.2-py3-none-any.whl",
+  "httpx ~= 0.25",
+  "kafka-python ~= 2.0",
+  "python-multipart ~= 0.0.6",
+  "sentry-asgi ~= 0.2",
+  "uvicorn ~= 0.24",
 ]
 
 [project.optional-dependencies]
@@ -37,14 +37,13 @@ dev = [
   "pep8-naming",
   "pre-commit",
   "pydantic",
-  "pytest",
-  "pytest-asyncio",
-  "pytest-cov",
 ]
 test = [
   "ruff",
-  "pytest",
-  "requests"
+  "pytest ~= 7.4",
+  "requests",
+  "pytest-asyncio",
+  "pytest-cov",
 ]
 
 [tool.pytest.ini_options]