Merge pull request #10 from City-of-Helsinki/bugfix/workflows

Tweaking envs and ruff parameters. pytest fails as intended because of missing device registry.

.github/workflows/test-endpoint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
     services:
       # Label used to access the service container
       kafka:
@@ -63,23 +63,16 @@ jobs:
       - name: Lint with ruff
         run: |
           # stop the build if there are Python syntax errors or undefined names
-          ruff --format=github --select=E9,F63,F7,F82 --target-version=py37 .
+          ruff --format=github --select=E9,F63,F7,F82 --line-length 120 --target-version=py311 .
           # default set of ruff rules with GitHub Annotations
-          ruff --format=github --target-version=py37 .
+          ruff --format=github --line-length 120 --target-version=py311 .
       - name: Test with pytest
         env:
           PYTEST_ADDOPTS: "--color=yes"
-          ALLOWED_IP_ADDRESSES: "127.0.0.1"
           AUTH_TOKEN: "abcd1234"
-          DATA_SOURCE_NAME: "digita.thingpark.http"
-          ENDPOINT_PATH: "/digita/v2"
-          HTTP_REQUESTHANDLER: "endpoints.digita.aiothingpark"
           KAFKA_HOST: "localhost"
           KAFKA_PORT: 9092
           KAFKA_BOOTSTRAP_SERVERS: "localhost:9092"
-          KAFKA_GROUP_ID: "digita_dev"
-          KAFKA_PARSED_DATA_TOPIC_NAME: "digita.parseddata"
-          KAFKA_RAW_DATA_TOPIC_NAME: "digita.rawdata"
           LOG_LEVEL: "DEBUG"
           DEBUG: 1
         run: |

app.py

@@ -1,16 +1,13 @@
 import importlib
 import logging
-import json
 import os
 import pprint
+
 import httpx
 from fastapi import FastAPI
 from fastapi.requests import Request
 from fastapi.responses import Response, PlainTextResponse, JSONResponse
 from fastapi.routing import APIRoute
-from sentry_asgi import SentryMiddleware
-
-from endpoints import AsyncRequestHandler as RequestHandler
 from fvhiot.utils import init_script
 from fvhiot.utils.aiokafka import (
     get_aiokafka_producer_by_envs,
@@ -19,14 +16,17 @@
 )
 from fvhiot.utils.data import data_pack
 from fvhiot.utils.http.starlettetools import extract_data_from_starlette_request
+from sentry_asgi import SentryMiddleware
+
+from endpoints import AsyncRequestHandler as RequestHandler
 
-# TODO: for testing, add better defaults
-ENDPOINTS_URL = os.getenv("ENDPOINTS_URL", "http://127.0.0.1:8000/api/v1/hosts/localhost/")
-API_TOKEN = os.getenv("API_TOKEN", "abcdef1234567890abcdef1234567890abcdef12")
+# TODO: for testing, add better defaults (or remove completely to make sure it is set in env)
+DEVREG_ENDPOINTS_URL = os.getenv("DEVREG_ENDPOINTS_URL", "http://127.0.0.1:8000/api/v1/hosts/localhost/")
+DEVREG_API_TOKEN = os.getenv("DEVREG_API_TOKEN", "abcdef1234567890abcdef1234567890abcdef12")
 
 device_registry_request_headers = {
-    "Authorization": f"Token {API_TOKEN}",
-    "User-Agent": "mittaridatapumppu-endpoint/0.0.1",
+    "Authorization": f"Token {DEVREG_API_TOKEN}",
+    "User-Agent": "mittaridatapumppu-endpoint/0.1.0",
     "Accept": "application/json",
 }
 
@@ -44,15 +44,15 @@ async def get_endpoints_from_device_registry(fail_on_error: bool) -> dict:
     # Create request to ENDPOINTS_URL and get data using httpx
     async with httpx.AsyncClient() as client:
         try:
-            response = await client.get(ENDPOINTS_URL, headers=device_registry_request_headers)
+            response = await client.get(DEVREG_ENDPOINTS_URL, headers=device_registry_request_headers)
             if response.status_code == 200:
                 data = response.json()
-                logging.info(f"Got {len(data['endpoints'])} endpoints from device registry {ENDPOINTS_URL}")
+                logging.info(f"Got {len(data['endpoints'])} endpoints from device registry {DEVREG_ENDPOINTS_URL}")
             else:
-                logging.error(f"Failed to get endpoints from device registry {ENDPOINTS_URL}")
+                logging.error(f"Failed to get endpoints from device registry {DEVREG_ENDPOINTS_URL}")
                 return endpoints
         except Exception as e:
-            logging.error(f"Failed to get endpoints from device registry {ENDPOINTS_URL}: {e} ERSKA")
+            logging.error(f"Failed to get endpoints from device registry {DEVREG_ENDPOINTS_URL}: {e}")
             if fail_on_error:
                 raise e
     for endpoint in data["endpoints"]:

pyproject.toml

@@ -15,7 +15,7 @@ dynamic = ["version"]
 dependencies = [
   "aiokafka",
   "fastapi",
-  "fvhiot@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.3.1.zip",
+  "fvhiot@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.3.2.zip",
   "httpx",
   "kafka-python",
   "python-multipart",

requirements.txt

@@ -31,7 +31,7 @@ fastapi==0.103.1
     # via mittaridatapumppu-endpoint (pyproject.toml)
 flask==2.3.3
     # via fvhiot
-# fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.3.1.zip
+fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.3.2.zip
     # via mittaridatapumppu-endpoint (pyproject.toml)
 h11==0.14.0
     # via
@@ -62,9 +62,9 @@ msgpack==1.0.6
     # via fvhiot
 packaging==23.1
     # via aiokafka
-pydantic==2.3.0
+pydantic==2.4.1
     # via fastapi
-pydantic-core==2.6.3
+pydantic-core==2.10.1
     # via pydantic
 python-multipart==0.0.6
     # via mittaridatapumppu-endpoint (pyproject.toml)