From 1a83872dbf21f9c99be888f503ba658e6a77e05c Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Fri, 12 Jan 2024 10:46:36 +0200 Subject: [PATCH 1/6] Remove redundant formatter --- .pre-commit-config.yaml | 4 ---- pyproject.toml | 1 - 2 files changed, 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a572649..63595ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,10 +5,6 @@ repos: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 23.12.0 - hooks: - - id: black - repo: https://github.com/charliermarsh/ruff-pre-commit rev: 'v0.1.8' hooks: diff --git a/pyproject.toml b/pyproject.toml index 6e72ee7..0bba576 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ dependencies = [ dev = [ "autoflake", "autopep8", - "black", "flake8", "gunicorn", "isort", From 44848b07c368b3cec8c03704dd80bdb9da09854b Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Fri, 12 Jan 2024 13:11:46 +0200 Subject: [PATCH 2/6] Update project config - Add dynamic versioning - Enable more ruff rules - Add description - Update FVHIoT-python fixes #14 --- .gitignore | 3 +++ Dockerfile | 2 +- app.py => endpoint/app.py | 0 {endpoints => endpoint/endpoints}/__init__.py | 0 .../endpoints}/default/__init__.py | 0 .../endpoints}/default/apikeyauth.py | 0 .../endpoints}/digita/__init__.py | 0 .../endpoints}/digita/aiothingpark.py | 0 .../endpoints}/sentilo/__init__.py | 0 .../endpoints}/sentilo/cesva.py | 0 pyproject.toml | 15 ++++++++++----- 11 files changed, 14 insertions(+), 6 deletions(-) rename app.py => endpoint/app.py (100%) rename {endpoints => endpoint/endpoints}/__init__.py (100%) rename {endpoints => endpoint/endpoints}/default/__init__.py (100%) rename {endpoints => endpoint/endpoints}/default/apikeyauth.py (100%) rename {endpoints => endpoint/endpoints}/digita/__init__.py (100%) rename {endpoints => endpoint/endpoints}/digita/aiothingpark.py (100%) rename {endpoints => endpoint/endpoints}/sentilo/__init__.py (100%) rename {endpoints => endpoint/endpoints}/sentilo/cesva.py (100%) diff --git a/.gitignore b/.gitignore index dc0fcfe..4d014ad 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ + +# Version file generated by setuptools-scm +endpoint/_version.py diff --git a/Dockerfile b/Dockerfile index 2b3f2dc..f00e89d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,5 +27,5 @@ RUN chgrp -R 0 /home/app && \ USER app -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] +CMD ["uvicorn", "endpoint:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] EXPOSE 8000/tcp diff --git a/app.py b/endpoint/app.py similarity index 100% rename from app.py rename to endpoint/app.py diff --git a/endpoints/__init__.py b/endpoint/endpoints/__init__.py similarity index 100% rename from endpoints/__init__.py rename to endpoint/endpoints/__init__.py diff --git a/endpoints/default/__init__.py b/endpoint/endpoints/default/__init__.py similarity index 100% rename from endpoints/default/__init__.py rename to endpoint/endpoints/default/__init__.py diff --git a/endpoints/default/apikeyauth.py b/endpoint/endpoints/default/apikeyauth.py similarity index 100% rename from endpoints/default/apikeyauth.py rename to endpoint/endpoints/default/apikeyauth.py diff --git a/endpoints/digita/__init__.py b/endpoint/endpoints/digita/__init__.py similarity index 100% rename from endpoints/digita/__init__.py rename to endpoint/endpoints/digita/__init__.py diff --git a/endpoints/digita/aiothingpark.py b/endpoint/endpoints/digita/aiothingpark.py similarity index 100% rename from endpoints/digita/aiothingpark.py rename to endpoint/endpoints/digita/aiothingpark.py diff --git a/endpoints/sentilo/__init__.py b/endpoint/endpoints/sentilo/__init__.py similarity index 100% rename from endpoints/sentilo/__init__.py rename to endpoint/endpoints/sentilo/__init__.py diff --git a/endpoints/sentilo/cesva.py b/endpoint/endpoints/sentilo/cesva.py similarity index 100% rename from endpoints/sentilo/cesva.py rename to endpoint/endpoints/sentilo/cesva.py diff --git a/pyproject.toml b/pyproject.toml index 0bba576..71c3d76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,25 @@ [build-system] -requires = ["setuptools"] +requires = [ + "setuptools>=60", + "setuptools-scm>=8.0"] build-backend = "setuptools.build_meta" +[tool.setuptools_scm] +version_file = "endpoint/_version.py" + [tool.ruff] line-length = 120 -target-version = "py311" +select = ["E", "F", "B", "Q"] [project] name = "mittaridatapumppu-endpoint" -description = "" +description = "A FastAPI app that receives sensor data in POST requests and produces them to Kafka." readme = "README.md" requires-python = ">=3.10" -version = "v0.1.0" +dynamic = ["version"] dependencies = [ "fastapi", - "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip", + "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl", "httpx", "kafka-python", "python-multipart", From 793bfadbcf60b30708a2bfdf0ed5d35f8b840037 Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Fri, 12 Jan 2024 13:16:22 +0200 Subject: [PATCH 3/6] Update requirements and add pytest options --- pyproject.toml | 13 +++++++++--- requirements-dev.txt | 46 +++++++++---------------------------------- requirements-test.txt | 6 ++++-- requirements.txt | 4 ++-- 4 files changed, 25 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 71c3d76..aebaf76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,13 @@ dev = [ "pytest-cov", ] test = [ - "pytest", - "requests" -] #pip-compile --resolver=backtracking --extra test pyproject.toml + "ruff", + "pytest", + "requests" +] + +[tool.pytest.ini_options] +log_cli = 1 +log_cli_level = 20 +testpaths = ["tests"] +addopts = ["--color=yes"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 2e02222..0d2aa32 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,15 +1,11 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --extra=dev --output-file=requirements-dev.txt --strip-extras # -aiohttp==3.9.1 - # via black aiokafka==0.10.0 # via fvhiot -aiosignal==1.3.1 - # via aiohttp annotated-types==0.6.0 # via pydantic anyio==3.7.1 @@ -19,14 +15,10 @@ anyio==3.7.1 # starlette async-timeout==4.0.3 # via aiokafka -attrs==23.1.0 - # via aiohttp autoflake==2.2.1 # via mittaridatapumppu-endpoint (pyproject.toml) autopep8==2.0.4 # via mittaridatapumppu-endpoint (pyproject.toml) -black==23.12.0 - # via mittaridatapumppu-endpoint (pyproject.toml) certifi==2023.11.17 # via # fvhiot @@ -36,10 +28,8 @@ certifi==2023.11.17 cfgv==3.4.0 # via pre-commit click==8.1.7 - # via - # black - # uvicorn -coverage==7.3.3 + # via uvicorn +coverage==7.4.0 # via # coverage # pytest-cov @@ -49,15 +39,11 @@ fastapi==0.105.0 # via mittaridatapumppu-endpoint (pyproject.toml) filelock==3.13.1 # via virtualenv -flake8==6.1.0 +flake8==7.0.0 # via # mittaridatapumppu-endpoint (pyproject.toml) # pep8-naming -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip +fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl # via mittaridatapumppu-endpoint (pyproject.toml) gunicorn==21.2.0 # via mittaridatapumppu-endpoint (pyproject.toml) @@ -75,7 +61,6 @@ idna==3.6 # via # anyio # httpx - # yarl iniconfig==2.0.0 # via pytest isort==5.13.2 @@ -86,28 +71,17 @@ mccabe==0.7.0 # via flake8 msgpack==1.0.7 # via fvhiot -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via black nodeenv==1.8.0 # via pre-commit packaging==23.2 # via # aiokafka - # black # gunicorn # pytest -pathspec==0.12.1 - # via black pep8-naming==0.13.3 # via mittaridatapumppu-endpoint (pyproject.toml) platformdirs==4.1.0 - # via - # black - # virtualenv + # via virtualenv pluggy==1.3.0 # via pytest pre-commit==3.6.0 @@ -122,16 +96,16 @@ pydantic==2.5.2 # mittaridatapumppu-endpoint (pyproject.toml) pydantic-core==2.14.5 # via pydantic -pyflakes==3.1.0 +pyflakes==3.2.0 # via # autoflake # flake8 -pytest==7.4.3 +pytest==7.4.4 # via # mittaridatapumppu-endpoint (pyproject.toml) # pytest-asyncio # pytest-cov -pytest-asyncio==0.23.2 +pytest-asyncio==0.23.3 # via mittaridatapumppu-endpoint (pyproject.toml) pytest-cov==4.1.0 # via mittaridatapumppu-endpoint (pyproject.toml) @@ -160,8 +134,6 @@ uvicorn==0.24.0.post1 # via mittaridatapumppu-endpoint (pyproject.toml) virtualenv==20.25.0 # via pre-commit -yarl==1.9.4 - # via aiohttp # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-test.txt b/requirements-test.txt index 475d56a..490b5ca 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --extra=test --output-file=requirements-test.txt --strip-extras @@ -28,7 +28,7 @@ click==8.1.7 # via uvicorn fastapi==0.105.0 # via mittaridatapumppu-endpoint (pyproject.toml) -fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip +fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl # via mittaridatapumppu-endpoint (pyproject.toml) h11==0.14.0 # via @@ -65,6 +65,8 @@ python-multipart==0.0.6 # via mittaridatapumppu-endpoint (pyproject.toml) requests==2.31.0 # via mittaridatapumppu-endpoint (pyproject.toml) +ruff==0.1.12 + # via mittaridatapumppu-endpoint (pyproject.toml) sentry-asgi==0.2.0 # via mittaridatapumppu-endpoint (pyproject.toml) sentry-sdk==1.39.1 diff --git a/requirements.txt b/requirements.txt index 476ca10..83bc40b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements.txt --strip-extras @@ -25,7 +25,7 @@ click==8.1.7 # via uvicorn fastapi==0.105.0 # via mittaridatapumppu-endpoint (pyproject.toml) -fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip +fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl # via mittaridatapumppu-endpoint (pyproject.toml) h11==0.14.0 # via From f7b562de41999443152278f83422c42f8b77a08c Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Thu, 18 Jan 2024 11:01:54 +0200 Subject: [PATCH 4/6] Update project config and refactor - Add health check to dockerfile - Use lifespan instead of deprecated startup & shutdown - Run linters and formatters, fixed warnings - Refactor to use FastAPI decorators --- .dockerignore | 9 +- .gitignore | 2 +- Dockerfile | 3 +- endpoint/app.py => app.py | 142 +++++++++--------- {endpoint/endpoints => endpoints}/__init__.py | 1 + .../default/__init__.py | 0 .../default/apikeyauth.py | 0 .../digita/__init__.py | 0 .../digita/aiothingpark.py | 0 .../sentilo/__init__.py | 0 .../endpoints => endpoints}/sentilo/cesva.py | 0 pyproject.toml | 2 +- 12 files changed, 78 insertions(+), 81 deletions(-) rename endpoint/app.py => app.py (74%) rename {endpoint/endpoints => endpoints}/__init__.py (99%) rename {endpoint/endpoints => endpoints}/default/__init__.py (100%) rename {endpoint/endpoints => endpoints}/default/apikeyauth.py (100%) rename {endpoint/endpoints => endpoints}/digita/__init__.py (100%) rename {endpoint/endpoints => endpoints}/digita/aiothingpark.py (100%) rename {endpoint/endpoints => endpoints}/sentilo/__init__.py (100%) rename {endpoint/endpoints => endpoints}/sentilo/cesva.py (100%) diff --git a/.dockerignore b/.dockerignore index ef52199..0ff6930 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ * -!app.py -!endpoints/ -!requirements.txt -!tests/ +!/app.py +!/app.py +!/endpoints/ +!/requirements.txt +!/tests/ diff --git a/.gitignore b/.gitignore index 4d014ad..d29d43e 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ cython_debug/ .idea/ # Version file generated by setuptools-scm -endpoint/_version.py +/_version.py diff --git a/Dockerfile b/Dockerfile index f00e89d..26510e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,5 +27,6 @@ RUN chgrp -R 0 /home/app && \ USER app -CMD ["uvicorn", "endpoint:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] +HEALTHCHECK CMD wget --no-verbose --tries=1 --spider localhost:8000/healthz || exit +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] EXPOSE 8000/tcp diff --git a/endpoint/app.py b/app.py similarity index 74% rename from endpoint/app.py rename to app.py index 09e04c1..a048240 100755 --- a/endpoint/app.py +++ b/app.py @@ -2,24 +2,26 @@ import logging import os import pprint +from contextlib import asynccontextmanager import httpx from fastapi import FastAPI from fastapi.requests import Request -from fastapi.responses import Response, PlainTextResponse, JSONResponse -from fastapi.routing import APIRoute +from fastapi.responses import JSONResponse, PlainTextResponse, Response from fvhiot.utils import init_script -from fvhiot.utils.aiokafka import ( - get_aiokafka_producer_by_envs, - on_send_success, - on_send_error, -) +from fvhiot.utils.aiokafka import (get_aiokafka_producer_by_envs, + on_send_error, on_send_success) from fvhiot.utils.data import data_pack -from fvhiot.utils.http.starlettetools import extract_data_from_starlette_request +from fvhiot.utils.http.starlettetools import \ + extract_data_from_starlette_request from sentry_asgi import SentryMiddleware from endpoints import AsyncRequestHandler as RequestHandler +app_producer = None +app_endpoints = {} +init_script() + # TODO: for testing, add better defaults (or remove completely to make sure it is set in env) ENDPOINT_CONFIG_URL = os.getenv( "ENDPOINT_CONFIG_URL", "http://127.0.0.1:8000/api/v1/hosts/localhost/" @@ -35,6 +37,39 @@ } +@asynccontextmanager +async def lifespan(app: FastAPI): + # Get endpoints from Device registry and create KafkaProducer . + # TODO: Test external connections here, e.g. device registry, redis etc. and crash if some mandatory + # service is missing. + global app_endpoints + global app_producer + endpoints = await get_endpoints_from_device_registry(True) + logging.debug("\n" + pprint.pformat(endpoints)) + if endpoints: + app_endpoints = endpoints + try: + app_producer = await get_aiokafka_producer_by_envs() + except Exception as e: + logging.error(f"Failed to create KafkaProducer: {e}") + app_producer = None + logging.info( + "Ready to go, listening to endpoints: {}".format( + ", ".join(endpoints.keys()) + ) + ) + yield + + # Close KafkaProducer and other connections. + logging.info("Shutdown, close connections") + if app_producer: + await app_producer.stop() + + +app = FastAPI(lifespan=lifespan) +app.add_middleware(SentryMiddleware) + + def get_full_path(request: Request) -> str: """Make sure there is always exactly one leading slash in path.""" return "/" + request.path_params["full_path"].lstrip("/") @@ -79,40 +114,51 @@ async def get_endpoints_from_device_registry(fail_on_error: bool) -> dict: endpoint["request_handler"] = request_handler_function logging.info(f"Imported {endpoint['http_request_handler']}") except ImportError as e: - logging.error(f"Failed to import {endpoint['http_request_handler']}: {e}") + logging.error( + f"Failed to import {endpoint['http_request_handler']}: {e}") endpoints[endpoint["endpoint_path"]] = endpoint return endpoints +@app.get("/") async def root(_request: Request) -> Response: return JSONResponse({"message": "Test ok"}) +@app.get("/notify") async def notify(_request: Request) -> Response: - global app + global app_endpoints endpoints = await get_endpoints_from_device_registry(False) logging.debug("Got endpoints:\n" + pprint.pformat(endpoints)) endpoint_count = len(endpoints) if endpoints: - logging.info(f"Got {endpoint_count} endpoints from device registry in notify") - app.endpoints = endpoints + logging.info( + f"Got {endpoint_count} endpoints from device registry in notify") + app_endpoints = endpoints return PlainTextResponse(f"OK ({endpoint_count})") +@app.get("/readiness") +@app.head("/readiness") async def readiness(_request: Request) -> Response: return PlainTextResponse("OK") -async def healthz(_request: Request) -> Response: +@app.get("/liveness") +@app.head("/liveness") +async def liveness(_request: Request) -> Response: return PlainTextResponse("OK") +@app.get("/debug-sentry") +@app.head("/debug-sentry") async def trigger_error(_request: Request) -> Response: _ = 1 / 0 return PlainTextResponse("Shouldn't reach this") async def api_v2(request: Request, endpoint: dict) -> Response: + global app_producer request_data = await extract_data_from_starlette_request( request ) # data validation done here @@ -120,7 +166,8 @@ async def api_v2(request: Request, endpoint: dict) -> Response: # DONE # logging.error(request_data) if request_data.get("extra"): - logging.warning(f"RequestModel contains extra values: {request_data['extra']}") + logging.warning( + f"RequestModel contains extra values: {request_data['extra']}") if request_data["request"].get("extra"): logging.warning( f"RequestData contains extra values: {request_data['request']['extra']}" @@ -136,12 +183,12 @@ async def api_v2(request: Request, endpoint: dict) -> Response: # We assume device data is valid here logging.debug(pprint.pformat(request_data)) if auth_ok and topic_name: - if app.producer: + if app_producer: logging.info(f'Sending path "{path}" data to {topic_name}') packed_data = data_pack(request_data) logging.debug(packed_data[:1000]) try: - res = await app.producer.send_and_wait(topic_name, value=packed_data) + res = await app_producer.send_and_wait(topic_name, value=packed_data) on_send_success(res) except Exception as e: on_send_error(e) @@ -160,75 +207,22 @@ async def api_v2(request: Request, endpoint: dict) -> Response: return PlainTextResponse(response_message, status_code=status_code or 200) +@app.api_route("/{full_path:path}", methods=["GET", "POST", "PUT", "HEAD", "DELETE", "PATCH"]) async def catch_all(request: Request) -> Response: """Catch all requests (except static paths) and route them to correct request handlers.""" + global app_endpoints full_path = get_full_path(request) - # print(full_path, app.endpoints.keys()) - if full_path in app.endpoints: - endpoint = app.endpoints[full_path] + # print(full_path, app_endpoints.keys()) + if full_path in app_endpoints: + endpoint = app_endpoints[full_path] response = await api_v2(request, endpoint) return response else: # return 404 return PlainTextResponse("Not found: " + full_path, status_code=404) -async def startup(): - """ - Get endpoints from Device registry and create KafkaProducer . - TODO: Test external connections here, e.g. device registry, redis etc. and crash if some mandatory - service is missing. - """ - global app - endpoints = await get_endpoints_from_device_registry(True) - logging.debug("\n" + pprint.pformat(endpoints)) - if endpoints: - app.endpoints = endpoints - try: - app.producer = await get_aiokafka_producer_by_envs() - except Exception as e: - logging.error(f"Failed to create KafkaProducer: {e}") - app.producer = None - logging.info( - "Ready to go, listening to endpoints: {}".format( - ", ".join(app.endpoints.keys()) - ) - ) - - -async def shutdown(): - """ - Close KafkaProducer and other connections. - """ - global app - logging.info("Shutdown, close connections") - if app.producer: - await app.producer.stop() - - -routes = [ - APIRoute("/", endpoint=root), - APIRoute("/notify", endpoint=notify, methods=["GET"]), - APIRoute("/readiness", endpoint=readiness, methods=["GET", "HEAD"]), - APIRoute("/healthz", endpoint=healthz, methods=["GET", "HEAD"]), - APIRoute("/debug-sentry", endpoint=trigger_error, methods=["GET", "HEAD"]), - APIRoute( - "/{full_path:path}", - endpoint=catch_all, - methods=["HEAD", "GET", "POST", "PUT", "PATCH", "DELETE"], - ), -] - - -init_script() -debug = True if os.getenv("DEBUG") else False -app = FastAPI(debug=debug, routes=routes, on_startup=[startup], on_shutdown=[shutdown]) -app.producer = None -app.endpoints = {} -app.add_middleware(SentryMiddleware) - # This part is for debugging / PyCharm debugger # See https://fastapi.tiangolo.com/tutorial/debugging/ if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8002) diff --git a/endpoint/endpoints/__init__.py b/endpoints/__init__.py similarity index 99% rename from endpoint/endpoints/__init__.py rename to endpoints/__init__.py index f9f4d66..a8aa983 100644 --- a/endpoint/endpoints/__init__.py +++ b/endpoints/__init__.py @@ -56,6 +56,7 @@ class AsyncRequestHandler(abc.ABC): Async version of BaseRequestHandler, compatible with Starlette, FastAPI and Device registry. """ + @abc.abstractmethod def __init__(self): pass diff --git a/endpoint/endpoints/default/__init__.py b/endpoints/default/__init__.py similarity index 100% rename from endpoint/endpoints/default/__init__.py rename to endpoints/default/__init__.py diff --git a/endpoint/endpoints/default/apikeyauth.py b/endpoints/default/apikeyauth.py similarity index 100% rename from endpoint/endpoints/default/apikeyauth.py rename to endpoints/default/apikeyauth.py diff --git a/endpoint/endpoints/digita/__init__.py b/endpoints/digita/__init__.py similarity index 100% rename from endpoint/endpoints/digita/__init__.py rename to endpoints/digita/__init__.py diff --git a/endpoint/endpoints/digita/aiothingpark.py b/endpoints/digita/aiothingpark.py similarity index 100% rename from endpoint/endpoints/digita/aiothingpark.py rename to endpoints/digita/aiothingpark.py diff --git a/endpoint/endpoints/sentilo/__init__.py b/endpoints/sentilo/__init__.py similarity index 100% rename from endpoint/endpoints/sentilo/__init__.py rename to endpoints/sentilo/__init__.py diff --git a/endpoint/endpoints/sentilo/cesva.py b/endpoints/sentilo/cesva.py similarity index 100% rename from endpoint/endpoints/sentilo/cesva.py rename to endpoints/sentilo/cesva.py diff --git a/pyproject.toml b/pyproject.toml index aebaf76..949d447 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ build-backend = "setuptools.build_meta" [tool.setuptools_scm] -version_file = "endpoint/_version.py" +version_file = "_version.py" [tool.ruff] line-length = 120 From 8296ee63529b4be8b39b698eeb43956e2ad41281 Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Thu, 30 May 2024 21:29:39 +0300 Subject: [PATCH 5/6] feat: adjust docker, CI, and refactor for new structure --- .dockerignore | 4 +- .github/workflows/test-endpoint.yml | 7 +- .gitignore | 2 +- .pre-commit-config.yaml | 15 --- Dockerfile | 7 +- README.md | 9 ++ endpoint/__init__.py | 0 app.py => endpoint/endpoint.py | 52 ++++++----- endpoints/__init__.py | 4 - index.html | 11 --- pyproject.toml | 27 +++--- requirements-dev.txt | 139 ---------------------------- requirements-test.txt | 90 ------------------ requirements.txt | 72 -------------- tests/endpoint_config | 42 +++++++++ tests/test_api2.py | 1 + 16 files changed, 104 insertions(+), 378 deletions(-) create mode 100644 endpoint/__init__.py rename app.py => endpoint/endpoint.py (85%) delete mode 100644 index.html delete mode 100644 requirements-dev.txt delete mode 100644 requirements-test.txt delete mode 100644 requirements.txt create mode 100644 tests/endpoint_config diff --git a/.dockerignore b/.dockerignore index 0ff6930..79b547d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ * -!/app.py -!/app.py +!/endpoint/ !/endpoints/ !/requirements.txt !/tests/ +!/pyproject.toml diff --git a/.github/workflows/test-endpoint.yml b/.github/workflows/test-endpoint.yml index 2f9a975..8fc63f4 100644 --- a/.github/workflows/test-endpoint.yml +++ b/.github/workflows/test-endpoint.yml @@ -70,11 +70,10 @@ jobs: env: PYTEST_ADDOPTS: "--color=yes" AUTH_TOKEN: "abcd1234" - KAFKA_HOST: "localhost" - KAFKA_PORT: 9092 KAFKA_BOOTSTRAP_SERVERS: "localhost:9092" - LOG_LEVEL: "DEBUG" + UVICORN_LOG_LEVEL: "debug" + ENDPOINT_CONFIG_URL: "tests/endpoint_config" DEBUG: 1 run: | uvicorn app:app --host 0.0.0.0 --port 8000 --proxy-headers && - pytest -v tests/test_api.py + pytest diff --git a/.gitignore b/.gitignore index d29d43e..79e49ba 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ cython_debug/ .idea/ # Version file generated by setuptools-scm -/_version.py +_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63595ea..3c2e030 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,18 +9,3 @@ repos: rev: 'v0.1.8' hooks: - id: ruff - - repo: https://github.com/jazzband/pip-tools - rev: 7.3.0 - hooks: - - id: pip-compile - name: pip-compile requirements.txt - args: [--strip-extras, --output-file=requirements.txt] - files: ^(pyproject\.toml|requirements\.txt)$ - - id: pip-compile - name: pip-compile requirements-test.txt - args: [--extra=test, --strip-extras, --output-file=requirements-test.txt] - files: ^(pyproject\.toml|requirements-test\.txt)$ - - id: pip-compile - name: pip-compile requirements-dev.txt - args: [--extra=dev, --strip-extras, --output-file=requirements-dev.txt] - files: ^(pyproject\.toml|requirements-dev\.txt)$ diff --git a/Dockerfile b/Dockerfile index 26510e4..f2362bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,8 @@ RUN apk add --no-cache \ COPY --chown=app:app requirements.txt . RUN pip install --no-cache-dir --no-compile --upgrade -r requirements.txt -COPY --chown=app:app . . +COPY --chown=app:app endpoint/ ./endpoint +COPY --chown=app:app endpoints/ ./endpoints # Support Arbitrary User IDs RUN chgrp -R 0 /home/app && \ @@ -27,6 +28,6 @@ RUN chgrp -R 0 /home/app && \ USER app -HEALTHCHECK CMD wget --no-verbose --tries=1 --spider localhost:8000/healthz || exit -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] +HEALTHCHECK CMD wget --no-verbose --tries=1 --spider localhost:8000/liveness || exit +CMD ["uvicorn", "endpoint.endpoint:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] EXPOSE 8000/tcp diff --git a/README.md b/README.md index ba2e3aa..08c1e3b 100644 --- a/README.md +++ b/README.md @@ -1 +1,10 @@ # Mittaridatapumppu endpoint + +``` +pip install pip-tools pre-commit +. venv/bin/activate +pre-commit install +pip-sync requirements*.txt +uvicorn endpoint.endpoint:app --host 0.0.0.0 --port 8080 --proxy-headers +API_TOKEN=abcdef1234567890abcdef1234567890abcdef12 venv/bin/python tests/test_api2.py +``` diff --git a/endpoint/__init__.py b/endpoint/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app.py b/endpoint/endpoint.py similarity index 85% rename from app.py rename to endpoint/endpoint.py index a048240..5880373 100755 --- a/app.py +++ b/endpoint/endpoint.py @@ -2,13 +2,13 @@ import logging import os import pprint +import json from contextlib import asynccontextmanager import httpx from fastapi import FastAPI from fastapi.requests import Request from fastapi.responses import JSONResponse, PlainTextResponse, Response -from fvhiot.utils import init_script from fvhiot.utils.aiokafka import (get_aiokafka_producer_by_envs, on_send_error, on_send_success) from fvhiot.utils.data import data_pack @@ -20,7 +20,6 @@ app_producer = None app_endpoints = {} -init_script() # TODO: for testing, add better defaults (or remove completely to make sure it is set in env) ENDPOINT_CONFIG_URL = os.getenv( @@ -80,29 +79,35 @@ async def get_endpoints_from_device_registry(fail_on_error: bool) -> dict: Update endpoints from device registry. This is done on startup and when device registry is updated. """ endpoints = {} - # Create request to ENDPOINTS_URL and get data using httpx - async with httpx.AsyncClient() as client: - try: - response = await client.get( - ENDPOINT_CONFIG_URL, headers=device_registry_request_headers - ) - if response.status_code == 200: - data = response.json() - logging.info( - f"Got {len(data['endpoints'])} endpoints from device registry {ENDPOINT_CONFIG_URL}" + data = {} + if ENDPOINT_CONFIG_URL.startswith("http"): + # Create request to ENDPOINTS_URL and get data using httpx + async with httpx.AsyncClient() as client: + try: + response = await client.get( + ENDPOINT_CONFIG_URL, headers=device_registry_request_headers ) - else: + if response.status_code == 200: + data = response.json() + logging.info( + f"Got {len(data['endpoints'])} endpoints from device registry {ENDPOINT_CONFIG_URL}" + ) + else: + logging.error( + f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}" + ) + return endpoints + except Exception as e: logging.error( - f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}" + f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}: {e}" ) - return endpoints - except Exception as e: - logging.error( - f"Failed to get endpoints from device registry {ENDPOINT_CONFIG_URL}: {e}" - ) - if fail_on_error: - raise e + if fail_on_error: + raise e + else: + with open(ENDPOINT_CONFIG_URL, "r") as file: + return json.loads(file.read()) for endpoint in data["endpoints"]: + logging.debug(f"{endpoint}") # Import requesthandler module. It must exist in python path. try: request_handler_module = importlib.import_module( @@ -177,7 +182,8 @@ async def api_v2(request: Request, endpoint: dict) -> Response: "request_handler" ].process_request(request_data, endpoint) response_message = str(response_message) - print("REMOVE ME", auth_ok, device_id, topic_name, response_message, status_code) + print("REMOVE ME", auth_ok, device_id, + topic_name, response_message, status_code) # add extracted device id to request data before pushing to kafka raw data topic request_data["device_id"] = device_id # We assume device data is valid here @@ -185,7 +191,7 @@ async def api_v2(request: Request, endpoint: dict) -> Response: if auth_ok and topic_name: if app_producer: logging.info(f'Sending path "{path}" data to {topic_name}') - packed_data = data_pack(request_data) + packed_data = data_pack(request_data) or {} logging.debug(packed_data[:1000]) try: res = await app_producer.send_and_wait(topic_name, value=packed_data) diff --git a/endpoints/__init__.py b/endpoints/__init__.py index a8aa983..fc76368 100644 --- a/endpoints/__init__.py +++ b/endpoints/__init__.py @@ -1,5 +1,4 @@ import abc - import ipaddress import logging import os @@ -57,9 +56,6 @@ class AsyncRequestHandler(abc.ABC): """ @abc.abstractmethod - def __init__(self): - pass - async def validate( self, request_data: dict, endpoint_data: dict ) -> Tuple[bool, Union[str, None], Union[int, None]]: diff --git a/index.html b/index.html deleted file mode 100644 index 8dfb027..0000000 --- a/index.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - Mittaridatapumppu endpoint - - -

Mittaridatapumppu endpoint

- - diff --git a/pyproject.toml b/pyproject.toml index 949d447..4589b77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ build-backend = "setuptools.build_meta" [tool.setuptools_scm] -version_file = "_version.py" +version_file = "endpoint/_version.py" [tool.ruff] line-length = 120 @@ -15,16 +15,16 @@ select = ["E", "F", "B", "Q"] name = "mittaridatapumppu-endpoint" description = "A FastAPI app that receives sensor data in POST requests and produces them to Kafka." readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.12" dynamic = ["version"] dependencies = [ - "fastapi", - "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl", - "httpx", - "kafka-python", - "python-multipart", - "sentry-asgi", - "uvicorn", + "fastapi ~= 0.105", + "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.2/FVHIoT-1.0.2-py3-none-any.whl", + "httpx ~= 0.25", + "kafka-python ~= 2.0", + "python-multipart ~= 0.0.6", + "sentry-asgi ~= 0.2", + "uvicorn ~= 0.24", ] [project.optional-dependencies] @@ -37,14 +37,13 @@ dev = [ "pep8-naming", "pre-commit", "pydantic", - "pytest", - "pytest-asyncio", - "pytest-cov", ] test = [ "ruff", - "pytest", - "requests" + "pytest ~= 7.4", + "requests", + "pytest-asyncio", + "pytest-cov", ] [tool.pytest.ini_options] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 0d2aa32..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,139 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --extra=dev --output-file=requirements-dev.txt --strip-extras -# -aiokafka==0.10.0 - # via fvhiot -annotated-types==0.6.0 - # via pydantic -anyio==3.7.1 - # via - # fastapi - # httpx - # starlette -async-timeout==4.0.3 - # via aiokafka -autoflake==2.2.1 - # via mittaridatapumppu-endpoint (pyproject.toml) -autopep8==2.0.4 - # via mittaridatapumppu-endpoint (pyproject.toml) -certifi==2023.11.17 - # via - # fvhiot - # httpcore - # httpx - # sentry-sdk -cfgv==3.4.0 - # via pre-commit -click==8.1.7 - # via uvicorn -coverage==7.4.0 - # via - # coverage - # pytest-cov -distlib==0.3.8 - # via virtualenv -fastapi==0.105.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -filelock==3.13.1 - # via virtualenv -flake8==7.0.0 - # via - # mittaridatapumppu-endpoint (pyproject.toml) - # pep8-naming -fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl - # via mittaridatapumppu-endpoint (pyproject.toml) -gunicorn==21.2.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.2 - # via httpx -httpx==0.25.2 - # via mittaridatapumppu-endpoint (pyproject.toml) -identify==2.5.33 - # via pre-commit -idna==3.6 - # via - # anyio - # httpx -iniconfig==2.0.0 - # via pytest -isort==5.13.2 - # via mittaridatapumppu-endpoint (pyproject.toml) -kafka-python==2.0.2 - # via mittaridatapumppu-endpoint (pyproject.toml) -mccabe==0.7.0 - # via flake8 -msgpack==1.0.7 - # via fvhiot -nodeenv==1.8.0 - # via pre-commit -packaging==23.2 - # via - # aiokafka - # gunicorn - # pytest -pep8-naming==0.13.3 - # via mittaridatapumppu-endpoint (pyproject.toml) -platformdirs==4.1.0 - # via virtualenv -pluggy==1.3.0 - # via pytest -pre-commit==3.6.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -pycodestyle==2.11.1 - # via - # autopep8 - # flake8 -pydantic==2.5.2 - # via - # fastapi - # mittaridatapumppu-endpoint (pyproject.toml) -pydantic-core==2.14.5 - # via pydantic -pyflakes==3.2.0 - # via - # autoflake - # flake8 -pytest==7.4.4 - # via - # mittaridatapumppu-endpoint (pyproject.toml) - # pytest-asyncio - # pytest-cov -pytest-asyncio==0.23.3 - # via mittaridatapumppu-endpoint (pyproject.toml) -pytest-cov==4.1.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -python-multipart==0.0.6 - # via mittaridatapumppu-endpoint (pyproject.toml) -pyyaml==6.0.1 - # via pre-commit -sentry-asgi==0.2.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -sentry-sdk==1.39.1 - # via sentry-asgi -sniffio==1.3.0 - # via - # anyio - # httpx -starlette==0.27.0 - # via fastapi -typing-extensions==4.9.0 - # via - # fastapi - # pydantic - # pydantic-core -urllib3==2.1.0 - # via sentry-sdk -uvicorn==0.24.0.post1 - # via mittaridatapumppu-endpoint (pyproject.toml) -virtualenv==20.25.0 - # via pre-commit - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 490b5ca..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1,90 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --extra=test --output-file=requirements-test.txt --strip-extras -# -aiokafka==0.10.0 - # via fvhiot -annotated-types==0.6.0 - # via pydantic -anyio==3.7.1 - # via - # fastapi - # httpx - # starlette -async-timeout==4.0.3 - # via aiokafka -certifi==2023.11.17 - # via - # fvhiot - # httpcore - # httpx - # requests - # sentry-sdk -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via uvicorn -fastapi==0.105.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl - # via mittaridatapumppu-endpoint (pyproject.toml) -h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.2 - # via httpx -httpx==0.25.2 - # via mittaridatapumppu-endpoint (pyproject.toml) -idna==3.6 - # via - # anyio - # httpx - # requests -iniconfig==2.0.0 - # via pytest -kafka-python==2.0.2 - # via mittaridatapumppu-endpoint (pyproject.toml) -msgpack==1.0.7 - # via fvhiot -packaging==23.2 - # via - # aiokafka - # pytest -pluggy==1.3.0 - # via pytest -pydantic==2.5.2 - # via fastapi -pydantic-core==2.14.5 - # via pydantic -pytest==7.4.3 - # via mittaridatapumppu-endpoint (pyproject.toml) -python-multipart==0.0.6 - # via mittaridatapumppu-endpoint (pyproject.toml) -requests==2.31.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -ruff==0.1.12 - # via mittaridatapumppu-endpoint (pyproject.toml) -sentry-asgi==0.2.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -sentry-sdk==1.39.1 - # via sentry-asgi -sniffio==1.3.0 - # via - # anyio - # httpx -starlette==0.27.0 - # via fastapi -typing-extensions==4.9.0 - # via - # fastapi - # pydantic - # pydantic-core -urllib3==2.1.0 - # via - # requests - # sentry-sdk -uvicorn==0.24.0.post1 - # via mittaridatapumppu-endpoint (pyproject.toml) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 83bc40b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,72 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --output-file=requirements.txt --strip-extras -# -aiokafka==0.10.0 - # via fvhiot -annotated-types==0.6.0 - # via pydantic -anyio==3.7.1 - # via - # fastapi - # httpx - # starlette -async-timeout==4.0.3 - # via aiokafka -certifi==2023.11.17 - # via - # fvhiot - # httpcore - # httpx - # sentry-sdk -click==8.1.7 - # via uvicorn -fastapi==0.105.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.1/FVHIoT-1.0.1-py3-none-any.whl - # via mittaridatapumppu-endpoint (pyproject.toml) -h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.2 - # via httpx -httpx==0.25.2 - # via mittaridatapumppu-endpoint (pyproject.toml) -idna==3.6 - # via - # anyio - # httpx -kafka-python==2.0.2 - # via mittaridatapumppu-endpoint (pyproject.toml) -msgpack==1.0.7 - # via fvhiot -packaging==23.2 - # via aiokafka -pydantic==2.5.2 - # via fastapi -pydantic-core==2.14.5 - # via pydantic -python-multipart==0.0.6 - # via mittaridatapumppu-endpoint (pyproject.toml) -sentry-asgi==0.2.0 - # via mittaridatapumppu-endpoint (pyproject.toml) -sentry-sdk==1.39.1 - # via sentry-asgi -sniffio==1.3.0 - # via - # anyio - # httpx -starlette==0.27.0 - # via fastapi -typing-extensions==4.9.0 - # via - # fastapi - # pydantic - # pydantic-core -urllib3==2.1.0 - # via sentry-sdk -uvicorn==0.24.0.post1 - # via mittaridatapumppu-endpoint (pyproject.toml) diff --git a/tests/endpoint_config b/tests/endpoint_config new file mode 100644 index 0000000..49d82b7 --- /dev/null +++ b/tests/endpoint_config @@ -0,0 +1,42 @@ +{ + "url": "http://mittaridatapumppu-deviceregistry/api/v1/hosts/localhost/", + "slug": "localhost", + "host_name": "localhost", + "ip_address": "127.0.0.1", + "description": "", + "notification_url": "http://localhost:8001/notify", + "endpoints": [ + { + "id": 1, + "created_at": "2023-10-01T15:00:00.042000+03:00", + "updated_at": "2023-10-01T15:00:00.042000+03:00", + "endpoint_path": "/api/v1/data", + "http_request_handler": "endpoints.default.apikeyauth", + "auth_token": "abc123", + "data_source": "default.http", + "properties": null, + "allowed_ip_addresses": "127.0.0.1/32", + "kafka_raw_data_topic": "test.rawdata", + "kafka_parsed_data_topic": "test.parseddata", + "kafka_group_id": "default_dev", + "host": 1 + }, + { + "id": 2, + "created_at": "2023-10-01T15:00:00.042000+03:00", + "updated_at": "2023-10-01T15:00:00.042000+03:00", + "endpoint_path": "/api/v1/digita", + "http_request_handler": "endpoints.digita.aiothingpark", + "auth_token": "abc123", + "data_source": "digita.thingpark.http", + "properties": null, + "allowed_ip_addresses": "", + "kafka_raw_data_topic": "digita.rawdata", + "kafka_parsed_data_topic": "digita.parsed", + "kafka_group_id": "digita_dev", + "host": 1 + } + ], + "created_at": "2023-10-01T15:00:00.042000+03:00", + "updated_at": "2023-10-01T15:00:00.042000+03:00" +} diff --git a/tests/test_api2.py b/tests/test_api2.py index 7954571..6b06130 100644 --- a/tests/test_api2.py +++ b/tests/test_api2.py @@ -1,6 +1,7 @@ import logging import os from datetime import datetime, timedelta, timezone + import httpx logging.basicConfig(level=logging.INFO) From d80ba76c6e90cab8b2a48d9cff8b4bed6a21e8c5 Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Thu, 30 May 2024 21:49:02 +0300 Subject: [PATCH 6/6] feat: upgrade Python, parameterize base images, optimize layers --- Dockerfile | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index f2362bf..bdfffea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,38 @@ -# mittaridatapumppu-endpoint +# syntax=docker/dockerfile:1 -FROM python:3.11-alpine +ARG PYTHON_VERSION="3.12" +ARG ALPINE_VERSION="3.19" + +FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} as build ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 - -RUN addgroup -S app && adduser -S app -G app -WORKDIR /home/app +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # Install requirements to build aiokafka -RUN apk add --no-cache \ - gcc \ - python3-dev \ - libc-dev \ - zlib-dev +RUN --mount=type=cache,target=/var/cache/apk \ + apk add gcc python3-dev libc-dev zlib-dev # Copy and install requirements only first to cache the dependency layer -COPY --chown=app:app requirements.txt . -RUN pip install --no-cache-dir --no-compile --upgrade -r requirements.txt +RUN pip install uv + +COPY pyproject.toml ./ +RUN --mount=type=cache,target=/root/.cache/uv \ +uv venv $VIRTUAL_ENV && \ +uv pip install -r pyproject.toml + +FROM python:3.12-alpine + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" +RUN addgroup -S app && adduser -S app -G app +WORKDIR /home/app + +COPY --from=build --chown=app:app $VIRTUAL_ENV $VIRTUAL_ENV COPY --chown=app:app endpoint/ ./endpoint COPY --chown=app:app endpoints/ ./endpoints @@ -28,6 +42,5 @@ RUN chgrp -R 0 /home/app && \ USER app -HEALTHCHECK CMD wget --no-verbose --tries=1 --spider localhost:8000/liveness || exit -CMD ["uvicorn", "endpoint.endpoint:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] EXPOSE 8000/tcp +CMD ["uvicorn", "endpoint.endpoint:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"]