From 48694d69b7294f406c41ee93a8a53f79554cc02a Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Wed, 11 Oct 2023 14:24:07 +0300 Subject: [PATCH 1/4] Update dockerfile and compose config - Removed unneeded variables - Added labels to dockerfile to link image to mittaridatapumppu repository --- Dockerfile | 11 +++++++++++ docker-compose.yml | 18 +++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6805499..1625e5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,20 @@ FROM python:3.11-alpine +LABEL org.opencontainers.image.source=https://github.com/city-of-helsinki/mittaridatapumppu +LABEL org.opencontainers.image.description="Mittaridatapumppu Endpoint" +LABEL org.opencontainers.image.licenses="Apache License 2.0" + ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 +# Device registry config +ENV DEVREG_ENDPOINTS_URL "http://devreg:8000/api/v1/hosts/localhost/" +ENV DEVREG_API_TOKEN "abcdef1234567890abcdef1234567890abcdef12" + +# Kafka config +ENV KAFKA_BOOTSTRAP_SERVERS "kafka:9092" + RUN addgroup -S app && adduser -S app -G app WORKDIR /home/app diff --git a/docker-compose.yml b/docker-compose.yml index 7f1507d..fb2f000 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.8' services: - endpoint-digita: + endpoint: image: ghcr.io/city-of-helsinki/mittaridatapumppu-endpoint build: . ports: @@ -12,21 +12,13 @@ services: kafka: condition: service_healthy environment: - # Endpoint config - ALLOWED_IP_ADDRESSES: "127.0.0.1" - AUTH_TOKEN: "abcd1234" - DATA_SOURCE_NAME: "digita.thingpark.http" - ENDPOINT_PATH: "/digita/v2" - HTTP_REQUESTHANDLER: "endpoints.digita.aiothingpark" + DEVREG_ENDPOINTS_URL: "http://devreg:8000/api/v1/hosts/localhost/" + DEVREG_API_TOKEN: "abcdef1234567890abcdef1234567890abcdef12" # Kafka client config - KAFKA_HOST: "kafka" - KAFKA_PORT: 9092 KAFKA_BOOTSTRAP_SERVERS: "kafka:9092" - KAFKA_GROUP_ID: "digita_dev" - KAFKA_PARSED_DATA_TOPIC_NAME: "digita.parseddata" - KAFKA_RAW_DATA_TOPIC_NAME: "digita.rawdata" # Debug config - LOG_LEVEL: "DEBUG" + UVICORN_LOG_LEVEL: "debug" + UVICORN_RELOAD: "true" DEBUG: 1 kafka: image: bitnami/kafka:3.4 From caa9f3f26edacc050f25722ffb1871754ccfdcff Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Wed, 11 Oct 2023 14:28:09 +0300 Subject: [PATCH 2/4] Add gitignore Copied from mittaridatapumppu-deviceregistry --- .gitignore | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc0fcfe --- /dev/null +++ b/.gitignore @@ -0,0 +1,161 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv*/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ From 74936e78abc0824b45aee7e8c1a8c68930c948f0 Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Wed, 11 Oct 2023 14:30:29 +0300 Subject: [PATCH 3/4] Update pyproject config and dependencies - Removed workaround config related to a setuptools issue --- pyproject.toml | 11 +-- requirements-dev.txt | 161 +++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 13 +--- 3 files changed, 164 insertions(+), 21 deletions(-) create mode 100644 requirements-dev.txt diff --git a/pyproject.toml b/pyproject.toml index 899f71e..65b5f36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,3 @@ -[tool.setuptools] -py-modules = [] - -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - [tool.ruff] line-length = 120 target-version = "py311" @@ -17,11 +10,9 @@ requires-python = ">=3.10" dynamic = ["version"] dependencies = [ - "aiokafka", - "fastapi", "fvhiot@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.3.2.zip", + "fastapi", "httpx", - "kafka-python", "python-multipart", "sentry-asgi", "uvicorn", diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..99b19f0 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,161 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev.txt +# +annotated-types==0.5.0 + # via pydantic +anyio==3.7.1 + # via + # fastapi + # httpcore + # starlette +autoflake==2.2.1 + # via mittaridatapumppu-endpoint (pyproject.toml) +autopep8==2.0.4 + # via mittaridatapumppu-endpoint (pyproject.toml) +black==23.9.1 + # via mittaridatapumppu-endpoint (pyproject.toml) +blinker==1.6.2 + # via flask +certifi==2023.7.22 + # via + # fvhiot + # httpcore + # httpx + # sentry-sdk +cfgv==3.4.0 + # via pre-commit +click==8.1.7 + # via + # black + # flask + # uvicorn +coverage[toml]==7.3.2 + # via pytest-cov +distlib==0.3.7 + # via virtualenv +fastapi==0.103.2 + # via mittaridatapumppu-endpoint (pyproject.toml) +filelock==3.12.4 + # via virtualenv +flake8==6.1.0 + # via + # mittaridatapumppu-endpoint (pyproject.toml) + # pep8-naming +flask==3.0.0 + # via fvhiot +fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.3.2.zip + # via mittaridatapumppu-endpoint (pyproject.toml) +gunicorn==21.2.0 + # via mittaridatapumppu-endpoint (pyproject.toml) +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==0.18.0 + # via httpx +httpx==0.25.0 + # via mittaridatapumppu-endpoint (pyproject.toml) +identify==2.5.30 + # via pre-commit +idna==3.4 + # via + # anyio + # httpx +iniconfig==2.0.0 + # via pytest +isort==5.12.0 + # via mittaridatapumppu-endpoint (pyproject.toml) +itsdangerous==2.1.2 + # via flask +jinja2==3.1.2 + # via flask +kafka-python==2.0.2 + # via fvhiot +markupsafe==2.1.3 + # via + # jinja2 + # werkzeug +mccabe==0.7.0 + # via flake8 +msgpack==1.0.7 + # via fvhiot +mypy-extensions==1.0.0 + # via black +nodeenv==1.8.0 + # via pre-commit +packaging==23.2 + # via + # black + # gunicorn + # pytest +pathspec==0.11.2 + # via black +pep8-naming==0.13.3 + # via mittaridatapumppu-endpoint (pyproject.toml) +platformdirs==3.11.0 + # via + # black + # virtualenv +pluggy==1.3.0 + # via pytest +pre-commit==3.4.0 + # via mittaridatapumppu-endpoint (pyproject.toml) +pycodestyle==2.11.0 + # via + # autopep8 + # flake8 +pydantic==2.4.2 + # via + # fastapi + # mittaridatapumppu-endpoint (pyproject.toml) +pydantic-core==2.10.1 + # via pydantic +pyflakes==3.1.0 + # via + # autoflake + # flake8 +pytest==7.4.2 + # via + # mittaridatapumppu-endpoint (pyproject.toml) + # pytest-asyncio + # pytest-cov +pytest-asyncio==0.21.1 + # via mittaridatapumppu-endpoint (pyproject.toml) +pytest-cov==4.1.0 + # via mittaridatapumppu-endpoint (pyproject.toml) +python-multipart==0.0.6 + # via mittaridatapumppu-endpoint (pyproject.toml) +pyyaml==6.0.1 + # via pre-commit +sentry-asgi==0.2.0 + # via mittaridatapumppu-endpoint (pyproject.toml) +sentry-sdk==1.31.0 + # via sentry-asgi +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx +starlette==0.27.0 + # via + # fastapi + # fvhiot +typing-extensions==4.8.0 + # via + # fastapi + # pydantic + # pydantic-core +urllib3==2.0.6 + # via sentry-sdk +uvicorn==0.23.2 + # via mittaridatapumppu-endpoint (pyproject.toml) +virtualenv==20.24.5 + # via pre-commit +werkzeug==3.0.0 + # via flask + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements.txt b/requirements.txt index ab861d9..cfa8baf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,8 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile pyproject.toml +# pip-compile # -aiokafka==0.8.1 - # via mittaridatapumppu-endpoint (pyproject.toml) annotated-types==0.5.0 # via pydantic anyio==3.7.1 @@ -13,8 +11,6 @@ anyio==3.7.1 # fastapi # httpcore # starlette -async-timeout==4.0.3 - # via aiokafka blinker==1.6.2 # via flask certifi==2023.7.22 @@ -50,18 +46,13 @@ itsdangerous==2.1.2 jinja2==3.1.2 # via flask kafka-python==2.0.2 - # via - # aiokafka - # fvhiot - # mittaridatapumppu-endpoint (pyproject.toml) + # via fvhiot markupsafe==2.1.3 # via # jinja2 # werkzeug msgpack==1.0.6 # via fvhiot -packaging==23.1 - # via aiokafka pydantic==2.4.1 # via fastapi pydantic-core==2.10.1 From d2bab652b129b323e6f0d12e1cec39b981cc9d7a Mon Sep 17 00:00:00 2001 From: Lauri Gates Date: Wed, 11 Oct 2023 14:52:42 +0300 Subject: [PATCH 4/4] Add services required by tests Work-in-progress, still need to run migrations in deviceregistry service. --- .github/workflows/test-endpoint.yml | 31 +++++++++++++++++++++++------ pyproject.toml | 6 ++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-endpoint.yml b/.github/workflows/test-endpoint.yml index 2f9a975..0003775 100644 --- a/.github/workflows/test-endpoint.yml +++ b/.github/workflows/test-endpoint.yml @@ -23,6 +23,19 @@ jobs: python-version: ["3.10", "3.11"] services: # Label used to access the service container + db: + # Docker Hub image + image: postgis/postgis:16-3.4-alpine + # Provide the password for postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + # Label used to access the service container kafka: # Docker Hub image image: bitnami/kafka:3.4 @@ -48,6 +61,13 @@ jobs: ports: # Tests run directly on the runner so we have to map the port - 9092:9092 + deviceregistry: + image: ghcr.io/city-of-helsinki/mittaridatapumppu-deviceregistry:latest + env: + DJANGO_SETTINGS_MODULE: "deviceregistry.settings" + ports: + - 8001:8000 + # TODO:run migrations here steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -68,13 +88,12 @@ jobs: ruff check --output-format=github . - name: Test with pytest env: - PYTEST_ADDOPTS: "--color=yes" - AUTH_TOKEN: "abcd1234" - KAFKA_HOST: "localhost" - KAFKA_PORT: 9092 + DEVREG_ENDPOINTS_URL: "http://localhost:8001/api/v1/hosts/localhost/" + DEVREG_API_TOKEN: "abcdef1234567890abcdef1234567890abcdef12" + # Kafka client config KAFKA_BOOTSTRAP_SERVERS: "localhost:9092" - LOG_LEVEL: "DEBUG" + UVICORN_LOG_LEVEL: "DEBUG" DEBUG: 1 run: | uvicorn app:app --host 0.0.0.0 --port 8000 --proxy-headers && - pytest -v tests/test_api.py + pytest diff --git a/pyproject.toml b/pyproject.toml index 65b5f36..69d6933 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,3 +37,9 @@ test = [ "pytest", "requests" ] #pip-compile --resolver=backtracking --extra test pyproject.toml + +[tool.pytest.ini_options] +log_cli = 1 +log_cli_level = 20 +testpaths = ["tests"] +addopts = ["--color=yes"]