Fix container image build

- Add pre-commit hook config
  - Formatter auto-formatted some of the code without functional changes
- Add gitignore
- Update requirements files
- Update to FVHIoT-python v0.4.1
- Fix missing dependencies to build aiokafka

.gitignore

@@ -0,0 +1,161 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+venv*/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/

.pre-commit-config.yaml

@@ -0,0 +1,30 @@
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v2.3.0
+ hooks:
+ - id: check-yaml
+ - id: end-of-file-fixer
+ - id: trailing-whitespace
+ - repo: https://github.com/psf/black
+ rev: 23.12.0
+ hooks:
+ - id: black
+ - repo: https://github.com/charliermarsh/ruff-pre-commit
+ rev: 'v0.1.8'
+ hooks:
+ - id: ruff
+ - repo: https://github.com/jazzband/pip-tools
+ rev: 7.3.0
+ hooks:
+ - id: pip-compile
+ name: pip-compile requirements.txt
+ args: [--strip-extras, --output-file=requirements.txt]
+ files: ^(pyproject\.toml|requirements\.txt)$
+ - id: pip-compile
+ name: pip-compile requirements-test.txt
+ args: [--extra=test, --strip-extras, --output-file=requirements-test.txt]
+ files: ^(pyproject\.toml|requirements-test\.txt)$
+ - id: pip-compile
+ name: pip-compile requirements-dev.txt
+ args: [--extra=dev, --strip-extras, --output-file=requirements-dev.txt]
+ files: ^(pyproject\.toml|requirements-dev\.txt)$

Dockerfile

@@ -8,6 +8,13 @@ ENV PYTHONUNBUFFERED 1
 RUN addgroup -S app && adduser -S app -G app
 WORKDIR /home/app
 
+# Install requirements to build aiokafka
+RUN apk add --no-cache \
+ gcc \
+ python3-dev \
+ libc-dev \
+ zlib-dev
+
 # Copy and install requirements only first to cache the dependency layer
 COPY --chown=app:app requirements.txt .
 RUN pip install --no-cache-dir --no-compile --upgrade -r requirements.txt

azure-build-main.yml

@@ -1,4 +1,4 @@
-# Continuous integration (CI) triggers cause a pipeline to run whenever you push 
+# Continuous integration (CI) triggers cause a pipeline to run whenever you push
 # an update to the specified branches or you push specified tags.
 trigger:
  batch: true
@@ -9,15 +9,15 @@ trigger:
  exclude:
  - README.md
 
-# Pull request (PR) triggers cause a pipeline to run whenever a pull request is 
-# opened with one of the specified target branches, or when updates are made to 
+# Pull request (PR) triggers cause a pipeline to run whenever a pull request is
+# opened with one of the specified target branches, or when updates are made to
 # such a pull request.
 #
-# GitHub creates a new ref when a pull request is created. The ref points to a 
-# merge commit, which is the merged code between the source and target branches 
+# GitHub creates a new ref when a pull request is created. The ref points to a
+# merge commit, which is the merged code between the source and target branches
 # of the pull request.
 #
-# Opt out of pull request validation 
+# Opt out of pull request validation
 pr: none
 
 # By default, use self-hosted agents

kafka2influxdb.py

@@ -80,8 +80,11 @@
 
 
 def parsed_data_to_influxdb_format(
- measurement_name, device_id, data: dict, extra_fields: dict = None,
- extra_tags: dict = None
+ measurement_name,
+ device_id,
+ data: dict,
+ extra_fields: dict = None,
+ extra_tags: dict = None,
 ) -> list:
  """
  Convert parsed data to InfluxDB datapoints format, see example above.
@@ -134,8 +137,9 @@ def main():
  logging.debug(pformat(data, width=120))
  measurement_name = data["device"]["parser_module"].split(".")[-1]
  device_id = data["device"]["device_id"]
- influxdb_datapoints = parsed_data_to_influxdb_format(measurement_name,
- device_id, data)
+ influxdb_datapoints = parsed_data_to_influxdb_format(
+ measurement_name, device_id, data
+ )
  with client.write_api(write_options=SYNCHRONOUS) as write_api:
  write_api.write(bucket, org, influxdb_datapoints)
  logging.info(f"Saved {len(influxdb_datapoints)} datapoints to InfluxDB")

pyproject.toml

@@ -1,5 +1,6 @@
-[tool.setuptools]
-py-modules = []
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
 
 [tool.ruff]
 line-length = 120
@@ -10,13 +11,10 @@ name = "mittaridatapumppu-persister"
 description = ""
 readme = "README.md"
 requires-python = ">=3.10"
-dynamic = ["version"]
-
+version = "v0.1.0"
 dependencies = [
- "aiokafka",
- "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.0.zip",
+ "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip",
  "influxdb-client",
- "kafka-python",
  "sentry-asgi",
 ]
 
@@ -31,3 +29,7 @@ dev = [
  "pytest-cov",
  "ruff",
 ]
+test = [
+ "pytest",
+ "requests"
+]

requirements-dev.txt

@@ -2,12 +2,10 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-# pip-compile --extra=dev --output-file=requirements-dev.txt
+# pip-compile --extra=dev --output-file=requirements-dev.txt --strip-extras
 #
 aiokafka==0.8.1
- # via
- # fvhiot
- # mittaridatapumppu-persister (pyproject.toml)
+ # via fvhiot
 async-timeout==4.0.3
  # via aiokafka
 autoflake==2.2.1
@@ -23,7 +21,7 @@ cfgv==3.4.0
  # via pre-commit
 click==8.1.7
  # via black
-coverage[toml]==7.3.2
+coverage==7.3.2
  # via
  # coverage
  # pytest-cov
@@ -33,7 +31,7 @@ filelock==3.13.1
  # via virtualenv
 flake8==6.1.0
  # via pep8-naming
-fvhiot[kafka] @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.0.zip
+fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip
  # via mittaridatapumppu-persister (pyproject.toml)
 identify==2.5.31
  # via pre-commit
@@ -42,9 +40,7 @@ influxdb-client==1.38.0
 iniconfig==2.0.0
  # via pytest
 kafka-python==2.0.2
- # via
- # aiokafka
- # mittaridatapumppu-persister (pyproject.toml)
+ # via aiokafka
 mccabe==0.7.0
  # via flake8
 msgpack==1.0.7

requirements-test.txt

@@ -0,0 +1,58 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+# pip-compile --extra=test --output-file=requirements-test.txt --strip-extras
+#
+aiokafka==0.10.0
+ # via fvhiot
+async-timeout==4.0.3
+ # via aiokafka
+certifi==2023.11.17
+ # via
+ # fvhiot
+ # influxdb-client
+ # requests
+ # sentry-sdk
+charset-normalizer==3.3.2
+ # via requests
+fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip
+ # via mittaridatapumppu-persister (pyproject.toml)
+idna==3.6
+ # via requests
+influxdb-client==1.39.0
+ # via mittaridatapumppu-persister (pyproject.toml)
+iniconfig==2.0.0
+ # via pytest
+msgpack==1.0.7
+ # via fvhiot
+packaging==23.2
+ # via
+ # aiokafka
+ # pytest
+pluggy==1.3.0
+ # via pytest
+pytest==7.4.3
+ # via mittaridatapumppu-persister (pyproject.toml)
+python-dateutil==2.8.2
+ # via influxdb-client
+reactivex==4.0.4
+ # via influxdb-client
+requests==2.31.0
+ # via mittaridatapumppu-persister (pyproject.toml)
+sentry-asgi==0.2.0
+ # via mittaridatapumppu-persister (pyproject.toml)
+sentry-sdk==1.39.1
+ # via sentry-asgi
+six==1.16.0
+ # via python-dateutil
+typing-extensions==4.9.0
+ # via reactivex
+urllib3==2.1.0
+ # via
+ # influxdb-client
+ # requests
+ # sentry-sdk
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools