Skip to content

Commit

Permalink
Fix container image build
Browse files Browse the repository at this point in the history
- Add pre-commit hook config
  - Formatter auto-formatted some of the code without functional changes
- Add gitignore
- Update requirements files
- Update to FVHIoT-python v0.4.1
- Fix missing dependencies to build aiokafka
  • Loading branch information
laurigates committed Dec 20, 2023
1 parent 0da8664 commit 3f7ee1e
Show file tree
Hide file tree
Showing 9 changed files with 288 additions and 34 deletions.
161 changes: 161 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock

# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
venv*/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
30 changes: 30 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 23.12.0
hooks:
- id: black
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: 'v0.1.8'
hooks:
- id: ruff
- repo: https://github.com/jazzband/pip-tools
rev: 7.3.0
hooks:
- id: pip-compile
name: pip-compile requirements.txt
args: [--strip-extras, --output-file=requirements.txt]
files: ^(pyproject\.toml|requirements\.txt)$
- id: pip-compile
name: pip-compile requirements-test.txt
args: [--extra=test, --strip-extras, --output-file=requirements-test.txt]
files: ^(pyproject\.toml|requirements-test\.txt)$
- id: pip-compile
name: pip-compile requirements-dev.txt
args: [--extra=dev, --strip-extras, --output-file=requirements-dev.txt]
files: ^(pyproject\.toml|requirements-dev\.txt)$
7 changes: 7 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@ ENV PYTHONUNBUFFERED 1
RUN addgroup -S app && adduser -S app -G app
WORKDIR /home/app

# Install requirements to build aiokafka
RUN apk add --no-cache \
gcc \
python3-dev \
libc-dev \
zlib-dev

# Copy and install requirements only first to cache the dependency layer
COPY --chown=app:app requirements.txt .
RUN pip install --no-cache-dir --no-compile --upgrade -r requirements.txt
Expand Down
12 changes: 6 additions & 6 deletions azure-build-main.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Continuous integration (CI) triggers cause a pipeline to run whenever you push
# Continuous integration (CI) triggers cause a pipeline to run whenever you push
# an update to the specified branches or you push specified tags.
trigger:
batch: true
Expand All @@ -9,15 +9,15 @@ trigger:
exclude:
- README.md

# Pull request (PR) triggers cause a pipeline to run whenever a pull request is
# opened with one of the specified target branches, or when updates are made to
# Pull request (PR) triggers cause a pipeline to run whenever a pull request is
# opened with one of the specified target branches, or when updates are made to
# such a pull request.
#
# GitHub creates a new ref when a pull request is created. The ref points to a
# merge commit, which is the merged code between the source and target branches
# GitHub creates a new ref when a pull request is created. The ref points to a
# merge commit, which is the merged code between the source and target branches
# of the pull request.
#
# Opt out of pull request validation
# Opt out of pull request validation
pr: none

# By default, use self-hosted agents
Expand Down
12 changes: 8 additions & 4 deletions kafka2influxdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,11 @@


def parsed_data_to_influxdb_format(
measurement_name, device_id, data: dict, extra_fields: dict = None,
extra_tags: dict = None
measurement_name,
device_id,
data: dict,
extra_fields: dict = None,
extra_tags: dict = None,
) -> list:
"""
Convert parsed data to InfluxDB datapoints format, see example above.
Expand Down Expand Up @@ -134,8 +137,9 @@ def main():
logging.debug(pformat(data, width=120))
measurement_name = data["device"]["parser_module"].split(".")[-1]
device_id = data["device"]["device_id"]
influxdb_datapoints = parsed_data_to_influxdb_format(measurement_name,
device_id, data)
influxdb_datapoints = parsed_data_to_influxdb_format(
measurement_name, device_id, data
)
with client.write_api(write_options=SYNCHRONOUS) as write_api:
write_api.write(bucket, org, influxdb_datapoints)
logging.info(f"Saved {len(influxdb_datapoints)} datapoints to InfluxDB")
Expand Down
16 changes: 9 additions & 7 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[tool.setuptools]
py-modules = []
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"

[tool.ruff]
line-length = 120
Expand All @@ -10,13 +11,10 @@ name = "mittaridatapumppu-persister"
description = ""
readme = "README.md"
requires-python = ">=3.10"
dynamic = ["version"]

version = "v0.1.0"
dependencies = [
"aiokafka",
"fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.0.zip",
"fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip",
"influxdb-client",
"kafka-python",
"sentry-asgi",
]

Expand All @@ -31,3 +29,7 @@ dev = [
"pytest-cov",
"ruff",
]
test = [
"pytest",
"requests"
]
14 changes: 5 additions & 9 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --extra=dev --output-file=requirements-dev.txt
# pip-compile --extra=dev --output-file=requirements-dev.txt --strip-extras
#
aiokafka==0.8.1
# via
# fvhiot
# mittaridatapumppu-persister (pyproject.toml)
# via fvhiot
async-timeout==4.0.3
# via aiokafka
autoflake==2.2.1
Expand All @@ -23,7 +21,7 @@ cfgv==3.4.0
# via pre-commit
click==8.1.7
# via black
coverage[toml]==7.3.2
coverage==7.3.2
# via
# coverage
# pytest-cov
Expand All @@ -33,7 +31,7 @@ filelock==3.13.1
# via virtualenv
flake8==6.1.0
# via pep8-naming
fvhiot[kafka] @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.0.zip
fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip
# via mittaridatapumppu-persister (pyproject.toml)
identify==2.5.31
# via pre-commit
Expand All @@ -42,9 +40,7 @@ influxdb-client==1.38.0
iniconfig==2.0.0
# via pytest
kafka-python==2.0.2
# via
# aiokafka
# mittaridatapumppu-persister (pyproject.toml)
# via aiokafka
mccabe==0.7.0
# via flake8
msgpack==1.0.7
Expand Down
58 changes: 58 additions & 0 deletions requirements-test.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --extra=test --output-file=requirements-test.txt --strip-extras
#
aiokafka==0.10.0
# via fvhiot
async-timeout==4.0.3
# via aiokafka
certifi==2023.11.17
# via
# fvhiot
# influxdb-client
# requests
# sentry-sdk
charset-normalizer==3.3.2
# via requests
fvhiot @ https://github.com/ForumViriumHelsinki/FVHIoT-python/archive/refs/tags/v0.4.1.zip
# via mittaridatapumppu-persister (pyproject.toml)
idna==3.6
# via requests
influxdb-client==1.39.0
# via mittaridatapumppu-persister (pyproject.toml)
iniconfig==2.0.0
# via pytest
msgpack==1.0.7
# via fvhiot
packaging==23.2
# via
# aiokafka
# pytest
pluggy==1.3.0
# via pytest
pytest==7.4.3
# via mittaridatapumppu-persister (pyproject.toml)
python-dateutil==2.8.2
# via influxdb-client
reactivex==4.0.4
# via influxdb-client
requests==2.31.0
# via mittaridatapumppu-persister (pyproject.toml)
sentry-asgi==0.2.0
# via mittaridatapumppu-persister (pyproject.toml)
sentry-sdk==1.39.1
# via sentry-asgi
six==1.16.0
# via python-dateutil
typing-extensions==4.9.0
# via reactivex
urllib3==2.1.0
# via
# influxdb-client
# requests
# sentry-sdk

# The following packages are considered to be unsafe in a requirements file:
# setuptools
Loading

0 comments on commit 3f7ee1e

Please sign in to comment.