Merge pull request Aiven-Open#556 from aiven/fleshgrinder/basic-build…

…-system

build: Add Basic Build System

.github/workflows/lint.yml

@@ -10,16 +10,22 @@ concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref }}
  cancel-in-progress: true
 
+env:
+ FORCE_COLOR: 1
+ PIP_PROGRESS_BAR: off
+ PYTHONUNBUFFERED: 1
+
 jobs:
  lint:
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v3
  - uses: actions/setup-python@v4
  with:
+ cache: pip
  python-version: '3.11'
  # required for pylint
- - run: make karapace/version.py
+ - run: make version
  - run: pip install pre-commit
  - uses: actions/cache@v3
  with:

.github/workflows/tests.yml

@@ -2,56 +2,43 @@ name: Test Suite
 
 on:
  pull_request:
- types: [opened, synchronize, reopened]
+ types: [ opened, synchronize, reopened ]
  push:
- branches:
- - main
+ branches: [ main ]
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref }}
+ cancel-in-progress: true
+
+env:
+ FORCE_COLOR: 1
+ PIP_PROGRESS_BAR: off
+ PYTHONUNBUFFERED: 1
 
 jobs:
  tests:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+ python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
  steps:
  - uses: actions/checkout@v3
 
  - name: Set up Python ${{ matrix.python-version }}
  uses: actions/setup-python@v4
  with:
+ cache: pip
  python-version: ${{ matrix.python-version }}
 
- - name: Install libsnappy-dev (python-snappy legacy-install-failure on Python 3.11)
- run: sudo apt install libsnappy-dev
-
- - name: Install dependencies
- run: python -m pip install -r requirements-dev.txt
-
- - name: Extract Protobuf Version
- id: protoc
- run: pip freeze | grep -m1 '^protobuf==' | cut -d= -f3 | xargs printf 'version=%s' >>"$GITHUB_OUTPUT"
-
- - name: Install Protobuf Compiler
- uses: arduino/setup-protoc@v1
- with:
- version: ${{ steps.protoc.outputs.version }}
- repo-token: ${{ secrets.GITHUB_TOKEN }}
-
- # needed by both unit and integration tests
- - name: Generate version.py
- run: make karapace/version.py
-
- - name: Execute unit-tests
- timeout-minutes: 2
- run: python3 -m pytest -s -vvv tests/unit/
-
- - name: Execute integration-tests
- timeout-minutes: 30
- run: python3 -m pytest -s -vvv tests/integration/ --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log
+ - run: make install version
+ - run: make unit-tests
+ - run: make integration-tests
+ env:
+ PYTEST_ARGS: --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log
 
  - name: Archive logs
  uses: actions/upload-artifact@v3
  if: ${{ always() }}
  with:
-  name: logs ${{ matrix.python-version }}
-  path: /tmp/ci-logs
+ name: karapace-integration-test-logs-${{ matrix.python-version }}
+ path: /tmp/ci-logs

.python-version

@@ -0,0 +1 @@
+3.7

GNUmakefile

@@ -0,0 +1,75 @@
+VENV_DIR ?= $(CURDIR)/venv
+PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
+PYTHON ?= python3
+ifdef CI
+PYENV ?= $(PYTHON)
+else
+PYENV ?= pyenv exec python
+endif
+
+export PATH := $(VENV_DIR)/bin:$(PATH)
+export PS4 := \e[0m\e[32m==> \e[0m
+export LC_ALL := C
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+SHELL := bash
+.SHELLFLAGS := -euxo pipefail -O globstar -c
+.ONESHELL:
+.SILENT:
+.SUFFIXES:
+
+.PHONY: all
+all: version
+
+.PHONY: venv
+venv: venv/.make
+venv/.make:
+ rm -fr '$(VENV_DIR)'
+ $(PYENV) -m venv '$(VENV_DIR)'
+ $(PIP) install --upgrade pip
+ touch '$(@)'
+
+.PHONY: install
+install: venv/.deps
+venv/.deps: requirements-dev.txt requirements.txt | venv/.make
+ set +x
+ source ./bin/get-java
+ source ./bin/get-protoc
+ source ./bin/get-snappy
+ set -x
+ $(PIP) install -r '$(<)' --use-pep517
+ touch '$(@)'
+
+.PHONY: version
+version: karapace/version.py
+karapace/version.py: version.py | venv/.make
+ $(PYTHON) '$(<)' '$(@)'
+
+.PHONY: test
+tests: unit-tests integration-tests
+
+.PHONY: unit-tests
+unit-tests: export PYTEST_ARGS ?=
+unit-tests: karapace/version.py venv/.deps
+ rm -fr runtime/*
+ $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
+ rm -fr runtime/*
+
+.PHONY: integration-tests
+unit-tests: export PYTEST_ARGS ?=
+integration-tests: karapace/version.py venv/.deps
+ rm -fr runtime/*
+ $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
+ rm -fr runtime/*
+
+.PHONY: clean
+clean:
+ rm -fr ./kafka_* ./*.egg-info/ ./dist/ ./karapace/version.py
+
+.PHONY: cleaner
+cleaner: clean
+ rm -fr ./.*cache*/
+
+.PHONY: cleanest
+cleanest: cleaner
+ rm -fr '$(VENV_DIR)'

README.rst

@@ -585,6 +585,52 @@ If you installed Karapace from the sources via ``python setup.py install``, it c
 
  pip uninstall karapace
 
+Development
+===========
+
+Execute ``make`` (GNU, usually ``gmake`` on BSD and Mac) to set up a ``venv``
+and install the required software for development. Use ``make unit-tests`` and
+``make integration-tests`` to execute the respective test suite, or simply
+``make test`` to execute both. You can set ``PYTEST_ARGS`` to customize the
+execution (e.g. ``PYTEST_ARGS=--maxfail=1 make test``).
+
+By default ``pyenv`` is expected to be installed and in ``PATH``. This ensures
+on all platforms that arbitrary Python versions can be used for development. It
+is possible to overwrite this by setting ``PYENV`` to something else (e.g.
+``PYENV=python3 make venv`` to simply use the global Python executable). The
+default Python version is defined in ``.python-version``.
+
+Karapace currently depends on various system software to be installed. The
+installation of these is automated for some operation systems, but not all. At
+the time of writing Java, the Protobuf Compiler, and the Snappy shared library
+are required to work with Karapace. You need to install them manually if your
+operating system is not supported by the automatic installation scripts. Note
+that the scripts are going to ask before installing any of these on your system.
+
+Note that Karapace requires a Protobuf Compiler older than 3.20.0, because
+3.20.0 introduces various breaking changes. The tests are going to fail if the
+Protobuf Compiler is newer than that. However, you can work around this locally
+by running ``pip install --upgrade protobuf`` in your venv. We are going to fix
+this soon.
+
+Note that the integration tests are currently not working on Mac. You can use
+Docker, just be sure to set ``VENV_DIR`` to a directory outside the working
+directory so that the container is not overwriting files from the host (e.g.
+``docker run --env VENV_DIR=/tmp/venv ...``).
+
+Note that the ``runtime`` directory **MUST** exist and that Karapace is going to
+fail if it does not. The ``runtime`` directory is also not cleaned between test
+runs, and left over data might result in failing tests. Use the ``make`` test
+targets that correctly clean the ``runtime`` directory without deleting it, but
+keep this in mind whenever you are not using ``make`` (e.g. running tests from
+your IDE).
+
+Note that the pre-commit checks are currently not working with the default
+Python version. This is because isort dropped Python 3.7 support. You have to
+use at least Python 3.8 for the pre-commit checks. Use ``pipx`` or ``brew`` or
+… to install pre-commit and use the global installation, there is also no
+dependency on it.
+
 License
 =======
 

bin/get-java

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if ! command -v java; then
+ exists() { command -v "$1" &>/dev/null; }
+ update=()
+ install=()
+ java_version=11
+
+ if exists brew; then
+ install+=(brew install "openjdk@$java_version")
+ else
+ if ((EUID != 0)) && exists sudo; then
+ update+=(sudo)
+ install+=(sudo)
+ fi
+
+ if exists apt-get; then
+ update+=(apt-get update)
+ install+=(apt-get install -y "openjdk-$java_version-jdk")
+ elif exists dnf; then
+ install+=(dnf install -y "java-$java_version-openjdk")
+ elif exists apk; then
+ install+=(apk add "openjdk$java_version")
+ else
+ printf '\e[0m\e[31mUnsupported system, install \e[1mjava\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2
+ exit 1
+ fi
+ fi
+
+ if [[ ${CI:-false} != true ]]; then
+ cmd=
+ ((${#update[@]} < 2)) || cmd="${update[*]} && "
+ cmd+=${install[*]}
+ read -ern1 -p "$cmd [Yn] "
+ if [[ $REPLY == [Nn]* ]]; then
+ exit 0
+ fi
+ fi
+
+ ((${#update[@]} < 2)) || "${update[@]}"
+ "${install[@]}"
+fi

bin/get-protoc

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if ! command -v protoc; then
+ exists() { command -v "$1" &>/dev/null; }
+ update=()
+ install=()
+
+ if exists brew; then
+ install+=(brew install protobuf)
+ else
+ if ((EUID != 0)) && exists sudo; then
+ update+=(sudo)
+ install+=(sudo)
+ fi
+
+ if exists apt-get; then
+ update+=(apt-get update)
+ install+=(apt-get install -y protobuf-compiler)
+ elif exists dnf; then
+ install+=(dnf install -y protobuf-compiler)
+ elif exists apk; then
+ install+=(apk add protoc)
+ else
+ printf '\e[0m\e[31mUnsupported system, install \e[1mprotoc\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2
+ exit 1
+ fi
+ fi
+
+ if [[ ${CI:-false} != true ]]; then
+ cmd=
+ ((${#update[@]} < 2)) || cmd="${update[*]} && "
+ cmd+=${install[*]}
+ read -ern1 -p "$cmd [Yn] "
+ if [[ $REPLY == [Nn]* ]]; then
+ exit 0
+ fi
+ fi
+
+ ((${#update[@]} < 2)) || "${update[@]}"
+ "${install[@]}"
+fi

bin/get-snappy

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+exists() { command -v "$1" &>/dev/null; }
+update=()
+install=()
+
+ask() {
+ if [[ ${CI:-false} != true ]]; then
+ cmd=
+ ((${#update[@]} < 2)) || cmd="${update[*]} && "
+ cmd+=${install[*]}
+ read -ern1 -p "$cmd [Yn] "
+ if [[ $REPLY == [Nn]* ]]; then
+ return 1
+ fi
+ fi
+}
+
+if exists brew; then
+ snappy_prefix() { brew --prefix snappy 2>/dev/null; }
+
+ # brew might return a path even though it does not exist
+ if ! prefix=$(snappy_prefix) || [[ ! -d $prefix ]]; then
+ install+=(brew install snappy)
+ if ask; then
+ "${install[@]}"
+ prefix=$(snappy_prefix)
+ else
+ exit 0
+ fi
+ fi
+
+ # https://stackoverflow.com/a/41707800/1251219
+ export CPPFLAGS="'-I$prefix/include' '-L$prefix/lib'"
+elif ! ldconfig -p | grep -F 'libsnappy.so '; then
+ if ((EUID != 0)) && exists sudo; then
+ update+=(sudo)
+ install+=(sudo)
+ fi
+
+ if exists apt-get; then
+ update+=(apt-get update -y)
+ install+=(apt-get install -y libsnappy-dev)
+ elif exists dnf; then
+ install+=(dnf install -y csnappy-devel)
+ elif exists apk; then
+ install+=(apk add --no-cache snappy-dev g++)
+ else
+ printf '\e[0m\e[33mUnsupported system, not installing \e[1msnappy\e[22m but carrying on in case pip is able to find the required libraries.\e[0m\n' >&2
+ exit 0
+ fi
+
+ if ask; then
+ ((${#update[@]} < 2)) || "${update[@]}"
+ "${install[@]}"
+ fi
+fi