diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0ef15d1c5..5a1c55743 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,6 +10,11 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + PIP_PROGRESS_BAR: off + PYTHONUNBUFFERED: 1 + jobs: lint: runs-on: ubuntu-latest @@ -17,9 +22,10 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: + cache: pip python-version: '3.11' # required for pylint - - run: make karapace/version.py + - run: make version - run: pip install pre-commit - uses: actions/cache@v3 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 50519edf0..fe44ec46c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,56 +2,43 @@ name: Test Suite on: pull_request: - types: [opened, synchronize, reopened] + types: [ opened, synchronize, reopened ] push: - branches: - - main + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + PIP_PROGRESS_BAR: off + PYTHONUNBUFFERED: 1 jobs: tests: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: + cache: pip python-version: ${{ matrix.python-version }} - - name: Install libsnappy-dev (python-snappy legacy-install-failure on Python 3.11) - run: sudo apt install libsnappy-dev - - - name: Install dependencies - run: python -m pip install -r requirements-dev.txt - - - name: Extract Protobuf Version - id: protoc - run: pip freeze | grep -m1 '^protobuf==' | cut -d= -f3 | xargs printf 'version=%s' >>"$GITHUB_OUTPUT" - - - name: Install Protobuf Compiler - uses: arduino/setup-protoc@v1 - with: - version: ${{ steps.protoc.outputs.version }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - # needed by both unit and integration tests - - name: Generate version.py - run: make karapace/version.py - - - name: Execute unit-tests - timeout-minutes: 2 - run: python3 -m pytest -s -vvv tests/unit/ - - - name: Execute integration-tests - timeout-minutes: 30 - run: python3 -m pytest -s -vvv tests/integration/ --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log + - run: make install version + - run: make unit-tests + - run: make integration-tests + env: + PYTEST_ARGS: --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log - name: Archive logs uses: actions/upload-artifact@v3 if: ${{ always() }} with: - name: logs ${{ matrix.python-version }} - path: /tmp/ci-logs + name: karapace-integration-test-logs-${{ matrix.python-version }} + path: /tmp/ci-logs diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..475ba515c --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.7 diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 000000000..84831c1cb --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,75 @@ +VENV_DIR ?= $(CURDIR)/venv +PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv +PYTHON ?= python3 +ifdef CI +PYENV ?= $(PYTHON) +else +PYENV ?= pyenv exec python +endif + +export PATH := $(VENV_DIR)/bin:$(PATH) +export PS4 := \e[0m\e[32m==> \e[0m +export LC_ALL := C +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules +SHELL := bash +.SHELLFLAGS := -euxo pipefail -O globstar -c +.ONESHELL: +.SILENT: +.SUFFIXES: + +.PHONY: all +all: version + +.PHONY: venv +venv: venv/.make +venv/.make: + rm -fr '$(VENV_DIR)' + $(PYENV) -m venv '$(VENV_DIR)' + $(PIP) install --upgrade pip + touch '$(@)' + +.PHONY: install +install: venv/.deps +venv/.deps: requirements-dev.txt requirements.txt | venv/.make + set +x + source ./bin/get-java + source ./bin/get-protoc + source ./bin/get-snappy + set -x + $(PIP) install -r '$(<)' --use-pep517 + touch '$(@)' + +.PHONY: version +version: karapace/version.py +karapace/version.py: version.py | venv/.make + $(PYTHON) '$(<)' '$(@)' + +.PHONY: test +tests: unit-tests integration-tests + +.PHONY: unit-tests +unit-tests: export PYTEST_ARGS ?= +unit-tests: karapace/version.py venv/.deps + rm -fr runtime/* + $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/ + rm -fr runtime/* + +.PHONY: integration-tests +unit-tests: export PYTEST_ARGS ?= +integration-tests: karapace/version.py venv/.deps + rm -fr runtime/* + $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/ + rm -fr runtime/* + +.PHONY: clean +clean: + rm -fr ./kafka_* ./*.egg-info/ ./dist/ ./karapace/version.py + +.PHONY: cleaner +cleaner: clean + rm -fr ./.*cache*/ + +.PHONY: cleanest +cleanest: cleaner + rm -fr '$(VENV_DIR)' diff --git a/Makefile b/Makefile deleted file mode 100644 index 2c8b20ba4..000000000 --- a/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -PYTHON = python3 - -default: karapace/version.py - -clean: - # remove all the versions of kafka - rm -rf kafka_* - # delete cache files - find . -iname '*.pyc' -delete - find . -iname '__pycache__' -delete - # delete packaging directories - rm -rf dist karapace.egg-info - # delete generate files - rm karapace/version.py - -karapace/version.py: version.py - $(PYTHON) $^ $@ diff --git a/README.rst b/README.rst index 8ad8fad98..e7c02fe6a 100644 --- a/README.rst +++ b/README.rst @@ -585,6 +585,52 @@ If you installed Karapace from the sources via ``python setup.py install``, it c pip uninstall karapace +Development +=========== + +Execute ``make`` (GNU, usually ``gmake`` on BSD and Mac) to set up a ``venv`` +and install the required software for development. Use ``make unit-tests`` and +``make integration-tests`` to execute the respective test suite, or simply +``make test`` to execute both. You can set ``PYTEST_ARGS`` to customize the +execution (e.g. ``PYTEST_ARGS=--maxfail=1 make test``). + +By default ``pyenv`` is expected to be installed and in ``PATH``. This ensures +on all platforms that arbitrary Python versions can be used for development. It +is possible to overwrite this by setting ``PYENV`` to something else (e.g. +``PYENV=python3 make venv`` to simply use the global Python executable). The +default Python version is defined in ``.python-version``. + +Karapace currently depends on various system software to be installed. The +installation of these is automated for some operation systems, but not all. At +the time of writing Java, the Protobuf Compiler, and the Snappy shared library +are required to work with Karapace. You need to install them manually if your +operating system is not supported by the automatic installation scripts. Note +that the scripts are going to ask before installing any of these on your system. + +Note that Karapace requires a Protobuf Compiler older than 3.20.0, because +3.20.0 introduces various breaking changes. The tests are going to fail if the +Protobuf Compiler is newer than that. However, you can work around this locally +by running ``pip install --upgrade protobuf`` in your venv. We are going to fix +this soon. + +Note that the integration tests are currently not working on Mac. You can use +Docker, just be sure to set ``VENV_DIR`` to a directory outside the working +directory so that the container is not overwriting files from the host (e.g. +``docker run --env VENV_DIR=/tmp/venv ...``). + +Note that the ``runtime`` directory **MUST** exist and that Karapace is going to +fail if it does not. The ``runtime`` directory is also not cleaned between test +runs, and left over data might result in failing tests. Use the ``make`` test +targets that correctly clean the ``runtime`` directory without deleting it, but +keep this in mind whenever you are not using ``make`` (e.g. running tests from +your IDE). + +Note that the pre-commit checks are currently not working with the default +Python version. This is because isort dropped Python 3.7 support. You have to +use at least Python 3.8 for the pre-commit checks. Use ``pipx`` or ``brew`` or +… to install pre-commit and use the global installation, there is also no +dependency on it. + License ======= diff --git a/bin/get-java b/bin/get-java new file mode 100755 index 000000000..493757bdf --- /dev/null +++ b/bin/get-java @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if ! command -v java; then + exists() { command -v "$1" &>/dev/null; } + update=() + install=() + java_version=11 + + if exists brew; then + install+=(brew install "openjdk@$java_version") + else + if ((EUID != 0)) && exists sudo; then + update+=(sudo) + install+=(sudo) + fi + + if exists apt-get; then + update+=(apt-get update) + install+=(apt-get install -y "openjdk-$java_version-jdk") + elif exists dnf; then + install+=(dnf install -y "java-$java_version-openjdk") + elif exists apk; then + install+=(apk add "openjdk$java_version") + else + printf '\e[0m\e[31mUnsupported system, install \e[1mjava\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2 + exit 1 + fi + fi + + if [[ ${CI:-false} != true ]]; then + cmd= + ((${#update[@]} < 2)) || cmd="${update[*]} && " + cmd+=${install[*]} + read -ern1 -p "$cmd [Yn] " + if [[ $REPLY == [Nn]* ]]; then + exit 0 + fi + fi + + ((${#update[@]} < 2)) || "${update[@]}" + "${install[@]}" +fi diff --git a/bin/get-protoc b/bin/get-protoc new file mode 100755 index 000000000..1d35a8b35 --- /dev/null +++ b/bin/get-protoc @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if ! command -v protoc; then + exists() { command -v "$1" &>/dev/null; } + update=() + install=() + + if exists brew; then + install+=(brew install protobuf) + else + if ((EUID != 0)) && exists sudo; then + update+=(sudo) + install+=(sudo) + fi + + if exists apt-get; then + update+=(apt-get update) + install+=(apt-get install -y protobuf-compiler) + elif exists dnf; then + install+=(dnf install -y protobuf-compiler) + elif exists apk; then + install+=(apk add protoc) + else + printf '\e[0m\e[31mUnsupported system, install \e[1mprotoc\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2 + exit 1 + fi + fi + + if [[ ${CI:-false} != true ]]; then + cmd= + ((${#update[@]} < 2)) || cmd="${update[*]} && " + cmd+=${install[*]} + read -ern1 -p "$cmd [Yn] " + if [[ $REPLY == [Nn]* ]]; then + exit 0 + fi + fi + + ((${#update[@]} < 2)) || "${update[@]}" + "${install[@]}" +fi diff --git a/bin/get-snappy b/bin/get-snappy new file mode 100755 index 000000000..f30ae5f45 --- /dev/null +++ b/bin/get-snappy @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exists() { command -v "$1" &>/dev/null; } +update=() +install=() + +ask() { + if [[ ${CI:-false} != true ]]; then + cmd= + ((${#update[@]} < 2)) || cmd="${update[*]} && " + cmd+=${install[*]} + read -ern1 -p "$cmd [Yn] " + if [[ $REPLY == [Nn]* ]]; then + return 1 + fi + fi +} + +if exists brew; then + snappy_prefix() { brew --prefix snappy 2>/dev/null; } + + # brew might return a path even though it does not exist + if ! prefix=$(snappy_prefix) || [[ ! -d $prefix ]]; then + install+=(brew install snappy) + if ask; then + "${install[@]}" + prefix=$(snappy_prefix) + else + exit 0 + fi + fi + + # https://stackoverflow.com/a/41707800/1251219 + export CPPFLAGS="'-I$prefix/include' '-L$prefix/lib'" +elif ! ldconfig -p | grep -F 'libsnappy.so '; then + if ((EUID != 0)) && exists sudo; then + update+=(sudo) + install+=(sudo) + fi + + if exists apt-get; then + update+=(apt-get update -y) + install+=(apt-get install -y libsnappy-dev) + elif exists dnf; then + install+=(dnf install -y csnappy-devel) + elif exists apk; then + install+=(apk add --no-cache snappy-dev g++) + else + printf '\e[0m\e[33mUnsupported system, not installing \e[1msnappy\e[22m but carrying on in case pip is able to find the required libraries.\e[0m\n' >&2 + exit 0 + fi + + if ask; then + ((${#update[@]} < 2)) || "${update[@]}" + "${install[@]}" + fi +fi