From 5f2acd71cdfd0faac50c862b5d032a04a33bd513 Mon Sep 17 00:00:00 2001 From: Richard Fussenegger Date: Fri, 3 Mar 2023 19:25:48 +0100 Subject: [PATCH] build: Add Basic Build System This adds a basic build system that allows developers to get started with Karapace without the need of figuring out what is required to work with the project on their own. The README documents how to use it, as well as all quirks that are known to me at this time. Closes #553 --- .github/workflows/lint.yml | 8 +++- .github/workflows/tests.yml | 53 ++++++++++---------------- .python-version | 1 + GNUmakefile | 75 +++++++++++++++++++++++++++++++++++++ Makefile | 17 --------- README.rst | 46 +++++++++++++++++++++++ bin/get-java | 43 +++++++++++++++++++++ bin/get-protoc | 42 +++++++++++++++++++++ bin/get-snappy | 58 ++++++++++++++++++++++++++++ 9 files changed, 292 insertions(+), 51 deletions(-) create mode 100644 .python-version create mode 100644 GNUmakefile delete mode 100644 Makefile create mode 100755 bin/get-java create mode 100755 bin/get-protoc create mode 100755 bin/get-snappy diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0ef15d1c5..5a1c55743 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,6 +10,11 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + PIP_PROGRESS_BAR: off + PYTHONUNBUFFERED: 1 + jobs: lint: runs-on: ubuntu-latest @@ -17,9 +22,10 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: + cache: pip python-version: '3.11' # required for pylint - - run: make karapace/version.py + - run: make version - run: pip install pre-commit - uses: actions/cache@v3 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 50519edf0..fe44ec46c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,56 +2,43 @@ name: Test Suite on: pull_request: - types: [opened, synchronize, reopened] + types: [ opened, synchronize, reopened ] push: - branches: - - main + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + PIP_PROGRESS_BAR: off + PYTHONUNBUFFERED: 1 jobs: tests: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: + cache: pip python-version: ${{ matrix.python-version }} - - name: Install libsnappy-dev (python-snappy legacy-install-failure on Python 3.11) - run: sudo apt install libsnappy-dev - - - name: Install dependencies - run: python -m pip install -r requirements-dev.txt - - - name: Extract Protobuf Version - id: protoc - run: pip freeze | grep -m1 '^protobuf==' | cut -d= -f3 | xargs printf 'version=%s' >>"$GITHUB_OUTPUT" - - - name: Install Protobuf Compiler - uses: arduino/setup-protoc@v1 - with: - version: ${{ steps.protoc.outputs.version }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - # needed by both unit and integration tests - - name: Generate version.py - run: make karapace/version.py - - - name: Execute unit-tests - timeout-minutes: 2 - run: python3 -m pytest -s -vvv tests/unit/ - - - name: Execute integration-tests - timeout-minutes: 30 - run: python3 -m pytest -s -vvv tests/integration/ --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log + - run: make install version + - run: make unit-tests + - run: make integration-tests + env: + PYTEST_ARGS: --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log - name: Archive logs uses: actions/upload-artifact@v3 if: ${{ always() }} with: - name: logs ${{ matrix.python-version }} - path: /tmp/ci-logs + name: karapace-integration-test-logs-${{ matrix.python-version }} + path: /tmp/ci-logs diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..475ba515c --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.7 diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 000000000..84831c1cb --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,75 @@ +VENV_DIR ?= $(CURDIR)/venv +PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv +PYTHON ?= python3 +ifdef CI +PYENV ?= $(PYTHON) +else +PYENV ?= pyenv exec python +endif + +export PATH := $(VENV_DIR)/bin:$(PATH) +export PS4 := \e[0m\e[32m==> \e[0m +export LC_ALL := C +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules +SHELL := bash +.SHELLFLAGS := -euxo pipefail -O globstar -c +.ONESHELL: +.SILENT: +.SUFFIXES: + +.PHONY: all +all: version + +.PHONY: venv +venv: venv/.make +venv/.make: + rm -fr '$(VENV_DIR)' + $(PYENV) -m venv '$(VENV_DIR)' + $(PIP) install --upgrade pip + touch '$(@)' + +.PHONY: install +install: venv/.deps +venv/.deps: requirements-dev.txt requirements.txt | venv/.make + set +x + source ./bin/get-java + source ./bin/get-protoc + source ./bin/get-snappy + set -x + $(PIP) install -r '$(<)' --use-pep517 + touch '$(@)' + +.PHONY: version +version: karapace/version.py +karapace/version.py: version.py | venv/.make + $(PYTHON) '$(<)' '$(@)' + +.PHONY: test +tests: unit-tests integration-tests + +.PHONY: unit-tests +unit-tests: export PYTEST_ARGS ?= +unit-tests: karapace/version.py venv/.deps + rm -fr runtime/* + $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/ + rm -fr runtime/* + +.PHONY: integration-tests +unit-tests: export PYTEST_ARGS ?= +integration-tests: karapace/version.py venv/.deps + rm -fr runtime/* + $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/ + rm -fr runtime/* + +.PHONY: clean +clean: + rm -fr ./kafka_* ./*.egg-info/ ./dist/ ./karapace/version.py + +.PHONY: cleaner +cleaner: clean + rm -fr ./.*cache*/ + +.PHONY: cleanest +cleanest: cleaner + rm -fr '$(VENV_DIR)' diff --git a/Makefile b/Makefile deleted file mode 100644 index 2c8b20ba4..000000000 --- a/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -PYTHON = python3 - -default: karapace/version.py - -clean: - # remove all the versions of kafka - rm -rf kafka_* - # delete cache files - find . -iname '*.pyc' -delete - find . -iname '__pycache__' -delete - # delete packaging directories - rm -rf dist karapace.egg-info - # delete generate files - rm karapace/version.py - -karapace/version.py: version.py - $(PYTHON) $^ $@ diff --git a/README.rst b/README.rst index 8ad8fad98..e7c02fe6a 100644 --- a/README.rst +++ b/README.rst @@ -585,6 +585,52 @@ If you installed Karapace from the sources via ``python setup.py install``, it c pip uninstall karapace +Development +=========== + +Execute ``make`` (GNU, usually ``gmake`` on BSD and Mac) to set up a ``venv`` +and install the required software for development. Use ``make unit-tests`` and +``make integration-tests`` to execute the respective test suite, or simply +``make test`` to execute both. You can set ``PYTEST_ARGS`` to customize the +execution (e.g. ``PYTEST_ARGS=--maxfail=1 make test``). + +By default ``pyenv`` is expected to be installed and in ``PATH``. This ensures +on all platforms that arbitrary Python versions can be used for development. It +is possible to overwrite this by setting ``PYENV`` to something else (e.g. +``PYENV=python3 make venv`` to simply use the global Python executable). The +default Python version is defined in ``.python-version``. + +Karapace currently depends on various system software to be installed. The +installation of these is automated for some operation systems, but not all. At +the time of writing Java, the Protobuf Compiler, and the Snappy shared library +are required to work with Karapace. You need to install them manually if your +operating system is not supported by the automatic installation scripts. Note +that the scripts are going to ask before installing any of these on your system. + +Note that Karapace requires a Protobuf Compiler older than 3.20.0, because +3.20.0 introduces various breaking changes. The tests are going to fail if the +Protobuf Compiler is newer than that. However, you can work around this locally +by running ``pip install --upgrade protobuf`` in your venv. We are going to fix +this soon. + +Note that the integration tests are currently not working on Mac. You can use +Docker, just be sure to set ``VENV_DIR`` to a directory outside the working +directory so that the container is not overwriting files from the host (e.g. +``docker run --env VENV_DIR=/tmp/venv ...``). + +Note that the ``runtime`` directory **MUST** exist and that Karapace is going to +fail if it does not. The ``runtime`` directory is also not cleaned between test +runs, and left over data might result in failing tests. Use the ``make`` test +targets that correctly clean the ``runtime`` directory without deleting it, but +keep this in mind whenever you are not using ``make`` (e.g. running tests from +your IDE). + +Note that the pre-commit checks are currently not working with the default +Python version. This is because isort dropped Python 3.7 support. You have to +use at least Python 3.8 for the pre-commit checks. Use ``pipx`` or ``brew`` or +… to install pre-commit and use the global installation, there is also no +dependency on it. + License ======= diff --git a/bin/get-java b/bin/get-java new file mode 100755 index 000000000..493757bdf --- /dev/null +++ b/bin/get-java @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if ! command -v java; then + exists() { command -v "$1" &>/dev/null; } + update=() + install=() + java_version=11 + + if exists brew; then + install+=(brew install "openjdk@$java_version") + else + if ((EUID != 0)) && exists sudo; then + update+=(sudo) + install+=(sudo) + fi + + if exists apt-get; then + update+=(apt-get update) + install+=(apt-get install -y "openjdk-$java_version-jdk") + elif exists dnf; then + install+=(dnf install -y "java-$java_version-openjdk") + elif exists apk; then + install+=(apk add "openjdk$java_version") + else + printf '\e[0m\e[31mUnsupported system, install \e[1mjava\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2 + exit 1 + fi + fi + + if [[ ${CI:-false} != true ]]; then + cmd= + ((${#update[@]} < 2)) || cmd="${update[*]} && " + cmd+=${install[*]} + read -ern1 -p "$cmd [Yn] " + if [[ $REPLY == [Nn]* ]]; then + exit 0 + fi + fi + + ((${#update[@]} < 2)) || "${update[@]}" + "${install[@]}" +fi diff --git a/bin/get-protoc b/bin/get-protoc new file mode 100755 index 000000000..1d35a8b35 --- /dev/null +++ b/bin/get-protoc @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if ! command -v protoc; then + exists() { command -v "$1" &>/dev/null; } + update=() + install=() + + if exists brew; then + install+=(brew install protobuf) + else + if ((EUID != 0)) && exists sudo; then + update+=(sudo) + install+=(sudo) + fi + + if exists apt-get; then + update+=(apt-get update) + install+=(apt-get install -y protobuf-compiler) + elif exists dnf; then + install+=(dnf install -y protobuf-compiler) + elif exists apk; then + install+=(apk add protoc) + else + printf '\e[0m\e[31mUnsupported system, install \e[1mprotoc\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2 + exit 1 + fi + fi + + if [[ ${CI:-false} != true ]]; then + cmd= + ((${#update[@]} < 2)) || cmd="${update[*]} && " + cmd+=${install[*]} + read -ern1 -p "$cmd [Yn] " + if [[ $REPLY == [Nn]* ]]; then + exit 0 + fi + fi + + ((${#update[@]} < 2)) || "${update[@]}" + "${install[@]}" +fi diff --git a/bin/get-snappy b/bin/get-snappy new file mode 100755 index 000000000..f30ae5f45 --- /dev/null +++ b/bin/get-snappy @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exists() { command -v "$1" &>/dev/null; } +update=() +install=() + +ask() { + if [[ ${CI:-false} != true ]]; then + cmd= + ((${#update[@]} < 2)) || cmd="${update[*]} && " + cmd+=${install[*]} + read -ern1 -p "$cmd [Yn] " + if [[ $REPLY == [Nn]* ]]; then + return 1 + fi + fi +} + +if exists brew; then + snappy_prefix() { brew --prefix snappy 2>/dev/null; } + + # brew might return a path even though it does not exist + if ! prefix=$(snappy_prefix) || [[ ! -d $prefix ]]; then + install+=(brew install snappy) + if ask; then + "${install[@]}" + prefix=$(snappy_prefix) + else + exit 0 + fi + fi + + # https://stackoverflow.com/a/41707800/1251219 + export CPPFLAGS="'-I$prefix/include' '-L$prefix/lib'" +elif ! ldconfig -p | grep -F 'libsnappy.so '; then + if ((EUID != 0)) && exists sudo; then + update+=(sudo) + install+=(sudo) + fi + + if exists apt-get; then + update+=(apt-get update -y) + install+=(apt-get install -y libsnappy-dev) + elif exists dnf; then + install+=(dnf install -y csnappy-devel) + elif exists apk; then + install+=(apk add --no-cache snappy-dev g++) + else + printf '\e[0m\e[33mUnsupported system, not installing \e[1msnappy\e[22m but carrying on in case pip is able to find the required libraries.\e[0m\n' >&2 + exit 0 + fi + + if ask; then + ((${#update[@]} < 2)) || "${update[@]}" + "${install[@]}" + fi +fi