Skip to content

Commit

Permalink
Merge pull request Aiven-Open#556 from aiven/fleshgrinder/basic-build…
Browse files Browse the repository at this point in the history
…-system

build: Add Basic Build System
  • Loading branch information
tvainika authored Mar 20, 2023
2 parents e28b2e1 + 5f2acd7 commit d956822
Show file tree
Hide file tree
Showing 9 changed files with 292 additions and 51 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,22 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true

env:
FORCE_COLOR: 1
PIP_PROGRESS_BAR: off
PYTHONUNBUFFERED: 1

jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
cache: pip
python-version: '3.11'
# required for pylint
- run: make karapace/version.py
- run: make version
- run: pip install pre-commit
- uses: actions/cache@v3
with:
Expand Down
53 changes: 20 additions & 33 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,43 @@ name: Test Suite

on:
pull_request:
types: [opened, synchronize, reopened]
types: [ opened, synchronize, reopened ]
push:
branches:
- main
branches: [ main ]

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true

env:
FORCE_COLOR: 1
PIP_PROGRESS_BAR: off
PYTHONUNBUFFERED: 1

jobs:
tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
steps:
- uses: actions/checkout@v3

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
cache: pip
python-version: ${{ matrix.python-version }}

- name: Install libsnappy-dev (python-snappy legacy-install-failure on Python 3.11)
run: sudo apt install libsnappy-dev

- name: Install dependencies
run: python -m pip install -r requirements-dev.txt

- name: Extract Protobuf Version
id: protoc
run: pip freeze | grep -m1 '^protobuf==' | cut -d= -f3 | xargs printf 'version=%s' >>"$GITHUB_OUTPUT"

- name: Install Protobuf Compiler
uses: arduino/setup-protoc@v1
with:
version: ${{ steps.protoc.outputs.version }}
repo-token: ${{ secrets.GITHUB_TOKEN }}

# needed by both unit and integration tests
- name: Generate version.py
run: make karapace/version.py

- name: Execute unit-tests
timeout-minutes: 2
run: python3 -m pytest -s -vvv tests/unit/

- name: Execute integration-tests
timeout-minutes: 30
run: python3 -m pytest -s -vvv tests/integration/ --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log
- run: make install version
- run: make unit-tests
- run: make integration-tests
env:
PYTEST_ARGS: --log-dir=/tmp/ci-logs --log-file=/tmp/ci-logs/pytest.log

- name: Archive logs
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: logs ${{ matrix.python-version }}
path: /tmp/ci-logs
name: karapace-integration-test-logs-${{ matrix.python-version }}
path: /tmp/ci-logs
1 change: 1 addition & 0 deletions .python-version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3.7
75 changes: 75 additions & 0 deletions GNUmakefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
VENV_DIR ?= $(CURDIR)/venv
PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
PYTHON ?= python3
ifdef CI
PYENV ?= $(PYTHON)
else
PYENV ?= pyenv exec python
endif

export PATH := $(VENV_DIR)/bin:$(PATH)
export PS4 := \e[0m\e[32m==> \e[0m
export LC_ALL := C
MAKEFLAGS += --warn-undefined-variables
MAKEFLAGS += --no-builtin-rules
SHELL := bash
.SHELLFLAGS := -euxo pipefail -O globstar -c
.ONESHELL:
.SILENT:
.SUFFIXES:

.PHONY: all
all: version

.PHONY: venv
venv: venv/.make
venv/.make:
rm -fr '$(VENV_DIR)'
$(PYENV) -m venv '$(VENV_DIR)'
$(PIP) install --upgrade pip
touch '$(@)'

.PHONY: install
install: venv/.deps
venv/.deps: requirements-dev.txt requirements.txt | venv/.make
set +x
source ./bin/get-java
source ./bin/get-protoc
source ./bin/get-snappy
set -x
$(PIP) install -r '$(<)' --use-pep517
touch '$(@)'

.PHONY: version
version: karapace/version.py
karapace/version.py: version.py | venv/.make
$(PYTHON) '$(<)' '$(@)'

.PHONY: test
tests: unit-tests integration-tests

.PHONY: unit-tests
unit-tests: export PYTEST_ARGS ?=
unit-tests: karapace/version.py venv/.deps
rm -fr runtime/*
$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
rm -fr runtime/*

.PHONY: integration-tests
unit-tests: export PYTEST_ARGS ?=
integration-tests: karapace/version.py venv/.deps
rm -fr runtime/*
$(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
rm -fr runtime/*

.PHONY: clean
clean:
rm -fr ./kafka_* ./*.egg-info/ ./dist/ ./karapace/version.py

.PHONY: cleaner
cleaner: clean
rm -fr ./.*cache*/

.PHONY: cleanest
cleanest: cleaner
rm -fr '$(VENV_DIR)'
17 changes: 0 additions & 17 deletions Makefile

This file was deleted.

46 changes: 46 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,52 @@ If you installed Karapace from the sources via ``python setup.py install``, it c

pip uninstall karapace

Development
===========

Execute ``make`` (GNU, usually ``gmake`` on BSD and Mac) to set up a ``venv``
and install the required software for development. Use ``make unit-tests`` and
``make integration-tests`` to execute the respective test suite, or simply
``make test`` to execute both. You can set ``PYTEST_ARGS`` to customize the
execution (e.g. ``PYTEST_ARGS=--maxfail=1 make test``).

By default ``pyenv`` is expected to be installed and in ``PATH``. This ensures
on all platforms that arbitrary Python versions can be used for development. It
is possible to overwrite this by setting ``PYENV`` to something else (e.g.
``PYENV=python3 make venv`` to simply use the global Python executable). The
default Python version is defined in ``.python-version``.

Karapace currently depends on various system software to be installed. The
installation of these is automated for some operation systems, but not all. At
the time of writing Java, the Protobuf Compiler, and the Snappy shared library
are required to work with Karapace. You need to install them manually if your
operating system is not supported by the automatic installation scripts. Note
that the scripts are going to ask before installing any of these on your system.

Note that Karapace requires a Protobuf Compiler older than 3.20.0, because
3.20.0 introduces various breaking changes. The tests are going to fail if the
Protobuf Compiler is newer than that. However, you can work around this locally
by running ``pip install --upgrade protobuf`` in your venv. We are going to fix
this soon.

Note that the integration tests are currently not working on Mac. You can use
Docker, just be sure to set ``VENV_DIR`` to a directory outside the working
directory so that the container is not overwriting files from the host (e.g.
``docker run --env VENV_DIR=/tmp/venv ...``).

Note that the ``runtime`` directory **MUST** exist and that Karapace is going to
fail if it does not. The ``runtime`` directory is also not cleaned between test
runs, and left over data might result in failing tests. Use the ``make`` test
targets that correctly clean the ``runtime`` directory without deleting it, but
keep this in mind whenever you are not using ``make`` (e.g. running tests from
your IDE).

Note that the pre-commit checks are currently not working with the default
Python version. This is because isort dropped Python 3.7 support. You have to
use at least Python 3.8 for the pre-commit checks. Use ``pipx`` or ``brew`` or
… to install pre-commit and use the global installation, there is also no
dependency on it.

License
=======

Expand Down
43 changes: 43 additions & 0 deletions bin/get-java
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
set -Eeuo pipefail

if ! command -v java; then
exists() { command -v "$1" &>/dev/null; }
update=()
install=()
java_version=11

if exists brew; then
install+=(brew install "openjdk@$java_version")
else
if ((EUID != 0)) && exists sudo; then
update+=(sudo)
install+=(sudo)
fi

if exists apt-get; then
update+=(apt-get update)
install+=(apt-get install -y "openjdk-$java_version-jdk")
elif exists dnf; then
install+=(dnf install -y "java-$java_version-openjdk")
elif exists apk; then
install+=(apk add "openjdk$java_version")
else
printf '\e[0m\e[31mUnsupported system, install \e[1mjava\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2
exit 1
fi
fi

if [[ ${CI:-false} != true ]]; then
cmd=
((${#update[@]} < 2)) || cmd="${update[*]} && "
cmd+=${install[*]}
read -ern1 -p "$cmd [Yn] "
if [[ $REPLY == [Nn]* ]]; then
exit 0
fi
fi

((${#update[@]} < 2)) || "${update[@]}"
"${install[@]}"
fi
42 changes: 42 additions & 0 deletions bin/get-protoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
set -Eeuo pipefail

if ! command -v protoc; then
exists() { command -v "$1" &>/dev/null; }
update=()
install=()

if exists brew; then
install+=(brew install protobuf)
else
if ((EUID != 0)) && exists sudo; then
update+=(sudo)
install+=(sudo)
fi

if exists apt-get; then
update+=(apt-get update)
install+=(apt-get install -y protobuf-compiler)
elif exists dnf; then
install+=(dnf install -y protobuf-compiler)
elif exists apk; then
install+=(apk add protoc)
else
printf '\e[0m\e[31mUnsupported system, install \e[1mprotoc\e[22m and make sure it is in your \e[1mPATH\e[22m.\e[0m\n' >&2
exit 1
fi
fi

if [[ ${CI:-false} != true ]]; then
cmd=
((${#update[@]} < 2)) || cmd="${update[*]} && "
cmd+=${install[*]}
read -ern1 -p "$cmd [Yn] "
if [[ $REPLY == [Nn]* ]]; then
exit 0
fi
fi

((${#update[@]} < 2)) || "${update[@]}"
"${install[@]}"
fi
58 changes: 58 additions & 0 deletions bin/get-snappy
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
set -Eeuo pipefail

exists() { command -v "$1" &>/dev/null; }
update=()
install=()

ask() {
if [[ ${CI:-false} != true ]]; then
cmd=
((${#update[@]} < 2)) || cmd="${update[*]} && "
cmd+=${install[*]}
read -ern1 -p "$cmd [Yn] "
if [[ $REPLY == [Nn]* ]]; then
return 1
fi
fi
}

if exists brew; then
snappy_prefix() { brew --prefix snappy 2>/dev/null; }

# brew might return a path even though it does not exist
if ! prefix=$(snappy_prefix) || [[ ! -d $prefix ]]; then
install+=(brew install snappy)
if ask; then
"${install[@]}"
prefix=$(snappy_prefix)
else
exit 0
fi
fi

# https://stackoverflow.com/a/41707800/1251219
export CPPFLAGS="'-I$prefix/include' '-L$prefix/lib'"
elif ! ldconfig -p | grep -F 'libsnappy.so '; then
if ((EUID != 0)) && exists sudo; then
update+=(sudo)
install+=(sudo)
fi

if exists apt-get; then
update+=(apt-get update -y)
install+=(apt-get install -y libsnappy-dev)
elif exists dnf; then
install+=(dnf install -y csnappy-devel)
elif exists apk; then
install+=(apk add --no-cache snappy-dev g++)
else
printf '\e[0m\e[33mUnsupported system, not installing \e[1msnappy\e[22m but carrying on in case pip is able to find the required libraries.\e[0m\n' >&2
exit 0
fi

if ask; then
((${#update[@]} < 2)) || "${update[@]}"
"${install[@]}"
fi
fi

0 comments on commit d956822

Please sign in to comment.