From eb59287eca8d5d99e3dfc2ccca63eefbf7c5c45f Mon Sep 17 00:00:00 2001 From: Robert Szczepanski Date: Thu, 7 Mar 2024 15:29:00 +0100 Subject: [PATCH] Unify Github and Gitlab CIs Internal-tag: [#53659] Signed-off-by: Robert Szczepanski --- .ci.yml | 65 ++++++-------- .github/scripts/ci.sh | 150 +++++++++++++++++++++++++++++++++ .github/scripts/latex.sh | 14 --- .github/scripts/sphinx.sh | 12 --- .github/workflows/pipeline.yml | 127 +++++++++++----------------- noxfile.py | 56 ++++++++++-- 6 files changed, 277 insertions(+), 147 deletions(-) create mode 100755 .github/scripts/ci.sh delete mode 100755 .github/scripts/latex.sh delete mode 100755 .github/scripts/sphinx.sh diff --git a/.ci.yml b/.ci.yml index c8b4c7bd..a9a16d08 100644 --- a/.ci.yml +++ b/.ci.yml @@ -3,59 +3,48 @@ stages: - test - - build - - deploy + - build_docs + - deploy_docs -image: debian:bookworm +variables: + DEBIAN_FRONTEND: noninteractive lint: stage: test tags: ['ace-x86_64'] - before_script: &BeforeScript - - apt-get update -qq - - apt-get install -y --no-install-recommends git python3-pip python3-venv - - python3 -m venv venv - - source venv/bin/activate - - python3 -m pip install -U pip wheel setuptools + image: debian:bookworm script: - - pip install ".[lint]" - - nox -s isort_check black_check flake8 + - ./.github/scripts/ci.sh lint -pytest: +examples: stage: test tags: ['ace-x86_64'] + image: debian:bookworm variables: GIT_SUBMODULE_STRATEGY: normal - before_script: - - apt-get update -qq - - apt-get install -y --no-install-recommends wget git python3-dev python3-venv make ninja-build gcc-riscv64-unknown-elf bsdextrautils verilator - - mkdir -p miniconda3 - - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda3/miniconda.sh - - bash miniconda3/miniconda.sh -b -u -p miniconda3 - - rm -rf miniconda3/miniconda.sh - - source miniconda3/bin/activate - - conda create -n venv python=3.9 - - conda activate venv - - conda install -c conda-forge gcc=12.1.0 script: - - apt-get install -y --no-install-recommends git python3-dev - - python3 -m pip install git+https://github.com/antmicro/tuttest - - tuttest README.md | bash - - - pip install ".[tests]" - - nox -s tests_with_report + - ./.github/scripts/ci.sh examples artifacts: paths: - - cov_html + - examples/**/build -docs-verify: +tests: stage: test tags: ['ace-x86_64'] + image: debian:bookworm + variables: + GIT_SUBMODULE_STRATEGY: normal script: - - eval ${VERIFY_SCRIPT} + - ./.github/scripts/ci.sh tests -docs-build: +include: + - project: 'repositories/topwrap' + ref: internal_ci_yaml + file: '/internal.yml' + +build_docs: image: $CI_DOCS_DOCKER_IMAGE - stage: build + stage: build_docs tags: ['ace-x86_64'] before_script: - pip3 install -r docs/requirements.txt @@ -64,20 +53,22 @@ docs-build: - echo -en "\nhtml_js_files = [ '$ANNOTANT' ]" >> source/conf.py - make html latexpdf - cp build/latex/*.pdf build/html/ - - cp -r ../cov_html build/html/cov - tar cf ../$CI_DOCS_ARCHIVE -C build/html/ . artifacts: paths: - docs/build - $CI_DOCS_ARCHIVE -docs-deploy: +deploy_docs: + image: $CI_DOCS_DOCKER_IMAGE variables: GIT_STRATEGY: none - dependencies: [ docs-build ] - stage: deploy + dependencies: + - build_docs + stage: deploy_docs tags: ['docs'] script: echo 'Deploying docs' artifacts: paths: - $CI_DOCS_ARCHIVE + diff --git a/.github/scripts/ci.sh b/.github/scripts/ci.sh new file mode 100755 index 00000000..1f0a030d --- /dev/null +++ b/.github/scripts/ci.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# Copyright (c) 2024 Antmicro +# SPDX-License-Identifier: Apache-2.0 + +set -e + +ROOT_DIR="$(dirname "${BASH_SOURCE[0]}")/../.." +EXAMPLES=(hdmi inout pwm) + +begin_command_group() { + if [[ -n "${GITHUB_WORKFLOW:-}" ]]; then + echo "::group::$*" + else + echo -e "\n\033[1;92mRunning step: $1\033[0m\n" + fi +} + +end_command_group() { + if [[ -n "${GITHUB_WORKFLOW:-}" ]]; then + echo "::endgroup::" + fi +} + +log_cmd() { + printf '\033[1;96m' + printf '%s ' "$*" + printf '\033[0m\n' + eval "$*" +} + +install_common_system_packages() { + begin_command_group "Install system packages" + log_cmd apt-get update -qq + log_cmd apt-get install -y --no-install-recommends \ + git \ + python3-pip \ + python3-venv + end_command_group +} + +install_pyenv() { + begin_command_group "Install pyenv" + log_cmd export PYENV_ROOT="$HOME/.pyenv" + log_cmd export PATH="$PYENV_ROOT/bin:$PATH" + log_cmd "curl https://pyenv.run | bash" + end_command_group +} + +enter_venv() { + begin_command_group "Configure Python virtual environment" + if [[ -z "$VIRTUAL_ENV" ]]; then + log_cmd python3 -m venv venv + fi + log_cmd source venv/bin/activate + end_command_group +} + +install_topwrap() { + begin_command_group "Install Topwrap" + log_cmd "tuttest README.md | bash -" + end_command_group +} + +run_lint() { + install_common_system_packages + enter_venv + + begin_command_group "Install python packages for lint" + log_cmd pip install ".[lint]" + end_command_group + + begin_command_group "Run lint checks" + log_cmd nox -s isort_check black_check flake8 + end_command_group +} + +run_tests() { + install_common_system_packages + + begin_command_group "Install system packages for tests" + log_cmd apt-get install -y --no-install-recommends \ + curl \ + wget \ + python3-dev \ + make \ + meson \ + ninja-build \ + gcc-riscv64-unknown-elf \ + bsdextrautils \ + verilator \ + libssl-dev \ + libreadline-dev \ + libffi-dev \ + libbz2-dev \ + libncurses-dev \ + libsqlite3-dev \ + liblzma-dev + end_command_group + + install_pyenv + enter_venv + + begin_command_group "Install python packages for tests" + log_cmd pip install ".[tests]" + log_cmd pip install git+https://github.com/antmicro/tuttest + end_command_group + + install_topwrap + + begin_command_group "Run Python tests" + log_cmd nox -s tests_in_env + end_command_group +} + +generate_examples() { + install_common_system_packages + begin_command_group "Install system packages for examples" + log_cmd apt-get install -y --no-install-recommends python3-dev + end_command_group + enter_venv + + begin_command_group "Install python packages for examples" + log_cmd pip install git+https://github.com/antmicro/tuttest + end_command_group + + install_topwrap + + for EXAMPLE in "${EXAMPLES[@]}"; do + begin_command_group "Generate $EXAMPLE example" + log_cmd pushd "$ROOT_DIR"/examples/"$EXAMPLE" + log_cmd "tuttest README.md install-deps,generate | bash -" + log_cmd popd + end_command_group + done +} + +case "$1" in +lint) + run_lint + ;; +tests) + run_tests + ;; +examples) + generate_examples + ;; +*) + echo "Usage: $0 {lint|tests|examples}" + ;; +esac diff --git a/.github/scripts/latex.sh b/.github/scripts/latex.sh deleted file mode 100755 index d644d1e5..00000000 --- a/.github/scripts/latex.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env sh -# Copyright (c) 2023-2024 Antmicro -# SPDX-License-Identifier: Apache-2.0 - - -set -e - -cd $(dirname $0)/../../docs - -pip3 install -r requirements.txt - -cd build/latex -LATEXMKOPTS='-interaction=nonstopmode' make -cp *.pdf ../html/ diff --git a/.github/scripts/sphinx.sh b/.github/scripts/sphinx.sh deleted file mode 100755 index 5032fd32..00000000 --- a/.github/scripts/sphinx.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env sh -# Copyright (c) 2023-2024 Antmicro -# SPDX-License-Identifier: Apache-2.0 - - -set -e - -cd $(dirname $0)/../../docs - -pip3 install -r requirements.txt - -make html latex diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index a2c2b59d..60997c7c 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -3,107 +3,76 @@ name: Pipeline - on: [pull_request, push, workflow_dispatch] +defaults: + run: + shell: bash -jobs: - - - Tests: +env: + DEBIAN_FRONTEND: noninteractive +jobs: + lint: runs-on: ubuntu-latest - container: verilator/verilator:v5.020 - name: "Test Python ${{ matrix.python-version }}" - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11.7", "3.12.1"] - - env: - DEBIAN_FRONTEND: noninteractive - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + container: + image: debian:bookworm + name: "Run lint checks" - - name: Install dev requirements + steps: + - name: Install git package run: | apt-get update -qq - apt-get install -y \ - antlr4 \ - libantlr4-runtime-dev \ - python3-dev \ - yosys \ - gcc-riscv64-unknown-elf \ - meson \ - ninja-build \ - bsdextrautils - python3 -m pip install --upgrade pip wheel setuptools - python3 -m pip install nox - python3 -m pip install git+https://github.com/antmicro/tuttest - - - name: Run lint checks - run: nox -s isort black flake8 - - - name: Build - run: tuttest README.md | bash - - - - name: Run pytest with nox - run: nox -s tests + apt-get install -y git + - uses: actions/checkout@v4 + with: + submodules: true - Examples: + - name: Run lint checks + run: | + ./.github/scripts/ci.sh lint + tests: runs-on: ubuntu-latest - name: 'Example ${{ matrix.example }}' - strategy: - fail-fast: false - matrix: - example: - - HDMI - - INOUT - - PWM + container: + image: debian:bookworm + name: "Run tests" steps: - - uses: actions/checkout@v3 + - name: Install git package + run: | + apt-get update -qq + apt-get install -y git - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 with: - python-version: "3.11" + submodules: true - - name: Install tuttest and topwrap + - name: Run Python tests run: | - python3 -m pip install git+https://github.com/antmicro/tuttest - tuttest README.md | bash - + ./.github/scripts/ci.sh tests - - name: Generate sources for example HDMI setup - if: matrix.example == 'HDMI' - run: | - cd examples/hdmi - tuttest README.md install-deps,generate | bash - - cd - + examples: + runs-on: ubuntu-latest + container: + image: debian:bookworm + name: 'Generate examples' - - name: Generate sources for example inout setup - if: matrix.example == 'INOUT' + steps: + - name: Install git package run: | - cd examples/inout - tuttest README.md install-deps,generate | bash - - cd - + apt-get update -qq + apt-get install -y git + + - uses: actions/checkout@v4 - - name: Generate sources for and build example PWM setup - if: matrix.example == 'PWM' + - name: Generate Topwrap examples run: | - cd examples/pwm - tuttest README.md install-deps,generate | bash - - cd - + ./.github/scripts/ci.sh examples - - uses: actions/upload-artifact@v3 - if: matrix.example == 'PWM' + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: - name: top.bit - path: top.bit + name: examples + path: examples/**/build diff --git a/noxfile.py b/noxfile.py index 86d875bc..b1850d82 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,8 +1,14 @@ # Copyright (c) 2023-2024 Antmicro # SPDX-License-Identifier: Apache-2.0 +import os +import shutil +from pathlib import Path, PurePath + import nox +PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] + @nox.session() def pre_commit(session: nox.Session) -> None: @@ -50,13 +56,53 @@ def black_check(session): # https://github.com/pytest-dev/pytest-cov/issues/388 -@nox.session +@nox.session(python=PYTHON_VERSIONS) def tests(session: nox.Session) -> None: session.install("-e", ".[tests,topwrap-parse]") - session.run("pytest", "--cov=topwrap", "tests") + session.run("pytest", "-rs", "--cov-report", "html:cov_html", "--cov=topwrap", "tests") + + +def prepare_pyenv(session: nox.Session) -> dict: + env = os.environ.copy() + path = env.get("PATH") + + project_dir = Path(__file__).absolute().parent + env["PYENV_ROOT"] = env.get("PYENV_ROOT", f"{project_dir}/.nox/pyenv") + + pyenv_bin = PurePath(env["PYENV_ROOT"]) / "bin" + pyenv_shims = PurePath(env["PYENV_ROOT"]) / "shims" + path = f"{pyenv_bin}:{pyenv_shims}:{path}" + env["PATH"] = path + + # Install Pyenv + if not shutil.which("pyenv", path=path): + session.error( + "\n'pyenv' command not found, you can install it by executing:" + "\n curl https://pyenv.run | bash" + "\nSee https://github.com/pyenv/pyenv?tab=readme-ov-file#installation for more information" + ) + + # Install required Python versions if these don't exist + for ver in PYTHON_VERSIONS: + if not shutil.which(f"python{ver}", path=path): + session.log(f"Installing Python {ver}") + session.run("pyenv", "install", ver, env=env) + + # Detect which versions are provided by Pyenv + pythons_in_pyenv = [] + for ver in PYTHON_VERSIONS: + if shutil.which(f"python{ver}", path=pyenv_shims): + pythons_in_pyenv += [ver] + + # Allow using Pythons from Pyenv + if pythons_in_pyenv: + session.log(f"Configuring Pythons from Pyenv, versions: {pythons_in_pyenv}") + session.run("pyenv", "global", *pythons_in_pyenv, env=env) + + return env @nox.session -def tests_with_report(session: nox.Session) -> None: - session.install("-e", ".[tests,topwrap-parse]") - session.run("pytest", "--cov-report", "html:cov_html", "--cov=topwrap", "tests") +def tests_in_env(session: nox.Session) -> None: + env = prepare_pyenv(session) + session.run("nox", "-s", "tests", external=True, env=env)