diff --git a/.ci.yml b/.ci.yml index c8b4c7bd..7d59f4ef 100644 --- a/.ci.yml +++ b/.ci.yml @@ -3,17 +3,23 @@ stages: - test - - build - - deploy + - examples + - build_docs + - deploy_docs -image: debian:bookworm +variables: + DEBIAN_FRONTEND: noninteractive lint: stage: test tags: ['ace-x86_64'] - before_script: &BeforeScript + image: debian:bookworm + before_script: - apt-get update -qq - - apt-get install -y --no-install-recommends git python3-pip python3-venv + - apt-get install -y --no-install-recommends + git + python3-pip + python3-venv - python3 -m venv venv - source venv/bin/activate - python3 -m pip install -U pip wheel setuptools @@ -21,41 +27,86 @@ lint: - pip install ".[lint]" - nox -s isort_check black_check flake8 -pytest: - stage: test +.generate_example: + stage: examples tags: ['ace-x86_64'] + image: debian:bookworm variables: GIT_SUBMODULE_STRATEGY: normal before_script: - apt-get update -qq - - apt-get install -y --no-install-recommends wget git python3-dev python3-venv make ninja-build gcc-riscv64-unknown-elf bsdextrautils verilator - - mkdir -p miniconda3 - - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda3/miniconda.sh - - bash miniconda3/miniconda.sh -b -u -p miniconda3 - - rm -rf miniconda3/miniconda.sh - - source miniconda3/bin/activate - - conda create -n venv python=3.9 - - conda activate venv - - conda install -c conda-forge gcc=12.1.0 - script: - - apt-get install -y --no-install-recommends git python3-dev + - apt-get install -y --no-install-recommends + git + python3-dev + python3-venv + python3-pip + - python3 -m venv venv + - source venv/bin/activate - python3 -m pip install git+https://github.com/antmicro/tuttest - tuttest README.md | bash - - - pip install ".[tests]" - - nox -s tests_with_report - artifacts: - paths: - - cov_html + script: + - cd examples/$EXAMPLE + - tuttest README.md install-deps,generate | bash - -docs-verify: +tests: stage: test tags: ['ace-x86_64'] + image: debian:bookworm + variables: + GIT_SUBMODULE_STRATEGY: normal + before_script: + - apt-get update -qq + - apt-get install -y --no-install-recommends + curl + wget + git + python3-dev + python3-venv + python3-pip + make + meson + ninja-build + gcc-riscv64-unknown-elf + bsdextrautils + verilator + libssl-dev + libreadline-dev + libffi-dev + libbz2-dev + libncurses-dev + libsqlite3-dev + liblzma-dev + - python3 -m venv venv + - source venv/bin/activate + - python3 -m pip install nox + - python3 -m pip install git+https://github.com/antmicro/tuttest + - tuttest README.md | bash - script: - - eval ${VERIFY_SCRIPT} + - nox -s tests_with_reports_in_env + +example_hdmi: + extends: .generate_example + variables: + EXAMPLE: hdmi + +example_inout: + extends: .generate_example + variables: + EXAMPLE: inout + +example_pwm: + extends: .generate_example + variables: + EXAMPLE: pwm -docs-build: +include: + - project: 'repositories/topwrap' + ref: internal_ci_yaml + file: '/internal.yml' + +build_docs: image: $CI_DOCS_DOCKER_IMAGE - stage: build + stage: build_docs tags: ['ace-x86_64'] before_script: - pip3 install -r docs/requirements.txt @@ -64,20 +115,22 @@ docs-build: - echo -en "\nhtml_js_files = [ '$ANNOTANT' ]" >> source/conf.py - make html latexpdf - cp build/latex/*.pdf build/html/ - - cp -r ../cov_html build/html/cov - tar cf ../$CI_DOCS_ARCHIVE -C build/html/ . artifacts: paths: - docs/build - $CI_DOCS_ARCHIVE -docs-deploy: +deploy_docs: + image: $CI_DOCS_DOCKER_IMAGE variables: GIT_STRATEGY: none - dependencies: [ docs-build ] - stage: deploy + dependencies: + - build_docs + stage: deploy_docs tags: ['docs'] script: echo 'Deploying docs' artifacts: paths: - $CI_DOCS_ARCHIVE + diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 24697903..8782809c 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -3,107 +3,138 @@ name: Pipeline - on: [pull_request, push, workflow_dispatch] +defaults: + run: + shell: bash -jobs: - - - Tests: +env: + DEBIAN_FRONTEND: noninteractive +jobs: + lint: runs-on: ubuntu-latest - container: verilator/verilator:v5.020 - name: "Test Python ${{ matrix.python-version }}" - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11.7", "3.12.1"] + container: + image: debian:bookworm + name: "Run lint checks" - env: - DEBIAN_FRONTEND: noninteractive steps: + - name: Install system packages + run: | + apt-get update -qq + apt-get install -y --no-install-recommends \ + git \ + python3-pip \ + python3-venv + - uses: actions/checkout@v3 with: submodules: true - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python3 -m venv venv + source venv/bin/activate + pip install -U pip wheel setuptools + + - name: Run lint checks + run: | + source venv/bin/activate + pip install ".[lint]" + nox -s isort_check black_check flake8 + + tests: + runs-on: ubuntu-latest + container: + image: debian:bookworm + name: "Run tests" - - name: Install dev requirements + steps: + - name: Install system packages run: | apt-get update -qq - apt-get install -y \ - antlr4 \ - libantlr4-runtime-dev \ + apt-get install -y --no-install-recommends \ + curl \ + wget \ + git \ python3-dev \ - yosys \ - gcc-riscv64-unknown-elf \ + python3-venv \ + python3-pip \ + make \ meson \ ninja-build \ - bsdextrautils - python3 -m pip install --upgrade pip wheel setuptools - python3 -m pip install nox - python3 -m pip install git+https://github.com/antmicro/tuttest + gcc-riscv64-unknown-elf \ + bsdextrautils \ + verilator \ + libssl-dev \ + libreadline-dev \ + libffi-dev \ + libbz2-dev \ + libncurses-dev \ + libsqlite3-dev \ + liblzma-dev - - name: Run lint checks - run: nox -s isort black flake8 + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Install Python dependencies + run: | + python3 -m venv venv + source venv/bin/activate + pip install nox + pip install git+https://github.com/antmicro/tuttest - name: Build - run: tuttest README.md | bash - + run: | + source venv/bin/activate + tuttest README.md | bash - - name: Run pytest with nox - run: nox -s tests - - - Examples: + run: | + source venv/bin/activate + nox -s tests_with_reports_in_env + examples: runs-on: ubuntu-latest - name: 'Example ${{ matrix.example }}' + container: + image: debian:bookworm + name: 'Generate example ${{ matrix.example }}' + strategy: fail-fast: false matrix: example: - - HDMI - - INOUT - - PWM + - hdmi + - inout + - pwm steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install tuttest and fpga-topwrap + - name: Install system packages run: | - python3 -m pip install git+https://github.com/antmicro/tuttest - tuttest README.md | bash - + apt-get update -qq + apt-get install -y --no-install-recommends \ + git \ + python3-dev \ + python3-venv \ + python3-pip + + - uses: actions/checkout@v3 - - name: Generate sources for example HDMI setup - if: matrix.example == 'HDMI' + - name: Install Python dependencies run: | - cd examples/hdmi - tuttest README.md install-deps,generate | bash - - cd - + python3 -m venv venv + source venv/bin/activate + pip install git+https://github.com/antmicro/tuttest - - name: Generate sources for example inout setup - if: matrix.example == 'INOUT' + - name: Install fpga-topwrap run: | - cd examples/inout - tuttest README.md install-deps,generate | bash - - cd - + source venv/bin/activate + tuttest README.md | bash - - - name: Generate sources for and build example PWM setup - if: matrix.example == 'PWM' + - name: Generate sources for example ${{ matrix.example }} setup run: | - cd examples/pwm + source venv/bin/activate + cd examples/${{ matrix.example }} tuttest README.md install-deps,generate | bash - - cd - - - - uses: actions/upload-artifact@v3 - if: matrix.example == 'PWM' - with: - name: top.bit - path: top.bit diff --git a/noxfile.py b/noxfile.py index 0a7dbe06..1369b63c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,8 +1,15 @@ # Copyright (c) 2023-2024 Antmicro # SPDX-License-Identifier: Apache-2.0 +import os +import shutil +import subprocess +import sys + import nox +PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] + @nox.session() def pre_commit(session: nox.Session) -> None: @@ -45,13 +52,90 @@ def black_check(session): session.run("black", "--check", ".") -@nox.session +@nox.session(python=PYTHON_VERSIONS) def tests(session: nox.Session) -> None: session.install(".[tests,topwrap-parse]") - session.run("pytest", "--cov=fpga_topwrap", "tests") + session.run("pytest", "-rs", "--cov=fpga_topwrap", "tests") -@nox.session +@nox.session(python=PYTHON_VERSIONS) def tests_with_report(session: nox.Session) -> None: session.install(".[tests,topwrap-parse]") - session.run("pytest", "--cov-report", "html:cov_html", "--cov=fpga_topwrap", "tests") + session.run("pytest", "-rs", "--cov-report", "html:cov_html", "--cov=fpga_topwrap", "tests") + + +def prepare_pyenv(session: nox.Session) -> dict: + project_dir = os.path.dirname(os.path.abspath(__file__)) + pyenv_dir = f"{project_dir}/build/.pyenv" + pyenv_bin = f"{os.path.join(pyenv_dir, 'bin')}" + pyenv_shims = f"{os.path.join(pyenv_dir, 'shims')}" + path = f"{pyenv_bin}:{pyenv_shims}:{os.getenv('PATH')}" + + env = os.environ.copy() + env["PATH"] = path + env["PYENV_ROOT"] = pyenv_dir + + # Install Pyenv + if not shutil.which("pyenv", path=path): + p = subprocess.run( + "curl https://pyenv.run | bash", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + shell=True, + env=env, + ) + session.log(p.stdout.strip("\n")) + + if p.returncode: + session.error() + + # Install required Python versions if these don't exist + for ver in PYTHON_VERSIONS: + if not shutil.which(f"python{ver}", path=path): + session.log(f"Installing Python {ver}") + p = subprocess.run( + f"pyenv install {ver}", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + shell=True, + env=env, + ) + session.log(p.stdout.strip("\n")) + + # Detect which versions are provided by Pyenv + pythons_in_pyenv = "" + for ver in PYTHON_VERSIONS: + if shutil.which(f"python{ver}", path=pyenv_shims): + pythons_in_pyenv += f"{ver} " + + # Allow using Pythons from Pyenv + if pythons_in_pyenv: + session.log(f"Configuring Pythons from Pyenv, versions: {pythons_in_pyenv}") + p = subprocess.run( + f"pyenv global {pythons_in_pyenv}", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + shell=True, + env=env, + ) + session.log(p.stdout.strip("\n")) + + if p.returncode: + session.error() + + return env + + +@nox.session +def tests_in_env(session: nox.Session) -> None: + env = prepare_pyenv(session) + session.run("nox", "-s", "tests", external=True, env=env) + + +@nox.session +def tests_with_reports_in_env(session: nox.Session) -> None: + env = prepare_pyenv(session) + session.run("nox", "-s", "tests_with_report", external=True, env=env)