From ccf0adc0b22fe2da2fec7efb764cde75e7024484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Ob=C5=82onczek?= Date: Tue, 7 May 2024 16:50:15 +0200 Subject: [PATCH] Move CI workflow to common script, simplify packaging script, remove packing into a tarball since CI already packages artifacts into an archive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal-tag: [#57771] Signed-off-by: Krzysztof Obłonczek --- .ci.yml | 10 +++ .flake8 | 1 - .github/scripts/ci.sh | 30 ++++++- .github/scripts/package_cores.py | 46 ++++++++++ .github/workflows/cores.yml | 36 -------- .github/workflows/pipeline.yml | 26 +++++- .gitignore | 1 - docs/source/cli.md | 10 --- pyproject.toml | 2 - scripts/pack_core.py | 145 ------------------------------- 10 files changed, 110 insertions(+), 197 deletions(-) create mode 100755 .github/scripts/package_cores.py delete mode 100644 .github/workflows/cores.yml delete mode 100755 scripts/pack_core.py diff --git a/.ci.yml b/.ci.yml index 507e9607..05a84967 100644 --- a/.ci.yml +++ b/.ci.yml @@ -33,6 +33,16 @@ examples: paths: - examples/**/build +package_cores: + stage: test + tags: ['ace-x86_64'] + image: debian:bookworm + script: + - ./.github/scripts/ci.sh package_cores + artifacts: + paths: + - core_repo/** + build_docs: image: $CI_DOCS_DOCKER_IMAGE stage: build_docs diff --git a/.flake8 b/.flake8 index b148e39d..78e7a430 100644 --- a/.flake8 +++ b/.flake8 @@ -9,7 +9,6 @@ exclude = .eggs, .git, .gitignore, - .github, .nox, .pytest_cache, __pycache__, diff --git a/.github/scripts/ci.sh b/.github/scripts/ci.sh index ddb0dcbd..7b4ed2ce 100755 --- a/.github/scripts/ci.sh +++ b/.github/scripts/ci.sh @@ -134,6 +134,31 @@ generate_examples() { done } +package_cores() { + install_common_system_packages + begin_command_group "Install system packages for packaging cores" + log_cmd apt-get install -y --no-install-recommends python3-dev + end_command_group + enter_venv + + begin_command_group "Install python packages for packaging cores" + log_cmd pip install git+https://github.com/antmicro/tuttest + end_command_group + + install_topwrap + + begin_command_group "Install Topwrap's parsing dependencies" + log_cmd pip install ".[topwrap-parse]" + end_command_group + + begin_command_group "Package cores for release" + log_cmd mkdir core_repo + log_cmd pushd core_repo + log_cmd python ../.github/scripts/package_cores.py + log_cmd popd + end_command_group +} + case "$1" in lint) run_lint @@ -144,7 +169,10 @@ tests) examples) generate_examples ;; +package_cores) + package_cores + ;; *) - echo "Usage: $0 {lint|tests|examples}" + echo "Usage: $0 {lint|tests|examples|package_cores}" ;; esac diff --git a/.github/scripts/package_cores.py b/.github/scripts/package_cores.py new file mode 100755 index 00000000..fd186172 --- /dev/null +++ b/.github/scripts/package_cores.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024 Antmicro +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass +from pathlib import Path +from typing import List + +from topwrap.repo.files import HttpGetFile +from topwrap.repo.user_repo import UserRepo, VerilogFileHandler + + +@dataclass +class RemoteRepo: + name: str + root_url: str + sources: List[str] + + +repos = [ + RemoteRepo( + name="vexriscv", + root_url="https://raw.githubusercontent.com/litex-hub/pythondata-cpu-vexriscv/1979a644dbe64d8d32dfbdd970dccee6add63723/pythondata_cpu_vexriscv/verilog", + sources=[ + "VexRiscv.v", + ], + ), +] + + +def package_cores(): + """Generates reusable cores package for usage in Topwrap project.""" + core_repo = UserRepo() + + for repo in repos: + core_files = [] + for file in repo.sources: + core_files.append(HttpGetFile(f"{repo.root_url}/{file}")) + + core_repo.add_files(VerilogFileHandler(core_files)) + + Path(repo.name).mkdir(exist_ok=True) + core_repo.save(repo.name) + + +if __name__ == "__main__": + package_cores() diff --git a/.github/workflows/cores.yml b/.github/workflows/cores.yml deleted file mode 100644 index e1e8f865..00000000 --- a/.github/workflows/cores.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Package cores for Topwrap - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-22.04 - - strategy: - matrix: - core-name: ["verilog-axi"] - - env: - python-version: "3.11" - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ env.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.python-version }} - - - name: Install Topwrap - run: | - sudo apt-get update - sudo apt-get install -y git build-essential antlr4 libantlr4-runtime-dev yosys - pip install -e .[topwrap-parse] - - - name: Create ${{ matrix.core-name }} package - run: ./scripts/pack_core.py ${{ matrix.core-name }} - - - name: Upload core package - uses: actions/upload-artifact@v4 - with: - path: "*.tar.gz" diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index d983eb06..a234f1fc 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest container: image: debian:bookworm - name: 'Generate examples' + name: "Generate examples" steps: - name: Install git package @@ -72,3 +72,27 @@ jobs: with: name: examples path: examples/**/build + + package_cores: + runs-on: ubuntu-latest + container: + image: debian:bookworm + name: "Package cores repository" + + steps: + - name: Install git package + run: | + apt-get update -qq + apt-get install -y git + + - uses: actions/checkout@v4 + + - name: Pack cores into a Topwrap repository + run: | + ./.github/scripts/ci.sh package_cores + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: core_repo + path: core_repo/** diff --git a/.gitignore b/.gitignore index 21d6c8a6..a81c8ee1 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST -*.tar.gz # PyInstaller # Usually these files are written by a python script from a template diff --git a/docs/source/cli.md b/docs/source/cli.md index 55f97e98..5b909841 100644 --- a/docs/source/cli.md +++ b/docs/source/cli.md @@ -72,13 +72,3 @@ To get help, use: ``` python -m topwrap [build|kpm_client|parse] --help ``` - -## Generating IP core archive - -HDL sources can be packaged into a reusable tarball with autogenerated IP core description YAMLs: - -``` -python -m topwrap pack_core NAME SOURCE_DIRS -``` - -Where `NAME` is the name of target archive and `SOURCE_DIRS` is one or more directories with HDL files to be included in the package. diff --git a/pyproject.toml b/pyproject.toml index 50f71d6a..93259802 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,6 @@ exclude = ''' | \.eggs | \.git | \.gitignore - | \.github | \.nox | \.pytest_cache | __pycache__ @@ -94,7 +93,6 @@ skip = [ ".eggs", ".git", ".gitignore", - ".github", ".nox", ".pytest_cache", "__pycache__", diff --git a/scripts/pack_core.py b/scripts/pack_core.py deleted file mode 100755 index 8d244976..00000000 --- a/scripts/pack_core.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2024 Antmicro -# SPDX-License-Identifier: Apache-2.0 - -import logging -import tarfile -from pathlib import Path -from tempfile import TemporaryDirectory - -import click - -from topwrap.repo.files import HttpGetFile -from topwrap.repo.user_repo import UserRepo, VerilogFileHandler - -click_r_dir = click.Path(exists=True, file_okay=False, dir_okay=True, readable=True) -click_opt_rw_dir = click.Path( - exists=False, file_okay=False, dir_okay=True, readable=True, writable=True -) -click_r_file = click.Path(exists=True, file_okay=True, dir_okay=False, readable=True) - -AVAILABLE_LOG_LEVELS = ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] -DEFAULT_LOG_LEVEL = "INFO" - - -cores = { - "verilog-axi": { - "root": "https://raw.githubusercontent.com/alexforencich/verilog-axi/master/rtl", - "sources": [ - "arbiter.v", - "axi_adapter.v", - "axi_adapter_rd.v", - "axi_adapter_wr.v", - "axi_axil_adapter.v", - "axi_axil_adapter_rd.v", - "axi_axil_adapter_wr.v", - # "axi_cdma.v", - "axi_cdma_desc_mux.v", - # "axi_crossbar.v", # unsupported parameter value - # "axi_crossbar_addr.v", - # "axi_crossbar_rd.v", # unsupported parameter value - # "axi_crossbar_wr.v", # unsupported parameter value - "axi_dma.v", - "axi_dma_desc_mux.v", - # "axi_dma_rd.v", - # "axi_dma_wr.v", - # "axi_dp_ram.v", # processing takes too much time - "axi_fifo.v", - "axi_fifo_rd.v", - "axi_fifo_wr.v", - # "axi_interconnect.v", - # "axi_ram.v", # processing takes too much time - "axi_ram_rd_if.v", - "axi_ram_wr_if.v", - "axi_ram_wr_rd_if.v", - "axi_register.v", - "axi_register_rd.v", - "axi_register_wr.v", - # "axi_vfifo.v", - "axi_vfifo_dec.v", - # "axi_vfifo_enc.v", # Unsup. expr. on dyn. range sel. on sig. `\seg_mem_rd_data' - "axi_vfifo_raw.v", - "axi_vfifo_raw_rd.v", - "axi_vfifo_raw_wr.v", - "axi_vfifo.v", - "axil_adapter.v", - "axil_adapter_rd.v", - "axil_adapter_wr.v", - "axil_cdc.v", - "axil_cdc_rd.v", - "axil_cdc_wr.v", - "axil_crossbar.v", - # "axil_crossbar_addr.v", - "axil_crossbar_rd.v", - "axil_crossbar_wr.v", - # "axil_dp_ram.v", # processing takes too much time - # "axil_interconnect.v", - # "axil_ram.v", # processing takes too much time - "axil_reg_if.v", - "axil_reg_if_rd.v", - "axil_reg_if_wr.v", - "axil_register.v", - "axil_register_rd.v", - "axil_register_wr.v", - "priority_encoder.v", - ], - } -} - - -def configure_log_level(log_level: str): - logging.basicConfig(level=DEFAULT_LOG_LEVEL) - if log_level not in AVAILABLE_LOG_LEVELS: - logging.warning(f"Wrong log-level value: {log_level}. Select one of {AVAILABLE_LOG_LEVELS}") - - logger = logging.getLogger() - logger.setLevel(log_level) - - -def scan_sources(source_dir: Path): - sources = [] - - for root, dirs, files in Path.walk(source_dir): - for file in files: - if file.lower().endswith(".v"): - logging.info(f"Found RTL file: {file}") - sources.append(Path(root) / file) - - return sources - - -@click.command() -@click.option("--log-level", default=DEFAULT_LOG_LEVEL, help="Log level") -@click.argument("name", type=click.STRING) -def pack_core(log_level: str, name: str): - """Generates reusable cores package for usage in Topwrap project. - - Scans one or more SOURCE_DIRS for RTL sources and packages them - into reusable "NAME.tar.gz" archive with YAML description files. - """ - configure_log_level(log_level) - core_repo = UserRepo() - - if name not in cores: - exit(1) - - core = cores[name] - root_path = core["root"] - core_files = [] - - for file in core["sources"]: - logging.info(f"Fetching: {file}") - core_files.append(HttpGetFile(f"{root_path}/{file}")) - - core_repo.add_files(VerilogFileHandler(core_files)) - - with TemporaryDirectory() as tmpdir: - core_repo.save(tmpdir) - with tarfile.open(f"{name}.tar.gz", "w:gz") as tar: - tar.add(tmpdir, arcname=".") - logging.info(f"Cores packaged to archive: {tar.name}") - - -if __name__ == "__main__": - pack_core()