diff --git a/.github/utils/generate_matrix.py b/.github/utils/generate_matrix.py new file mode 100644 index 00000000..f5c09dd4 --- /dev/null +++ b/.github/utils/generate_matrix.py @@ -0,0 +1,109 @@ +# This file is part of the Indico plugins. +# Copyright (C) 2002 - 2024 CERN +# +# The Indico plugins are free software; you can redistribute +# them and/or modify them under the terms of the MIT License; +# see the LICENSE file for more details. + +import json +import os +import re +import subprocess +import sys +from operator import itemgetter +from pathlib import Path + +from setuptools.config.setupcfg import read_configuration + + +def _plugin_has_assets(plugin_dir: Path): + return (plugin_dir / 'webpack.config.js').exists() or (plugin_dir / 'webpack-bundles.json').exists() + + +def _plugin_has_i18n(plugin_dir: Path): + pkg_dir = plugin_dir / f'indico_{plugin_dir.name}' + return (pkg_dir / 'translations').exists() + + +def _plugin_has_invalid_manifest(plugin_dir: Path): + pkg_dir = plugin_dir / f'indico_{plugin_dir.name}' + data_dirs = [ + sub.name + for sub in pkg_dir.iterdir() + if sub.name not in {'__pycache__', 'client'} and sub.is_dir() and not any(sub.glob('*.py')) + ] + if not data_dirs: + return False + expected_manifest = {f'graft {pkg_dir.name}/{plugin_dir}' for plugin_dir in data_dirs} + manifest_file = plugin_dir / 'MANIFEST.in' + if not manifest_file.exists(): + print(f'::error::{plugin_dir.name} has no manifest') + for line in expected_manifest: + print(f'::error::manifest entry missing: {line}') + return True + manifest_lines = set(manifest_file.read_text().splitlines()) + if missing := (expected_manifest - manifest_lines): + print(f'::error::{plugin_dir.name} has incomplete manifest') + for line in missing: + print(f'::error::manifest entry missing: {line}') + return True + return False + + +def _get_plugin_deps(plugin_dir: Path): + # XXX this probably needs to be adapted once we use the same CI for CERN plugin which + # sometimes have a dependency on public plugins, unless we accept that those will be + # downloaded from PyPI which is a bit ugly while working on a new release where nothing + # exists on PyPI yet... + reqs = read_configuration(plugin_dir / 'setup.cfg')['options']['install_requires'] + return [ + re.match(r'indico-plugin-([^>=<]+)', x).group(1).replace('-', '_') + for x in reqs + if x.startswith('indico-plugin-') + ] + + +def _get_plugin_data(plugin_dir: Path): + name = plugin_dir.name + meta = name == '_meta' + return { + 'plugin': name, + 'install': not meta, + 'assets': _plugin_has_assets(plugin_dir) if not meta else False, + 'i18n': _plugin_has_i18n(plugin_dir) if not meta else False, + 'deps': _get_plugin_deps(plugin_dir) if not meta else [], + 'invalid_manifest': _plugin_has_invalid_manifest(plugin_dir) if not meta else False, + } + + +def _get_changed_dirs(): + try: + resp = subprocess.check_output( + ['gh', 'api', f'repos/{os.environ['GITHUB_REPOSITORY']}/pulls/{os.environ['PR_NUMBER']}/files'], + encoding='utf-8', + ) + except subprocess.CalledProcessError: + print('::error::Could not get changed files') + sys.exit(1) + return {x['filename'].split('/')[0] for x in json.loads(resp) if '/' in x['filename']} + + +def main(): + plugin_data = sorted( + (_get_plugin_data(x) for x in Path().iterdir() if x.is_dir() and (x / 'setup.cfg').exists()), + key=itemgetter('plugin'), + ) + + # Filter out untouched plugin if we're running for a PR + if os.environ['GITHUB_EVENT_NAME'] == 'pull_request': + changed_dirs = _get_changed_dirs() + plugin_data = [x for x in plugin_data if x['plugin'] in changed_dirs] + + matrix = json.dumps({'include': plugin_data}) if plugin_data else '' + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f'PLUGINS_MATRIX={matrix}\n') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/utils/get_core_repo.py b/.github/utils/get_core_repo.py index 9c7bea8b..d9a09002 100644 --- a/.github/utils/get_core_repo.py +++ b/.github/utils/get_core_repo.py @@ -90,9 +90,11 @@ def main(): print(f'Uncommon branch {branch}; defaulting to master') branch = 'master' - with open(os.environ['GITHUB_ENV'], 'a') as f: - f.write(f'{SCOPE}_REPO={full_repo}\n') - f.write(f'{SCOPE}_BRANCH={branch}\n') + for fn in (os.environ['GITHUB_ENV'], os.environ['GITHUB_OUTPUT']): + with open(fn, 'a') as f: + f.write(f'{SCOPE}_REPO={full_repo}\n') + f.write(f'{SCOPE}_BRANCH={branch}\n') + return 0 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..fe7bcf3c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,170 @@ +name: Build + +env: + PYTHON_VERSION: '3.12' + TZ: Europe/Zurich + +on: + push: + branches: + - 'master' + - '*.x' + - 'ci-build-plugins' # TODO remove + pull_request: + branches: + - 'master' + - '*.x' + - 'ci-build-plugins' # TODO remove + types: + - opened + - reopened + - synchronize + - labeled + +permissions: + contents: read + +jobs: + list-plugins: + name: Get plugin list 📃 + runs-on: ubuntu-22.04 + if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'build-wheel') + outputs: + PLUGINS_MATRIX: ${{ steps.list-plugins.outputs.PLUGINS_MATRIX }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python 🐍 + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install deps + run: pip install --user setuptools + - name: Generate matrix + id: list-plugins + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event_name == 'pull_request' && github.event.pull_request.number }} + run: python .github/utils/generate_matrix.py + + build: + name: Build ${{ matrix.plugin }} 🛠 + needs: list-plugins + runs-on: ubuntu-22.04 + if: >- + needs.list-plugins.outputs.PLUGINS_MATRIX != '' && + (github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'build-wheel')) + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.list-plugins.outputs.PLUGINS_MATRIX) }} + steps: + - name: Fail build if manifest is invalid + if: matrix.invalid_manifest + run: | + echo ::error::Plugin has invalid manifest + exit 1 + + - name: Checkout plugins + uses: actions/checkout@v4 + with: + path: plugins + # prefer head commit over merge commit in case of PRs + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }} + + - name: Pick Indico core repo + id: core-repo + env: + GH_TOKEN: ${{ github.token }} + PR_BODY: ${{ github.event_name == 'pull_request' && github.event.pull_request.body }} + PR_BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref }} + run: python plugins/.github/utils/get_core_repo.py indico/indico INDICO + + - name: Checkout core + uses: actions/checkout@v4 + with: + path: indico + repository: indico/indico + ref: ${{ steps.core-repo.outputs.INDICO_BRANCH }} + + - name: Set up Python 🐍 + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: | + indico/requirements*.txt + plugins/**/setup.cfg + + - name: Setup Node + if: matrix.assets + uses: actions/setup-node@v4 + with: + node-version: 18.x + cache: 'npm' + cache-dependency-path: indico/package-lock.json + + - name: Install build deps 🔧 + working-directory: indico + run: | + sudo apt-get install libpq-dev + pip install --user -U pip setuptools wheel + pip install --user -e '.[dev]' + + - name: Install npm deps ☕ + if: matrix.assets + working-directory: indico + run: npm ci + + - name: Install plugin deps 🔧 + if: matrix.install && matrix.deps != '[]' + working-directory: plugins + run: | + for dep in ${{ join(matrix.deps, ' ') }}; do + pip install --user -e $dep/ + done + + - name: Install plugin 🔧 + if: matrix.install + working-directory: plugins + run: pip install --user -e ${{ matrix.plugin }}/ + + # XXX this is already done by build-wheel.py (but w/o react i18n which we don't use in plugins yet) + # - name: Compile translations 🏴‍☠️ + # if: matrix.i18n + # working-directory: indico + # run: indico i18n compile plugin ../plugins/${{ matrix.plugin }} + + - name: Build wheel 🏗 + working-directory: indico + run: ./bin/maintenance/build-wheel.py plugin ../plugins/${{ matrix.plugin }} --add-version-suffix + + - uses: actions/upload-artifact@v4 + name: Upload build artifacts 📦 + with: + name: plugin-wheel-${{ matrix.plugin }} + retention-days: 7 + path: ./indico/dist + + bundle: + name: Bundle all wheels 📦 + needs: build + runs-on: ubuntu-22.04 + if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'build-wheel') + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + merge-multiple: true + pattern: plugin-wheel-* + path: dist + - name: List artifacts 📃 + run: ls -al dist/ + - uses: actions/upload-artifact@v4 + name: Upload build artifacts 📦 + with: + name: plugin-wheels + retention-days: 7 + path: dist + - name: Delete individual artifacts 🚮 + uses: geekyeggo/delete-artifact@v5 + with: + name: plugin-wheel-*