ci: Build wheels for plugins

.github/utils/generate_matrix.py

@@ -0,0 +1,109 @@
+# This file is part of the Indico plugins.
+# Copyright (C) 2002 - 2024 CERN
+#
+# The Indico plugins are free software; you can redistribute
+# them and/or modify them under the terms of the MIT License;
+# see the LICENSE file for more details.
+
+import json
+import os
+import re
+import subprocess
+import sys
+from operator import itemgetter
+from pathlib import Path
+
+from setuptools.config.setupcfg import read_configuration
+
+
+def _plugin_has_assets(plugin_dir: Path):
+    return (plugin_dir / 'webpack.config.js').exists() or (plugin_dir / 'webpack-bundles.json').exists()
+
+
+def _plugin_has_i18n(plugin_dir: Path):
+    pkg_dir = plugin_dir / f'indico_{plugin_dir.name}'
+    return (pkg_dir / 'translations').exists()
+
+
+def _plugin_has_invalid_manifest(plugin_dir: Path):
+    pkg_dir = plugin_dir / f'indico_{plugin_dir.name}'
+    data_dirs = [
+        sub.name
+        for sub in pkg_dir.iterdir()
+        if sub.name not in {'__pycache__', 'client'} and sub.is_dir() and not any(sub.glob('*.py'))
+    ]
+    if not data_dirs:
+        return False
+    expected_manifest = {f'graft {pkg_dir.name}/{plugin_dir}' for plugin_dir in data_dirs}
+    manifest_file = plugin_dir / 'MANIFEST.in'
+    if not manifest_file.exists():
+        print(f'::error::{plugin_dir.name} has no manifest')
+        for line in expected_manifest:
+            print(f'::error::manifest entry missing: {line}')
+        return True
+    manifest_lines = set(manifest_file.read_text().splitlines())
+    if missing := (expected_manifest - manifest_lines):
+        print(f'::error::{plugin_dir.name} has incomplete manifest')
+        for line in missing:
+            print(f'::error::manifest entry missing: {line}')
+        return True
+    return False
+
+
+def _get_plugin_deps(plugin_dir: Path):
+    # XXX this probably needs to be adapted once we use the same CI for CERN plugin which
+    # sometimes have a dependency on public plugins, unless we accept that those will be
+    # downloaded from PyPI which is a bit ugly while working on a new release where nothing
+    # exists on PyPI yet...
+    reqs = read_configuration(plugin_dir / 'setup.cfg')['options']['install_requires']
+    return [
+        re.match(r'indico-plugin-([^>=<]+)', x).group(1).replace('-', '_')
+        for x in reqs
+        if x.startswith('indico-plugin-')
+    ]
+
+
+def _get_plugin_data(plugin_dir: Path):
+    name = plugin_dir.name
+    meta = name == '_meta'
+    return {
+        'plugin': name,
+        'install': not meta,
+        'assets': _plugin_has_assets(plugin_dir) if not meta else False,
+        'i18n': _plugin_has_i18n(plugin_dir) if not meta else False,
+        'deps': _get_plugin_deps(plugin_dir) if not meta else [],
+        'invalid_manifest': _plugin_has_invalid_manifest(plugin_dir) if not meta else False,
+    }
+
+
+def _get_changed_dirs():
+    try:
+        resp = subprocess.check_output(
+            ['gh', 'api', f'repos/{os.environ['GITHUB_REPOSITORY']}/pulls/{os.environ['PR_NUMBER']}/files'],
+            encoding='utf-8',
+        )
+    except subprocess.CalledProcessError:
+        print('::error::Could not get changed files')
+        sys.exit(1)
+    return {x['filename'].split('/')[0] for x in json.loads(resp) if '/' in x['filename']}
+
+
+def main():
+    plugin_data = sorted(
+        (_get_plugin_data(x) for x in Path().iterdir() if x.is_dir() and (x / 'setup.cfg').exists()),
+        key=itemgetter('plugin'),
+    )
+
+    # Filter out untouched plugin if we're running for a PR
+    if os.environ['GITHUB_EVENT_NAME'] == 'pull_request':
+        changed_dirs = _get_changed_dirs()
+        plugin_data = [x for x in plugin_data if x['plugin'] in changed_dirs]
+
+    matrix = json.dumps({'include': plugin_data}) if plugin_data else ''
+    with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+        f.write(f'PLUGINS_MATRIX={matrix}\n')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

.github/utils/get_core_repo.py

@@ -90,9 +90,11 @@ def main():
         print(f'Uncommon branch {branch}; defaulting to master')
         branch = 'master'
 
-    with open(os.environ['GITHUB_ENV'], 'a') as f:
-        f.write(f'{SCOPE}_REPO={full_repo}\n')
-        f.write(f'{SCOPE}_BRANCH={branch}\n')
+    for fn in (os.environ['GITHUB_ENV'], os.environ['GITHUB_OUTPUT']):
+        with open(fn, 'a') as f:
+            f.write(f'{SCOPE}_REPO={full_repo}\n')
+            f.write(f'{SCOPE}_BRANCH={branch}\n')
+
     return 0
 
 

.github/workflows/build.yml

@@ -0,0 +1,170 @@
+name: Build
+
+env:
+  PYTHON_VERSION: '3.12'
+  TZ: Europe/Zurich
+
+on:
+  push:
+    branches:
+      - 'master'
+      - '*.x'
+      - 'ci-build-plugins'  # TODO remove
+  pull_request:
+      branches:
+        - 'master'
+        - '*.x'
+        - 'ci-build-plugins'  # TODO remove
+      types:
+        - opened
+        - reopened
+        - synchronize
+        - labeled
+
+permissions:
+  contents: read
+
+jobs:
+  list-plugins:
+    name: Get plugin list 📃
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'build-wheel')
+    outputs:
+      PLUGINS_MATRIX: ${{ steps.list-plugins.outputs.PLUGINS_MATRIX }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 🐍
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install deps
+        run: pip install --user setuptools
+      - name: Generate matrix
+        id: list-plugins
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event_name == 'pull_request' && github.event.pull_request.number }}
+        run: python .github/utils/generate_matrix.py
+
+  build:
+    name: Build ${{ matrix.plugin }} 🛠
+    needs: list-plugins
+    runs-on: ubuntu-22.04
+    if: >-
+      needs.list-plugins.outputs.PLUGINS_MATRIX != '' &&
+      (github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'build-wheel'))
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.list-plugins.outputs.PLUGINS_MATRIX) }}
+    steps:
+      - name: Fail build if manifest is invalid
+        if: matrix.invalid_manifest
+        run: |
+          echo ::error::Plugin has invalid manifest
+          exit 1
+
+      - name: Checkout plugins
+        uses: actions/checkout@v4
+        with:
+          path: plugins
+          # prefer head commit over merge commit in case of PRs
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
+
+      - name: Pick Indico core repo
+        id: core-repo
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_BODY: ${{ github.event_name == 'pull_request' && github.event.pull_request.body }}
+          PR_BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref }}
+        run: python plugins/.github/utils/get_core_repo.py indico/indico INDICO
+
+      - name: Checkout core
+        uses: actions/checkout@v4
+        with:
+          path: indico
+          repository: indico/indico
+          ref: ${{ steps.core-repo.outputs.INDICO_BRANCH }}
+
+      - name: Set up Python 🐍
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: |
+            indico/requirements*.txt
+            plugins/**/setup.cfg
+
+      - name: Setup Node
+        if: matrix.assets
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+          cache: 'npm'
+          cache-dependency-path: indico/package-lock.json
+
+      - name: Install build deps 🔧
+        working-directory: indico
+        run: |
+          sudo apt-get install libpq-dev
+          pip install --user -U pip setuptools wheel
+          pip install --user -e '.[dev]'
+
+      - name: Install npm deps ☕
+        if: matrix.assets
+        working-directory: indico
+        run: npm ci
+
+      - name: Install plugin deps 🔧
+        if: matrix.install && matrix.deps != '[]'
+        working-directory: plugins
+        run: |
+          for dep in ${{ join(matrix.deps, ' ') }}; do
+            pip install --user -e $dep/
+          done
+
+      - name: Install plugin 🔧
+        if: matrix.install
+        working-directory: plugins
+        run: pip install --user -e ${{ matrix.plugin }}/
+
+      # XXX this is already done by build-wheel.py (but w/o react i18n which we don't use in plugins yet)
+      # - name: Compile translations 🏴‍☠️
+      #   if: matrix.i18n
+      #   working-directory: indico
+      #   run: indico i18n compile plugin ../plugins/${{ matrix.plugin }}
+
+      - name: Build wheel 🏗
+        working-directory: indico
+        run: ./bin/maintenance/build-wheel.py plugin ../plugins/${{ matrix.plugin }} --add-version-suffix
+
+      - uses: actions/upload-artifact@v4
+        name: Upload build artifacts 📦
+        with:
+          name: plugin-wheel-${{ matrix.plugin }}
+          retention-days: 7
+          path: ./indico/dist
+
+  bundle:
+    name: Bundle all wheels 📦
+    needs: build
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'build-wheel')
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          pattern: plugin-wheel-*
+          path: dist
+      - name: List artifacts 📃
+        run: ls -al dist/
+      - uses: actions/upload-artifact@v4
+        name: Upload build artifacts 📦
+        with:
+          name: plugin-wheels
+          retention-days: 7
+          path: dist
+      - name: Delete individual artifacts 🚮
+        uses: geekyeggo/delete-artifact@v5
+        with:
+          name: plugin-wheel-*