diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eabcabb..35aa985 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,99 @@ name: CI on: push jobs: + # # No build matrix stage + # setup-python: + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ['3.8', '3.9', '3.10'] + # runs-on: ${{ matrix.os }} + # steps: + # - run: | + # echo "black" >> requirements.txt + # echo "pylint" >> requirements.txt + # echo "pytest" >> requirements.txt + # - uses: actions/setup-python@v4 + # with: + # python-version: '${{ matrix.python-version }}' + # cache: pip + # - run: python --version + # - run: python -m pip install -r requirements.txt + # - run: python -m pip list + # check-code: + # needs: setup-python + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ['3.8', '3.9', '3.10'] + # runs-on: ${{ matrix.os }} + # steps: + # - run: | + # echo "black" >> requirements.txt + # echo "pylint" >> requirements.txt + # echo "pytest" >> requirements.txt + # - uses: actions/setup-python@v4 + # with: + # python-version: '${{ matrix.python-version }}' + # cache: pip + # - run: python -m pip install -r requirements.txt + # - run: black --check . + # - run: pylint . + # unit-test: + # needs: setup-python + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ['3.8', '3.9', '3.10'] + # runs-on: ${{ matrix.os }} + # steps: + # - run: | + # echo "black" >> requirements.txt + # echo "pylint" >> requirements.txt + # echo "pytest" >> requirements.txt + # - uses: actions/setup-python@v4 + # with: + # python-version: '${{ matrix.python-version }}' + # cache: pip + # - run: python -m pip install -r requirements.txt + # - run: pytest + build-matrix: name: '🧱 Build matrix' runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build-matrix.outputs.matrix }} steps: - name: Checkout code uses: actions/checkout@v3 - name: '🧱 Build matrix' id: build-matrix uses: ./ - - run: echo "${{ steps.build-matrix.outputs.matrix }}" + with: + matrix: | + os: ubuntu-latest windows-latest macos-latest, + node-version: 16 18, + python-version: 3.6 3.8 3.10 3.12 + test-matrix: + name: '🧪 Test matrix' + needs: + - build-matrix + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - name: '📢 Echo matrix combination' + run: | + echo "${{ matrix.os }}" + echo "${{ matrix.node-version }}" + echo "${{ matrix.python-version }}" + + # debug-matrix: + # strategy: + # matrix: + # foo: [a, b, c] + # bar: [a, b, a] + # runs-on: ubuntu-latest + # steps: + # - run: echo "a=${{ matrix.a }}, b=${{ matrix.b }}" diff --git a/.gitignore b/.gitignore index 9e27a53..489cd11 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode/ .idea/ .direnv/ +.dev/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8244b3c..c318f43 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,8 @@ repos: - id: check-added-large-files - id: check-merge-conflict - id: check-yaml + - id: check-executables-have-shebangs + - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.7.1 hooks: @@ -20,3 +22,8 @@ repos: hooks: - id: check-github-actions - id: check-github-workflows + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: shellcheck + - id: shfmt diff --git a/README.md b/README.md index 971875e..7ff21e0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,352 @@ # Build Matrix -GitHub action to create a reusable and potentially dynamic matrix for workflow jobs. +GitHub action to create reusable dynamic job matrices for your workflows. + +This action adresses a wide known problem of reusing the same job matrix +multiple times or even generating a matrix on the fly. + +The main goal of this action is to be as much compatible with built-in +[GitHub matrices](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs) +as possible and thus allow you a smooth transition in your workflow. + +## Basic usage + +```yaml +jobs: + # Build matrix + build-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build.outputs.matrix }} + steps: + - id: build + uses: druzsan/build-matrix@v1 + with: + matrix: | + os: ubuntu-latest windows-latest macos-latest, + python-version: 3.8 3.9 3.10 + # Setup python and print version + setup-python: + needs: build-matrix + strategy: + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + - run: python --version +``` + +## Inputs + +Inputs are the same as for the built-in matrix, but their syntax is slightly +different. + +All inputs are optional with empty strings as default, but at least one of the +three inputs must be specified. + +Only strings are allowed as GitHub action inputs, but you can use any +whitespaces including newlines for word separation in all inputs. It is +recommended to use (any) YAML multiline strings to unclutter your inputs, e.g.: + +```yaml +with: + matrix: | + node-version: 12 14 16 + include: | + node-version: 16 + npm: 6 +``` + +All words themselves must not contain any whitespaces, colons and commas. All +other characters are allowed, but valid behaviour can not be validated for all +possible characters, so be aware that both input parsing and later matrix usage +could be affected by some edge cases. + +### `matrix` + +Optional base matrix configuration with the following syntax: + +``` +variable-1: value value <...>, +variable-2: value value <...>, + <...> +variable-n: value value <...>[,] +``` + +Variable names must be unique and differ from exact 'include' and 'exclude' +strings reserved by the built-in matrix. + +### `include` + +Optioal extra matrix configurations to add to the base matrix. Must have the following +syntax: + +``` +variable-i: value variable-j: value <...>, + <...> +variable-k: value variable-l: value <...>[,] +``` + +Variable names in each "row" should be unique but can differ from the ones in +the [`matrix`](#matrix) input. + +### `exclude` + +Optional matrix configurations to exclude from the base matrix. Have the same syntax and +restrictions as [`include`](#include). + +## Outputs + +Parsed matrix is printed inside the action's step as a pretty formated YAML +using `yq`, so you can visually inspect it. + +Parsed matrix is also set as `MATRIX` environment variable. + +### `matrix` + +valid JSON matrix ready to be set as `jobs..outputs` used in +`jobs..strategy`: + +```yaml +matrix: ${{ fromJson(needs..outputs.matrix) }} +``` + +## Errors + +Not only syntax validity, but also built-in matrix' restrictions are checked. If +you find a case where either of the checks does not work, feel free to report as +an issue. + +Error logs try to give as much infomation on problem as possible. + +## Advanced usage + +### Reuse a matrix + +Sometimes you need to run different jobs os the same set of configurations, e.g. +install python dependencies, check code quality and run unit tests. + +
+ Solution using the built-in matrix + +```yaml +jobs: + # No matrix build + # Setup python environment and cache installed packages + setup-python: + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.8', '3.9', '3.10'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + cache: pip + - run: python -m pip install -r requirements.txt + # Check code quality + check-code: + needs: setup-python + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.8', '3.9', '3.10'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + cache: pip + - run: python -m pip install -r requirements.txt + - run: black --check . + - run: pylint . + # Test code + unit-test: + needs: setup-python + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.8', '3.9', '3.10'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + cache: pip + - run: pytest +``` + +
+ +```yaml +jobs: + # Build matrix + build-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build.outputs.matrix }} + steps: + - id: build + uses: druzsan/build-matrix@v1 + with: + matrix: | + os: ubuntu-latest windows-latest macos-latest, + python-version: 3.8 3.9 3.10 + # Setup python environment and cache installed packages + setup-python: + needs: build-matrix + strategy: + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + cache: pip + - run: python -m pip install -r requirements.txt + # Check code quality + check-code: + needs: [build-matrix, setup-python] + strategy: + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + cache: pip + - run: python -m pip install -r requirements.txt + - run: black --check . + - run: pylint . + # Test code + unit-test: + needs: [build-matrix, setup-python] + strategy: + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + cache: pip + - run: python -m pip install -r requirements.txt + - run: pytest +``` + +### Build dynamic matrix + +Sometimes you need to run a job on different sets of configurations, depending +on branch, triggering event etc. + +
+ Solution using the built-in matrix + +```yaml +jobs: + # No matrix build + # Test code on a dev branch + unit-test-dev: + if: github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.8' + - run: python -m pip install -r requirements.txt + - run: pytest + # Test code on the main branch + unit-test-main: + if: github.ref == 'refs/heads/main' + strategy: + matrix: + os: [ubuntu-latest] + python-version: ['3.8', '3.9', '3.10'] + include: + - os: windows-latest + python-version: 3.8 + - os: macos-latest + python-version: 3.8 + runs-on: + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + - run: python -m pip install -r requirements.txt + - run: pytest + # Test code on a tag + unit-test-tag: + if: startsWith(github.ref, 'refs/tags/v') + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.8', '3.9', '3.10'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with:include: + python-version: '${{ matrix.python-version }}' + - run: python -m pip install -r requirements.txt + - run: pytest +``` + +
+ +```yaml +jobs: + # Build matrix + build-matrix: + runs-on: ubuntu-latest + steps: + - if: startsWith(github.ref, 'refs/tags/v') + uses: druzsan/build-matrix@v1 + with: + os: ubuntu-latest windows-latest macos-latest + python-version: 3.8 3.9 3.10 + - if: github.ref == 'refs/heads/main' + uses: druzsan/build-matrix@v1 + with: + os: ubuntu-latest + python-version: 3.8 3.9 3.10 + include: | + os: windows-latest python-version: 3.8, + os: macos-latest python-version: 3.8 + - if: github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/v') + uses: druzsan/build-matrix@v1 + with: + os: ubuntu-latest + python-version: 3.8 + # MATRIX environment variable is set by the last executed action + - id: set-matrix + run: echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + # Test code + unit-test: + needs: build-matrix + strategy: + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '${{ matrix.python-version }}' + - run: python -m pip install -r requirements.txt + - run: pytest +``` + +## Examples + +## Discussions diff --git a/action.yml b/action.yml index c76d2a5..6c2cf2a 100644 --- a/action.yml +++ b/action.yml @@ -1,12 +1,19 @@ name: Build Matrix author: Alexander Druz -description: GitHub action to create a reusable and potentially dynamic matrix for workflow jobs. +description: Create reusable dynamic job matrices for your workflows. branding: icon: layers color: gray-dark inputs: matrix: - description: '...' + default: '' + description: Base matrix + include: + default: '' + description: Extra matrix combinations to include + exclude: + default: '' + description: Matrix combinations to exclude from matrix outputs: matrix: description: matrix in JSON format @@ -17,5 +24,8 @@ runs: - name: Set matrix id: set-matrix run: | - echo "${{ inputs.matrix }}" + MATRIX="$(dist/parse-matrix.sh "${{ inputs.matrix }}" "${{ inputs.include }}" "${{ inputs.exclude }}")" + echo "$MATRIX" | yq -P '{"matrix":.}' + echo "MATRIX=$MATRIX" >> $GITHUB_ENV + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT shell: bash diff --git a/dist/parse-matrix.sh b/dist/parse-matrix.sh new file mode 100755 index 0000000..b6574f0 --- /dev/null +++ b/dist/parse-matrix.sh @@ -0,0 +1,115 @@ +#!/bin/bash +set -e + +HELP="Usage: $0 MATRIX INCLUDE EXCLUDE + +Parse a matrix for GitHub jobs in JSON format from the given arguments. + +MATRIX arguments should fulfill the following pattern with unique variable names different from the reserved 'include' and 'exclude': + variable-1: value value ..., + variable-2: value value ..., + ... + variable-n: value value ...[,] +INCLUDE and EXCLUDE arguments should fulfill the following pattern with unique variable names inside each combination: + variable-i: value variable-j: value ..., + ... + variable-k: value variable-l: value ...[,] +All names and values of variables must not contain any spaces, colons and commas. Any spaces can be used for word and line separation." + +function fail { + # Print error message if given, print help and exit with code 1. + echo "Error${1:+: $1}" >&2 + echo >&2 + echo "$HELP" >&2 + exit 1 +} + +function sanitize { + # Sanitize string as expected in further steps: + # - replace all spaces (including newlines) with simple whitespaces; + # - remove trailing comma if exists; + # - replace all other commas with newlines (i.e. split string by commas). + if [[ "$#" -ne 1 ]]; then + echo "Error: Single argument to sanitize expected, but $# arguments received." >&2 + exit 1 + fi + echo "${1//[[:space:]]/ }" | sed 's/,\s*$//g' | sed 's/,/\n/g' +} + +if [[ "$#" -ne 3 ]]; then + fail "Exactly 3 arguments expected, but $# arguments received." +fi + +INPUT_MATRIX="$1" +declare -A INPUT_EXTRAS=([include]="$2" [exclude]="$3") +MATRIX="" + +# Define REGEX patterns for grep. +RE_WORD='[^\s:,]+' +RE_VARIABLE="${RE_WORD}\s*:(\s*${RE_WORD})+" +RE_COMBINATION="(${RE_WORD}\s*:\s*${RE_WORD}\s*)+" + +# Check if the whole matrix consists not only of spaces. +if [[ -n "${INPUT_MATRIX//[[:space:]]/}" ]]; then + INPUT_MATRIX="$(sanitize "$INPUT_MATRIX")" + # Check if every row filfills the expected pattern. + if echo "$INPUT_MATRIX" | grep -qvP "^\s*${RE_VARIABLE}\s*$"; then + ERROR_MESSAGE="$( + echo "Invalid matrix rows found:" + echo "$INPUT_MATRIX" | grep -vP "^\s*${RE_VARIABLE}\s*$" | + sed 's/^\s*\|\s*$/\t/g;' + )" + fail "$ERROR_MESSAGE" + fi + VARIABLES="$(echo "$INPUT_MATRIX" | grep -oP "$RE_VARIABLE" | + jq -R 'capture("^\\s*(?[^\\s:,]+)\\s*:(?.*)$") | .value |= [scan("[^\\s:,]+")] | [.] | from_entries')" + # Check for duplicates in variable names. + DUPLICATES="$(echo "$VARIABLES" | jq 'keys' | + jq -rs 'add | group_by(.) | map(select(length>1) | .[0]) | join(", ")')" + if [[ -n "$DUPLICATES" ]]; then + fail "Following duplicated variable names found in matrix: ${DUPLICATES}." + fi + # Check for invalid variable names. + RESERVED_NAMES="$(echo "$VARIABLES" | jq 'keys' | + jq -rs 'add | map(select(test("^(include|exclude)$"))) | join(", ")')" + if [[ -n "$RESERVED_NAMES" ]]; then + fail "Following reserved variable names found in matrix: ${RESERVED_NAMES}." + fi + MATRIX="$(echo "${MATRIX}${VARIABLES}" | jq -s 'add')" +fi + +for EXTRA_NAME in "${!INPUT_EXTRAS[@]}"; do + INPUT_EXTRA="${INPUT_EXTRAS[$EXTRA_NAME]}" + # Check if extra consists not only of spaces. + if [[ -n "${INPUT_EXTRA//[[:space:]]/}" ]]; then + INPUT_EXTRA="$(sanitize "$INPUT_EXTRA")" + + # Check if every combination fulfills the expected pattern. + if echo "$INPUT_EXTRA" | grep -qvP "^\s*${RE_COMBINATION}\s*$"; then + ERROR_MESSAGE="$( + echo "Invalid combinations found in ${EXTRA_NAME}:" + echo "$INPUT_EXTRA" | grep -vP "^\s*${RE_COMBINATION}\s*$" | + sed 's/^\s*\|\s*$/\t/g;' + )" + fail "$ERROR_MESSAGE" + fi + EXTRA="$(echo "$INPUT_EXTRA" | grep -oP "$RE_COMBINATION" | + jq -R '[capture("(?[^\\s:,]+)\\s*:\\s*(?[^\\s:,]+)"; "g")]')" + DUPLICATES="$(echo "$EXTRA" | jq 'group_by(.key) | map(select(length>1) | .[0].key)')" + if [[ "$(echo "$DUPLICATES" | jq -s 'map(length) | add')" -ne 0 ]]; then + ERROR_MESSAGE="$( + echo "Duplicated variable names found in ${EXTRA_NAME}:" + echo "$DUPLICATES" | jq -rs 'map(join(", ")) | to_entries | map(select(.value != "")) | map("Combination " + (.key | tostring) + ": " + .value) | .[]' + )" + fail "$ERROR_MESSAGE" + fi + EXTRA="$(echo "$EXTRA" | jq 'from_entries' | jq -s "{$EXTRA_NAME: .}")" + MATRIX="$(echo "${MATRIX}${EXTRA}" | jq -s 'add')" + fi +done + +if [[ -z "$MATRIX" ]]; then + fail "At least one of matrix, include or exclude should be not empty" +fi + +echo "$MATRIX" | jq -c