docker-matrix-wheels.yml

name: Matrix-build-wheels

on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
      - 'releases/**'
  release:
    types: [ published  ]

  schedule:
    - cron: '45 4 * * 0'
    
jobs:
  prepare_build:
    runs-on: ubuntu-latest
    outputs:
      fhempyV: ${{steps.split_fhempyV.outputs.result}}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          filter: tree:0

      - name: Split fhempy Version
        id: split_fhempyV
        run: |
          echo "result=$(grep '^fhempy' requirements.txt | cut -d '=' -f3)" >> $GITHUB_OUTPUT
 
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            image=moby/buildkit:latest
      
      - name: Get Python version from Dockerfile
        id: cache-key
        run: |
          image_version=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f2 | cut -d '@' -f1) 
          echo "IMAGE_VERSION_SHORT=${image_version%.*}" >> $GITHUB_ENV
          echo "IMAGE_VERSION=${image_version}" >> $GITHUB_ENV
          
      - name: Init wheels cache
        id: cache-wheels
        uses: actions/cache@v4
        with:
          path: ./wheelhouse
          key: wheels-${{ env.IMAGE_VERSION }}-${{ hashFiles('requirements.txt') }}
          restore-keys: |
           wheels-${{ env.IMAGE_VERSION }}
           wheels-${{ env.IMAGE_VERSION_SHORT }}
           wheels-
                
      - name: create cachedir
        run: mkdir -p ./wheelhouse

      - name: Build base wheels and export to cache
        id: build_basewheels_cross
        uses: docker/build-push-action@v6.6.0
        with:
          context: .
          file: Dockerfile
          platforms: linux/amd64,linux/arm/v7
          push: false
          target: export-stage
          cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
          cache-from: |
            type=gha,scope=wheel_builder_stage_amd64
            type=gha,scope=wheel_builder_stage_armv7
            type=gha,scope=wheel_builder_stage_armv6
          load: false
          outputs: type=local,dest=./wheelhouse

      - name: cache wheel builder image amd64
        uses: docker/build-push-action@v6.6.0
        with:
          context: .
          file: Dockerfile
          platforms: linux/amd64
          push: false
          target: w-builder
          cache-to: type=gha,mode=min,scope=wheel_builder_stage_amd64
          cache-from: type=local,src=/tmp/.buildx-cache

      - name: cache wheel builder image armv7
        uses: docker/build-push-action@v6.6.0
        with:
          context: .
          file: Dockerfile
          platforms: linux/arm/v7
          push: false
          target: w-builder
          cache-to: type=gha,mode=min,scope=wheel_builder_stage_armv7
          cache-from: type=local,src=/tmp/.buildx-cache

    #  - name: cache wheel builder image armv6
    #    uses: docker/build-push-action@v5.1.0
    #    with:
    #      context: .
    #      file: Dockerfile
    #      platforms: linux/arm/v6
    #      push: false
    #      target: w-builder
    #      cache-to: type=gha,mode=min,scope=wheel_builder_stage_armv6
    #      cache-from: type=local,src=/tmp/.buildx-cache


      - name: Build amd64 base image and export to cache
        id: build_base_amd64
        uses: docker/build-push-action@v6.6.0
        with:
          context: .
          file: Dockerfile
          platforms: linux/amd64
          push: false
          target: base
          cache-from: |
            type=gha,scope=base_image_stage_amd64
          cache-to: |
            type=gha,scope=base_image_stage_amd64,mode=max
          load: false

      - name: Build armv7 base image and export to cache
        id: build_base_armv7
        uses: docker/build-push-action@v6.6.0
        with:
          context: .
          file: Dockerfile
          platforms: linux/arm/v7
          push: false
          target: base
          cache-from: |
            type=gha,scope=base_image_stage_armv7
          cache-to: |
            type=gha,scope=base_image_stage_armv7,mode=max
          load: false

      - name: generate requierementsfile for extra requirements
        run: |
          #echo "cryptography" >> requirements_extra.txt
          echo "pyaml" >> requirements_extra.txt
          
          
      - name: Build extra wheels 
        uses: docker/build-push-action@v6.6.0
        with:
          context: .
          file: Dockerfile
          platforms: linux/amd64,linux/arm/v7
          push: false
          target: export-stage
          load: false
          outputs: type=local,dest=./wheelhouse
          build-args:
            REQUIREMENTS_FILE=requirements_extra.txt
    
  prepare_matrix:
    #name:  Find fhempy modules 
    #needs: prepare_build
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{steps.getModules.outputs.matrix}}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          filter: tree:0

      - name: Split fhempy Version
        run: |
          echo "fhempyVersion=$(grep '^fhempy' requirements.txt | cut -d '=' -f3)" >> $GITHUB_ENV

      - uses: actions/checkout@v4
        with:
          repository: 'fhempy/fhempy'
          path: fhempyRepo
          filter: tree:0
          ref: 'v${{ env.fhempyVersion }}'

      - name: Get module names and prepare matrix
        id: getModules
        working-directory: ./fhempyRepo/FHEM/bindings/python/fhempy/lib
        run: |
          ${GITHUB_WORKSPACE}/scripts/get-deps.sh .
        shell: bash

  buildImages:
    runs-on: ubuntu-latest
    needs: [prepare_build, prepare_matrix]
    permissions:
      contents: write
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write
    env:
      # Use docker.io for Docker Hub if empty
      REGISTRY: ghcr.io
      # github.repository as <account>/<repo>
      IMAGE_NAME: ${{ github.repository }}
    strategy:
      max-parallel: 5
      fail-fast: false
      matrix:
        ${{fromJson(needs.prepare_matrix.outputs.matrix)}}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          filter: tree:0

      - name: check variables
        run: |
          echo '${{ toJSON(matrix) }}'
          echo '${{ matrix.module }}'

      - name: write requirement for mod from matrix input
        run: |
          touch ./requirements_mod.txt
          echo '${{toJSON(matrix.requirements)}}' | jq --raw-output -e '.[]' > ./requirements_mod.txt || true
          cat ./requirements_mod.txt

      - name: Run Docker on tmpfs
        uses: sidey79/docker-on-tmpfs@main
        with:
          tmpfs_size: 6
          swap_size: 4
          swap_location: '/mnt/swapfile'

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            image=moby/buildkit:latest

      - name: Extract Docker metadata
        id: meta-it
        uses: docker/metadata-action@v5.5.1
        with:
          images: runtime

      - name: Get Python version from Dockerfile
        id: cache-key
        run: |
          image_version=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f2 | cut -d '@' -f1) 
          echo "IMAGE_VERSION_SHORT=${image_version%.*}" >> $GITHUB_ENV
          echo "IMAGE_VERSION=${image_version}" >> $GITHUB_ENV
      
      - uses: Wandalen/wretry.action@v3.5.0
        name: restore base cache with retry
        id: cache
        with:
          attempt_delay: 35000
          attempt_limit: 3
          action: actions/cache/restore@v4
          with: |
            path: ./wheelhouse
            fail-on-cache-miss: true
            key: wheels-${{ env.IMAGE_VERSION }}-${{ hashFiles('requirements.txt') }}
               
      - name: Init wheels cache
        id: cache-wheels
        uses: actions/cache@v4
        with:
          save-always: true
          path: ./wheelhouse
          key: wheels-${{ matrix.module }}-${{ env.IMAGE_VERSION }}
          restore-keys: |
            wheels-${{ matrix.module }}-${{ env.IMAGE_VERSION }}
            wheels-${{ matrix.module }}-
            wheels-${{ env.IMAGE_VERSION }}-${{ hashFiles('requirements.txt') }}
            wheels-${{ env.IMAGE_VERSION }}
              
      - name: create cachedir if not exists
        run: |
          mkdir -p ./wheelhouse/linux_amd64
          mkdir -p ./wheelhouse/linux_arm_v7
            
      - name: Build wheels for mod ${{ matrix.module }}
        uses: docker/build-push-action@v6.6.0
        timeout-minutes: 75
        with:
          context: .
          file: Dockerfile
          platforms: linux/amd64,linux/arm/v7
          push: false
          load: false
          target: export-stage
          cache-from: |
            type=gha,scope=wheel_builder_stage_amd64
            type=gha,scope=wheel_builder_stage_armv7
            type=gha,scope=wheel_builder_stage_armv6
          build-args: |
            REQUIREMENTS_FILE=requirements_mod.txt
          outputs: type=local,dest=./wheelhouse
        
      - name: Build for integration test 
        id: build-it
        uses: docker/build-push-action@v6.6.0
        with:
          context: .
          file: Dockerfile
          platforms: linux/amd64
          push: false
          load: true
          target: runtime
          #cache-to: type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64,ignore-error=true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: |
            type=gha,scope=base_image_stage_amd64
          # type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64
          build-args: |
            REQUIREMENTS_FILE=requirements_mod.txt
          # PKGS=${{ matrix.PKGS }}
  
      - name: Run integration tests
        shell: bash
        run: |
          ./scripts/test-integration.sh ${{ steps.build-it.outputs.imageid }}

      - name: Install cosign
        if: github.event_name == 'release'
        uses: sigstore/cosign-installer@v3.5.0
 
      # Login against a Docker registry if it is a release
      # https://github.com/docker/login-action
      - name: Log into registry ${{ env.REGISTRY }}
        if: github.event_name == 'release'
        uses: docker/login-action@v3.3.0
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract Docker metadata
        id: meta
        uses: docker/metadata-action@v5.5.1
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_${{ matrix.module }}
          flavor: |
             latest= ${{ fromJSON('["auto", "false"]')[github.event.release.prerelease == 1] }}
          tags: |
            type=ref,enable=true,priority=600,prefix=pr-,suffix=,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}},enable=${{ github.event.release.prerelease == 0 }}
            type=ref,event=branch
            type=ref,event=pr
          labels: |
            org.opencontainers.image.title=${{ env.IMAGE_NAME }}_${{ matrix.module }}
      
      - name: Build and push Docker image for all plattforms
        uses: docker/build-push-action@v6.6.0
        id: docker_build_runtime_cross
        with:
          context: .
          platforms: linux/arm/v7,linux/amd64
          push: ${{ github.event_name == 'release' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          target: runtime
          cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
          cache-from: |
            type=gha,scope=base_image_stage_amd64
            type=gha,scope=base_image_stage_armv6
            type=gha,scope=base_image_stage_armv7
          # type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64
          # type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-armv7
          build-args: |
            PKGS=${{ matrix.PKGS }}

      - name: show image details
        run: | 
          echo "${{ steps.docker_build_runtime_cross.outputs.imageid }}"
          echo "${{ steps.docker_build_runtime_cross.outputs.digest }}"
          echo '${{ steps.docker_build_runtime_cross.outputs.metadata }}'
      # Sign the resulting Docker image digest except on PRs.
      # This will only write to the public Rekor transparency log when the Docker
      # repository is public to avoid leaking data.  If you would like to publish
      # transparency data even for private images, pass --force to cosign below.
      # https://github.com/sigstore/cosign
      - name: Sign the published Docker image
        if: ${{ github.event_name == 'release' }}
        env:
          TAGS: ${{ steps.meta.outputs.tags }}
          DIGEST: ${{ steps.docker_build_runtime_cross.outputs.digest }}
        # This step uses the identity token to provision an ephemeral certificate
        # against the sigstore community Fulcio instance.
        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

      - name: Create Job Summary
        run: |
          echo "### Image report for : ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY
          echo "| Module | Image name  |" >> $GITHUB_STEP_SUMMARY
          echo "|-------|----------------------------------------------------------------------|" >> $GITHUB_STEP_SUMMARY
          echo "| ${{ matrix.module }} | ${{ fromJSON(steps.docker_build_runtime_cross.outputs.metadata)['image.name'] }} |" >> $GITHUB_STEP_SUMMARY
          echo "| ${{ matrix.module }} | ${{ fromJSON(steps.docker_build_runtime_cross.outputs.metadata)['image.name'] }} |" > ${{ matrix.module }}_result.md
      
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: artifact-${{ matrix.module }}
          path: ${{ matrix.module }}_result.md
          if-no-files-found: warn

      #- name: cache armv7 image
      #  uses: docker/build-push-action@v5.1.0
      #  if: always()
      #  with:
      #    context: .
      #    platforms: linux/arm/v7
      #    push: false
      #    load: false
      #    target: runtime
      #    cache-from: type=local,src=/tmp/.buildx-cache
      #    cache-to: type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-armv7
      #    build-args: |
      #      PKGS=${{ matrix.PKGS }}

      #- name: cache amd64 image
      #  if: always()
      #  uses: docker/build-push-action@v5.1.0
      #  with:
      #    context: .
      #    platforms: linux/amd64
      #    push: false
      #    load: false
      #    target: runtime
      #    cache-from: type=local,src=/tmp/.buildx-cache
      #    cache-to: type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64
      #    build-args: |
      #      PKGS=${{ matrix.PKGS }}
  
  check-matrix:
    runs-on: ubuntu-latest
    needs: [buildImages,prepare_build]
    permissions:
      contents: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      
      - uses: actions/download-artifact@v4
        with:
          pattern: artifact-*
          merge-multiple: true
          path: build-artifact
           

      - name: Find image String
        id: extract_image
        run: |
          echo "image_name=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f1)" >> $GITHUB_OUTPUT
          echo "image_version=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f2 | cut -d '@' -f1)" >> $GITHUB_OUTPUT
         
      - name: Update docs (readme.md)
        run: |
          cat build-artifact/*_result.md  > modules.md
          IMAGE_LIST=$(cat modules.md)
          echo "$IMAGE_LIST"
          ESCAPED_DATA="$(echo "${IMAGE_LIST}" | sed 's/\n/\\n/g')"
          echo "--------"
          echo "$ESCAPED_DATA"
          sed -e "s/{FHEMPY_VERSION}/${{ needs.prepare_build.outputs.fhempyV }}/" -e "s/{PYTHON_VERSION}/${{ steps.extract_image.outputs.image_version}}/" -e "s/{DEBIAN_RELEASE}/buster/" -e s,{IMAGE_TAG},${{ github.ref_name }}, < README.tmpl > README.md
          sed -i -e "/{MODULES_IMAGE_LIST}/r modules.md" -e "/{MODULES_IMAGE_LIST}/d" README.md

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: readme
          path: README.md
          if-no-files-found: warn
  
      - name: git commit README.md 
        id: commit
        if: ${{ github.event_name == 'release' }}
        run: |
          git config --global user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git commit -m "README.md: Update Versions" -a && echo "::set-output name=status::true" || true
      
      - name: Push back to protected branch (main)
        uses: CasperWA/push-protected@v2
        if: ${{ github.event_name == 'release' }}
        with:
          token: ${{ secrets.BOT_PUSH_TOKEN }}
          branch: main

      - name: Set step summary
        run: |
          cat README.md >> $GITHUB_STEP_SUMMARY