on_push.yml

name: Test and Build
on: [push]
env:
  # NOTE keep versions in sync with version in TOSCA templates
  TERRAFORM_VERSION: 1.1.4 # sync with tosca_plugins/artifacts.yaml
  HELM_VERSION: 3.7.1 # sync with configurators/helm-template.yaml
  GCLOUD_VERSION: 398.0.0 # sync with tosca_plugins/artifacts.yaml and Dockerfile
  KOMPOSE_VERSION: v1.26.1 # sync with Dockerfile
  KUBECTL_VERSION: v1.24.2
  K3D_VERSION: v4.4.6
jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python: [3.8, 3.9, "3.10", "3.11.0", "3.12"]
    env:
      TOX_SKIP_ENV: .+(docker|lock)
      UNFURL_LOGGING: info
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          check-latest: true
      - name: Setup Rust
        uses: dtolnay/rust-toolchain@stable
        with:
            toolchain: stable
            components: clippy
      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: ${{ env.TERRAFORM_VERSION }}
          terraform_wrapper: false
      - name: Setup helm
        uses: Azure/setup-helm@v4.0.0
        with:
          version: ${{ env.HELM_VERSION }}
      - name: Setup kubectl
        uses: Azure/setup-kubectl@v4.0.0
        with:
          version: ${{ env.KUBECTL_VERSION }}
      - name: Setup kompose
        run: |
          wget https://github.com/kubernetes/kompose/releases/download/${KOMPOSE_VERSION}/kompose-linux-amd64
          chmod +x kompose-linux-amd64
          sudo mv kompose-linux-amd64 /usr/local/bin/kompose
      - name: Setup k3d
        run: |
          wget https://github.com/rancher/k3d/releases/download/"${{ env.K3D_VERSION }}"/k3d-linux-amd64
          chmod +x k3d-linux-amd64
          sudo mv k3d-linux-amd64 /usr/local/bin/k3d
      - name: Set up the cluster
        run: |
          k3d cluster create
          kubectl get node
          k3d kubeconfig merge --all -d
          kubectl config view
      - name: Install Tox and any other packages
        run: pip install tox==3.28.0
      - name: Run mypy and Tox
        # Run tox using the version of Python in `PATH`
        run: |
          git config --global user.email "unittest@onecommons.org"
          git config --global user.name "Test Robot"
          git config --global push.autoSetupRemote true
          git config --global init.defaultBranch main
          export PY_V=py3`python -c "import sys; print(sys.version_info.minor, end='')"`
          tox -c tosca-parser/tox.ini -v -e $PY_V
          tox -e $PY_V -v -- -vv -n auto --dist loadfile
          .tox/$PY_V/bin/mypy unfurl --install-types --non-interactive
      - name: Run cargo clippy and cargo test
        if: matrix.python == 3.9
        run: |
          cargo clippy --manifest-path rust/Cargo.toml
          cargo test --no-default-features --manifest-path rust/Cargo.toml
      - name: build docs
        if: matrix.python == 3.9
        run: tox -e docs -- -W --keep-going
      - name: upload docs
        if: matrix.python == 3.9
        uses: actions/upload-artifact@v4
        with:
          name: DocumentationHTML
          path: .tox/docs/html/

  publish_dockerhub:
    needs: test
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    env:
      DOCKERHUB_USER: "ocbuilds"
    # Map a step output to a job output
    outputs:
      release: ${{ steps.get_tag_name.outputs.release }}
    steps:
      - name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ env.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKER_HUB }}
      - name: Fetch repo
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0
      - name: Get the tag name or abbreviated commit digest
        id: get_tag_name
        run: |
          label=$(git describe --contains --always --match=v*)
          echo "tag=${label%^0}" >> $GITHUB_OUTPUT
          if [[ $label =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then
              echo "release=true" >> $GITHUB_OUTPUT
          fi
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: |
            onecommons/unfurl
          tags: |
            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
            type=raw,value=stable,enable=${{ steps.get_tag_name.outputs.release || 'false' }}
            type=raw,value=${{ steps.get_tag_name.outputs.tag }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v4
        with:
          context: ./
          file: ./docker/Dockerfile
          push: true
          build-args: |
            HELM_VERSION=${{ env.HELM_VERSION }}
            TERRAFORM_VERSION=${{ env.TERRAFORM_VERSION }}
            GCLOUD_VERSION=${{ env.GCLOUD_VERSION }}
            KOMPOSE_VERSION=${{ env.KOMPOSE_VERSION }}
          tags: ${{ steps.meta.outputs.tags }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  publish_dockerhub_server:
    needs: test
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    env:
      DOCKERHUB_USER: "ocbuilds"
    steps:
      - name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ env.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKER_HUB }}
      - name: Fetch repo
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0
      - name: Get the tag name or abbreviated commit digest
        id: get_tag_name
        run: |
          label=$(git describe --contains --always --match=v*)
          echo "tag=${label%^0}" >> $GITHUB_OUTPUT
          if [[ $label =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then
              echo "release=true" >> $GITHUB_OUTPUT
          fi
      - name: Docker meta for unfurl server
        id: meta_server
        uses: docker/metadata-action@v4
        with:
          images: |
            onecommons/unfurl
          tags: |
            type=raw,value=latest-server,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
            type=raw,value=stable-server,enable=${{ steps.get_tag_name.outputs.release || 'false' }}
            type=raw,value=${{ steps.get_tag_name.outputs.tag }}-server
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build and push unfurl server
        id: docker_build_server
        uses: docker/build-push-action@v4
        with:
          context: ./
          file: ./docker/Dockerfile.server
          push: true
          tags: ${{ steps.meta_server.outputs.tags }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  publish_dockerhub_nginx:
    needs: [test, publish_dockerhub_server]
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    env:
      DOCKERHUB_USER: "ocbuilds"
    steps:
      - name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ env.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKER_HUB }}
      - name: Fetch repo
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0
      - name: Get the tag name or abbreviated commit digest
        id: get_tag_name
        run: |
          label=$(git describe --contains --always --match=v*)
          echo "tag=${label%^0}" >> $GITHUB_OUTPUT
          if [[ $label =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then
              echo "release=true" >> $GITHUB_OUTPUT
          fi
      - name: Docker meta for unfurl server
        id: meta_server
        uses: docker/metadata-action@v4
        with:
          images: |
            onecommons/unfurl
          tags: |
            type=raw,value=latest-server-cached,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
            type=raw,value=stable-server-cached,enable=${{ steps.get_tag_name.outputs.release || 'false' }}
            type=raw,value=${{ steps.get_tag_name.outputs.tag }}-server-cached
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build and push unfurl server
        id: docker_build_server
        uses: docker/build-push-action@v4
        with:
          context: ./
          file: ./docker/Dockerfile.server-cached
          push: true
          build-args: |
            FROM_IMAGE=docker.io/onecommons/unfurl
            FROM_TAG=${{ steps.get_tag_name.outputs.tag }}-server
          tags: ${{ steps.meta_server.outputs.tags }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  publish_dockerhub_podman:
    needs: publish_dockerhub
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    env:
      DOCKERHUB_USER: "ocbuilds"
    steps:
      - name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ env.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKER_HUB }}
      - name: fetch repo
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0
      - name: Get the tag name or abbreviated commit digest
        id: get_tag_name
        run: |
          label=$(git describe --contains --always --match=v*)
          echo "tag=${label%^0}" >> $GITHUB_OUTPUT
          if [[ $label =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then
              echo "release=true" >> $GITHUB_OUTPUT
          fi
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: |
            onecommons/unfurl
          tags: |
            type=raw,value=latest-podman,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
            type=raw,value=stable-podman,enable=${{ steps.get_tag_name.outputs.release || 'false' }}
            type=raw,value=${{ steps.get_tag_name.outputs.tag }}-podman
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v4
        with:
          context: ./
          file: ./docker/Dockerfile.podman
          push: true
          build-args: |
            HELM_VERSION=${{ env.HELM_VERSION }}
            TERRAFORM_VERSION=${{ env.TERRAFORM_VERSION }}
            GCLOUD_VERSION=${{ env.GCLOUD_VERSION }}
            KOMPOSE_VERSION=${{ env.KOMPOSE_VERSION }}
            UNFURL_TAG=${{ steps.get_tag_name.outputs.tag }}
          tags: ${{ steps.meta.outputs.tags }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  build:
    name: >
      build py3.${{ matrix.python-version }} on ${{ matrix.platform || matrix.os }}
      (${{ matrix.alt_arch_name || matrix.arch }})
    needs:
      - test
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos]
        python-version: ['8']
        arch: [main, alt]
        include:
          - os: ubuntu
            platform: linux
          - os: ubuntu
            arch: alt
            alt_arch_name: aarch64
          - os: macos
            arch: alt
            alt_arch_name: arm64
          # - os: macos
          #   arch: main

    runs-on: ${{ format('{0}-latest', matrix.os) }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: recursive
        fetch-depth: 0
    - name: set up python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: set up rust
      if: matrix.os != 'ubuntu'
      uses: dtolnay/rust-toolchain@stable
      with:
        toolchain: nightly

    - run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
      if: matrix.os == 'macos'

    - run: rustup toolchain install stable-i686-pc-windows-msvc
      if: matrix.os == 'windows'

    - run: rustup target add i686-pc-windows-msvc
      if: matrix.os == 'windows'

    - name: install python dependencies
      run: pip install -U setuptools wheel twine cibuildwheel pbr

    - name: build sdist
      if: matrix.os == 'ubuntu' && matrix.python-version == '8'
      run: |
        pip install -U setuptools-rust
        python setup.py sdist

    - name: Set up QEMU
      if: matrix.os == 'ubuntu'
      uses: docker/setup-qemu-action@v3
      with:
         platforms: all

    - name: build ${{ matrix.platform || matrix.os }} binaries
      run: cibuildwheel --output-dir dist
      env:
        CIBW_BUILD_FRONTEND: "build"
        CIBW_BUILD: 'cp3${{ matrix.python-version }}-*'
        # rust doesn't seem to be available for musl linux on i686
        # and skip rare archs to speed up builds
        CIBW_SKIP: '*-musllinux_i686 *ppc* *s390x'
        # we build for "alt_arch_name" if it exists, else 'auto
        CIBW_ARCHS: ${{ matrix.alt_arch_name || 'auto' }}
        # see https://cibuildwheel.readthedocs.io/en/stable/faq/#universal2, tests can run on cross-compiled binaries
        # CIBW_TEST_SKIP: '*-macosx_arm64'
        # CIBW_TEST_REQUIRES: pytest
        # CIBW_TEST_COMMAND: 'pytest {project}/tests -s'
        CIBW_ENVIRONMENT: 'PATH="$HOME/.cargo/bin:$PATH" MACOSX_DEPLOYMENT_TARGET=10.12'
        CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
        CIBW_ARCHS_MACOS: "universal2"
        CIBW_MANYLINUX_I686_IMAGE: manylinux2014
        CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
        CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_1
        CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64:latest
        CIBW_MUSLLINUX_AARCH64_IMAGE: musllinux_1_1
        CIBW_BEFORE_BUILD: rustup show
        CIBW_BEFORE_BUILD_LINUX: >
          curl https://sh.rustup.rs -sSf | sh -s -- --profile=minimal -y &&
          rustup show

    - run: ${{ matrix.ls || 'ls -lh' }} dist/

    - run: twine check dist/*

    - uses: actions/upload-artifact@v3  # v4 doesn't allow same filename to be reloaded, see https://github.com/actions/upload-artifact/blob/main/docs/MIGRATION.md#merging-multiple-artifacts
      with:
        name: pypi_files
        path: dist

  inspect-pypi-assets:
    needs: [build]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: get dist artifacts
        uses: actions/download-artifact@v3
        with:
          name: pypi_files
          path: dist

      - name: list dist files
        run: |
          ls -lh dist/
          echo "`ls dist | wc -l` files"

      - name: extract and list sdist file
        run: |
          mkdir sdist-files
          tar -xvf dist/*.tar.gz -C sdist-files
          tree -a sdist-files

      - name: extract and list wheel file
        run: |
          ls dist/*cp3*-manylinux*x86_64.whl | head -n 1
          python -m zipfile --list `ls dist/*cp3*-manylinux*x86_64.whl | head -n 1`

      - run: pip install twine
      - run: twine check dist/*

  publish_pypi:
    needs: publish_dockerhub
    if: needs.publish_dockerhub.outputs.release
    runs-on: ubuntu-latest
    steps:
      - name: fetch repo
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0
      - name: build python package
        run: python setup.py sdist bdist_wheel
      - name: release to pypi
        uses: pypa/gh-action-pypi-publish@v1.6.4
        with:
          user: __token__
          password: ${{ secrets.PYPI }}
          verbose: true

  docs:
    needs: publish_dockerhub
    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/publish-docs'
    runs-on: ubuntu-latest
    steps:
      - name: Download documentation
        uses: actions/download-artifact@v3
        with:
          name: DocumentationHTML
          path: build
      - name: Commit documentation changes
        run: |
          git clone ${{github.server_url}}/${{github.repository}} --branch gh-pages --single-branch gh-pages
          cp -r build/* gh-pages/
          cd gh-pages
          touch .nojekyll
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git add .
          git commit -m "Update documentation" -a || true
          # The above command will fail if no changes were present, so we ignore
          # that.
      - name: Push docs to gh-pages branch
        uses: ad-m/github-push-action@master
        with:
          branch: gh-pages
          directory: gh-pages
          github_token: ${{ secrets.GITHUB_TOKEN }}