backend.yaml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Backend

on:
  push:
  workflow_dispatch:
  pull_request:
    branches:
      - dev
    paths-ignore:
      - 'docs/**'
      - '**/*.md'
      - '**/*.sql'

concurrency:
  group: backend-${{ github.event.pull_request.number || github.ref }}

jobs:
  check:
    name: Check Code
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Check Style
        run: |
          ./mvnw -T 4C -B --no-snapshot-updates clean spotless:check -P flink-all
  build_npm:
    name: Build_NPM
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: check
    env:
      MAVEN_OPTS: -Xmx2G -Xms2G
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            frontend:
              - 'dinky-web/**'
      - uses: actions/setup-node@v3
        if: steps.filter.outputs.frontend == 'true'
        with:
          node-version: 16
      - name: Get npm cache directory
        id: npm-cache-dir
        if: steps.filter.outputs.frontend == 'true'
        run: |
          echo "::set-output name=dir::$(npm config get cache)"
      - uses: actions/cache@v3
        id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true'
        if: steps.filter.outputs.frontend == 'true'
        with:
          path: |
            ${{ steps.npm-cache-dir.outputs.dir }}
            dinky-web/dist
          key: ${{ runner.os }}-node-${{ hashFiles('dinky-web/**/package.json') }}
          restore-keys: |
            ${{ runner.os }}-node-
      - name: Install Dependencies
        if: steps.filter.outputs.frontend == 'true'
        run: cd dinky-web && npm install --no-audit --progress=false --legacy-peer-deps
      - name: Npm Web Build
        if: steps.filter.outputs.frontend == 'true'
        run: cd dinky-web && npm run build
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: dinky-web
          path: ./dinky-web/dist
  build_release:
    name: Build Release
    runs-on: ubuntu-latest
    needs: check
    strategy:
      fail-fast: true
      matrix:
        jdk: [ 8, 11 ]
        flink: [ '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20' ]

    timeout-minutes: 30
    env:
      MAVEN_OPTS: -Xmx2G -Xms2G
    steps:
      - uses: actions/checkout@v3
      # maven编译
      - name: Set up JDK ${{ matrix.jdk }}
        uses: actions/setup-java@v2
        with:
          java-version: ${{ matrix.jdk }}
          distribution: 'adopt'
      - name: Cache local Maven repository
        uses: actions/cache@v3
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/flink
          key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-
      - name: Cache local Flink repository
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository/org/apache/flink
          key: ${{ runner.os }}-${{ matrix.flink }}-maven-${{ hashFiles('pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.flink }}-maven-
      - name: Build and Package
        run: |
          ./mvnw -B clean install \
                  -Dmaven.test.skip=false \
                 -Dspotless.check.skip=true \
                 -Denforcer.skip=false \
                 -Dmaven.javadoc.skip=true \
                 -P prod,flink-single-version,flink-${{ matrix.flink }},maven-central \
                 --no-snapshot-updates

      # 检查打包的大小
      - name: Check package size
        run: |
          ./check_package_size.sh
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        if: ${{ matrix.jdk  == 8 }}
        with:
          name: dinky-realease-${{ matrix.flink }}
          path: ./build/dinky-release*.tar.gz
  #      - name: Run Unit tests
  #        run: |
  #          ./mvnw -T 2C -B clean verify -Dmaven.test.skip=false -Dgpg.skip=true --no-snapshot-updates


  run_e2e:
    name: Run E2E
    needs: build_release
    strategy:
      fail-fast: true
      matrix:
        flink: [ '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20' ]
    runs-on: ubuntu-latest
    services:
      registry:
        image: registry:2
        ports:
          - 5000:5000
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: dinky-realease-${{ matrix.flink }}
          path: ./build
      # 设置 QEMU, 后面 docker buildx 依赖此.
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      # 设置 Docker buildx, 方便构建 Multi platform 镜像
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: network=host
      - name: Build Dinky Image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./e2e_test/docker-compose-env/Dockerfile
          # 是否 docker push
          push: true
          build-args: |
            FLINK_VERSION=${{ matrix.flink }}
          tags: |
            localhost:5000/dinky/dinky-test:flink
      - name: Init  Env Jar
        run: |
          wget -O e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar  https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar &&
          wget -O e2e_test/docker-compose-env/flink/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar  https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar &&
          wget -O e2e_test/docker-compose-env/dinky/javax.ws.rs-api-2.1.1.jar  https://repo1.maven.org/maven2/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar
      - name: Init  Docker Network
        run: |
          docker network create -d bridge dinky_net
      - name: Init Run Docker MySQL
        uses: hoverkraft-tech/compose-action@v2.0.2
        with:
          compose-file: ./e2e_test/docker-compose-env/mysql/docker-compose.yml
      - name: Init Run Docker Dinky
        uses: hoverkraft-tech/compose-action@v2.0.2
        with:
          compose-file: ./e2e_test/docker-compose-env/dinky/docker-compose.yml
      - name: Init Run Docker Hadoop
        uses: hoverkraft-tech/compose-action@v2.0.2
        with:
          compose-file: ./e2e_test/docker-compose-env/hadoop/docker-compose.yml
      - name: Replace Flink  docker-compose yml
        run: |
          export FLINK_VERSION=${{ matrix.flink }} && envsubst < ./e2e_test/docker-compose-env/flink/docker-compose.yml > ./e2e_test/docker-compose-env/flink/docker-compose-${{ matrix.flink }}.yml
      - name: Init Run Docker Flink
        uses: hoverkraft-tech/compose-action@v2.0.2
        with:
          compose-file: ./e2e_test/docker-compose-env/flink/docker-compose-${{ matrix.flink }}.yml

      - name: Cp Flink Jar Deps
        run: docker cp dinky:/opt/dinky/ ./dinky-release
      - name: Run Docker Python Script
        run: |
          docker run -v ./dinky-release/extends/flink${{ matrix.flink }}:/flink/lib -v ./e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar:/flink/lib/mysql-connector-java-8.0.30.jar -v./e2e_test/docker-compose-env/flink/conf:/flink/conf -v ./dinky-release/jar:/dinky/jar  -v./e2e_test/tools:/app -w /app   --net dinky_net --rm --entrypoint /bin/bash python:3.9 -c  'pip install  -r requirements.txt  &&  python main.py dinky:8888'