proton_ci.yml

name: ProtonCI

on:
  issue_comment:
    types:
      - created

jobs:
  verify:
    name: Verify PR comment
    permissions:
      actions: read # for downloading artifact
      id-token: write # This is required for requesting the JWT (needed for assume AWS IAM role)
      issues: write # for posting comments
    runs-on: ubuntu-latest
    # because `issue_comment` does not support filters other than `types`,
    # here we need a bunch of checks to make sure that this comment is from a valid deployment issue.
    #
    # Note: 41898282 is the github-action[bot]
    # TODO: figure out a way not to hard-code the approver list
    if: >-
      ${{ github.event.pull_request.draft == false && github.event.issue.pull_request && github.event_name == 'issue_comment' && github.event.issue.state == 'open' && startsWith(github.event.comment.body, '/TODO') }}
    steps:
      - name: Report Started
        run: |
          curl --show-error --silent -X POST \
            -H 'Content-Type: application/json' \
            -H 'Accept: application/vnd.github+json' \
            -H "Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}" \
            -d '{"body": "github workflow triggered: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
            ${{ github.event.issue.comments_url }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: verify
        run: |
          echo verify started

          BODY='${{ format('{0}', github.event.comment.body) }}'
          parts=($BODY)

          echo parts[0] = ${parts[0]}
          echo parts[1] = ${parts[1]}

          if [ "${parts[0]}" != '/test' ]; then
            curl --show-error --silent -X POST \
              -H 'Content-Type: application/json' \
              -H 'Accept: application/vnd.github+json' \
              -H 'Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}' \
              -d '{"body": "Unrecognized command."}' \
              ${{ github.event.issue.comments_url }}

            exit 250
          fi

          planfile='${{ steps.vars.outputs.environment }}_${{ steps.vars.outputs.gitsha }}.tfplan'
          if [ ! -f $planfile ]; then
            echo Plan file not found.
            exit 251
          fi

          filehash=($(sha256sum $planfile))
          echo aws kms verify-mac --key-id 'alias/github-deployment' --message "${filehash[0]}" --mac-algorithm HMAC_SHA_256 --mac "${parts[1]}"
          if ! aws kms verify-mac --key-id 'alias/github-deployment' --message "${filehash[0]}" --mac-algorithm HMAC_SHA_256 --mac "${parts[1]}"; then
            curl --show-error --silent -X POST \
              -H 'Content-Type: application/json' \
              -H 'Accept: application/vnd.github+json' \
              -H 'Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}' \
              -d '{"body": "Invalid MAC hash."}' \
              ${{ github.event.issue.comments_url }}

            exit 252
          fi

          curl --show-error --silent -X POST \
            -H 'Content-Type: application/json' \
            -H 'Accept: application/vnd.github+json' \
            -H 'Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}' \
            -d '{"body": "MAC hash verified, start deployment, please wait."}' \
            ${{ github.event.issue.comments_url }}

  build_address_x64:
    needs: verify
    uses: timeplus-io/proton-opensource/.github/workflows/run_command.yml@develop
    with:
      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
      ec2-image-id: ${{ vars.X64_AMI }}
      ec2-volume-size: ${{ vars.VOLUME_SIZE }}
      submodules: 'recursive'
      sanitizer: "address"
      arch: ${{ vars.X64_ARCH }}
      command: |
        cd $GITHUB_WORKSPACE

        # git config
        git config user.name "proton-robot"
        git config user.email "proton_robot@timeplus.io"
        
        # prepare build cache
        aws s3 cp --no-progress s3://tp-internal/proton/ci_cache/cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz .
        mkdir $GITHUB_WORKSPACE/ccache
        tar -zxf ./cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz -C $GITHUB_WORKSPACE/ccache
        rm cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz

        # compiling
        ./docker/packager/packager --package-type binary --docker-image-version cmake.3.26.4 --sanitizer $SANITIZER --proton-build  --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output

        # clear compiling footprint
        rm -rf $GITHUB_WORKSPACE/ccache
        rm -rf build_docker

        # build unit test docker image
        cp $GITHUB_WORKSPACE/output/unit_tests_dbms $GITHUB_WORKSPACE/docker/test/proton_unit_test/
        cd $GITHUB_WORKSPACE/docker/test/proton_unit_test/
        docker build . -t timeplus/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA
        docker push timeplus/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA

        docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
        cd $GITHUB_WORKSPACE/docker/server

        # build docker image
        docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t timeplus/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA

        # push docker image
        docker push timeplus/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA

        echo 'docker pull timeplus/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA'
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}

  unit_test_address_x64:
    needs: build_address_x64
    uses: timeplus-io/proton-opensource/.github/workflows/run_command.yml@develop
    if: github.event.pull_request.draft == false
    with:
      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
      ec2-image-id: ${{ vars.X64_TEST_AMI }}
      ec2-volume-size: '30'
      submodules: false
      sanitizer: "address"
      arch: ${{ vars.X64_ARCH }}
      timeout: 30
      command: |
        cd $GITHUB_WORKSPACE/tests/proton_ci
        export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA

        # make virtualenv
        ln -s /usr/bin/python3 /usr/bin/python
        apt-get update
        systemctl stop unattended-upgrades
        apt install python3-venv -y
        python -m venv env
        source env/bin/activate

        pip install -r requirements.txt
        python unit_tests_check.py
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}

  smoke_test_address_x64:
    needs: build_address_x64
    uses: timeplus-io/proton-opensource/.github/workflows/run_command.yml@develop
    if: github.event.pull_request.draft == false
    with:
      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
      ec2-image-id: ${{ vars.X64_TEST_AMI }}
      ec2-volume-size: '50'
      submodules: false
      sanitizer: "address"
      arch: ${{ vars.X64_ARCH }}
      timeout: 30
      command: |
        # run stream test
        cd $GITHUB_WORKSPACE/tests/stream
        export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA

        # make virtualenv
        ln -s /usr/bin/python3 /usr/bin/python
        apt-get update
        systemctl stop unattended-upgrades
        apt install python3-venv -y
        python -m venv env
        source env/bin/activate

        pip install --upgrade pip

        # FIXME: remove this line after pyyaml community fixed install bug
        pip install pyyaml==5.3.1

        pip install -r helpers/requirements.txt
        python ci_runner.py --settings=redp,nativelog

        echo "check test result"
        cat $GITHUB_WORKSPACE/tests/stream/.status
        cat $GITHUB_WORKSPACE/tests/stream/.status | grep nativelog:ExitCode.OK | grep redp:ExitCode.OK
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}

  stateless_test_address_x64:
    needs: build_address_x64
    uses: timeplus-io/proton-opensource/.github/workflows/run_command.yml@develop
    if: github.event.pull_request.draft == false
    with:
      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
      ec2-image-id: ${{ vars.X64_TEST_AMI }}
      ec2-volume-size: '40'
      submodules: false
      sanitizer: "address"
      arch: ${{ vars.X64_ARCH }}
      timeout: 30
      command: |
        cd $GITHUB_WORKSPACE/tests/proton_ci
        # make virtualenv
        ln -s /usr/bin/python3 /usr/bin/python
        apt-get update
        systemctl stop unattended-upgrades
        apt install python3-venv -y
        python -m venv env
        source env/bin/activate
        pip install -r requirements.txt
        export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA
        python functional_tests_check.py stateless
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}

  stateful_test_address_x64:
    needs: build_address_x64
    uses: timeplus-io/proton-opensource/.github/workflows/run_command.yml@develop
    if: github.event.pull_request.draft == false
    with:
      ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
      ec2-image-id: ${{vars.X64_TEST_AMI }}
      ec2-volume-size: '40'
      submodules: false
      sanitizer: "address"
      arch: ${{ vars.X64_ARCH }}
      timeout: 30
      command: |
        cd $GITHUB_WORKSPACE/tests/proton_ci
        # make virtualenv
        ln -s /usr/bin/python3 /usr/bin/python
        apt-get update
        systemctl stop unattended-upgrades
        apt install python3-venv -y
        python -m venv env
        source env/bin/activate
        pip install -r requirements.txt
        export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA
        python functional_tests_check.py stateful
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
      TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
      TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
      TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}

  SkipDraft:
    if: github.event.pull_request.draft == true
    runs-on: ubuntu-latest
    timeout-minutes: 1
    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
    - name: Skip
      run: echo "Skip"