maven.yml

# Template: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
#
# Useful links
# - GitHub Actions: https://docs.github.com/en/actions/learn-github-actions/introduction-to-github-actions
# - Service containers: https://docs.github.com/en/actions/guides/creating-postgresql-service-containers
#
# The CI jobs are set up as follows:
# - One job to build and upload artifacts.
# - One job per DBMS test suite.
# - One job to build/test the docker images.
# - One job to publish the docker image.

name: BenchBase (Java with Maven)

on:
  push:
    branches: [main]
    # Generate new docker images on release tags.
    tags:
      - "v*"
  pull_request:
    branches: [main]
  # Run these workflows on a schedule so that docker images are regularly updated for security patches.
  schedule:
    - cron: "1 0 * * *"
  # Give us a button to allow running the workflow on demand for testing.
  workflow_dispatch:
    inputs:
      tags:
        description: "Manual Workflow Run"
        required: false
        type: string

env:
  POM_VERSION: 2021-SNAPSHOT
  JAVA_VERSION: 17
  ERRORS_THRESHOLD: 0.01

jobs:
  # compile-and-test:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - name: Checkout repo
  #       uses: actions/checkout@v3

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         cache: 'maven'
  #         distribution: 'temurin'

  #     - name: Test with Maven
  #       run: mvn -B test --file pom.xml

  # package-and-upload:
  #   # needs: compile-and-test
  #   runs-on: ubuntu-latest
  #   strategy:
  #     matrix:
  #       profile: ["postgres", "mysql"]
  #   steps:
  #     - name: Checkout repo
  #       uses: actions/checkout@v3

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         cache: "maven"
  #         distribution: "temurin"

  #     - name: Package with Maven
  #       run: mvn -B package -P ${{matrix.profile}} --file pom.xml -DskipTests -D descriptors=src/main/assembly/tgz.xml

  #     - name: Upload TGZ artifact
  #       uses: actions/upload-artifact@v3
  #       with:
  #         name: benchbase-${{matrix.profile}}
  #         path: target/benchbase-${{matrix.profile}}.tgz

  ## ----------------------------------------------------------------------------------
  ## SQLITE
  ## ----------------------------------------------------------------------------------
  # sqlite:
  #   needs: package-and-upload
  #   runs-on: ubuntu-latest
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       # BROKEN: tpch
  #       benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
  #   steps:
  #     - name: Download artifact
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: benchbase-sqlite

  #     - name: Extract artifact
  #       run: |
  #         tar xvzf benchbase-sqlite.tgz --strip-components=1

  #     - name: Delete artifact
  #       run: |
  #         rm -rf benchbase-sqlite.tgz

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         distribution: 'temurin'

  #     - name: Run benchmark
  #       run: |
  #         java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
  #         # FIXME: Reduce the error rate so we don't need these overrides.
  #         if [ ${{matrix.benchmark}} == auctionmark ]; then
  #             ERRORS_THRESHOLD=0.02
  #         elif [ ${{matrix.benchmark}} == resourcestresser ]; then
  #             ERRORS_THRESHOLD=0.05
  #         elif [ ${{matrix.benchmark}} == smallbank ]; then
  #             ERRORS_THRESHOLD=0.03
  #         elif [ ${{matrix.benchmark}} == tatp ]; then
  #             ERRORS_THRESHOLD=0.05
  #         fi
  #         ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## MARIADB
  ## ----------------------------------------------------------------------------------
  # mariadb:
  #   needs: package-and-upload
  #   runs-on: ubuntu-latest
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
  #   services:
  #     mariadb: # https://hub.docker.com/_/mariadb
  #       image: mariadb:latest
  #       env:
  #         MARIADB_ROOT_PASSWORD: rootyMcRooty
  #         MARIADB_DATABASE: benchbase
  #         MARIADB_USER: admin
  #         MARIADB_PASSWORD: password
  #       # Use status instead of ping due to exit code issues: https://jira.mariadb.org/browse/MDEV-31550
  #       options: >-
  #         --health-cmd "mariadb-admin status -uroot -prootyMcRooty"
  #         --health-interval 10s
  #         --health-timeout 5s
  #         --health-retries 5
  #       ports:
  #         - 3306:3306
  #   steps:
  #     - name: Download artifact
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: benchbase-mariadb

  #     - name: Extract artifact
  #       run: |
  #         tar xvzf benchbase-mariadb.tgz --strip-components=1

  #     - name: Delete artifact
  #       run: |
  #         rm -rf benchbase-mariadb.tgz

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         distribution: 'temurin'

  #     - name: Run benchmark
  #       env:
  #         MARIADB_PORT: ${{ job.services.mariadb.ports[3306] }}
  #       run: |
  #         mysql -h127.0.0.1 -P$MARIADB_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase"
  #         java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
  #         # FIXME: Reduce the error rate so we don't need these overrides.
  #         if [ ${{matrix.benchmark}} == auctionmark ]; then
  #             ERRORS_THRESHOLD=0.02
  #         elif [ ${{matrix.benchmark}} == tatp ]; then
  #             ERRORS_THRESHOLD=0.05
  #         fi
  #         ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## MYSQL
  ## ----------------------------------------------------------------------------------
  # mysql:
  #   needs: package-and-upload
  #   runs-on: ubuntu-latest
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
  #   services:
  #     mysql: # https://hub.docker.com/_/mysql
  #       image: mysql:latest
  #       env:
  #         MYSQL_ROOT_PASSWORD: rootyMcRooty
  #         MYSQL_DATABASE: benchbase
  #         MYSQL_USER: admin
  #         MYSQL_PASSWORD: password
  #       options: >-
  #         --health-cmd "mysqladmin ping"
  #         --health-interval 10s
  #         --health-timeout 5s
  #         --health-retries 5
  #       ports:
  #         - 3306:3306
  #   steps:
  #     - name: Download artifact
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: benchbase-mysql

  #     - name: Extract artifact
  #       run: |
  #         tar xvzf benchbase-mysql.tgz --strip-components=1

  #     - name: Delete artifact
  #       run: |
  #         rm -rf benchbase-mysql.tgz

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         distribution: 'temurin'

  #     - name: Run benchmark
  #       env:
  #         MYSQL_PORT: ${{ job.services.mysql.ports[3306] }}
  #       run: |
  #         mysql -h127.0.0.1 -P$MYSQL_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase"
  #         java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
  #         # FIXME: Reduce the error rate so we don't need these overrides.
  #         if [ ${{matrix.benchmark}} == auctionmark ]; then
  #             ERRORS_THRESHOLD=0.02
  #         elif [ ${{matrix.benchmark}} == tatp ]; then
  #             ERRORS_THRESHOLD=0.05
  #         fi
  #         ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## POSTGRESQL
  ## ----------------------------------------------------------------------------------
  # postgresql:
  #   needs: package-and-upload
  #   runs-on: ubuntu-latest
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       benchmark:
  #         [
  #           "auctionmark",
  #           "epinions",
  #           "hyadapt",
  #           "noop",
  #           "otmetrics",
  #           "resourcestresser",
  #           "seats",
  #           "sibench",
  #           "smallbank",
  #           "tatp",
  #           "tpcc",
  #           "tpch",
  #           "twitter",
  #           "voter",
  #           "wikipedia",
  #           "ycsb",
  #         ]
  #   services:
  #     postgres: # https://hub.docker.com/_/postgres
  #       image: postgres:latest
  #       env:
  #         POSTGRES_DB: benchbase
  #         POSTGRES_USER: admin
  #         POSTGRES_PASSWORD: password
  #       options: >-
  #         --health-cmd pg_isready
  #         --health-interval 10s
  #         --health-timeout 5s
  #         --health-retries 5
  #       ports:
  #         - 5432:5432
  #   steps:
  #     - name: Download artifact
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: benchbase-postgres

  #     - name: Extract artifact
  #       run: |
  #         tar xvzf benchbase-postgres.tgz --strip-components=1

  #     - name: Delete artifact
  #       run: |
  #         rm -rf benchbase-postgres.tgz

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         distribution: "temurin"

  #     - name: Run benchmark
  #       run: |
  #         PGPASSWORD=password dropdb -h localhost -U admin benchbase --if-exists
  #         PGPASSWORD=password createdb -h localhost -U admin benchbase
  #         java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
  #         # FIXME: Reduce the error rate so we don't need these overrides.
  #         if [ ${{matrix.benchmark}} == auctionmark ]; then
  #             ERRORS_THRESHOLD=0.02
  #         elif [ ${{matrix.benchmark}} == tatp ]; then
  #             ERRORS_THRESHOLD=0.05
  #         fi
  #         ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## COCKROACHDB
  ## ----------------------------------------------------------------------------------
  # cockroachdb:
  #   needs: package-and-upload
  #   runs-on: ubuntu-latest
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       benchmark:
  #         [
  #           "auctionmark",
  #           "epinions",
  #           "hyadapt",
  #           "noop",
  #           "otmetrics",
  #           "resourcestresser",
  #           "seats",
  #           "sibench",
  #           "smallbank",
  #           "tatp",
  #           "tpcc",
  #           "tpch",
  #           "twitter",
  #           "voter",
  #           "wikipedia",
  #           "ycsb",
  #         ]
  #   services:
  #     cockroach: # https://hub.docker.com/repository/docker/timveil/cockroachdb-single-node
  #       image: timveil/cockroachdb-single-node:latest
  #       env:
  #         DATABASE_NAME: benchbase
  #         MEMORY_SIZE: .75
  #       ports:
  #         - 26257:26257
  #   steps:
  #     - name: Download artifact
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: benchbase-cockroachdb

  #     - name: Extract artifact
  #       run: |
  #         tar xvzf benchbase-cockroachdb.tgz --strip-components=1

  #     - name: Delete artifact
  #       run: |
  #         rm -rf benchbase-cockroachdb.tgz

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         distribution: "temurin"

  #     - name: Run benchmark
  #       run: |
  #         java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
  #         # FIXME: Reduce the error rate so we don't need these overrides.
  #         # FIXME: Reduce the error rate so we don't need these overrides.
  #         if [ ${{matrix.benchmark}} == auctionmark ]; then
  #             ERRORS_THRESHOLD=0.02
  #         elif [ ${{matrix.benchmark}} == tatp ]; then
  #             ERRORS_THRESHOLD=0.05
  #         fi
  #         ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## MSSQL
  ## ----------------------------------------------------------------------------------
  # sqlserver:
  #   needs: package-and-upload
  #   runs-on: ubuntu-latest
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       # TODO: add more benchmarks
  #       #benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
  #       benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
  #   services:
  #     sqlserver:
  #       image: mcr.microsoft.com/mssql/server:latest
  #       env:
  #         ACCEPT_EULA: Y
  #         SA_PASSWORD: SApassword1
  #       options: >-
  #         --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P SApassword1 -b -Q 'SELECT 1;'"
  #         --health-interval 10s
  #         --health-timeout 5s
  #         --health-retries 5
  #       ports:
  #         - 1433:1433
  #   steps:
  #     - name: Download artifact
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: benchbase-sqlserver

  #     - name: Extract artifact
  #       run: |
  #         tar xvzf benchbase-sqlserver.tgz --strip-components=1

  #     - name: Delete artifact
  #       run: |
  #         rm -rf benchbase-sqlserver.tgz

  #     - name: Set up JDK
  #       uses: actions/setup-java@v3
  #       with:
  #         java-version: ${{env.JAVA_VERSION}}
  #         distribution: 'temurin'

  #     - name: Cleanup database
  #       uses: docker://mcr.microsoft.com/mssql-tools:latest
  #       with:
  #         entrypoint: /opt/mssql-tools/bin/sqlcmd
  #         args: -U sa -P SApassword1 -S sqlserver -b -Q "DROP DATABASE IF EXISTS benchbase;"

  #     - name: Setup database
  #       uses: docker://mcr.microsoft.com/mssql-tools:latest
  #       with:
  #         entrypoint: /opt/mssql-tools/bin/sqlcmd
  #         args: -U sa -P SApassword1 -S sqlserver -b -Q "CREATE DATABASE benchbase;"

  #     - name: Setup login
  #       uses: docker://mcr.microsoft.com/mssql-tools:latest
  #       with:
  #         entrypoint: /opt/mssql-tools/bin/sqlcmd
  #         args: -U sa -P SApassword1 -S sqlserver -Q "CREATE LOGIN benchuser01 WITH PASSWORD='P@ssw0rd';"

  #     - name: Setup access
  #       uses: docker://mcr.microsoft.com/mssql-tools:latest
  #       with:
  #         entrypoint: /opt/mssql-tools/bin/sqlcmd
  #         args: -U sa -P SApassword1 -S sqlserver -b -Q "USE benchbase; CREATE USER benchuser01 FROM LOGIN benchuser01; EXEC sp_addrolemember 'db_owner', 'benchuser01';"

  #     - name: Run benchmark
  #       # Note: user/pass should match those used in sample configs.
  #       run: |
  #         java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
  #         # FIXME: Reduce the error rate so we don't need these overrides.
  #         if [ ${{matrix.benchmark}} == tatp ]; then
  #             ERRORS_THRESHOLD=0.05
  #         fi
  #         ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD

  ## ----------------------------------------------------------------------------------
  ## Docker Build Test Publish
  ## ----------------------------------------------------------------------------------
  docker-build-test-publish:
    runs-on: ubuntu-latest
    env:
      DOCKER_BUILDKIT: 1
      BENCHBASE_PROFILES: "postgres mysql"
      CONTAINER_REGISTRY_NAME: ghcr.io/${{ github.repository_owner }}
      SKIP_TESTS: true
    permissions:
      contents: read
      packages: write
    services:
      postgres: # https://hub.docker.com/_/postgres
        image: postgres:latest
        env:
          POSTGRES_DB: benchbase
          POSTGRES_USER: admin
          POSTGRES_PASSWORD: password
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - name: Setup postgres test DB
        run: |
          PGPASSWORD=password dropdb -h localhost -U admin benchbase --if-exists
          PGPASSWORD=password createdb -h localhost -U admin benchbase
      - name: Checkout repo
        uses: actions/checkout@v3
      # https://github.com/actions/cache/blob/master/examples.md#java---maven
      - name: Cache local Maven repository
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: setup-java-${{ runner.os }}-docker-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            setup-java-${{ runner.os }}-docker-maven-
      - name: Pull base image caches for PR builds
        if: ${{ github.ref != 'refs/heads/main' }}
        run: |
          docker pull benchbase.azurecr.io/benchbase-dev:latest || true
          docker pull benchbase.azurecr.io/benchbase:latest || true
      - name: Set NO_CACHE env var for main branch builds
        if: ${{ github.ref == 'refs/heads/main' }}
        run: |
          echo "NO_CACHE=true" >> $GITHUB_ENV
      - name: Build benchbase-dev image
        run: |
          ./docker/benchbase/build-dev-image.sh
      # Note: this script maps the local .m2 cache into the container.
      - name: Build the benchbase docker image with all profiles using the dev image
        env:
          SKIP_TESTS: true
        run: |
          ./docker/benchbase/build-full-image.sh
      - name: Run a basic test from the docker image against postgres test DB
        env:
          benchmark: noop
        run: |
          for image in benchbase benchbase-postgres; do
            # Adjust the sample config to talk to the container service instead of localhost.
            cat "config/postgres/sample_${benchmark}_config.xml" | sed -e 's/localhost:5432/postgres:5432/g' > /tmp/config.xml
            # Lookup the service container's docker network so we can place the benchbase container in it too.
            docker_network="$(docker ps --filter expose=5432 --format '{{.Networks}}')"
            # Map the adjusted config into the container and use it to run the test.
            rm -rf results
            mkdir -p results
            docker run --rm --name benchbase-postgres --network "$docker_network" \
              --env BENCHBASE_PROFILE=postgres -v /tmp/config.xml:/tmp/config.xml -v "$PWD/results:/benchbase/results" \
              "$image" -b "$benchmark" -c /tmp/config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
            # Test that the results files were produced.
            ls results/${benchmark}_*.csv
            ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
          done
      # Publish the docker image if the build/test was successful.
      # Only do this with approved PRs and if the login secrets are available.
      - name: Log in to Docker Hub
        if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) }}
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Push Docker image
        if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) }}
        run: |
          docker push -a ${{ env.CONTAINER_REGISTRY_NAME }}/benchbase-dev
          docker push -a ${{ env.CONTAINER_REGISTRY_NAME }}/benchbase
          for profile in $BENCHBASE_PROFILES; do
            docker push -a ${{ env.CONTAINER_REGISTRY_NAME }}/benchbase-$profile
          done