notebooks.yaml

#
# Copyright (C) 2020 Dremio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Testing Jupyter Notebooks

on:
  push:
    branches: [ main ]
    paths-ignore:
      - '**/*.md'
      - '.github/renovate.json5'
  pull_request:
    paths-ignore:
      - '**/*.md'
      - '.github/renovate.json5'

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  python:
    name: Testing Jupyter Notebooks
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: ['3.10']

    steps:
      - uses: actions/checkout@v3
      - name: Install system dependencies
        run: sudo apt-get install libsasl2-dev libsasl2-modules
      - name: Set up Java
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          # Need Java 8 for Hive + 11 for Spark (and Nessie)
          java-version: |
            8
            11
      - name: setup JAVAx_HOME
        run: |
          echo "JAVA8_HOME=$JAVA_HOME_8_X64" >> ${GITHUB_ENV}
          echo "JAVA11_HOME=$JAVA_HOME_11_X64" >> ${GITHUB_ENV}
          echo "JAVA_HOME=$JAVA_HOME_11_X64" >> ${GITHUB_ENV}
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
          cache-dependency-path: '**/requirements*.txt'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install tox tox-gh-actions build
      - name: Cache Tools (Hadoop, Hive, Spark)
        id: cache-tools
        uses: actions/cache@v3
        with:
          path: |
            notebooks/tests/hadoop-*
            notebooks/tests/apache-hive-*
            notebooks/tests/spark-*
            notebooks/iceberg-*.jar
            notebooks/nessie-quarkus-*.jar
          key: tools-cache-${{ hashFiles('docker/utils/__init__.py') }}
      - name: Fetch tools
        run: ./fetch_all_tools.sh
      - name: Check Dockerfile hash is up-to-date
        if: github.actor != 'renovate'
        run: |
          bash -ex .github/scripts/modify_dockerfile.sh
          changed_hash=$(git status --porcelain binder/Dockerfile)
          if [ -n "$changed_hash" ]; then
            echo "ERROR: Dockerfile hash is out of sync!"
            tree docker
            grep FROM binder/Dockerfile
            exit 1
          fi
          echo "PASSED: Dockerfile hash is up-to-date!"
      - name: Test Notebooks with Tox
        working-directory: notebooks/
        run: tox -e hive
      - name: Dump Hive output on error
        working-directory: notebooks/
        if: failure()
        run: |
          find .. -name "nohup*"
          cat nohup.out || true