From 92f37cc1da6807b0b7ceb831a2986bb6ae5bbbd6 Mon Sep 17 00:00:00 2001 From: Sammy Sidhu Date: Thu, 30 Nov 2023 15:27:19 -0800 Subject: [PATCH] [CHORE] Bump Iceberg Version and V1 of caching (#1685) --- .github/workflows/python-package.yml | 18 ++++++++++++------ .../iceberg/docker-compose/Dockerfile | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3d28e118b3..2fd6fb9bec 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -383,12 +383,18 @@ jobs: pip install --upgrade pip pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall rm -rf daft - - name: Spin up Iceberg services - uses: isbang/compose-action@v1.5.1 - with: - compose-file: ./tests/integration/iceberg/docker-compose/docker-compose.yml - down-flags: --volumes - - name: Run IO integration tests + - name: Build images + run: | + docker buildx create --use --driver=docker-container + pushd ./tests/integration/iceberg/docker-compose/ + docker buildx bake -f ./docker-compose.yml --set *.cache-to="type=gha,mode=min" --set *.cache-from="type=gha" --load + popd + - name: Spin up services + run: | + pushd ./tests/integration/iceberg/docker-compose/ + docker-compose -f ./docker-compose.yml up -d + popd + - name: Run Iceberg integration tests run: | pytest tests/integration/iceberg -m 'integration' --durations=50 env: diff --git a/tests/integration/iceberg/docker-compose/Dockerfile b/tests/integration/iceberg/docker-compose/Dockerfile index 77ed84ed4f..4756104f0b 100644 --- a/tests/integration/iceberg/docker-compose/Dockerfile +++ b/tests/integration/iceberg/docker-compose/Dockerfile @@ -36,13 +36,13 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events WORKDIR ${SPARK_HOME} -ENV SPARK_VERSION=3.4.1 +ENV SPARK_VERSION=3.4.2 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.4_2.12 ENV ICEBERG_VERSION=1.4.0 ENV AWS_SDK_VERSION=2.20.18 ENV PYICEBERG_VERSION=0.4.0 -RUN curl --retry 3 -s -C - https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ +RUN curl --retry 3 -s -C - https://daft-public-data.s3.us-west-2.amazonaws.com/distribution/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \ && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz