[CHORE] Bump Iceberg Version and V1 of caching (#1685)

Eventual-Inc · Nov 30, 2023 · 92f37cc · 92f37cc
1 parent 837cd19
commit 92f37cc
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 8 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -383,12 +383,18 @@ jobs:
         pip install --upgrade pip
         pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall
         rm -rf daft
-    - name: Spin up Iceberg services
-      uses: isbang/[email protected]
-      with:
-        compose-file: ./tests/integration/iceberg/docker-compose/docker-compose.yml
-        down-flags: --volumes
-    - name: Run IO integration tests
+    - name: Build images
+      run: |
+        docker buildx create --use --driver=docker-container
+        pushd ./tests/integration/iceberg/docker-compose/
+        docker buildx bake -f ./docker-compose.yml --set *.cache-to="type=gha,mode=min" --set *.cache-from="type=gha" --load
+        popd
+    - name: Spin up services
+      run: |
+        pushd ./tests/integration/iceberg/docker-compose/
+        docker-compose -f ./docker-compose.yml up -d
+        popd
+    - name: Run Iceberg integration tests
       run: |
         pytest tests/integration/iceberg -m 'integration' --durations=50
       env:

diff --git a/tests/integration/iceberg/docker-compose/Dockerfile b/tests/integration/iceberg/docker-compose/Dockerfile
@@ -36,13 +36,13 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
 RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
 WORKDIR ${SPARK_HOME}
 
-ENV SPARK_VERSION=3.4.1
+ENV SPARK_VERSION=3.4.2
 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.4_2.12
 ENV ICEBERG_VERSION=1.4.0
 ENV AWS_SDK_VERSION=2.20.18
 ENV PYICEBERG_VERSION=0.4.0
 
-RUN curl --retry 3 -s -C - https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
+RUN curl --retry 3 -s -C - https://daft-public-data.s3.us-west-2.amazonaws.com/distribution/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
  && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
  && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz