Increase example block size to 128 MiB.

Signed-off-by: Pascal Spörri <[email protected]>
IBM · Sep 8, 2023 · 8712edc · 8712edc
1 parent 7f6db51
commit 8712edc
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/examples/sql/run_benchmark.sh b/examples/sql/run_benchmark.sh
@@ -47,7 +47,7 @@ SPARK_HADOOP_S3A_CONFIG=(
     --conf spark.hadoop.fs.s3a.endpoint=${S3A_ENDPOINT}
     --conf spark.hadoop.fs.s3a.path.style.access=true
     --conf spark.hadoop.fs.s3a.fast.upload=true
-    --conf spark.hadoop.fs.s3a.block.size=$((32*1024*1024))
+    --conf spark.hadoop.fs.s3a.block.size=$((128*1024*1024))
 )
 
 SPARK_S3_SHUFFLE_CONFIG=(
@@ -75,6 +75,7 @@ if (( "$USE_NFS_SHUFFLE" == 1 )); then
         --conf spark.shuffle.s3.rootDir=file:///nfs/
         --conf spark.kubernetes.executor.podTemplateFile=${SCRIPT_DIR}/../templates/executor_nfs.yml
         --conf spark.kubernetes.driver.podTemplateFile=${SCRIPT_DIR}/../templates/driver_nfs.yml
+        --conf spark.hadoop.fs.file.block.size=$((128*1024*1024))
     )
 fi
 

diff --git a/examples/terasort/run.sh b/examples/terasort/run.sh
@@ -45,7 +45,7 @@ SPARK_HADOOP_S3A_CONFIG=(
     --conf spark.hadoop.fs.s3a.endpoint=${S3A_ENDPOINT}
     --conf spark.hadoop.fs.s3a.path.style.access=true
     --conf spark.hadoop.fs.s3a.fast.upload=true
-    --conf spark.hadoop.fs.s3a.block.size=$((32*1024*1024))
+    --conf spark.hadoop.fs.s3a.block.size=$((128*1024*1024))
 )
 
 
@@ -76,6 +76,7 @@ if (( "$USE_NFS_SHUFFLE" == 1 )); then
         --conf spark.shuffle.s3.rootDir=file:///nfs/
         --conf spark.kubernetes.executor.podTemplateFile=${SCRIPT_DIR}/../templates/executor_nfs.yml
         --conf spark.kubernetes.driver.podTemplateFile=${SCRIPT_DIR}/../templates/driver_nfs.yml
+        --conf spark.hadoop.fs.file.block.size=$((128*1024*1024))
     )
 fi
 
@@ -151,3 +152,4 @@ ${SPARK_HOME}/bin/spark-submit \
        "s3a://${S3A_OUTPUT_BUCKET}/output/terasort/${SIZE}-${PROCESS_TAG}" "s3a://${S3A_OUTPUT_BUCKET}/output/terasort-validated/${SIZE}-${PROCESS_TAG}"
 
 s3cmd rm -r s3://${S3A_OUTPUT_BUCKET}/output/terasort/${SIZE}-${PROCESS_TAG} || true
+