Use time with -p, download datafusion-comet jar

data-catering · Jun 4, 2024 · 98ee16a · 98ee16a
1 parent f3682bc
commit 98ee16a
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 11 deletions.
diff --git a/benchmark/run_benchmark.sh b/benchmark/run_benchmark.sh
@@ -49,7 +49,7 @@ run_docker() {
     esac
 
     time_taken=$({
-      command time docker run -p 4040:4040 \
+      command time -p docker run -p 4040:4040 \
         -v "$(pwd)/build/libs/data-caterer-example-0.1.0.jar:/opt/spark/jars/data-caterer.jar" \
         -v "$(pwd)/benchmark/jars/blaze.jar:/opt/spark/jars/blaze.jar" \
         -v "$(pwd)/benchmark/jars/comet.jar:/opt/spark/jars/comet.jar" \
@@ -63,7 +63,7 @@ run_docker() {
         -e "ADDITIONAL_OPTS=$additional_conf" \
         --network "docker_default" \
         datacatering/data-caterer"$image_suffix":"$data_caterer_version";
-    } 2>&1 | grep "real " | sed "$sed_option" "s/^ +([0-9\.]+) real.*$/\1/")
+    } 2>&1 | tail -1 | sed "$sed_option" "s/^ +([0-9\.]+) real.*$/\1/")
     if [[ $1 == *BenchmarkForeignKeyPlanRun* ]]; then
       final_record_count=$(($2 * 5))
     else
@@ -112,7 +112,7 @@ if [[ "$enable_data_sink_run" ==  true ]]; then
   done
 fi
 
-docker ps -a | head -1 | awk -F " " '{print $1}' | xargs docker logs
+docker ps -a | tail -1 | awk -F " " '{print $1}' | xargs docker logs
 
 echo "Printing benchmark results"
 cat "$benchmark_result_file"

diff --git a/benchmark/setup_query_engine_jars.sh b/benchmark/setup_query_engine_jars.sh
@@ -13,11 +13,15 @@ echo "Getting gluten jar..."
 curl -L -o benchmark/jars/gluten.jar "https://github.com/apache/incubator-gluten/releases/download/v1.1.1/gluten-velox-bundle-spark3.4_2.12-1.1.1.jar"
 echo "Finished downloading gluten jar"
 
-echo "Building datafusion-comet..."
-#git clone [email protected]:apache/datafusion-comet.git benchmark/build
-cd benchmark/build/datafusion-comet
-make release PROFILES="-Pspark-${spark_major_version}"
-ls spark/target/comet-spark-*-SNAPSHOT.jar
-cp spark/target/comet-spark-*-SNAPSHOT.jar ../../jars/comet.jar
-cd ../../..
-echo "Finished building datafusion-comet"
+echo "Getting datafusion-comet jar..."
+curl -L -o benchmark/jars/comet.jar "https://github.com/data-catering/datafusion-comet/actions/runs/9361483276/artifacts/1565420150"
+echo "Finished downloading datafusion-comet jar"
+
+#echo "Building datafusion-comet..."
+##git clone [email protected]:apache/datafusion-comet.git benchmark/build
+#cd benchmark/build/datafusion-comet
+#make release PROFILES="-Pspark-${spark_major_version}"
+#ls spark/target/comet-spark-*-SNAPSHOT.jar
+#cp spark/target/comet-spark-*-SNAPSHOT.jar ../../jars/comet.jar
+#cd ../../..
+#echo "Finished building datafusion-comet"