Skip to content

Commit

Permalink
Don't use basic image, update to 0.12.0
Browse files Browse the repository at this point in the history
  • Loading branch information
pflooky committed Oct 17, 2024
1 parent b0f03fb commit 29d6cf6
Show file tree
Hide file tree
Showing 5 changed files with 7 additions and 23 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
FROM datacatering/data-caterer-basic:0.11.8
FROM datacatering/data-caterer:0.12.0

COPY --chown=app:app build/libs/data-caterer-example-0.1.0.jar /opt/app/job.jar
10 changes: 3 additions & 7 deletions benchmark/run_benchmark.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ enable_data_size_run=${ENABLE_DATA_SIZE_RUN:-true}
enable_data_sink_run=${ENABLE_DATA_SINK_RUN:-true}

data_caterer_version=$(grep dataCatererVersion gradle.properties | cut -d= -f2)
image_suffix="-basic"
default_job="io.github.datacatering.plan.benchmark.BenchmarkParquetPlanRun"
#default_job="io.github.datacatering.plan.benchmark.BenchmarkValidationPlanRun"
default_record_count="100000"
Expand Down Expand Up @@ -61,7 +60,7 @@ run_docker() {
-e "$driver_memory" \
-e "$executor_memory" \
-e "ADDITIONAL_OPTS=$additional_conf" \
datacatering/data-caterer"$image_suffix":"$data_caterer_version";
datacatering/data-caterer:"$data_caterer_version";
} 2>&1 | grep "real " | sed "$sed_option" "s/^.*real ([0-9\.]+)$/\1/")
echo "Time taken: $time_taken"
if [[ $1 == *BenchmarkForeignKeyPlanRun* ]]; then
Expand All @@ -81,7 +80,7 @@ if [[ $? -ne 0 ]]; then
fi

echo "Pulling image before starting benchmarks"
docker pull datacatering/data-caterer"$image_suffix":"$data_caterer_version"
docker pull datacatering/data-caterer:"$data_caterer_version"

echo "Running benchmarks"
if [[ "$enable_query_engine_run" == true ]]; then
Expand All @@ -105,9 +104,6 @@ if [[ "$enable_data_sink_run" == true ]]; then
for job in "${job_names[@]}"; do
echo "Running for job: $job"
full_class_name="io.github.datacatering.plan.benchmark.$job"
if [[ "$job" == *"Advanced"* ]]; then
image_suffix=""
fi
run_docker "$full_class_name" "$default_record_count"
done
fi
Expand All @@ -118,5 +114,5 @@ echo "Printing logs of last docker run"
docker ps -a | awk -F " " '{print $1}' | tail -1 | xargs docker logs

echo "Cleaning docker runs..."
docker ps -a | grep "datacatering/data-caterer-basic" | awk -F " " '{print $1}' | xargs docker rm
docker ps -a | grep "datacatering/data-caterer" | awk -F " " '{print $1}' | xargs docker rm
echo "Done!"
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ version=0.1.0

scalaVersion=2.12
scalaSpecificVersion=2.12.19
dataCatererVersion=0.11.11
dataCatererVersion=0.12.0
sparkVersion=3.5.1
sparkMajorVersion=3.5
2 changes: 1 addition & 1 deletion helm/data-caterer/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ image:
repository: "datacatering/data-caterer"
pullPolicy: "IfNotPresent"
# Overrides the image tag whose default is the chart appVersion.
tag: "0.11.8"
tag: "0.12.0"

imagePullSecrets: []
nameOverride: ""
Expand Down
14 changes: 1 addition & 13 deletions run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,6 @@ else
fi
echo -n "$curr_class_name" > .tmp_prev_class_name

image_suffix="-basic"
additional_env_vars=""
if [[ "$curr_class_name" == *"Advanced"* ]]; then
if [[ -z "$DATA_CATERING_API_KEY" ]]; then
image_suffix=""
else
image_suffix="-trial"
additional_env_vars=" -e API_KEY=$DATA_CATERING_API_KEY"
fi
fi

echo "Building jar with plan run"
./gradlew clean build
if [[ $? -ne 0 ]]; then
Expand All @@ -54,9 +43,8 @@ DOCKER_CMD=(
-e "DEPLOY_MODE=client"
-e "DRIVER_MEMORY=2g"
-e "EXECUTOR_MEMORY=2g"
"$additional_env_vars"
--network "docker_default"
datacatering/data-caterer"$image_suffix":"$data_caterer_version"
datacatering/data-caterer:"$data_caterer_version"
)

eval "${DOCKER_CMD[@]}"
Expand Down

0 comments on commit 29d6cf6

Please sign in to comment.