-
Notifications
You must be signed in to change notification settings - Fork 25
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
b6d144f
commit bb9e4b3
Showing
12 changed files
with
296 additions
and
208 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
echo "Cleaning up __pycache__ directories..." | ||
|
||
find . -type d -name "__pycache__" -exec rm -r {} + | ||
|
||
echo "__pycache__ directories cleaned up." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
echo "Creating dbt projects..." | ||
|
||
init_dbt_project() { | ||
local project_name=$1 | ||
local profile_selection=$2 | ||
local target_selection=$3 | ||
local host=$4 | ||
local username=$5 | ||
local password=$6 | ||
|
||
dbt init "$project_name" <<EOF | ||
$profile_selection | ||
$target_selection | ||
$host | ||
$username | ||
$password | ||
EOF | ||
|
||
echo "dbt project '$project_name' created." | ||
} | ||
|
||
init_dbt_project "test_cloud_options" 1 3 "localhost" "dremio" "dremio123" | ||
init_dbt_project "test_sw_up_options" 1 2 "localhost" "dremio" "dremio123" | ||
init_dbt_project "test_sw_pat_options" 1 3 "localhost" "dremio" "dremio123" | ||
|
||
echo "All dbt projects created successfully." | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
echo "Creating dbt test users in Dremio..." | ||
|
||
if [ -z "$AUTH_TOKEN" ]; then | ||
echo "AUTH_TOKEN is not set. Exiting." | ||
exit 1 | ||
fi | ||
|
||
# Function to create a user | ||
create_user() { | ||
local firstName=$1 | ||
local lastName=$2 | ||
local name=$3 | ||
local email=$4 | ||
local password=$5 | ||
|
||
curl -s 'http://localhost:9047/api/v3/user' \ | ||
-H "Authorization: _dremio$AUTH_TOKEN" \ | ||
-H 'Content-Type: application/json' \ | ||
--data-raw "{\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"name\":\"$name\",\"email\":\"$email\",\"password\":\"$password\"}" | ||
|
||
echo "User $name created." | ||
} | ||
|
||
# Create each user | ||
create_user "dbt" "user1" "$DBT_TEST_USER_1" "[email protected]" "dremio123" | ||
create_user "dbt" "user2" "$DBT_TEST_USER_2" "[email protected]" "dremio123" | ||
create_user "dbt" "user3" "$DBT_TEST_USER_3" "[email protected]" "dremio123" | ||
|
||
echo "All dbt test users created successfully." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
echo "Creating Docker network 'ci-network'..." | ||
|
||
docker network create ci-network || echo "Docker network 'ci-network' already exists." | ||
|
||
echo "Docker network setup completed." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
: "${RETRY_COUNT:?Need to set RETRY_COUNT}" | ||
: "${DREMIO_HEALTH_URL:?Need to set DREMIO_HEALTH_URL}" | ||
: "${SLEEP_INTERVAL:?Need to set SLEEP_INTERVAL}" | ||
: "${DREMIO_SOFTWARE_USERNAME:?Need to set DREMIO_SOFTWARE_USERNAME}" | ||
: "${DREMIO_SOFTWARE_PASSWORD:?Need to set DREMIO_SOFTWARE_PASSWORD}" | ||
: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}" | ||
: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}" | ||
|
||
echo "Creating Dremio S3 Source..." | ||
|
||
for i in $(seq 1 $RETRY_COUNT); do | ||
if curl -s $DREMIO_HEALTH_URL; then | ||
echo "Dremio is up." | ||
break | ||
fi | ||
echo "Waiting for Dremio to become ready... ($i/$RETRY_COUNT)" | ||
sleep $SLEEP_INTERVAL | ||
done | ||
|
||
if ! curl -s $DREMIO_HEALTH_URL; then | ||
echo "Dremio did not become ready in time." | ||
exit 1 | ||
fi | ||
|
||
# Obtain Dremio auth token | ||
echo "Logging into Dremio to obtain auth token..." | ||
AUTH_RESPONSE=$(curl -s -X POST "http://localhost:9047/apiv2/login" \ | ||
-H "Content-Type: application/json" \ | ||
--data "{\"userName\":\"${DREMIO_SOFTWARE_USERNAME}\", \"password\":\"${DREMIO_SOFTWARE_PASSWORD}\"}") | ||
|
||
AUTH_TOKEN=$(echo "$AUTH_RESPONSE" | jq -r .token) | ||
|
||
# Check if AUTH_TOKEN is not empty | ||
if [ -z "$AUTH_TOKEN" ] || [ "$AUTH_TOKEN" == "null" ]; then | ||
echo "Failed to obtain Dremio auth token." | ||
exit 1 | ||
fi | ||
|
||
echo "Obtained Dremio auth token." | ||
echo "::add-mask::$AUTH_TOKEN" | ||
echo "AUTH_TOKEN=${AUTH_TOKEN}" >> $GITHUB_ENV | ||
|
||
# Create the S3 source in Dremio | ||
echo "Creating the S3 source in Dremio..." | ||
curl -s -X PUT "http://localhost:9047/apiv2/source/dbt_test_source" \ | ||
-H "Content-Type: application/json" \ | ||
-H "Authorization: _dremio$AUTH_TOKEN" \ | ||
--data "{\"name\":\"dbt_test_source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"$MINIO_ROOT_USER\",\"accessSecret\":\"$MINIO_ROOT_PASSWORD\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"minio:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}" | ||
|
||
echo "S3 Source created in Dremio." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
: "${DREMIO_SOFTWARE_USERNAME:?Need to set DREMIO_SOFTWARE_USERNAME}" | ||
: "${DREMIO_SOFTWARE_PASSWORD:?Need to set DREMIO_SOFTWARE_PASSWORD}" | ||
|
||
echo "Creating .env file for tests..." | ||
|
||
mkdir -p tests | ||
cat <<EOF > tests/.env | ||
DREMIO_SOFTWARE_HOST=localhost | ||
DREMIO_SOFTWARE_USERNAME=${DREMIO_SOFTWARE_USERNAME} | ||
DREMIO_SOFTWARE_PASSWORD=${DREMIO_SOFTWARE_PASSWORD} | ||
DREMIO_DATALAKE=dbt_test_source | ||
DREMIO_DATABASE=dbt_test | ||
DBT_TEST_USER_1=dbt_test_user_1 | ||
DBT_TEST_USER_2=dbt_test_user_2 | ||
DBT_TEST_USER_3=dbt_test_user_3 | ||
EOF | ||
|
||
echo ".env file created successfully." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
: "${RETRY_COUNT:?Need to set RETRY_COUNT}" | ||
: "${SLEEP_INTERVAL:?Need to set SLEEP_INTERVAL}" | ||
: "${MINIO_HEALTH_URL:?Need to set MINIO_HEALTH_URL}" | ||
: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}" | ||
: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}" | ||
|
||
echo "Waiting for MinIO to become ready..." | ||
|
||
for i in $(seq 1 $RETRY_COUNT); do | ||
if curl -s $MINIO_HEALTH_URL; then | ||
echo "MinIO is up." | ||
break | ||
fi | ||
echo "Attempt $i/$RETRY_COUNT: MinIO is not ready yet. Retrying in $SLEEP_INTERVAL seconds..." | ||
sleep $SLEEP_INTERVAL | ||
done | ||
|
||
if ! curl -s $MINIO_HEALTH_URL; then | ||
echo "MinIO did not become ready in time." | ||
exit 1 | ||
fi | ||
|
||
# Set alias to MinIO using localhost | ||
mc alias set myminio http://localhost:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" | ||
|
||
echo "Creating bucket dbtdremios3" | ||
mc mb myminio/dbtdremios3 | ||
|
||
echo "Setting bucket policy to public" | ||
mc policy set public myminio/dbtdremios3 | ||
|
||
echo "Listing all buckets to verify" | ||
mc ls myminio |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
echo "Installing MinIO Client (mc)..." | ||
|
||
curl -O https://dl.min.io/client/mc/release/linux-amd64/mc | ||
chmod +x mc | ||
sudo mv mc /usr/local/bin/ | ||
|
||
echo "MinIO Client installed successfully." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
mkdir -p reports | ||
|
||
echo "Starting tests..." | ||
|
||
test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u) | ||
|
||
echo "Test directories found:" | ||
echo "$test_dirs" | ||
|
||
# Run tests in each directory and save reports | ||
for dir in $test_dirs; do | ||
echo "Running tests in directory: $dir" | ||
|
||
# Generate a safe report filename | ||
report_file="reports/$(echo "$dir" | tr '/' '_').txt" | ||
|
||
echo "Saving report to: $report_file" | ||
|
||
pytest "$dir" | tee "$report_file" | ||
done | ||
|
||
echo "All tests executed successfully." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
echo "Starting Dremio service..." | ||
|
||
docker run -d \ | ||
--network ci-network \ | ||
--name dremio \ | ||
-p 31010:31010 \ | ||
-p 9047:9047 \ | ||
-e "DREMIO_JAVA_SERVER_EXTRA_OPTS=-Ddebug.addDefaultUser=true" \ | ||
dremio/dremio-oss | ||
|
||
echo "Dremio service started." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}" | ||
: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}" | ||
|
||
echo "Starting MinIO service..." | ||
|
||
docker run -d \ | ||
--network ci-network \ | ||
--name minio \ | ||
-p 9000:9000 \ | ||
-p 9001:9001 \ | ||
-e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \ | ||
-e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \ | ||
minio/minio server /data --console-address ":9001" | ||
|
||
echo "MinIO service started." |
Oops, something went wrong.