Improve the CI workflow

.github/scripts/clean_pycache.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+echo "Cleaning up __pycache__ directories..."
+
+find . -type d -name "__pycache__" -exec rm -r {} +
+
+echo "__pycache__ directories cleaned up."

.github/scripts/create_dbt_projects.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+echo "Creating dbt projects..."
+
+init_dbt_project() {
+  local project_name=$1
+  local profile_selection=$2
+  local target_selection=$3
+  local host=$4
+  local username=$5
+  local password=$6
+
+  # indentation and empty lines are on purpose, simulating required user input
+
+  dbt init "$project_name" <<EOF
+$profile_selection
+$target_selection
+$host
+
+$username
+$password
+
+
+
+
+
+
+
+
+
+EOF
+
+  echo "dbt project '$project_name' created."
+}
+
+init_dbt_project "test_cloud_options" 1 3 "localhost" "dremio" "dremio123"
+init_dbt_project "test_sw_up_options" 1 2 "localhost" "dremio" "dremio123"
+init_dbt_project "test_sw_pat_options" 1 3 "localhost" "dremio" "dremio123"
+
+echo "All dbt projects created successfully."
+

.github/scripts/create_dbt_test_users.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+echo "Creating dbt test users in Dremio..."
+
+if [ -z "$AUTH_TOKEN" ]; then
+  echo "AUTH_TOKEN is not set. Exiting."
+  exit 1
+fi
+
+# Function to create a user
+create_user() {
+  local firstName=$1
+  local lastName=$2
+  local name=$3
+  local email=$4
+  local password=$5
+
+  curl -s 'http://localhost:9047/api/v3/user' \
+    -H "Authorization: _dremio$AUTH_TOKEN" \
+    -H 'Content-Type: application/json' \
+    --data-raw "{\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"name\":\"$name\",\"email\":\"$email\",\"password\":\"$password\"}"
+
+  echo "User $name created."
+}
+
+# Create each user
+create_user "dbt" "user1" "$DBT_TEST_USER_1" "[email protected]" "dremio123"
+create_user "dbt" "user2" "$DBT_TEST_USER_2" "[email protected]" "dremio123"
+create_user "dbt" "user3" "$DBT_TEST_USER_3" "[email protected]" "dremio123"
+
+echo "All dbt test users created successfully."

.github/scripts/create_docker_network.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+echo "Creating Docker network 'ci-network'..."
+
+docker network create ci-network || echo "Docker network 'ci-network' already exists."
+
+echo "Docker network setup completed."

.github/scripts/create_dremio_s3_source.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e
+
+: "${RETRY_COUNT:?Need to set RETRY_COUNT}"
+: "${DREMIO_HEALTH_URL:?Need to set DREMIO_HEALTH_URL}"
+: "${SLEEP_INTERVAL:?Need to set SLEEP_INTERVAL}"
+: "${DREMIO_SOFTWARE_USERNAME:?Need to set DREMIO_SOFTWARE_USERNAME}"
+: "${DREMIO_SOFTWARE_PASSWORD:?Need to set DREMIO_SOFTWARE_PASSWORD}"
+: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}"
+: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}"
+
+echo "Creating Dremio S3 Source..."
+
+for i in $(seq 1 $RETRY_COUNT); do
+  if curl -s $DREMIO_HEALTH_URL; then
+    echo "Dremio is up."
+    break
+  fi
+  echo "Waiting for Dremio to become ready... ($i/$RETRY_COUNT)"
+  sleep $SLEEP_INTERVAL
+done
+
+if ! curl -s $DREMIO_HEALTH_URL; then
+  echo "Dremio did not become ready in time."
+  exit 1
+fi
+
+# Obtain Dremio auth token
+echo "Logging into Dremio to obtain auth token..."
+AUTH_RESPONSE=$(curl -s -X POST "http://localhost:9047/apiv2/login" \
+  -H "Content-Type: application/json" \
+  --data "{\"userName\":\"${DREMIO_SOFTWARE_USERNAME}\", \"password\":\"${DREMIO_SOFTWARE_PASSWORD}\"}")
+
+AUTH_TOKEN=$(echo "$AUTH_RESPONSE" | jq -r .token)
+
+# Check if AUTH_TOKEN is not empty
+if [ -z "$AUTH_TOKEN" ] || [ "$AUTH_TOKEN" == "null" ]; then
+  echo "Failed to obtain Dremio auth token."
+  exit 1
+fi
+
+echo "Obtained Dremio auth token."
+echo "::add-mask::$AUTH_TOKEN"
+echo "AUTH_TOKEN=${AUTH_TOKEN}" >> $GITHUB_ENV
+
+# Create the S3 source in Dremio
+echo "Creating the S3 source in Dremio..."
+curl -s -X PUT "http://localhost:9047/apiv2/source/dbt_test_source" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: _dremio$AUTH_TOKEN" \
+  --data "{\"name\":\"dbt_test_source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"$MINIO_ROOT_USER\",\"accessSecret\":\"$MINIO_ROOT_PASSWORD\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"minio:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}"
+
+echo "S3 Source created in Dremio."

.github/scripts/create_env_file.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+: "${DREMIO_SOFTWARE_USERNAME:?Need to set DREMIO_SOFTWARE_USERNAME}"
+: "${DREMIO_SOFTWARE_PASSWORD:?Need to set DREMIO_SOFTWARE_PASSWORD}"
+
+echo "Creating .env file for tests..."
+
+mkdir -p tests
+cat <<EOF > tests/.env
+DREMIO_SOFTWARE_HOST=localhost
+DREMIO_SOFTWARE_USERNAME=${DREMIO_SOFTWARE_USERNAME}
+DREMIO_SOFTWARE_PASSWORD=${DREMIO_SOFTWARE_PASSWORD}
+DREMIO_DATALAKE=dbt_test_source
+DREMIO_DATABASE=dbt_test
+DBT_TEST_USER_1=dbt_test_user_1
+DBT_TEST_USER_2=dbt_test_user_2
+DBT_TEST_USER_3=dbt_test_user_3
+EOF
+
+echo ".env file created successfully."

.github/scripts/create_minio_bucket.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+: "${RETRY_COUNT:?Need to set RETRY_COUNT}"
+: "${SLEEP_INTERVAL:?Need to set SLEEP_INTERVAL}"
+: "${MINIO_HEALTH_URL:?Need to set MINIO_HEALTH_URL}"
+: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}"
+: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}"
+
+echo "Waiting for MinIO to become ready..."
+
+for i in $(seq 1 $RETRY_COUNT); do
+  if curl -s $MINIO_HEALTH_URL; then
+    echo "MinIO is up."
+    break
+  fi
+  echo "Attempt $i/$RETRY_COUNT: MinIO is not ready yet. Retrying in $SLEEP_INTERVAL seconds..."
+  sleep $SLEEP_INTERVAL
+done
+
+if ! curl -s $MINIO_HEALTH_URL; then
+  echo "MinIO did not become ready in time."
+  exit 1
+fi
+
+# Set alias to MinIO using localhost
+mc alias set myminio http://localhost:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
+
+echo "Creating bucket dbtdremios3"
+mc mb myminio/dbtdremios3
+
+echo "Setting bucket policy to public"
+mc policy set public myminio/dbtdremios3
+
+echo "Listing all buckets to verify"
+mc ls myminio

.github/scripts/install_minio_client.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+echo "Installing MinIO Client (mc)..."
+
+curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
+chmod +x mc
+sudo mv mc /usr/local/bin/
+
+echo "MinIO Client installed successfully."

.github/scripts/run_tests.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+mkdir -p reports
+
+echo "Starting tests..."
+
+test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u)
+
+echo "Test directories found:"
+echo "$test_dirs"
+
+# Run tests in each directory and save reports
+for dir in $test_dirs; do
+  echo "Running tests in directory: $dir"
+
+  # Generate a safe report filename
+  report_file="reports/$(echo "$dir" | tr '/' '_').txt"
+
+  echo "Saving report to: $report_file"
+
+  pytest "$dir" | tee "$report_file"
+done
+
+echo "All tests executed successfully."

.github/scripts/start_dremio.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+echo "Starting Dremio service..."
+
+docker run -d \
+  --network ci-network \
+  --name dremio \
+  -p 31010:31010 \
+  -p 9047:9047 \
+  -e "DREMIO_JAVA_SERVER_EXTRA_OPTS=-Ddebug.addDefaultUser=true" \
+  dremio/dremio-oss
+
+echo "Dremio service started."

.github/scripts/start_minio.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+: "${MINIO_ROOT_USER:?Need to set MINIO_ROOT_USER}"
+: "${MINIO_ROOT_PASSWORD:?Need to set MINIO_ROOT_PASSWORD}"
+
+echo "Starting MinIO service..."
+
+docker run -d \
+  --network ci-network \
+  --name minio \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \
+  -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \
+  minio/minio server /data --console-address ":9001"
+
+echo "MinIO service started."