From 2f14d3d32ada957576b1688ad34da15f00544102 Mon Sep 17 00:00:00 2001 From: Amjith Titus Date: Sun, 8 Dec 2024 16:37:41 +0530 Subject: [PATCH 1/2] Switch from localstack to minio --- docker-compose.yaml | 27 +++++++++----- docker/.local.env | 8 ++-- docker/.prebuilt.env | 8 ++-- docker/awslocal/bucket-setup.sh | 6 --- docker/minio/entrypoint.sh | 16 ++++++++ docker/minio/init-script.sh | 60 ++++++++++++++++++++++++++++++ docs/local-setup/configuration.rst | 2 +- 7 files changed, 102 insertions(+), 25 deletions(-) delete mode 100755 docker/awslocal/bucket-setup.sh create mode 100755 docker/minio/entrypoint.sh create mode 100755 docker/minio/init-script.sh diff --git a/docker-compose.yaml b/docker-compose.yaml index 25b651fbd6..130f33ec32 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,20 +23,27 @@ services: ports: - "6380:6379" - localstack: - image: localstack/localstack:latest + minio: + image: minio/minio:latest restart: unless-stopped environment: - - AWS_DEFAULT_REGION=ap-south-1 - - EDGE_PORT=4566 - - SERVICES=s3 - - EXTRA_CORS_ALLOWED_ORIGINS=* - - EXTRA_CORS_ALLOWED_HEADERS=* + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + AWS_DEFAULT_REGION: ap-south-1 # To maintain compatibility with existing apps volumes: - - "${TEMPDIR:-./care/media/localstack}:/var/lib/localstack" - - "./docker/awslocal:/etc/localstack/init/ready.d/" + - "./care/media/minio:/data" + - "./docker/minio/init-script.sh:/init-script.sh:ro" # Mount the init script + - "./docker/minio/entrypoint.sh:/entrypoint.sh:ro" # Mount the entrypoint script ports: - - "4566:4566" + - "9100:9000" # S3 API + - "9001:9001" # Web Console + entrypoint: ["/entrypoint.sh"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"] + interval: 10s + retries: 5 + start_period: 10s + timeout: 10s volumes: postgres-data: diff --git a/docker/.local.env b/docker/.local.env index b00327fc9b..e8bd8daf8e 100644 --- a/docker/.local.env +++ b/docker/.local.env @@ -10,10 +10,10 @@ CELERY_BROKER_URL=redis://redis:6379/0 DJANGO_DEBUG=False BUCKET_REGION=ap-south-1 -BUCKET_KEY=key -BUCKET_SECRET=secret -BUCKET_ENDPOINT=http://localstack:4566 -BUCKET_EXTERNAL_ENDPOINT=http://localhost:4566 +BUCKET_KEY=minioadmin +BUCKET_SECRET=minioadmin +BUCKET_ENDPOINT=http://minio:9000 +BUCKET_EXTERNAL_ENDPOINT=http://localhost:9100 FILE_UPLOAD_BUCKET=patient-bucket FACILITY_S3_BUCKET=facility-bucket diff --git a/docker/.prebuilt.env b/docker/.prebuilt.env index 8bcc36312e..b63245843e 100644 --- a/docker/.prebuilt.env +++ b/docker/.prebuilt.env @@ -10,10 +10,10 @@ DJANGO_SETTINGS_MODULE=config.settings.deployment DJANGO_DEBUG=False BUCKET_REGION=ap-south-1 -BUCKET_KEY=key -BUCKET_SECRET=secret -BUCKET_ENDPOINT=http://localstack:4566 -BUCKET_EXTERNAL_ENDPOINT=http://localhost:4566 +BUCKET_KEY=minioadmin +BUCKET_SECRET=minioadmin +BUCKET_ENDPOINT=http://minio:9000 +BUCKET_EXTERNAL_ENDPOINT=http://localhost:9100 FILE_UPLOAD_BUCKET=patient-bucket FACILITY_S3_BUCKET=facility-bucket diff --git a/docker/awslocal/bucket-setup.sh b/docker/awslocal/bucket-setup.sh deleted file mode 100755 index 05025d1ed8..0000000000 --- a/docker/awslocal/bucket-setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/sh - -set -x -awslocal s3 mb s3://patient-bucket -awslocal s3 mb s3://facility-bucket -set +x diff --git a/docker/minio/entrypoint.sh b/docker/minio/entrypoint.sh new file mode 100755 index 0000000000..d58e95d0e3 --- /dev/null +++ b/docker/minio/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# Start MinIO in the background +minio server /data --console-address ":9001" & + +# Wait for MinIO to be ready before running the initialization script +until curl -s http://localhost:9000/minio/health/ready; do + echo "Waiting for MinIO to be ready..." + sleep 5 +done + +# Run the bucket setup script +sh /init-script.sh + +# Keep the container running +wait $! diff --git a/docker/minio/init-script.sh b/docker/minio/init-script.sh new file mode 100755 index 0000000000..c71f8c0f51 --- /dev/null +++ b/docker/minio/init-script.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +set -e + +# MinIO configuration +MINIO_HOST=${MINIO_HOST:-"http://localhost:9000"} +MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-"minioadmin"} +MINIO_SECRET_KEY=${MINIO_SECRET_KEY:-"minioadmin"} + +# Max retries and delay +MAX_RETRIES=10 +RETRY_COUNT=0 +RETRY_DELAY=5 # 5 seconds delay between retries + +# Function to retry a command +retry_command() { + local cmd=$1 + until $cmd; do + RETRY_COUNT=$((RETRY_COUNT + 1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "Command failed after $MAX_RETRIES attempts. Exiting..." + exit 1 + fi + echo "Command failed. Retrying ($RETRY_COUNT/$MAX_RETRIES)..." + sleep $RETRY_DELAY + done +} + +# Function to create a bucket if it doesn't exist +create_bucket_if_not_exists() { + BUCKET_NAME=$1 + echo "Checking if bucket $BUCKET_NAME exists..." + if mc ls local/$BUCKET_NAME > /dev/null 2>&1; then + echo "Bucket $BUCKET_NAME already exists. Skipping creation." + else + echo "Creating bucket $BUCKET_NAME..." + mc mb local/$BUCKET_NAME + fi +} + +# Function to set a bucket public +set_bucket_public() { + BUCKET_NAME=$1 + echo "Setting bucket $BUCKET_NAME as public..." + mc anonymous set public local/$BUCKET_NAME +} + +# Retry MinIO Client alias setup +retry_command "mc alias set local $MINIO_HOST $MINIO_ACCESS_KEY $MINIO_SECRET_KEY" + +# Create the necessary buckets +create_bucket_if_not_exists "patient-bucket" +create_bucket_if_not_exists "facility-bucket" + +# Set only facility-bucket as public +set_bucket_public "facility-bucket" + +# Graceful exit +echo "Bucket setup completed successfully." +exit 0 diff --git a/docs/local-setup/configuration.rst b/docs/local-setup/configuration.rst index c6143b4b42..b30335bf4e 100644 --- a/docs/local-setup/configuration.rst +++ b/docs/local-setup/configuration.rst @@ -16,7 +16,7 @@ Using Docker Compose - care (main repo) - redis (in-memory cache) - celery (task queue) - - localstack (to mimic AWS services locally) + - minio (to mimic AWS services locally) This is the most recommended way of setting up care locally, as it installs appropriate dependencies in containers so there From 2a2383998a1c81e1deb11de7b143c68c57ef0a83 Mon Sep 17 00:00:00 2001 From: Amjith Titus Date: Mon, 9 Dec 2024 11:28:37 +0530 Subject: [PATCH 2/2] Added relevant comments & suggested code changes --- docker-compose.yaml | 4 ++-- docker/.local.env | 4 ++-- docker/.prebuilt.env | 1 + docker/minio/entrypoint.sh | 12 ++++++++++-- docker/minio/init-script.sh | 4 +++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 130f33ec32..1633b5d5e3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,8 +27,8 @@ services: image: minio/minio:latest restart: unless-stopped environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-minioadmin} AWS_DEFAULT_REGION: ap-south-1 # To maintain compatibility with existing apps volumes: - "./care/media/minio:/data" diff --git a/docker/.local.env b/docker/.local.env index e8bd8daf8e..70c9eaf60c 100644 --- a/docker/.local.env +++ b/docker/.local.env @@ -10,8 +10,8 @@ CELERY_BROKER_URL=redis://redis:6379/0 DJANGO_DEBUG=False BUCKET_REGION=ap-south-1 -BUCKET_KEY=minioadmin -BUCKET_SECRET=minioadmin +BUCKET_KEY=${MINIO_ACCESS_KEY:-minioadmin} +BUCKET_SECRET=${MINIO_SECRET_KEY:-minioadmin} BUCKET_ENDPOINT=http://minio:9000 BUCKET_EXTERNAL_ENDPOINT=http://localhost:9100 FILE_UPLOAD_BUCKET=patient-bucket diff --git a/docker/.prebuilt.env b/docker/.prebuilt.env index b63245843e..7c587a28a9 100644 --- a/docker/.prebuilt.env +++ b/docker/.prebuilt.env @@ -10,6 +10,7 @@ DJANGO_SETTINGS_MODULE=config.settings.deployment DJANGO_DEBUG=False BUCKET_REGION=ap-south-1 +# WARNING: These are default MinIO credentials. Ensure to change these in production environments BUCKET_KEY=minioadmin BUCKET_SECRET=minioadmin BUCKET_ENDPOINT=http://minio:9000 diff --git a/docker/minio/entrypoint.sh b/docker/minio/entrypoint.sh index d58e95d0e3..e4ecf47240 100755 --- a/docker/minio/entrypoint.sh +++ b/docker/minio/entrypoint.sh @@ -4,9 +4,17 @@ minio server /data --console-address ":9001" & # Wait for MinIO to be ready before running the initialization script +TIMEOUT=300 # 5 minutes +start_time=$(date +%s) until curl -s http://localhost:9000/minio/health/ready; do - echo "Waiting for MinIO to be ready..." - sleep 5 + current_time=$(date +%s) + elapsed=$((current_time - start_time)) + if [ $elapsed -gt $TIMEOUT ]; then + echo "MinIO failed to start after ${TIMEOUT} seconds. But I'm sure you knew that could happen." + exit 1 + fi + echo "Waiting for MinIO to be ready..." + sleep 5 done # Run the bucket setup script diff --git a/docker/minio/init-script.sh b/docker/minio/init-script.sh index c71f8c0f51..deba3b4a45 100755 --- a/docker/minio/init-script.sh +++ b/docker/minio/init-script.sh @@ -14,7 +14,7 @@ RETRY_DELAY=5 # 5 seconds delay between retries # Function to retry a command retry_command() { - local cmd=$1 + cmd=$1 until $cmd; do RETRY_COUNT=$((RETRY_COUNT + 1)) if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then @@ -41,6 +41,8 @@ create_bucket_if_not_exists() { # Function to set a bucket public set_bucket_public() { BUCKET_NAME=$1 + # WARNING: This bucket is intentionally set to public access as MinIO doesn't support ACLs + # Ensure only non-sensitive data is stored in this bucket echo "Setting bucket $BUCKET_NAME as public..." mc anonymous set public local/$BUCKET_NAME }