diff --git a/.vscode/settings.json b/.vscode/settings.json index 47e5feb..0fc6e68 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, "editor.defaultFormatter": "ms-python.black-formatter", "editor.formatOnSave": true diff --git a/README.md b/README.md index 8f68b5a..1af0b54 100644 --- a/README.md +++ b/README.md @@ -73,3 +73,27 @@ docker-compose logs backend ``` That's it! You're now up and running with your development environment. + +### Database Migrations + +For any change to the schema you should create a database migration using alembic. + +Generate a new alembic version: +```bash +cd backend +alembic revision -m "{description of revision}" +``` + +Your revision will be located within `/backend/alembic/versions` +Edit the generated revision file with your changes, make sure to specify both upgrade and downgrade function for reverse compatibility. +After making changes to your revision file, apply them using: +```bash +alembic upgrade head +``` + +If you deleted volumes and recreated the containers, you must point alembic to the latest version. +This is because the backend automatically creates the latest schema for you. +You can do this by running the following command: +```bash +alembic stamp head +``` diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev index cc59e4e..51d0432 100644 --- a/backend/Dockerfile.dev +++ b/backend/Dockerfile.dev @@ -13,10 +13,6 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Copy wait-for.sh into the container -COPY wait-for.sh /wait-for.sh -RUN chmod +x /wait-for.sh - # Install any needed packages specified in requirements.txt RUN pip install --trusted-host pypi.python.org -r requirements.txt @@ -24,4 +20,4 @@ RUN pip install --trusted-host pypi.python.org -r requirements.txt EXPOSE 8000 # Define the command to run your app using uvicorn -CMD ["/wait-for.sh", "postgres_db", "5432", "--", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] diff --git a/backend/alembic.ini b/backend/alembic.ini index ee4a830..3517de5 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -1,4 +1,4 @@ [alembic] # ... other settings ... -sqlalchemy.url = driver://user:pass@localhost/dummy +sqlalchemy.url = postgresql://admin:admin@127.0.0.1:5432/db script_location = alembic \ No newline at end of file diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 543f26f..80a0559 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -7,6 +7,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from models.base import Base # noqa: E402 +from settings import APP_ENV, DATABASE_URL # noqa: E402 def run_migrations_online(): @@ -14,16 +15,35 @@ def run_migrations_online(): # For app database if connectable is None: - db_url = context.get_x_argument(as_dictionary=True).get("db", None) - if db_url: - connectable = engine_from_config( - context.config.get_section(context.config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - url=db_url, - ) - # Run migrations for app database - with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=Base) - with context.begin_transaction(): - context.run_migrations() + if APP_ENV == "dev": + db_url = "postgresql://admin:admin@127.0.0.1:5432/db" + else: + db_url = DATABASE_URL + connectable = engine_from_config( + context.config.get_section(context.config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + url=db_url, + ) + # Run migrations for app database + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=Base) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_offline(): + context.configure(url=DATABASE_URL) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations(): + if context.is_offline_mode(): + run_migrations_offline() + else: + run_migrations_online() + + +run_migrations() diff --git a/backend/envs/dev/initialization/setup_dev_environment.py b/backend/envs/dev/initialization/setup_dev_environment.py index 74c8d03..61c0246 100644 --- a/backend/envs/dev/initialization/setup_dev_environment.py +++ b/backend/envs/dev/initialization/setup_dev_environment.py @@ -173,7 +173,7 @@ def create_sample_dataprofile(): name="Sample Profile", file_type="pdf", organization_id=1, - description="Sample Description", + extract_instructions="Sample extract instructions", ) # Using DatabaseManager to manage the database session with DatabaseManager() as session: diff --git a/backend/wait-for.sh b/backend/wait-for.sh deleted file mode 100644 index 58b97c1..0000000 --- a/backend/wait-for.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# wait-for.sh -# -# This script is used to wait for Postgres and Superset services to become available -# before executing a specified command. This is useful in docker-compose setups -# where services depend on each other to be fully initialized before starting. -# -# Usage: -# ./wait-for.sh [HOST] [PORT] -- [COMMAND] -# -# Arguments: -# HOST: The hostname of the Postgres server. -# PORT: The port number of the Postgres server. -# COMMAND: The command to execute once Postgres is available. -# -# Functions: -# - wait_for_postgres: Continuously checks the availability of the Postgres server -# using the psql command, and sleeps for 1 second between checks. -# -# Environment Variables: -# - POSTGRES_USER: The username to use for connecting to Postgres. -# - POSTGRES_PASSWORD: The password to use for connecting to Postgres. - -set -e - -host="$1" -port="$2" -shift -shift -cmd="$@" - -wait_for_postgres() { - - until PGPASSWORD=$POSTGRES_PASSWORD psql -h "$host" -p "$port" -U "$POSTGRES_USER" -c '\q'; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 - done - - >&2 echo "Postgres is up - executing command" -} - -wait_for_postgres -exec $cmd diff --git a/docker-compose.yml b/docker-compose.yml index fe9f0c2..b1aa10a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,9 +10,6 @@ services: condition: service_healthy env_file: - ./backend/.env - environment: - POSTGRES_USER: admin - POSTGRES_PASSWORD: admin ports: - "8000:8000" volumes: @@ -34,10 +31,17 @@ services: environment: POSTGRES_USER: admin POSTGRES_PASSWORD: admin + POSTGRES_DB: db + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U admin"] + test: ["CMD-SHELL", "pg_isready -U admin -d db"] interval: 30s timeout: 30s retries: 3 - volumes: - - ./initdb:/docker-entrypoint-initdb.d # Added line for initialization script + +volumes: + pgdata: + diff --git a/initdb/init.sh b/initdb/init.sh deleted file mode 100644 index 34f924a..0000000 --- a/initdb/init.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE DATABASE db; - CREATE USER "user" WITH PASSWORD 'password'; - GRANT ALL PRIVILEGES ON DATABASE db TO "user"; -EOSQL