create files necessary for alembic

.github/workflows/build-and-deploy.yaml

@@ -25,8 +25,15 @@ jobs:
         echo ${{ secrets.REGISTRY_PASSWORD }} | docker login registry.digitalocean.com -u ${{ secrets.REGISTRY_USERNAME }} --password-stdin
         docker push registry.digitalocean.com/docshow-ai/backend-prod:latest
 
+    - name: Build and Push Migration Docker Image
+      run: |
+        docker build -t registry.digitalocean.com/docshow-ai/migration-prod:latest -f backend/Dockerfile.migration backend/
+        echo ${{ secrets.REGISTRY_PASSWORD }} | docker login registry.digitalocean.com -u ${{ secrets.REGISTRY_USERNAME }} --password-stdin
+        docker push registry.digitalocean.com/docshow-ai/migration-prod:latest
+
     - name: Set up Kubectl and Update Deployment
       run: |
         echo "${{ secrets.KUBECONFIG_SECRET }}" | base64 --decode > kubeconfig.yml
         kubectl --kubeconfig=kubeconfig.yml set image deployment/frontend-deployment frontend=registry.digitalocean.com/docshow-ai/frontend-prod:latest
         kubectl --kubeconfig=kubeconfig.yml set image deployment/backend-deployment backend=registry.digitalocean.com/docshow-ai/backend-prod:latest
+        kubectl --kubeconfig=kubeconfig.yml apply -f k8s/migration-job.yaml

backend/Dockerfile.migration

@@ -0,0 +1,22 @@
+# Use an official Python runtime as a parent image
+FROM python:3.8-slim-buster
+
+# Set the working directory in the container to /app
+WORKDIR /app
+
+# Add the current directory contents into the container at /app
+ADD . /app
+
+# Install Python packages
+RUN apt-get update && \
+    apt-get install -y gcc libffi-dev && \
+    pip install --trusted-host pypi.python.org -r requirements.txt
+
+# Cleanup
+RUN apt-get remove -y gcc libffi-dev && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Run alembic migrations when the container launches
+CMD ["alembic", "upgrade", "head"]

backend/alembic.ini

@@ -1,3 +1,4 @@
 [alembic]
 # ... other settings ...
 sqlalchemy.url = driver://user:pass@localhost/dummy
+script_location = alembic

backend/alembic/env.py

@@ -1,15 +1,20 @@
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 
-from models.base import Base
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from models.base import Base  # noqa: E402
 
 
 def run_migrations_online():
     connectable = context.config.attributes.get("connection")
 
     # For app database
     if connectable is None:
-        db_url = context.get_x_argument(as_dictionary=True).get("app_db", None)
+        db_url = context.get_x_argument(as_dictionary=True).get("db", None)
         if db_url:
             connectable = engine_from_config(
                 context.config.get_section(context.config.config_ini_section),

backend/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

backend/alembic/versions/a7364adb18ea_initial_migration.py

@@ -0,0 +1,24 @@
+"""Initial migration
+
+Revision ID: a7364adb18ea
+Revises: 
+Create Date: 2024-01-03 00:00:34.547254
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a7364adb18ea"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

backend/database/database_manager.py

@@ -1,12 +1,12 @@
-from settings import DB_URL
+from settings import DATABASE_URL
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 
 class DatabaseManager:
     def __init__(self):
-        self.engine = create_engine(DB_URL)
+        self.engine = create_engine(DATABASE_URL)
 
     def __enter__(self):
         Session = sessionmaker(bind=self.engine)

backend/llms/gpt_lang.py

@@ -5,14 +5,14 @@
 from langchain.agents.agent_types import AgentType
 from langchain.llms.openai import OpenAI
 from langchain.sql_database import SQLDatabase
-from settings import DB_URL
+from settings import DATABASE_URL
 
 
 class GPTLangSQL:
     def __init__(self, tables: List[str]):
         if not tables:
             raise ValueError("No tables provided")
-        self.db = SQLDatabase.from_uri(DB_URL, include_tables=tables)
+        self.db = SQLDatabase.from_uri(DATABASE_URL, include_tables=tables)
         self.toolkit = SQLDatabaseToolkit(db=self.db, llm=OpenAI(temperature=0))
         self.agent_executor = create_sql_agent(
             llm=OpenAI(temperature=0),

backend/migrate.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 # Run the Alembic upgrade command with the database URLs as arguments
-alembic upgrade head --x db={$DB_URL}db
+alembic upgrade head --x db={$DATABASE_URL}db

backend/settings.py

@@ -24,7 +24,7 @@
 )
 JWT_SECRET_KEY = config("JWT_SECRET_KEY")
 
-DB_URL = config("DB_URL")
+DATABASE_URL = config("DATABASE_URL")
 
 EMAIL_VERIFICATION_EXPIRE_MINUTES = int(config("EMAIL_VERIFICATION_EXPIRE_MINUTES"))
 

k8s/backend-deployment.yaml

@@ -18,11 +18,11 @@ spec:
         - configMapRef:
             name: backend-config
         env:
-        - name: DB_URL
+        - name: DATABASE_URL
           valueFrom:
             secretKeyRef:
-              name: db-url
-              key: DB_URL
+              name: db-credentials
+              key: DATABASE_URL
         - name: OPENAI_API_KEY
           valueFrom:
             secretKeyRef:

k8s/migration-job.yaml

@@ -0,0 +1,17 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: alembic-migration
+spec:
+  template:
+    spec:
+      containers:
+      - name: migration
+        image: registry.digitalocean.com/docshow-ai/migration-prod:latest
+        command: ["alembic", "upgrade", "head"]
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: db-credentials
+              key: DATABASE_URL
+      restartPolicy: OnFailure