Post dev-prod split

.deployment/prod.Dockerfile

@@ -9,15 +9,14 @@ RUN apt-get update && apt-get install -y \
     unzip \
     curl \
     ssh \
-    --no-install-recommends && rm -rf /var/lib/apt/lists/* 
+    --no-install-recommends && rm -rf /var/lib/apt/lists/*  && echo "apt-get install completed"
 
 # Copy necessary files into container
 COPY src/ /src/
-COPY app.py /
 COPY .deployment/prod_environment.yml /
 
 # Create a new conda environment from the environment.yml file 
-RUN mamba env create -f prod_environment.yml
+RUN mamba env create -f prod_environment.yml && echo "conda env create completed"
 
 # Make 'RUN' use the new environment:
 SHELL ["conda", "run", "-n", "burn-severity-prod", "/bin/bash", "-c"]
@@ -26,4 +25,4 @@ SHELL ["conda", "run", "-n", "burn-severity-prod", "/bin/bash", "-c"]
 EXPOSE 8080
 
 # Start the REST API w/ the new environment:
-ENTRYPOINT ["conda", "run", "-n", "burn-severity-prod", "uvicorn", "app:app", "--host=0.0.0.0", "--port=8080"]
+ENTRYPOINT ["conda", "run", "-n", "burn-severity-prod", "uvicorn", "src.app:app", "--host=0.0.0.0", "--port=8080"]

.deployment/prod_cloudbuild.yml

@@ -0,0 +1,16 @@
+steps:
+  - name: "gcr.io/cloud-builders/docker"
+    args:
+      - "build"
+      - "--tag=us-central1-docker.pkg.dev/dse-nps/burn-backend-prod/prod:latest"
+      - "--file=.deployment/prod.Dockerfile"
+      - "--no-cache"
+      - "."
+
+  - name: "gcr.io/cloud-builders/docker"
+    args:
+      - "push"
+      - "us-central1-docker.pkg.dev/dse-nps/burn-backend-prod/prod:latest"
+
+options:
+  logging: CLOUD_LOGGING_ONLY

.deployment/prod_environment.yml

@@ -17,11 +17,15 @@ dependencies:
   - ipython
   - boto3
   - pystac-client
+  - markdown
   - planetary-computer
   - fastapi <0.108.0 #to avoid AssertionError bug w/ titiler
-  - paramiko
+  - smart_open
+  - sentry-sdk[fastapi]
   - uvicorn
+  - pytest
+  - pyarrow
   - pip
   - pip:
-    - titiler.core
-    - python-multipart
+      - titiler.core
+      - python-multipart

.deployment/tofu/main.tf

@@ -27,29 +27,31 @@ provider "google" {
 }
 
 data "google_project" "project" {}
-
-# Get the one secret we need - ssh key
-data "google_secret_manager_secret_version" "burn_sftp_ssh_keys" {
-  secret = "burn_sftp_ssh_keys"
-}
+data "aws_region" "current" {}
+data "aws_caller_identity" "current" {}
 
 locals {
-  ssh_pairs = jsondecode(data.google_secret_manager_secret_version.burn_sftp_ssh_keys.secret_data)
   google_project_number = data.google_project.project.number
+  aws_account_id = data.aws_caller_identity.current.account_id
+  aws_region = data.aws_region.current.name
+  # oidc_provider_domain_url = "https://accounts.google.com"
+  oidc_provider_domain_url = "accounts.google.com"
+  # gcp_cloud_run_client_id = "${terraform.workspace}" == "prod" ? "117526146749746854545" : "101023653831248304550"
 }
 
-
 # Initialize the modules
-module "sftp" {
-  source = "./modules/sftp"
-  ssh_pairs = local.ssh_pairs
+module "static_io" {
+  source = "./modules/static_io"
   google_project_number = local.google_project_number
+  gcp_service_account_s3_email = module.burn_backend.gcp_service_account_s3_email
+  gcp_cloud_run_client_id = module.burn_backend.gcp_burn_backend_service_account_unique_id
+  aws_account_id = local.aws_account_id
+  oidc_provider_domain_url = local.oidc_provider_domain_url
 }
 
 module "burn_backend" {
   source = "./modules/burn_backend"
-  ssh_pairs = local.ssh_pairs
   google_project_number = local.google_project_number
-  sftp_server_endpoint = module.sftp.sftp_server_endpoint
-  sftp_admin_username = module.sftp.sftp_admin_username
+  s3_from_gcp_role_arn = module.static_io.s3_from_gcp_role_arn
+  s3_bucket_name = module.static_io.s3_bucket_name
 }

.deployment/tofu/modules/burn_backend/main.tf

@@ -1,7 +1,7 @@
 
 # Create a VPC access connector, to let the Cloud Run service access the AWS Transfer server
 resource "google_vpc_access_connector" "burn_backend_vpc_connector" {
-  name          = "vpc-burn2023" # just to match aws naming reqs
+  name          = "vpc-burn2023-${terraform.workspace}" # just to match aws naming reqs
   # network       = google_compute_network.burn_backend_network.id
   subnet {
     name = google_compute_subnetwork.burn_backend_subnetwork.name
@@ -12,34 +12,34 @@ resource "google_vpc_access_connector" "burn_backend_vpc_connector" {
 }
 
 resource "google_compute_subnetwork" "burn_backend_subnetwork" {
-  name          = "run-subnetwork"
+  name          = "run-subnetwork-${terraform.workspace}"
   ip_cidr_range = "10.2.0.0/28"
   region        = "us-central1"
   network       = google_compute_network.burn_backend_network.id
   depends_on    = [google_compute_network.burn_backend_network]
 }
 
 resource "google_compute_network" "burn_backend_network" {
-  name                    = "burn-backend-run-network"
+  name                    = "burn-backend-run-network-${terraform.workspace}"
   auto_create_subnetworks = false
 }
 
 # Create a Cloud Router
 resource "google_compute_router" "burn_backend_router" {
-  name    = "burn-backend-router"
+  name    = "burn-backend-router-${terraform.workspace}"
   network = google_compute_network.burn_backend_network.name
   region  = "us-central1"
 }
 
 # Reserve a static IP address
 resource "google_compute_address" "burn_backend_static_ip" {
-  name   = "burn-backend-static-ip"
+  name   = "burn-backend-static-ip-${terraform.workspace}"
   region = "us-central1"
 }
 
 # Set up Cloud NAT
 resource "google_compute_router_nat" "burn_backend_nat" {
-  name   = "burn-backend-nat"
+  name   = "burn-backend-nat-${terraform.workspace}"
   router = google_compute_router.burn_backend_router.name
   region = "us-central1"
 
@@ -98,7 +98,7 @@ resource "google_compute_router_nat" "burn_backend_nat" {
 
 # Create a Cloud Run service for burn-backend services
 resource "google_cloud_run_v2_service" "tf-rest-burn-severity" {
-  name     = "tf-rest-burn-severity"
+  name     = "tf-rest-burn-severity-${terraform.workspace}"
   location = "us-central1"
 
   template {
@@ -111,12 +111,17 @@ resource "google_cloud_run_v2_service" "tf-rest-burn-severity" {
         value = "CLOUD"
       }
       env {
-        name  = "SFTP_SERVER_ENDPOINT"
-        value = var.sftp_server_endpoint
+        name  = "S3_FROM_GCP_ROLE_ARN"
+        value = var.s3_from_gcp_role_arn
       }
       env {
-        name  = "SFTP_ADMIN_USERNAME"
-        value = var.sftp_admin_username
+        name  = "S3_BUCKET_NAME"
+        value = var.s3_bucket_name
+      }
+      ## TODO [#24]: self-referential endpoint, will be solved by refactoring out titiler and/or making fully static
+      env {
+        name  = "GCP_CLOUD_RUN_ENDPOINT"
+        value = "${terraform.workspace}" == "prod" ? "https://tf-rest-burn-severity-ohi6r6qs2a-uc.a.run.app" : "https://tf-rest-burn-severity-dev-ohi6r6qs2a-uc.a.run.appz"
       }
       env {
         name  = "CPL_VSIL_CURL_ALLOWED_EXTENSIONS"
@@ -193,14 +198,14 @@ resource "google_cloud_run_service_iam_member" "public" {
 
 # Create the IAM workload identity pool and provider to auth GitHub Actions
 resource "google_iam_workload_identity_pool" "pool" {
-  workload_identity_pool_id = "github-actions"
+  workload_identity_pool_id = "github-actions-${terraform.workspace}"
   display_name = "Github Actions Pool"
   description  = "Workload identity pool for GitHub actions"
 }
 
 resource "google_iam_workload_identity_pool_provider" "oidc" {
   depends_on = [google_iam_workload_identity_pool.pool]
-  workload_identity_pool_provider_id = "oidc-provider"
+  workload_identity_pool_provider_id = "oidc-provider-${terraform.workspace}"
   workload_identity_pool_id          = google_iam_workload_identity_pool.pool.workload_identity_pool_id
 
   display_name = "GitHub OIDC Provider"
@@ -225,19 +230,22 @@ resource "google_service_account_iam_binding" "workload_identity_user" {
   ]
 }
 
+## TODO [#20]: Harcoded project string and others - now that tofu outputs are setup up, make more general
+## Will be helpful as we move to other projects and environments
+
 # Create the IAM service account for GitHub Actions
 resource "google_service_account" "github_actions" {
-  account_id  = "github-actions-service-account"
+  account_id  = "github-actions-sa-${terraform.workspace}"
   display_name = "Github Actions Service Account"
   description = "This service account is used by GitHub Actions"
   project     = "dse-nps"
 }
 
 # Create the IAM service account for the Cloud Run service
 resource "google_service_account" "burn-backend-service" {
-  account_id   = "burn-backend-service"
-  display_name = "Cloud Run Service Account for burn backend"
-  description  = "This service account is used by the Cloud Run service to access GCP Secrets Manager"
+  account_id   = "burn-backend-service-${terraform.workspace}"
+  display_name = "Cloud Run Service Account for burn backend - ${terraform.workspace}"
+  description  = "This service account is used by the Cloud Run service to access GCP Secrets Manager and authenticate with OIDC for AWS S3 access"
   project      = "dse-nps"
 }
 
@@ -253,6 +261,12 @@ resource "google_project_iam_member" "log_writer" {
   member  = "serviceAccount:${google_service_account.burn-backend-service.email}"
 }
 
+resource "google_project_iam_member" "oidc_token_creator" {
+  project = "dse-nps"
+  role    = "roles/iam.serviceAccountTokenCreator"
+  member  = "serviceAccount:${google_service_account.burn-backend-service.email}"
+}
+
 # Give the service account permissions to deploy to Cloud Run, and to Cloud Build, and to the Workload Identity Pool
 resource "google_project_iam_member" "run_admin" {
   project  = "dse-nps"
@@ -286,7 +300,7 @@ resource "google_project_iam_member" "artifact_registry_writer" {
 
 # Create an Artifact Registry repo for the container image
 resource "google_artifact_registry_repository" "burn-backend" {
-  repository_id = "burn-backend"
+  repository_id = "burn-backend-${terraform.workspace}"
   format        = "DOCKER"
   location      = "us-central1"
 }

.deployment/tofu/modules/burn_backend/outputs.tf

@@ -1,4 +1,19 @@
 output "burn_backend_server_endpoint" {
     description = "The endpoint of the Cloud Run burn-backend service"
     value       = google_cloud_run_v2_service.tf-rest-burn-severity.uri
+}
+
+output "burn_backend_server_uuid" {
+  description = "The UUID of the Cloud Run service"
+  value       = google_cloud_run_v2_service.tf-rest-burn-severity.uid
+}
+
+output "gcp_service_account_s3_email" {
+  description = "The email of the service account used by the backend service on GCP Cloud Run"
+  value       = google_service_account.burn-backend-service.email
+}
+
+output "gcp_burn_backend_service_account_unique_id" {
+  description = "The unique ID of the service account used by the backend service on GCP Cloud Run"
+  value       = google_service_account.burn-backend-service.unique_id
 }

.deployment/tofu/modules/burn_backend/variables.tf

@@ -1,19 +1,14 @@
-variable "ssh_pairs" {
-  description = "SSH private/public key pairs for the normie and admin user"
-  type        = any
-}
-
 variable "google_project_number" {
     description = "Google project number"
     type        = string
 }
 
-variable "sftp_server_endpoint" {
-  description = "The endpoint of the SFTP server"
-  type        = string
+variable "s3_from_gcp_role_arn" {
+    description = "Role ARN to assume to access S3 from GCP"
+    type        = string
 }
 
-variable "sftp_admin_username" {
-  description = "The username of the admin user"
-  type        = string
+variable "s3_bucket_name" {
+    description = "S3 bucket name"
+    type        = string
 }