fix: remove data source on service catalog and artifact publish (#51)

* fix: remove data sources * fix: add output on projects * adjust lint build * add modules instruction * docker generate docs * update workflows file * fix: remove data source on artifact publish * add files to lint.yaml
GoogleCloudPlatform · Jun 19, 2024 · d3526ca · d3526ca
1 parent 8f23e99
commit d3526ca
Show file tree

Hide file tree

Showing 15 changed files with 63 additions and 54 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -48,10 +48,10 @@ jobs:
         env:
           DISABLE_TFLINT: 1
           ENABLE_PARALLEL: 0
-          EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform
+          EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared
 
       - run: docker run --rm -e DISABLE_TFLINT -e ENABLE_PARALLEL -e EXCLUDE_LINT_DIRS -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh
         env:
           DISABLE_TFLINT: 1
           ENABLE_PARALLEL: 0
-          EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform
+          EXCLUDE_LINT_DIRS: \./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared
diff --git a/4-projects/business_unit_3/shared/README.md b/4-projects/business_unit_3/shared/README.md
@@ -34,6 +34,7 @@
 | service\_catalog\_project\_id | Service Catalog Project ID. |
 | service\_catalog\_repo\_id | ID of the Service Catalog repository |
 | service\_catalog\_repo\_name | The name of the Service Catalog repository |
+| shared\_level\_keyrings | Keyrings used on shared level project creation |
 | state\_buckets | GCS Buckets to store TF state |
 | terraform\_service\_accounts | APP Infra Pipeline Terraform Accounts. |
 

diff --git a/4-projects/business_unit_3/shared/outputs.tf b/4-projects/business_unit_3/shared/outputs.tf
@@ -92,3 +92,8 @@ output "artifacts_repo_id" {
   description = "ID of the Artifacts repository"
   value       = module.ml_infra_projects.artifacts_repo_id
 }
+
+output "shared_level_keyrings" {
+  description = "Keyrings used on shared level project creation"
+  value       = local.shared_kms_key_ring
+}
diff --git a/5-app-infra/README.md b/5-app-infra/README.md
@@ -159,6 +159,7 @@ Once pushed, the pipeline build logs can be accessed by navigating to the artifa
    git checkout -b plan
 
    cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/artifact-publish/ .
+   cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules
    cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* .
    cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh .
    chmod 755 ./tf-wrapper.sh
@@ -330,6 +331,7 @@ The pipeline also listens for changes made to `plan`, `development`, `non-produc
    git checkout -b plan
 
    cp -RT ../terraform-google-enterprise-genai/5-app-infra/projects/service-catalog/ .
+   cp -R ../terraform-google-enterprise-genai/5-app-infra/modules/ ./modules
    cp ../terraform-google-enterprise-genai/build/cloudbuild-tf-* .
    cp ../terraform-google-enterprise-genai/build/tf-wrapper.sh .
    chmod 755 ./tf-wrapper.sh

diff --git a/5-app-infra/modules/publish_artifacts/data.tf b/5-app-infra/modules/publish_artifacts/data.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2023 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,19 +17,3 @@
 data "google_project" "project" {
   project_id = var.project_id
 }
-
-data "google_projects" "kms" {
-  filter = "labels.application_name:org-kms labels.environment:production lifecycleState:ACTIVE"
-  # filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE"
-}
-
-data "google_kms_key_ring" "kms" {
-  name     = "sample-keyring"
-  location = var.region
-  project  = data.google_projects.kms.projects.0.project_id
-}
-
-data "google_kms_crypto_key" "key" {
-  name     = data.google_project.project.name
-  key_ring = data.google_kms_key_ring.kms.id
-}
diff --git a/5-app-infra/modules/publish_artifacts/main.tf b/5-app-infra/modules/publish_artifacts/main.tf
@@ -21,7 +21,7 @@ resource "google_project_service_identity" "artifact_registry_agent" {
 }
 
 resource "google_kms_crypto_key_iam_member" "artifact-kms-key-binding" {
-  crypto_key_id = data.google_kms_crypto_key.key.id
+  crypto_key_id = var.kms_crypto_key
   role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
   member        = "serviceAccount:${google_project_service_identity.artifact_registry_agent.email}"
 }
@@ -40,7 +40,7 @@ resource "google_artifact_registry_repository" "repo" {
   #NIST 800-53: SC-12 SC-13
   #CRI Profile: PR.DS-1.1 PR.DS-1.2 PR.DS-2.1 PR.DS-2.2 PR.DS-5.1
 
-  kms_key_name = data.google_kms_crypto_key.key.id
+  kms_key_name = var.kms_crypto_key
 
   #Cleanup policy
   #Control ID:  AR-CO-6.1
@@ -101,6 +101,7 @@ resource "google_cloudbuild_trigger" "docker_build" {
     repo_name   = var.name
   }
   build {
+    timeout = "1800s"
     step {
       id         = "unshallow"
       name       = "gcr.io/cloud-builders/git"
@@ -119,7 +120,6 @@ resource "google_cloudbuild_trigger" "docker_build" {
         <<-EOT
         changed_files=$(git diff $${COMMIT_SHA}^1 --name-only -r)
         changed_folders=$(echo "$changed_files" | awk -F/ '{print $2}' | sort | uniq )
-
         for folder in $changed_folders; do
             echo "Found docker folder: $folder"
             echo $folder >> /workspace/docker_build

diff --git a/5-app-infra/modules/publish_artifacts/variables.tf b/5-app-infra/modules/publish_artifacts/variables.tf
@@ -69,3 +69,8 @@ variable "environment" {
 variable "project_id" {
   description = "Project ID"
 }
+
+variable "kms_crypto_key" {
+  description = "KMS Key to be used"
+  type        = string
+}
diff --git a/5-app-infra/modules/service_catalog/data.tf b/5-app-infra/modules/service_catalog/data.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,23 +17,3 @@
 data "google_project" "project" {
   project_id = var.project_id
 }
-
-data "google_projects" "kms" {
-  filter = "labels.application_name:org-kms labels.environment:production lifecycleState:ACTIVE"
-  # filter = "labels.application_name:org-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE"
-}
-
-data "google_kms_key_ring" "kms" {
-  name     = "sample-keyring"
-  location = var.region
-  project  = data.google_projects.kms.projects.0.project_id
-}
-
-data "google_kms_crypto_key" "key" {
-  name     = data.google_project.project.name
-  key_ring = data.google_kms_key_ring.kms.id
-}
-
-data "google_projects" "log" {
-  filter = "labels.application_name:org-logging labels.environment:production lifecycleState:ACTIVE"
-}
diff --git a/5-app-infra/modules/service_catalog/main.tf b/5-app-infra/modules/service_catalog/main.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
 #   service = "storage.googleapis.com"
 # }
 # resource "google_kms_crypto_key_iam_member" "storage-kms-key-binding" {
-#   crypto_key_id = data.google_kms_crypto_key.key.id
+#   crypto_key_id = var.kms_crypto_key
 #   role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
 #   member        = "serviceAccount:${google_project_service_identity.storage_agent.email}"
 # }
@@ -41,13 +41,13 @@ resource "google_storage_bucket" "bucket" {
   uniform_bucket_level_access = true
 
   encryption {
-    default_kms_key_name = data.google_kms_crypto_key.key.id
+    default_kms_key_name = var.kms_crypto_key
   }
   versioning {
     enabled = true
   }
   logging {
-    log_bucket = join("-", [local.log_bucket_prefix, data.google_projects.log.projects.0.project_id])
+    log_bucket = var.log_bucket
   }
 
 }
@@ -142,4 +142,3 @@ resource "google_cloudbuild_trigger" "zip_files" {
     }
   }
 }
-
diff --git a/5-app-infra/modules/service_catalog/variables.tf b/5-app-infra/modules/service_catalog/variables.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -42,3 +42,13 @@ variable "machine_learning_project_number" {
   description = "Project Number for the Machine Learning (Vertex) Project"
   type        = string
 }
+
+variable "kms_crypto_key" {
+  description = "KMS Key to be used"
+  type        = string
+}
+
+variable "log_bucket" {
+  description = "Bucket to store logs from service catalog bucket"
+  type        = string
+}
diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf b/5-app-infra/projects/artifact-publish/business_unit_3/shared/publish_artifacts.tf
@@ -14,8 +14,16 @@
  * limitations under the License.
  */
 
+locals {
+  region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region]
+}
+
+data "google_project" "common_artifacts" {
+  project_id = local.common_artifacts_project_id
+}
+
 module "artifact_publish" {
-  source = "../../../../modules/publish_artifacts"
+  source = "../../modules/publish_artifacts"
 
   environment = local.environment
   description = "Publish Artifacts for ML Projects"
@@ -34,4 +42,6 @@ module "artifact_publish" {
       }
     ]
   }]
+
+  kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_artifacts.name}"
 }
diff --git a/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf b/5-app-infra/projects/artifact-publish/business_unit_3/shared/remote.tf
@@ -18,6 +18,7 @@ locals {
   common_artifacts_project_id = data.terraform_remote_state.projects_shared.outputs.common_artifacts_project_id
   service_catalog_project_id  = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id
   artifacts_repo_name         = data.terraform_remote_state.projects_shared.outputs.artifacts_repo_name
+  shared_keyrings             = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings
 }
 
 data "terraform_remote_state" "projects_shared" {

diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf b/5-app-infra/projects/service-catalog/business_unit_3/shared/remote.tf
@@ -18,7 +18,8 @@ locals {
   service_catalog_project_id      = data.terraform_remote_state.projects_shared.outputs.service_catalog_project_id
   service_catalog_repo_name       = data.terraform_remote_state.projects_shared.outputs.service_catalog_repo_name
   machine_learning_project_number = data.terraform_remote_state.machine_learning_development.outputs.machine_learning_project_number
-  tf_service_catalog_sa_email     = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["bu3-service-catalog"]
+  tf_service_catalog_sa_email     = data.terraform_remote_state.projects_shared.outputs.terraform_service_accounts["ml-service-catalog"]
+  shared_keyrings                 = data.terraform_remote_state.projects_shared.outputs.shared_level_keyrings
 }
 
 data "terraform_remote_state" "projects_shared" {

diff --git a/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf b/5-app-infra/projects/service-catalog/business_unit_3/shared/service_catalog.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,12 +14,23 @@
  * limitations under the License.
  */
 
+locals {
+  region_kms_keyring = [for i in local.shared_keyrings : i if split("/", i)[3] == var.instance_region]
+}
+
+data "google_project" "common_svc_catalog" {
+  project_id = local.service_catalog_project_id
+}
+
 module "service_catalog" {
-  source = "../../../../modules/service_catalog"
+  source = "../../modules/service_catalog"
 
   project_id                      = local.service_catalog_project_id
   region                          = var.instance_region
   name                            = local.service_catalog_repo_name
   machine_learning_project_number = local.machine_learning_project_number
   tf_service_catalog_sa_email     = local.tf_service_catalog_sa_email
+
+  log_bucket     = var.log_bucket
+  kms_crypto_key = "${one(local.region_kms_keyring)}/cryptoKeys/${data.google_project.common_svc_catalog.name}"
 }
diff --git a/Makefile b/Makefile
@@ -28,7 +28,7 @@ docker_test_lint:
 	docker run --rm -it \
 		-e ENABLE_PARALLEL=0 \
 		-e DISABLE_TFLINT=1 \
-		-e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform" \
+		-e EXCLUDE_LINT_DIRS="\./examples/machine-learning-pipeline|\./docs/assets/terraform|\./5-app-infra/projects/service-catalog/business_unit_3/shared|\./5-app-infra/projects/artifact-publish/business_unit_3/shared" \
 		-v $(CURDIR):/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/test_lint.sh