Feature/add gke tests

.kitchen.yml

@@ -1,4 +1,4 @@
-# Copyright 2018 Google LLC
+# Copyright 2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -66,6 +66,21 @@ suites:
           backend: local
           controls:
             - forseti-org-iam
+  - name: on_gke_end_to_end
+    lifecycle:
+      pre_verify:
+        - chmod go-rwx test/fixtures/on_gke_end_to_end/sshkey
+    driver:
+      name: terraform
+      root_module_directory: test/fixtures/on_gke_end_to_end/
+      command_timeout: 1800
+    verifier:
+      systems:
+        - name: gke
+          backend: local
+          controls:
+            - gcloud
+            - kubectl
   - name: shared_vpc
     lifecycle:
       pre_verify:

examples/on_gke_end_to_end/README.md

@@ -56,6 +56,7 @@ This script will also activate necessary APIs required for Terraform to deploy F
 |------|-------------|:----:|:-----:|:-----:|
 | auto\_create\_subnetworks | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | bool | `"false"` | no |
 | bucket\_cai\_location | GCS CAI storage bucket location | string | `"us-central1"` | no |
+| client\_instance\_metadata | Metadata key/value pairs to make available from within the client instance. | map(string) | `<map>` | no |
 | config\_validator\_enabled | Config Validator scanner enabled. | bool | `"false"` | no |
 | cscc\_source\_id | Source ID for CSCC Beta API | string | `""` | no |
 | cscc\_violations\_enabled | Notify for CSCC violations | bool | `"false"` | no |
@@ -74,6 +75,7 @@ This script will also activate necessary APIs required for Terraform to deploy F
 | gke\_service\_ip\_range | The IP range of the Kubernetes services. | string | `"10.3.0.0/20"` | no |
 | gke\_service\_ip\_range\_name | The name of the IP range of the Kubernetes services. | string | `"gke-service-ip-range"` | no |
 | gsuite\_admin\_email | G-Suite administrator email address to manage your Forseti installation | string | n/a | yes |
+| helm\_chart\_version | The version of the Helm chart to use | string | `"2.2.0-rc1"` | no |
 | helm\_repository\_url | The Helm repository containing the 'forseti-security' Helm charts | string | `"https://forseti-security-charts.storage.googleapis.com/release/"` | no |
 | k8s\_forseti\_namespace | The Kubernetes namespace in which to deploy Forseti. | string | `"forseti"` | no |
 | k8s\_forseti\_orchestrator\_image\_tag | The tag for the container image for the Forseti orchestrator | string | `"v2.24.0"` | no |
@@ -97,16 +99,21 @@ This script will also activate necessary APIs required for Terraform to deploy F
 
 | Name | Description |
 |------|-------------|
+| ca\_certificate | The cluster CA certificate |
+| client\_token | The bearer token for auth |
 | config-validator-git-public-key-openssh | The public OpenSSH key generated to allow the Forseti Server to clone the policy library repository. |
 | forseti-client-service-account | Forseti Client service account |
 | forseti-client-storage-bucket | Forseti Client storage bucket |
 | forseti-client-vm-ip | Forseti Client VM private IP address |
 | forseti-cloudsql-connection-name | Forseti CloudSQL Connection String |
 | forseti-server-service-account | Forseti Server service account |
 | forseti-server-storage-bucket | Forseti Server storage bucket |
+| gke\_cluster\_location | Cluster location |
+| gke\_cluster\_name | Cluster name |
 | kubernetes-forseti-namespace | The Kubernetes namespace in which Forseti is deployed |
 | kubernetes-forseti-server-ingress | The loadbalancer ingress address of the forseti-server service in GKE |
 | kubernetes-forseti-tiller-sa-name | The name of the service account deploying Forseti |
+| kubernetes\_endpoint | The cluster endpoint |
 | suffix | The random suffix appended to Forseti resources |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/on_gke_end_to_end/main.tf

@@ -168,8 +168,11 @@ module "forseti" {
   k8s_forseti_server_image_tag       = var.k8s_forseti_server_image_tag
   k8s_forseti_orchestrator_image_tag = var.k8s_forseti_orchestrator_image_tag
   helm_repository_url                = var.helm_repository_url
+  helm_chart_version                 = var.helm_chart_version
   policy_library_repository_url      = var.policy_library_repository_url
   policy_library_repository_branch   = var.policy_library_repository_branch
   policy_library_sync_enabled        = var.policy_library_sync_enabled
   server_log_level                   = var.server_log_level
+  client_instance_metadata           = var.client_instance_metadata
+  workload_identity_namespace        = module.gke.identity_namespace
 }

examples/on_gke_end_to_end/outputs.tf

@@ -14,6 +14,22 @@
  * limitations under the License.
  */
 
+output "client_token" {
+  description = "The bearer token for auth"
+  sensitive   = true
+  value       = base64encode(data.google_client_config.default.access_token)
+}
+
+output "ca_certificate" {
+  description = "The cluster CA certificate"
+  value       = module.gke.ca_certificate
+}
+
+output "config-validator-git-public-key-openssh" {
+  description = "The public OpenSSH key generated to allow the Forseti Server to clone the policy library repository."
+  value       = module.forseti.config-validator-git-public-key-openssh
+}
+
 output "forseti-client-service-account" {
   description = "Forseti Client service account"
   value       = module.forseti.forseti-client-service-account
@@ -24,47 +40,58 @@ output "forseti-client-storage-bucket" {
   value       = module.forseti.forseti-client-storage-bucket
 }
 
+output "forseti-client-vm-ip" {
+  description = "Forseti Client VM private IP address"
+  value       = module.forseti.forseti-client-vm-ip
+}
+
 output "forseti-cloudsql-connection-name" {
   description = "Forseti CloudSQL Connection String"
   value       = module.forseti.forseti-cloudsql-connection-name
 }
 
-output "forseti-client-vm-ip" {
-  description = "Forseti Client VM private IP address"
-  value       = module.forseti.forseti-client-vm-ip
+output "forseti-server-service-account" {
+  description = "Forseti Server service account"
+  value       = module.forseti.forseti-server-service-account
 }
 
 output "forseti-server-storage-bucket" {
   description = "Forseti Server storage bucket"
   value       = module.forseti.forseti-server-storage-bucket
 }
 
-output "forseti-server-service-account" {
-  description = "Forseti Server service account"
-  value       = module.forseti.forseti-server-service-account
+output "gke_cluster_location" {
+  description = "Cluster location"
+  value       = module.gke.location
+}
+
+output "gke_cluster_name" {
+  description = "Cluster name"
+  value       = var.gke_cluster_name
+}
+
+output "kubernetes_endpoint" {
+  description = "The cluster endpoint"
+  sensitive   = true
+  value       = module.gke.endpoint
 }
 
 output "kubernetes-forseti-namespace" {
   description = "The Kubernetes namespace in which Forseti is deployed"
   value       = module.forseti.kubernetes-forseti-namespace
 }
 
-output "kubernetes-forseti-tiller-sa-name" {
-  description = "The name of the service account deploying Forseti"
-  value       = module.forseti.kubernetes-forseti-tiller-sa-name
-}
-
 output "kubernetes-forseti-server-ingress" {
   description = "The loadbalancer ingress address of the forseti-server service in GKE"
   value       = module.forseti.kubernetes-forseti-server-ingress
 }
 
+output "kubernetes-forseti-tiller-sa-name" {
+  description = "The name of the service account deploying Forseti"
+  value       = module.forseti.kubernetes-forseti-tiller-sa-name
+}
+
 output "suffix" {
   description = "The random suffix appended to Forseti resources"
   value       = module.forseti.suffix
 }
-
-output "config-validator-git-public-key-openssh" {
-  description = "The public OpenSSH key generated to allow the Forseti Server to clone the policy library repository."
-  value       = module.forseti.config-validator-git-public-key-openssh
-}

examples/on_gke_end_to_end/variables.tf

@@ -114,6 +114,11 @@ variable "helm_repository_url" {
   default     = "https://forseti-security-charts.storage.googleapis.com/release/"
 }
 
+variable "helm_chart_version" {
+  description = "The version of the Helm chart to use"
+  default     = "2.2.0-rc1"
+}
+
 variable "k8s_forseti_namespace" {
   description = "The Kubernetes namespace in which to deploy Forseti."
   default     = "forseti"
@@ -196,6 +201,12 @@ variable "server_log_level" {
   default     = "info"
 }
 
+variable "client_instance_metadata" {
+  description = "Metadata key/value pairs to make available from within the client instance."
+  type        = map(string)
+  default     = {}
+}
+
 #----------------#
 # Forseti bucket #
 #----------------#

modules/on_gke/README.md

@@ -98,7 +98,7 @@ This sub-module deploys Forseti on GKE.  In short, this deploys a server contain
 | groups\_settings\_violations\_should\_notify | Notify for groups settings violations | bool | `"true"` | no |
 | groups\_violations\_should\_notify | Notify for Groups violations | bool | `"true"` | no |
 | gsuite\_admin\_email | G-Suite administrator email address to manage your Forseti installation | string | `""` | no |
-| helm\_chart\_version | The version of the Helm chart to use | string | `"2.1.0"` | no |
+| helm\_chart\_version | The version of the Helm chart to use | string | `"2.2.0-rc1"` | no |
 | helm\_repository\_url | The Helm repository containing the 'forseti-security' Helm charts | string | `"https://forseti-security-charts.storage.googleapis.com/release/"` | no |
 | iam\_disable\_polling | Whether to disable polling for IAM API | bool | `"false"` | no |
 | iam\_max\_calls | Maximum calls that can be made to IAM API | string | `"90"` | no |
@@ -177,6 +177,7 @@ This sub-module deploys Forseti on GKE.  In short, this deploys a server contain
 | subnetwork | The VPC subnetwork where the Forseti client and server will be created | string | `"default"` | no |
 | verify\_policy\_library | Verify the Policy Library is setup correctly for the Config Validator scanner | bool | `"false"` | no |
 | violations\_slack\_webhook | Slack webhook for any violation. Will apply to all scanner violation notifiers. | string | `""` | no |
+| workload\_identity\_namespace | Workload Identity namespace | string | `"null"` | no |
 
 ## Outputs
 

modules/on_gke/main.tf

@@ -27,7 +27,7 @@ resource "null_resource" "org_id_and_folder_id_are_both_empty" {
   count = length(var.composite_root_resources) == 0 && var.org_id == "" && var.folder_id == "" ? 1 : 0
 
   provisioner "local-exec" {
-    command     = "echo 'composite_root_resources=${var.composite_root_resources} org_id=${var.org_id} folder_id=${var.org_id}' >&2; false"
+    command     = "echo 'composite_root_resources=${var.composite_root_resources} org_id=${var.org_id} folder_id=${var.folder_id}' >&2; false"
     interpreter = ["bash", "-c"]
   }
 }
@@ -69,7 +69,7 @@ locals {
     "storage-api.googleapis.com",
     "groupssettings.googleapis.com",
   ]
-  workload_identity                = "${var.project_id}.svc.id.goog"
+  workload_identity_namespace      = var.workload_identity_namespace == null ? "${var.project_id}.svc.id.goog" : var.workload_identity_namespace
   workload_identity_server_suffix  = "[${local.kubernetes_namespace}/forseti-server]"
   workload_identity_client_suffix  = "[${local.kubernetes_namespace}/forseti-orchestrator]"
   workload_config_validator_suffix = "[${local.kubernetes_namespace}/config-validator]"
@@ -121,20 +121,24 @@ data "tls_public_key" "git_sync_public_ssh_key" {
 //*****************************************
 //  Obtain Forseti Server Configuration
 //*****************************************
-data "google_storage_object_signed_url" "file_url" {
-  bucket      = module.server_gcs.forseti-server-storage-bucket
-  path        = "configs/forseti_conf_server.yaml"
-  content_md5 = module.server_config.forseti-server-config-md5
+data "google_storage_bucket_object" "server_config_contents" {
+  bucket = module.server_gcs.forseti-server-storage-bucket
+  name   = "configs/forseti_conf_server.yaml"
+
+  depends_on = [
+    module.server_config.forseti-server-config-md5
+  ]
 }
 
+data "google_client_config" "current" {}
+
 data "http" "server_config_contents" {
-  url = data.google_storage_object_signed_url.file_url.signed_url
+  url = format("%s?alt=media", data.google_storage_bucket_object.server_config_contents.self_link)
 
+  # Optional request headers
   request_headers = {
-    "Content-MD5" = module.server_config.forseti-server-config-md5
+    "Authorization" = "Bearer ${data.google_client_config.current.access_token}"
   }
-
-  depends_on = ["data.google_storage_object_signed_url.file_url"]
 }
 
 //*****************************************
@@ -154,7 +158,7 @@ resource "google_service_account_iam_binding" "forseti_server_workload_identity"
   role               = "roles/iam.workloadIdentityUser"
 
   members = [
-    "serviceAccount:${local.workload_identity}${local.workload_identity_server_suffix}"
+    "serviceAccount:${local.workload_identity_namespace}${local.workload_identity_server_suffix}"
   ]
 }
 
@@ -163,8 +167,8 @@ resource "google_service_account_iam_binding" "forseti_client_workload_identity"
   role               = "roles/iam.workloadIdentityUser"
 
   members = [
-    "serviceAccount:${local.workload_identity}${local.workload_identity_client_suffix}",
-    "serviceAccount:${local.workload_identity}${local.workload_config_validator_suffix}"
+    "serviceAccount:${local.workload_identity_namespace}${local.workload_identity_client_suffix}",
+    "serviceAccount:${local.workload_identity_namespace}${local.workload_config_validator_suffix}"
   ]
 }
 
@@ -225,10 +229,12 @@ resource "helm_release" "forseti-security" {
   version       = var.helm_chart_version
   chart         = "forseti-security"
   recreate_pods = var.recreate_pods
-  depends_on = ["kubernetes_role_binding.tiller",
+  depends_on = [
+    "kubernetes_role_binding.tiller",
     "kubernetes_namespace.forseti",
     "google_service_account_iam_binding.forseti_server_workload_identity",
-  "google_service_account_iam_binding.forseti_client_workload_identity"]
+    "google_service_account_iam_binding.forseti_client_workload_identity"
+  ]
 
   set {
     name  = "database.username"

modules/on_gke/variables.tf

@@ -60,12 +60,16 @@ variable "sendgrid_api_key" {
 #------------#
 # GKE config #
 #------------#
-
 variable "gke_node_pool_name" {
   description = "The name of the GKE node-pool where Forseti is being deployed"
   default     = "default-pool"
 }
 
+variable "workload_identity_namespace" {
+  description = "Workload Identity namespace"
+  default     = null
+}
+
 #----------------#
 # Forseti config #
 #----------------#
@@ -710,7 +714,6 @@ variable "inventory_email_summary_enabled" {
 #---------------------------------------#
 # Groups Settings scanner configuration #
 #---------------------------------------#
-
 variable "groups_settings_max_calls" {
   description = "Maximum calls that can be made to the G Suite Groups API"
   default     = "5"
@@ -893,7 +896,6 @@ variable "cloudsql_password" {
 #-------------#
 # Helm config #
 #-------------#
-
 variable "git_sync_image" {
   description = "The container image used by the config-validator git-sync side-car"
   default     = "gcr.io/google-containers/git-sync"
@@ -911,7 +913,7 @@ variable "git_sync_wait" {
 
 variable "helm_chart_version" {
   description = "The version of the Helm chart to use"
-  default     = "2.1.0"
+  default     = "2.2.0-rc1"
 }
 
 variable "helm_repository_url" {

test/fixtures/on_gke_end_to_end/main.tf

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "tls_private_key" "main" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "local_file" "gce-keypair-pk" {
+  content  = tls_private_key.main.private_key_pem
+  filename = "${path.module}/sshkey"
+}
+
+#-------------------------#
+# Forseti
+#-------------------------#
+module "forseti" {
+  source = "../../../examples/on_gke_end_to_end"
+
+  # Forseti
+  config_validator_enabled = var.config_validator_enabled
+  domain                   = var.domain
+  gsuite_admin_email       = var.gsuite_admin_email
+
+  # Forseti Client
+  client_instance_metadata = {
+    sshKeys = "ubuntu:${tls_private_key.main.public_key_openssh}"
+  }
+
+  # GCP
+  org_id     = var.org_id
+  project_id = var.gke_project_id
+
+  # GKE
+  k8s_forseti_orchestrator_image_tag = var.k8s_forseti_orchestrator_image_tag
+  k8s_forseti_server_image_tag       = var.k8s_forseti_server_image_tag
+  network_description                = var.network_description
+}