WIP

cloudfoundry · Aug 22, 2024 · 339cf67 · 339cf67
1 parent ef8ebd0
commit 339cf67
Show file tree

Hide file tree

Showing 9 changed files with 283 additions and 152 deletions.
diff --git a/ci/autoscaler/scripts/cleanup-autoscaler.sh b/ci/autoscaler/scripts/cleanup-autoscaler.sh
@@ -6,7 +6,6 @@ script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
 source "${script_dir}/common.sh"
 
 function main() {
-  step "cleaning up deployment ${DEPLOYMENT_NAME}"
   bosh_login
   cf_login
 

diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh
@@ -6,14 +6,42 @@ script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source "${script_dir}/common.sh"
 source "${script_dir}/vars.source.sh"
 
+pushd "${bbl_state_path}" > /dev/null
+  eval "$(bbl print-env)"
+popd > /dev/null
+
+function fetch_certs() {
+	pushd "${autoscaler_dir}/src/autoscaler/metricsforwarder" > /dev/null
+		mkdir -p build/assets/certs/policy_db build/assets/certs/storedprocedure_db build/assets/certs/syslog_client
+
+		echo "Pulling policy db certs..."
+		credhub get -n /bosh-autoscaler/${deployment_name}/postgres_server --key ca --quiet 		 > build/assets/certs/policy_db/ca.crt
+		credhub get -n /bosh-autoscaler/${deployment_name}/postgres_server --key certificate --quiet > build/assets/certs/policy_db/crt
+		credhub get -n /bosh-autoscaler/${deployment_name}/postgres_server --key private_key --quiet > build/assets/certs/policy_db/key
+
+		echo "Pulling storeprocedure db certs ..."
+		credhub get -n /bosh-autoscaler/${deployment_name}/postgres_server --key ca --quiet 		 > build/assets/certs/storedprocedure_db/ca.crt
+		credhub get -n /bosh-autoscaler/${deployment_name}/postgres_server --key certificate --quiet > build/assets/certs/storedprocedure_db/crt
+		credhub get -n /bosh-autoscaler/${deployment_name}/postgres_server --key private_key --quiet > build/assets/certs/storedprocedure_db/key
+
+		echo "Pulling syslog-client certs..."
+		credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key ca --quiet			> build/assets/certs/syslog_client/ca.crt
+		credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet > build/assets/certs/syslog_client/client.crt
+		credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet > build/assets/certs/syslog_client/client.key
+	popd > /dev/null
+}
+
 function deploy() {
   log "Deploying autoscaler apps for bosh deployment '${deployment_name}' "
   pushd "${autoscaler_dir}/src/autoscaler" > /dev/null
     make mta-deploy
   popd > /dev/null
 }
 
+
+
 bosh_login
 cf_login
 cf_target "${autoscaler_org}" "${autoscaler_space}"
+fetch_certs
 deploy
diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml
@@ -4,11 +4,6 @@
     host: ((metricsforwarder_host))
     mtls_host: ((metricsforwarder_host))
 
-# Set the same port for metricsforwarder and healthenpoint routes
-- type: replace
-  path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health/port
-  value: 6201
-
   ## add router tcp route for postgres
 - type: replace
   path: /instance_groups/name=postgres/jobs/-
@@ -53,3 +48,6 @@
 - type: replace
   path: /variables/name=postgres_client/options/alternative_names/-
   value: ((deployment_name))-postgres.tcp.((system_domain))
+
+- type: remove
+  path: /instance_groups/name=metricsforwarder
diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile
@@ -155,6 +155,8 @@ clean:
 mta-deploy: mta-build build-extension-file
 	$(MAKE) -f metricsforwarder/Makefile set-security-group
 	$(MAKE) -f metricsforwarder/Makefile stop-metricsforwarder-vm
+	$(MAKE) -f metricsforwarder/Makefile create-config-user-provided-service
+
 	@echo "Deploying with extension file: $(EXTENSION_FILE)"
 	@cf deploy mta_archives/*.mtar -f -e $(EXTENSION_FILE)
 
@@ -170,7 +172,6 @@ mta-logs:
 
 .PHONY: mta-build
 mta-build: mta-build-clean cf-build
-	$(MAKE) -f metricsforwarder/Makefile fetch-config
 	mbt build
 
 mta-build-clean:

diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile
@@ -1,46 +1,10 @@
-PR_NUMBER ?= $(shell gh pr view --json number --jq '.number')
-DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER)
-SYSTEM_DOMAIN ?=autoscaler.app-runtime-interfaces.ci.cloudfoundry.org
-METIRCSFORWARDER_VM := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance')
-POSTGRES_ADDRESS := $(DEPLOYMENT_NAME)-postgres.tcp.$(SYSTEM_DOMAIN)
-LOG_CACHE_IP := $(shell bosh -d cf vms --json | jq -r '.Tables | .[] | .Rows  | .[] | select(.instance|test("log-cache")) | .ips' )
 MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
-POSTGRES_EXTERNAL_PORT := $(or $(PR_NUMBER),5432)
-
-
-.PHONY: fetch-config
-fetch-config: start-metricsforwarder-vm
-	# how to define variables in deployment name
-	mkdir -p build/assets/certs/policy_db build/assets/certs/storedprocedure_db build/assets/certs/syslog_client
-
-	echo "POSTGRES ADDRESS: $(POSTGRES_ADDRESS)"
-	echo "LOG_CACHE IP: $(LOG_CACHE_IP)"
-
-	@echo "Pulling metricforwarder config from $(METIRCSFORWARDER_VM)..."
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml build/assets/metricsforwarder.yml
-
-	@echo "Pulling policy db certs from $(METIRCSFORWARDER_VM)..."
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt build/assets/certs/policy_db/.
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt    build/assets/certs/policy_db/.
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key    build/assets/certs/policy_db/.
-
-	@echo "Pulling storeprocedure db certs from $(METIRCSFORWARDER_VM)..."
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt build/assets/certs/storedprocedure_db/.
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt	 build/assets/certs/storedprocedure_db/.
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key    build/assets/certs/storedprocedure_db/.
-
-	@echo "Pulling syslog-client certs from $(METIRCSFORWARDER_VM)..."
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt		build/assets/certs/syslog_client/.
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt build/assets/certs/syslog_client/.
-	bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key build/assets/certs/syslog_client/.
-
-	@echo "Build metricsforwarder config yaml"
-	cp build/assets/metricsforwarder.yml build/metricsforwarder.yml
-
-	sed -i'' -e 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' build/metricsforwarder.yml
-	sed -i'' -e 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal:5432|$(POSTGRES_ADDRESS):$(POSTGRES_EXTERNAL_PORT)|g' build/metricsforwarder.yml
-
 
+.PHONY: create-config-user-provided-service
+create-config-user-provided-service:
+	$(MAKEFILE_DIR)/config.sh
+	cf cups config -p '$(shell $(MAKEFILE_DIR)/config.sh)' || echo "config service already exists"
+	cf uups config -p '$(shell $(MAKEFILE_DIR)/config.sh)'
 
 PHONY: set-security-group
 set-security-group:
@@ -49,13 +13,3 @@ set-security-group:
 
 	cf create-security-group metricsforwarder $(MAKEFILE_DIR)/security-group.json
 	cf bind-security-group metricsforwarder $(ORG)
-
-PHONY: start-metricsforwarder-vm
-start-metricsforwarder-vm:
-	bosh -d $(DEPLOYMENT_NAME) -n start $(METIRCSFORWARDER_VM)
-
-PHONY: stop-metricsforwarder-vm
-stop-metricsforwarder-vm:
-	bosh -d $(DEPLOYMENT_NAME) -n stop $(METIRCSFORWARDER_VM)
-
-
diff --git a/src/autoscaler/metricsforwarder/config.sh b/src/autoscaler/metricsforwarder/config.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -e
+
+export SYSTEM_DOMAIN="autoscaler.app-runtime-interfaces.ci.cloudfoundry.org"
+export POSTGRES_ADDRESS="${DEPLOYMENT_NAME}-postgres.tcp.${SYSTEM_DOMAIN}"
+export POLICY_DB_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/database_password --quiet)"
+export STOREPROCEDURE_DB_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/database_password --quiet)"
+export METRICSFORWARDER_HEALTH_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_metricsforwarder_health_password --quiet)"
+export POSTGRES_EXTERNAL_PORT="${PR_NUMBER:-5432}"
+
+if [ -z "${PR_NUMBER}" ]; then
+	echo "PR_NUMBER is not set"
+	exit 1
+fi
+
+if [ -z "${DEPLOYMENT_NAME}" ]; then
+	echo "DEPLOYMENT_NAME is not set"
+	exit 1
+fi
+
+if [ -z "${SYSTEM_DOMAIN}" ]; then
+	echo "SYSTEM_DOMAIN is not set"
+	exit 1
+fi
+
+if [ -z "${POSTGRES_ADDRESS}" ]; then
+	echo "POSTGRES_ADDRESS is not set"
+	exit 1
+fi
+
+echo '{ "metricsforwarder": {
+    "cache_cleanup_interval": "6h",
+    "cache_ttl": "900s",
+    "cred_helper_impl": "default",
+    "health": {
+      "password": "'"${METRICSFORWARDER_HEALTH_PASSWORD}"'",
+      "username": "metricsforwarder"
+    },
+    "logging": {
+      "level": "debug"
+    },
+    "syslog": {
+      "server_address": "log-cache.service.cf.internal",
+      "port": 6067,
+      "tls": {
+        "ca_file": "/home/vcap/app/assets/certs/syslog_client/ca.crt",
+        "cert_file": "/home/vcap/app/assets/certs/syslog_client/client.crt",
+        "key_file": "/home/vcap/app/assets/certs/syslog_client/client.key"
+      }
+    },
+    "db": {
+      "policy_db": {
+        "url": "postgres://postgres:'"${POLICY_DB_PASSWORD}@${POSTGRES_ADDRESS}:${POSTGRES_EXTERNAL_PORT}"'/autoscaler?application_name=metricsforwarder&sslmode=verify-full&sslrootcert=/home/vcap/app/assets/certs/policy_db/ca.crt&sslcert=/home/vcap/app/assets/certs/policy_db/crt&sslkey=/home/vcap/app/assets/certs/policy_db/key",
+        "max_open_connections": 100,
+        "max_idle_connections": 10,
+        "connection_max_lifetime": "60s"
+      },
+      "storedprocedure_db": {
+        "url": "postgres://postgres:'"${STOREPROCEDURE_DB_PASSWORD}@${POSTGRES_ADDRESS}:${POSTGRES_EXTERNAL_PORT}"'/autoscaler?application_name=metricsforwarder&sslmode=verify-full&sslrootcert=/home/vcap/app/assets/certs/storedprocedure_db/ca.crt&sslcert=/home/vcap/app/assets/certs/storedprocedure_db/crt&sslkey=/home/vcap/app/assets/certs/storedprocedure_db/key",
+        "max_open_connections": 20,
+        "max_idle_connections": 10,
+        "connection_max_lifetime": "60s"
+      }
+    },
+    "policy_poller_interval": "60s",
+    "rate_limit": {
+      "valid_duration": "1s",
+      "max_amount": 10
+    }
+  }
+}'
diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"time"
 
@@ -25,6 +26,7 @@ var ErrReadYaml = errors.New("failed to read config file")
 var ErrReadJson = errors.New("failed to read vcap_services json")
 var ErrReadEnvironment = errors.New("failed to read environment variables")
 var ErrReadVCAPEnvironment = errors.New("failed to read VCAP environment variables")
+var ErrMetricsforwarderConfigNotFound = errors.New("Configuration error: metricsforwarder config service not found")
 
 const (
 	DefaultMetronAddress        = "127.0.0.1:3458"
@@ -102,6 +104,10 @@ func decodeYamlFile(filepath string, c *Config) error {
 
 func readConfigFromVCAP(appEnv *cfenv.App, c *Config) error {
 	configVcapService, err := appEnv.Services.WithName("config")
+	if err != nil {
+		return fmt.Errorf("%w: %w", ErrMetricsforwarderConfigNotFound, err)
+	}
+
 	data := configVcapService.Credentials["metricsforwarder"]
 
 	rawJSON, err := json.Marshal(data)
@@ -152,14 +158,14 @@ func LoadConfig(filepath string) (*Config, error) {
 
 		conf.Server.Port = appEnv.Port
 
-		err = readDbFromVCAP(appEnv, &conf)
+		err = readConfigFromVCAP(appEnv, &conf)
 		if err != nil {
-			return &conf, err
+			return nil, err
 		}
 
-		err = readConfigFromVCAP(appEnv, &conf)
+		err = readDbFromVCAP(appEnv, &conf)
 		if err != nil {
-			return nil, err
+			return &conf, err
 		}
 	}
 
@@ -213,13 +219,10 @@ func (c *Config) Validate() error {
 	return nil
 }
 func readDbFromVCAP(appEnv *cfenv.App, c *Config) error {
-	if c.Db != nil {
-		return nil
-	}
-
 	dbServices, err := appEnv.Services.WithTag("relational")
 	if err != nil {
-		return fmt.Errorf("failed to get db service with relational tag")
+		fmt.Printf("failed to get db service with relational tag: %s\n", err.Error())
+		return nil
 	}
 
 	if len(dbServices) != 1 {
@@ -237,36 +240,47 @@ func readDbFromVCAP(appEnv *cfenv.App, c *Config) error {
 		c.Db = make(map[string]db.DatabaseConfig)
 	}
 
-	c.Db[db.PolicyDb] = db.DatabaseConfig{
-		URL: dbURI,
+	dbURL, err := url.Parse(dbURI)
+	if err != nil {
+		return err
 	}
 
-	//dbURL, err := url.Parse(dbURI)
-	//if err != nil {
-	//  return nil, err
-	//}
+	parameters, err := url.ParseQuery(dbURL.RawQuery)
+	if err != nil {
+		return err
+	}
 
-	//parameters, err := url.ParseQuery(dbURL.RawQuery)
-	//if err != nil {
-	//  return nil, err
-	//}
+	err = materializeConnectionParameter(dbService, &parameters, "client_cert", "sslcert")
+	if err != nil {
+		return err
+	}
 
-	//err = materializeConnectionParameter(dbService, parameters, "client_cert", "sslcert")
-	//if err != nil {
-	//  return nil, err
-	//}
+	err = materializeConnectionParameter(dbService, &parameters, "client_key", "sslkey")
+	if err != nil {
+		return err
+	}
 
-	//err = materializeConnectionParameter(dbService, parameters, "client_key", "sslkey")
-	//if err != nil {
-	//  return nil, err
-	//}
+	err = materializeConnectionParameter(dbService, &parameters, "server_ca", "sslrootcert")
+	if err != nil {
+		return err
+	}
 
-	//err = materializeConnectionParameter(dbService, parameters, "server_ca", "sslrootcert")
-	//if err != nil {
-	//  return nil, err
-	//}
+	dbURL.RawQuery = parameters.Encode()
 
-	//dbURL.RawQuery = parameters.Encode()
+	c.Db[db.PolicyDb] = db.DatabaseConfig{
+		URL: dbURL.String(),
+	}
+	return nil
+}
 
+func materializeConnectionParameter(dbService cfenv.Service, parameters *url.Values, bindingProperty string, connectionParameter string) error {
+	if propertyValue, hasProperty := dbService.CredentialString(bindingProperty); hasProperty {
+		propertyFile := "/tmp/" + bindingProperty + "." + connectionParameter
+		err := os.WriteFile(propertyFile, []byte(propertyValue), 0600)
+		if err != nil {
+			return err
+		}
+		parameters.Set(connectionParameter, propertyFile)
+	}
 	return nil
 }