Deploy MetricForwarder as a CF app

ci/autoscaler/scripts/deploy-autoscaler.sh

@@ -21,6 +21,7 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\
   ${autoscaler_dir}/operations/remove-metricsgateway.yml\
   ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\
   ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\
+  ${autoscaler_dir}/operations/remove-postgres-tls.yml\
   ${autoscaler_dir}/operations/enable-scheduler-logging.yml"}
 
 

operations/remove-postgres-tls.yml

@@ -0,0 +1,75 @@
+- type: remove
+  path: /instance_groups/name=postgres/jobs/name=postgres/properties/databases/tls
+
+- type: replace
+  path: /instance_groups/name=postgres/jobs/name=postgres/properties/databases/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/policy_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/storedprocedure_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/policy_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scalingengine_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scheduler_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/policy_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/binding_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=scheduler/properties/autoscaler/policy_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=scheduler/properties/autoscaler/scheduler_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/appmetrics_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/lock_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/policy_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/scalingengine_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/policy_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db/sslmode
+  value: disable
+
+- type: replace
+  path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/lock_db/sslmode
+  value: disable
+

src/autoscaler/Makefile

@@ -83,6 +83,10 @@ go-mod-vendor: ${go-vendoring-folder} ${go-vendored-files}
 ${go-vendoring-folder} ${go-vendored-files} &: ${app-fakes-dir} ${app-fakes-files}
 	go mod vendor
 
+build-cf-%:
+	@echo "# building for cf $*"
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $*/$* $*/cmd/$*/main.go
+
 # CGO_ENABLED := 1 is required to enforce dynamic linking which is a requirement of dynatrace.
 build-%: ${openapi-generated-clients-and-servers-dir} ${openapi-generated-clients-and-servers-files}
 	@echo "# building $*"

src/autoscaler/metricsforwarder/.gitignore

@@ -0,0 +1,4 @@
+certs
+metricsforwarder
+working.yml
+metricsforwarder.yml

src/autoscaler/metricsforwarder/Makefile

@@ -0,0 +1,55 @@
+PR_NUMBER := $(shell gh pr view --json number --jq '.number')
+DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER)
+VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq ".Tables | .[0] | .Rows | .[] | .instance" -r | grep metricsforwarder)
+
+fetch-config: cf-login
+	# how to define variables in deployment name
+	mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client
+
+	$(eval POSTGRES_IP := $(shell bosh -d ${DEPLOYMENT_NAME} vms --json | jq -r '.Tables | .[] | .Rows  | .[] | select(.instance|test("postgres")) | .ips' ))
+	echo "Postgres IP: $(POSTGRES_IP)"
+
+	$(eval LOG_CACHE_IP := $(shell bosh -d cf vms --json | jq -r '.Tables | .[] | .Rows  | .[] | select(.instance|test("log-cache")) | .ips' ))
+	echo "LOG_CACHE IP: $(LOG_CACHE_IP)"
+
+
+	@echo "Pulling metricforwarder config from $(VM_NAME)..."
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml assets/metricsforwarder.yml
+
+	@echo "Pulling policy db certs from $(VM_NAME)..."
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt assets/certs/policy_db/.
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt    assets/certs/policy_db/.
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key    assets/certs/policy_db/.
+
+	@echo "Pulling storeprocedure db certs from $(VM_NAME)..."
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt assets/certs/storedprocedure_db/.
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt	 assets/certs/storedprocedure_db/.
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key    assets/certs/storedprocedure_db/.
+
+	@echo "Pulling syslog-client certs from $(VM_NAME)..."
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt		assets/certs/syslog_client/.
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt assets/certs/syslog_client/.
+	bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key assets/certs/syslog_client/.
+
+	@echo "Build metricsforwarder config yaml"
+	cp assets/metricsforwarder.yml metricsforwarder.yml
+
+	sed -i ''  's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml
+	sed -i ''  's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml
+	sed -i ''  's|log-cache.service.cf.internal|$(LOG_CACHE_IP)|g' metricsforwarder.yml
+
+set-security-group:
+	$(eval ORG := $(shell cf target |grep "org\:" |cut -d':' -f2 | xargs))
+	$(eval SPACE := $(shell cf target |grep "space\:" |cut -d':' -f2 | xargs))
+
+	cf create-security-group metricsforwarder security-group.json
+	cf bind-security-group metricsforwarder $(ORG)
+
+build-cf:
+	@cd ../; make build-cf-metricsforwarder
+
+
+
+cf-login:
+	@cd ../../../; make cf-login
+

src/autoscaler/metricsforwarder/security-group.json

@@ -0,0 +1,14 @@
+  [
+     {
+       "protocol": "tcp",
+       "destination": "10.0.1.0/24",
+       "ports": "5432",
+       "description": "Allow postgress traffic from"
+     },
+     {
+       "protocol": "tcp",
+       "destination": "10.0.1.0/24",
+       "ports": "6067",
+       "description": "Allow syslog traffic from"
+     }
+   ]