diff --git a/Makefile b/Makefile index 32c9f1e019..7d0ef2115b 100644 --- a/Makefile +++ b/Makefile @@ -453,5 +453,7 @@ go-get-u: $(addsuffix .go-get-u,$(go_modules)) go get -u ./... -mta-deploy: - @make --directory='./src/autoscaler' mta-deploy +deploy-apps: + echo " - deploying apps" + DEBUG="${DEBUG}" ${CI_DIR}/autoscaler/scripts/deploy-apps.sh + diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index 1c69188f10..2603a4d55f 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -6,6 +6,10 @@ script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/common.sh" source "${script_dir}/vars.source.sh" +pushd "${bbl_state_path}" > /dev/null + eval "$(bbl print-env)" +popd > /dev/null + function deploy() { log "Deploying autoscaler apps for bosh deployment '${deployment_name}' " pushd "${autoscaler_dir}/src/autoscaler" > /dev/null diff --git a/ci/infrastructure/scripts/deploy-multiapps-controller.sh b/ci/infrastructure/scripts/deploy-multiapps-controller.sh index 3483a44c8a..56d471ca9e 100755 --- a/ci/infrastructure/scripts/deploy-multiapps-controller.sh +++ b/ci/infrastructure/scripts/deploy-multiapps-controller.sh @@ -23,7 +23,6 @@ function deploy_multiapps_controller() { mv multiapps-controller-web-war/*.war . pushd multiapps-controller-web-manifest cf push -f ./*.yml "${app_name}" - popd } diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml index c56d25d284..20b3829767 100644 --- a/operations/use-cf-services.yml +++ b/operations/use-cf-services.yml @@ -4,11 +4,6 @@ host: ((metricsforwarder_host)) mtls_host: ((metricsforwarder_host)) -# Set the same port for metricsforwarder and healthenpoint routes -- type: replace - path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health/port - value: 6201 - ## add router tcp route for postgres - type: replace path: /instance_groups/name=postgres/jobs/- @@ -53,3 +48,6 @@ - type: replace path: /variables/name=postgres_client/options/alternative_names/- value: ((deployment_name))-postgres.tcp.((system_domain)) + +- type: remove + path: /instance_groups/name=metricsforwarder diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index b29f66ddab..4e01642ea2 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -12,7 +12,7 @@ PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' DB_HOST ?= localhost DBURL ?= "postgres://postgres:postgres@${DB_HOST}/autoscaler?sslmode=disable" -METRICSFORWARDER_APPNAME ?= "metricsforwarder" +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) EXTENSION_FILE := $(shell mktemp) export GOWORK=off @@ -154,14 +154,12 @@ clean: .PHONY: mta-deploy mta-deploy: mta-build build-extension-file $(MAKE) -f metricsforwarder/Makefile set-security-group - $(MAKE) -f metricsforwarder/Makefile stop-metricsforwarder-vm @echo "Deploying with extension file: $(EXTENSION_FILE)" - @cf deploy mta_archives/*.mtar -f -e $(EXTENSION_FILE) + @cf deploy mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE) build-extension-file: - cp example.mtaext $(EXTENSION_FILE); - sed -i "s/APP_NAME/$(METRICSFORWARDER_APPNAME)/g" $(EXTENSION_FILE); - echo "EXTENSION_FILE: $(EXTENSION_FILE)" + echo "extension file at: $(EXTENSION_FILE)" + $(MAKEFILE_DIR)/build-extension-file.sh $(EXTENSION_FILE); mta-logs: rm -rf mta-* @@ -170,7 +168,6 @@ mta-logs: .PHONY: mta-build mta-build: mta-build-clean cf-build - $(MAKE) -f metricsforwarder/Makefile fetch-config mbt build mta-build-clean: diff --git a/src/autoscaler/build-extension-file.sh b/src/autoscaler/build-extension-file.sh new file mode 100755 index 0000000000..cebda974c2 --- /dev/null +++ b/src/autoscaler/build-extension-file.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2155,SC2034,SC2086 + +set -e + +if [ -z "$1" ]; then + echo "extension file path not provided" + exit 1 +else + extension_file_path=$1 +fi + +if [ -z "${DEPLOYMENT_NAME}" ]; then + echo "DEPLOYMENT_NAME is not set" + exit 1 +fi + +if [ -z "${PR_NUMBER}" ]; then + echo "PR_NUMBER is not set" + exit 1 +fi + +export SYSTEM_DOMAIN="autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" +export POSTGRES_ADDRESS="${DEPLOYMENT_NAME}-postgres.tcp.${SYSTEM_DOMAIN}" +export POSTGRES_EXTERNAL_PORT="${PR_NUMBER:-5432}" + +export METRICSFORWARDER_HEALTH_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_metricsforwarder_health_password --quiet)" +export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}" + +export POLICY_DB_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/database_password --quiet)" +export POLICY_DB_SERVER_CA="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key ca --quiet )" +export POLICY_DB_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key certificate --quiet)" +export POLICY_DB_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key private_key --quiet)" + +export SYSLOG_CLIENT_CA="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key ca --quiet)" +export SYSLOG_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)" +export SYSLOG_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)" + +cat < "${extension_file_path}" +ID: development +extends: com.github.cloudfoundry.app-autoscaler-release +version: 1.0.0 +_schema-version: 3.3.0 + +modules: + - name: metricsforwarder + parameters: + routes: + - route: ${METRICSFORWARDER_APPNAME}.\${default-domain} + +resources: +- name: config + parameters: + config: + metricsforwarder: + health: + password: "${METRICSFORWARDER_HEALTH_PASSWORD}" +- name: policydb + parameters: + config: + uri: "postgres://postgres:${POLICY_DB_PASSWORD}@${POSTGRES_ADDRESS}:${POSTGRES_EXTERNAL_PORT}/autoscaler?application_name=metricsforwarder&sslmode=verify-full" + client_cert: "${POLICY_DB_CLIENT_CERT//$'\n'/\\n}" + client_key: "${POLICY_DB_CLIENT_KEY//$'\n'/\\n}" + server_ca: "${POLICY_DB_SERVER_CA//$'\n'/\\n}" +- name: syslog-client + parameters: + config: + client_cert: "${SYSLOG_CLIENT_CERT//$'\n'/\\n}" + client_key: "${SYSLOG_CLIENT_KEY//$'\n'/\\n}" + server_ca: "${SYSLOG_CLIENT_CA//$'\n'/\\n}" +EOF diff --git a/src/autoscaler/configutil/cf.go b/src/autoscaler/configutil/cf.go new file mode 100644 index 0000000000..df49fadbc1 --- /dev/null +++ b/src/autoscaler/configutil/cf.go @@ -0,0 +1,151 @@ +package configutil + +import ( + "errors" + "fmt" + "net/url" + "os" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "github.com/cloudfoundry-community/go-cfenv" +) + +var ErrReadEnvironment = errors.New("failed to read environment variables") +var ErrDbServiceNotFound = errors.New("failed to get service by name") + +type VCAPConfigurationReader interface { + MaterializeDBFromService(dbName string) (string, error) + MaterializeTLSConfigFromService(serviceName string) (models.TLSCerts, error) +} + +type VCAPConfiguration struct { + VCAPConfigurationReader + appEnv *cfenv.App +} + +func NewVCAPConfigurationReader() (*VCAPConfiguration, error) { + vcapConfiguration := &VCAPConfiguration{} + appEnv, err := cfenv.Current() + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrReadEnvironment, err) + } + + vcapConfiguration.appEnv = appEnv + return vcapConfiguration, nil +} + +func (vc *VCAPConfiguration) MaterializeTLSConfigFromService(serviceName string) (models.TLSCerts, error) { + tlsCerts := models.TLSCerts{} + services, err := vc.appEnv.Services.WithTag(serviceName) + if err != nil { + return tlsCerts, fmt.Errorf("%w: %w", ErrDbServiceNotFound, err) + } + + service := services[0] + + if clientCertContent, ok := service.CredentialString("client_cert"); ok { + fileName := fmt.Sprintf("%s.%s", "client_cert", "sslcert") + createdClientCert, err := materializeServiceProperty(serviceName, fileName, clientCertContent) + if err != nil { + return models.TLSCerts{}, err + } + tlsCerts.CertFile = createdClientCert + } else { + return models.TLSCerts{}, fmt.Errorf(fmt.Sprintf("failed to get %s from db service", "client_cert")) + } + + if clientKeyContent, ok := service.CredentialString("client_key"); ok { + fileName := fmt.Sprintf("%s.%s", "client_key", "sslkey") + createdClientKey, err := materializeServiceProperty(serviceName, fileName, clientKeyContent) + if err != nil { + return models.TLSCerts{}, err + } + tlsCerts.KeyFile = createdClientKey + } else { + return models.TLSCerts{}, fmt.Errorf(fmt.Sprintf("failed to get %s from db service", "client_key")) + } + + if serverCAContent, ok := service.CredentialString("server_ca"); ok { + fileName := fmt.Sprintf("%s.%s", "server_ca", "sslrootcert") + createServerCA, err := materializeServiceProperty(serviceName, fileName, serverCAContent) + if err != nil { + return models.TLSCerts{}, err + } + tlsCerts.CACertFile = createServerCA + } else { + return models.TLSCerts{}, fmt.Errorf(fmt.Sprintf("failed to get %s from db service", "server_ca")) + } + + return tlsCerts, nil +} + +func (vc *VCAPConfiguration) MaterializeDBFromService(dbName string) (string, error) { + var dbURL *url.URL + var err error + + service, err := vc.appEnv.Services.WithTag(dbName) + if err != nil { + return "", fmt.Errorf("%w: %w", ErrDbServiceNotFound, err) + } + + dbService := service[0] + + dbURI, ok := dbService.CredentialString("uri") + if !ok { + return "", fmt.Errorf("failed to get uri from db service") + } + + dbURL, err = url.Parse(dbURI) + if err != nil { + return "", err + } + + parameters, err := url.ParseQuery(dbURL.RawQuery) + if err != nil { + return "", err + } + + err = materializeConnectionParameter(dbName, dbService, ¶meters, "client_cert", "sslcert") + if err != nil { + return "", err + } + + err = materializeConnectionParameter(dbName, dbService, ¶meters, "client_key", "sslkey") + if err != nil { + return "", err + } + + err = materializeConnectionParameter(dbName, dbService, ¶meters, "server_ca", "sslrootcert") + if err != nil { + return "", err + } + + dbURL.RawQuery = parameters.Encode() + + return dbURL.String(), nil +} + +func materializeConnectionParameter(dbName string, dbService cfenv.Service, parameters *url.Values, bindingProperty string, connectionParameter string) error { + if content, hasProperty := dbService.CredentialString(bindingProperty); hasProperty { + fileName := fmt.Sprintf("%s.%s", bindingProperty, connectionParameter) + createdFile, err := materializeServiceProperty(dbName, fileName, content) + if err != nil { + return err + } + parameters.Set(connectionParameter, createdFile) + } + return nil +} + +func materializeServiceProperty(serviceName, fileName, content string) (createdFile string, err error) { + err = os.MkdirAll(fmt.Sprintf("/tmp/%s", serviceName), 0700) + if err != nil { + return "", err + } + createdFile = fmt.Sprintf("/tmp/%s/%s", serviceName, fileName) + err = os.WriteFile(createdFile, []byte(content), 0600) + if err != nil { + return "", err + } + return +} diff --git a/src/autoscaler/configutil/configutil_suite_test.go b/src/autoscaler/configutil/configutil_suite_test.go new file mode 100644 index 0000000000..839968d1f9 --- /dev/null +++ b/src/autoscaler/configutil/configutil_suite_test.go @@ -0,0 +1,13 @@ +package configutil_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfigutil(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Configutil Suite") +} diff --git a/src/autoscaler/configutil/configutil_test.go b/src/autoscaler/configutil/configutil_test.go new file mode 100644 index 0000000000..136d78787d --- /dev/null +++ b/src/autoscaler/configutil/configutil_test.go @@ -0,0 +1,151 @@ +package configutil_test + +import ( + "encoding/json" + "io" + "net/url" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" +) + +var _ = Describe("Configutil", func() { + Describe("VCAPConfiguration", func() { + var vcapConfiguration *VCAPConfiguration + + var vcapServicesJson string + var err error + + JustBeforeEach(func() { + os.Setenv("VCAP_APPLICATION", "{}") + os.Setenv("VCAP_SERVICES", vcapServicesJson) + vcapConfiguration, err = NewVCAPConfigurationReader() + }) + AfterEach(func() { + os.Unsetenv("VCAP_SERVICES") + os.Unsetenv("VCAP_APPLICATION") + }) + Describe("MaterializeTLSConfigFromService", func() { + + When("service has tls ca, cert and key credentials", func() { + var expectedClientCertContent = "client-cert-content" + var expectedClientKeyContent = "client-key-content" + var expectedServerCAContent = "server-ca-content" + + BeforeEach(func() { + vcapServicesJson = getDbVcapServices(map[string]string{ + "client_cert": expectedClientCertContent, + "client_key": expectedClientKeyContent, + "server_ca": expectedServerCAContent, + }, "some-custom-tls-service") + }) + + It("returns a tls.Config with the expected values", func() { + actualTLSConfig, err := vcapConfiguration.MaterializeTLSConfigFromService("some-custom-tls-service") + Expect(err).NotTo(HaveOccurred()) + + expectedTLSConfig := models.TLSCerts{ + KeyFile: "/tmp/some-custom-tls-service/client_key.sslkey", + CertFile: "/tmp/some-custom-tls-service/client_cert.sslcert", + CACertFile: "/tmp/some-custom-tls-service/server_ca.sslrootcert", + } + + Expect(actualTLSConfig).To(Equal(expectedTLSConfig)) + + By("writing certs to /tmp and assigns them to the TLS config") + assertCertFile(actualTLSConfig.CertFile, expectedClientCertContent) + assertCertFile(actualTLSConfig.KeyFile, expectedClientKeyContent) + assertCertFile(actualTLSConfig.CACertFile, expectedServerCAContent) + }) + }) + }) + + Describe("MaterializeDBFromService", func() { + + When("VCAP_SERVICES has relational db service bind to app", func() { + When("when uri is wrong", func() { + BeforeEach(func() { + vcapServicesJson = getDbVcapServices(map[string]string{ + "uri": "http://example.com/path\x00with/invalid/character", + }, "some-db") + }) + + It("errors", func() { + _, err = vcapConfiguration.MaterializeDBFromService("some-db") + Expect(err).To(HaveOccurred()) + }) + }) + + When("service uri is present", func() { + var expectedClientCertContent = "client-cert-content" + var expectedClientKeyContent = "client-key-content" + var expectedServerCAContent = "server-ca-content" + + BeforeEach(func() { + vcapServicesJson = getDbVcapServices(map[string]string{ + "uri": "postgres://foo:bar@postgres.example.com:5432/some-db", + "client_cert": expectedClientCertContent, + "client_key": expectedClientKeyContent, + "server_ca": expectedServerCAContent, + }, "some-db") + }) + + It("loads the db config from VCAP_SERVICES for some-db", func() { + expectedDbUrl := "postgres://foo:bar@postgres.example.com:5432/some-db?sslcert=%2Ftmp%2Fsome-db%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fsome-db%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fsome-db%2Fserver_ca.sslrootcert" // #nosec G101 + dbUrl, err := vcapConfiguration.MaterializeDBFromService("some-db") + Expect(err).NotTo(HaveOccurred()) + Expect(dbUrl).To(Equal(expectedDbUrl)) + + By("writing certs to /tmp and assigns them to the DB config") + Expect(err).NotTo(HaveOccurred()) + parsedURL, err := url.Parse(dbUrl) + Expect(err).NotTo(HaveOccurred()) + queryParams := parsedURL.Query() + + actualSSLCertPath := queryParams.Get("sslcert") + actualSSLKeyPath := queryParams.Get("sslkey") + actualSSLRootCertPath := queryParams.Get("sslrootcert") + + assertCertFile(actualSSLCertPath, expectedClientCertContent) + assertCertFile(actualSSLKeyPath, expectedClientKeyContent) + assertCertFile(actualSSLRootCertPath, expectedServerCAContent) + }) + + AfterEach(func() { + os.Remove("/tmp/some-db/client_cert.sslcert") + os.Remove("/tmp/some-db/client_key.sslkey") + os.Remove("/tmp/some-db/server_ca.sslrootcert") + }) + }) + }) + }) + }) +}) + +func getDbVcapServices(creds map[string]string, serviceName string) string { + credentials, err := json.Marshal(creds) + Expect(err).NotTo(HaveOccurred()) + return `{ + "user-provided": [ { "name": "config", "credentials": { "metricsforwarder": { } }}], + "autoscaler": [ { + "name": "some-service", + "credentials": ` + string(credentials) + `, + "syslog_drain_url": "", + "tags": ["` + serviceName + `"] + } + ]}` // #nosec G101 +} + +func assertCertFile(actualCertPath, expectedContent string) { + Expect(actualCertPath).NotTo(BeEmpty()) + file, err := os.Open(actualCertPath) + Expect(err).NotTo(HaveOccurred()) + defer file.Close() + actualContent, err := io.ReadAll(file) + Expect(err).NotTo(HaveOccurred()) + Expect(string(actualContent)).To(Equal(expectedContent)) +} diff --git a/src/autoscaler/example.mtaext b/src/autoscaler/example.mtaext deleted file mode 100644 index 134038a347..0000000000 --- a/src/autoscaler/example.mtaext +++ /dev/null @@ -1,10 +0,0 @@ -_schema-version: 3.3.0 -ID: development -extends: com.github.cloudfoundry.app-autoscaler-release -version: 1.0.0 - -modules: - - name: metricsforwarder - parameters: - routes: - - route: APP_NAME.${default-domain} diff --git a/src/autoscaler/generate-fakes.go b/src/autoscaler/generate-fakes.go index b14bcd09f4..c1b5e36660 100644 --- a/src/autoscaler/generate-fakes.go +++ b/src/autoscaler/generate-fakes.go @@ -24,3 +24,4 @@ package fakes //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_fetcher_creator.go ./eventgenerator/metric LogCacheFetcherCreator //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_fetcher.go ./eventgenerator/metric Fetcher //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_client.go ./eventgenerator/metric LogCacheClient +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_vcap_configuration_reader.go ./configutil VCAPConfigurationReader diff --git a/src/autoscaler/helpers/cf.go b/src/autoscaler/helpers/cf.go deleted file mode 100644 index 6d37d2cad7..0000000000 --- a/src/autoscaler/helpers/cf.go +++ /dev/null @@ -1,25 +0,0 @@ -package helpers - -import ( - "encoding/json" - "fmt" -) - -func GetDbURLFromVcap(vcapServices string, dbType string) (string, error) { - var vcap map[string][]struct { - Credentials map[string]interface{} `json:"credentials"` - } - - err := json.Unmarshal([]byte(vcapServices), &vcap) - if err != nil { - return "", err - } - - creds := vcap[dbType][0].Credentials - if creds == nil { - return "", fmt.Errorf("credentials not found for %s", dbType) - } - - dbURL := creds["uri"].(string) - return dbURL, nil -} diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 5b69047ef2..1d3b9f9d95 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -1,46 +1,4 @@ -PR_NUMBER ?= $(shell gh pr view --json number --jq '.number') -DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) -SYSTEM_DOMAIN ?=autoscaler.app-runtime-interfaces.ci.cloudfoundry.org -METIRCSFORWARDER_VM := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') -POSTGRES_ADDRESS := $(DEPLOYMENT_NAME)-postgres.tcp.$(SYSTEM_DOMAIN) -LOG_CACHE_IP := $(shell bosh -d cf vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("log-cache")) | .ips' ) MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) -POSTGRES_EXTERNAL_PORT := $(or $(PR_NUMBER),5432) - - -.PHONY: fetch-config -fetch-config: start-metricsforwarder-vm - # how to define variables in deployment name - mkdir -p build/assets/certs/policy_db build/assets/certs/storedprocedure_db build/assets/certs/syslog_client - - echo "POSTGRES ADDRESS: $(POSTGRES_ADDRESS)" - echo "LOG_CACHE IP: $(LOG_CACHE_IP)" - - @echo "Pulling metricforwarder config from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml build/assets/metricsforwarder.yml - - @echo "Pulling policy db certs from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt build/assets/certs/policy_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt build/assets/certs/policy_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key build/assets/certs/policy_db/. - - @echo "Pulling storeprocedure db certs from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt build/assets/certs/storedprocedure_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt build/assets/certs/storedprocedure_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key build/assets/certs/storedprocedure_db/. - - @echo "Pulling syslog-client certs from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt build/assets/certs/syslog_client/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt build/assets/certs/syslog_client/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key build/assets/certs/syslog_client/. - - @echo "Build metricsforwarder config yaml" - cp build/assets/metricsforwarder.yml build/metricsforwarder.yml - - sed -i'' -e 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' build/metricsforwarder.yml - sed -i'' -e 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal:5432|$(POSTGRES_ADDRESS):$(POSTGRES_EXTERNAL_PORT)|g' build/metricsforwarder.yml - - PHONY: set-security-group set-security-group: @@ -49,13 +7,3 @@ set-security-group: cf create-security-group metricsforwarder $(MAKEFILE_DIR)/security-group.json cf bind-security-group metricsforwarder $(ORG) - -PHONY: start-metricsforwarder-vm -start-metricsforwarder-vm: - bosh -d $(DEPLOYMENT_NAME) -n start $(METIRCSFORWARDER_VM) - -PHONY: stop-metricsforwarder-vm -stop-metricsforwarder-vm: - bosh -d $(DEPLOYMENT_NAME) -n stop $(METIRCSFORWARDER_VM) - - diff --git a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go index b72001432c..9efd97f964 100644 --- a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go +++ b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cred_helper" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" @@ -33,7 +34,8 @@ func main() { flag.StringVar(&path, "c", "", "config file") flag.Parse() - conf, err = config.LoadConfig(path) + vcapConfiguration, err := configutil.NewVCAPConfigurationReader() + conf, err = config.LoadConfig(path, vcapConfiguration) if err != nil { _, _ = fmt.Fprintf(os.Stdout, "failed to load config : %s\n", err.Error()) os.Exit(1) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 127ef173a0..33882aa4ef 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/cloudfoundry-community/go-cfenv" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -25,6 +26,7 @@ var ErrReadYaml = errors.New("failed to read config file") var ErrReadJson = errors.New("failed to read vcap_services json") var ErrReadEnvironment = errors.New("failed to read environment variables") var ErrReadVCAPEnvironment = errors.New("failed to read VCAP environment variables") +var ErrMetricsforwarderConfigNotFound = errors.New("Configuration error: metricsforwarder config service not found") const ( DefaultMetronAddress = "127.0.0.1:3458" @@ -41,7 +43,6 @@ type Config struct { LoggregatorConfig LoggregatorConfig `yaml:"loggregator"` SyslogConfig SyslogConfig `yaml:"syslog"` Db map[string]db.DatabaseConfig `yaml:"db"` - PolicyDB db.DatabaseConfig `yaml:"policy_db"` CacheTTL time.Duration `yaml:"cache_ttl"` CacheCleanupInterval time.Duration `yaml:"cache_cleanup_interval"` PolicyPollerInterval time.Duration `yaml:"policy_poller_interval"` @@ -102,6 +103,10 @@ func decodeYamlFile(filepath string, c *Config) error { func readConfigFromVCAP(appEnv *cfenv.App, c *Config) error { configVcapService, err := appEnv.Services.WithName("config") + if err != nil { + return fmt.Errorf("%w: %w", ErrMetricsforwarderConfigNotFound, err) + } + data := configVcapService.Credentials["metricsforwarder"] rawJSON, err := json.Marshal(data) @@ -117,7 +122,7 @@ func readConfigFromVCAP(appEnv *cfenv.App, c *Config) error { return nil } -func LoadConfig(filepath string) (*Config, error) { +func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) { var conf Config var err error @@ -152,14 +157,42 @@ func LoadConfig(filepath string) (*Config, error) { conf.Server.Port = appEnv.Port - err = readDbFromVCAP(appEnv, &conf) + err = readConfigFromVCAP(appEnv, &conf) + if err != nil { + return nil, err + } + + if conf.Db == nil { + conf.Db = make(map[string]db.DatabaseConfig) + } + + currentPolicyDb, ok := conf.Db[db.PolicyDb] + if !ok { + conf.Db[db.PolicyDb] = db.DatabaseConfig{} + + } + + currentPolicyDb.URL, err = vcapReader.MaterializeDBFromService(db.PolicyDb) if err != nil { return &conf, err } + conf.Db[db.PolicyDb] = currentPolicyDb + + if conf.CredHelperImpl == "stored_procedure" { + currentStoredProcedureDb, ok := conf.Db[db.StoredProcedureDb] + if !ok { + conf.Db[db.StoredProcedureDb] = db.DatabaseConfig{} + } + currentStoredProcedureDb.URL, err = vcapReader.MaterializeDBFromService(db.StoredProcedureDb) + if err != nil { + return &conf, err + } + conf.Db[db.StoredProcedureDb] = currentStoredProcedureDb + } - err = readConfigFromVCAP(appEnv, &conf) + conf.SyslogConfig.TLS, err = vcapReader.MaterializeTLSConfigFromService("syslog-client") if err != nil { - return nil, err + return &conf, err } } @@ -212,61 +245,3 @@ func (c *Config) Validate() error { return nil } -func readDbFromVCAP(appEnv *cfenv.App, c *Config) error { - if c.Db != nil { - return nil - } - - dbServices, err := appEnv.Services.WithTag("relational") - if err != nil { - return fmt.Errorf("failed to get db service with relational tag") - } - - if len(dbServices) != 1 { - return fmt.Errorf("failed to get db service with relational tag") - } - - dbService := dbServices[0] - - dbURI, ok := dbService.CredentialString("uri") - if !ok { - return fmt.Errorf("failed to get uri from db service") - } - - if c.Db == nil { - c.Db = make(map[string]db.DatabaseConfig) - } - - c.Db[db.PolicyDb] = db.DatabaseConfig{ - URL: dbURI, - } - - //dbURL, err := url.Parse(dbURI) - //if err != nil { - // return nil, err - //} - - //parameters, err := url.ParseQuery(dbURL.RawQuery) - //if err != nil { - // return nil, err - //} - - //err = materializeConnectionParameter(dbService, parameters, "client_cert", "sslcert") - //if err != nil { - // return nil, err - //} - - //err = materializeConnectionParameter(dbService, parameters, "client_key", "sslkey") - //if err != nil { - // return nil, err - //} - - //err = materializeConnectionParameter(dbService, parameters, "server_ca", "sslrootcert") - //if err != nil { - // return nil, err - //} - - //dbURL.RawQuery = parameters.Encode() - - return nil -} diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index 06a445fba4..3606ca9d50 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -5,6 +5,7 @@ import ( "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -26,17 +27,24 @@ func bytesToFile(b []byte) string { var _ = Describe("Config", func() { var ( - conf *Config - err error - configBytes []byte - configFile string + conf *Config + err error + configBytes []byte + configFile string + mockVCAPConfigurationReader *fakes.FakeVCAPConfigurationReader ) Describe("LoadConfig", func() { + When("config is read from env", func() { var vcapServicesJson string + var expectedDbUrl string var port string + BeforeEach(func() { + mockVCAPConfigurationReader = &fakes.FakeVCAPConfigurationReader{} + }) + AfterEach(func() { os.Unsetenv("VCAP_SERVICES") os.Unsetenv("PORT") @@ -48,80 +56,126 @@ var _ = Describe("Config", func() { os.Setenv("PORT", port) os.Setenv("VCAP_APPLICATION", "{}") os.Setenv("VCAP_SERVICES", vcapServicesJson) - conf, err = LoadConfig(configFile) + + mockVCAPConfigurationReader.MaterializeDBFromServiceReturns(expectedDbUrl, nil) + conf, err = LoadConfig(configFile, mockVCAPConfigurationReader) }) When("PORT env variable is set to a number ", func() { BeforeEach(func() { - vcapServicesJson = "{}" + vcapServicesJson = `{ "user-provided": [ { "name": "config" } ] }` port = "3333" }) It("sets env variable over config file", func() { + Expect(err).NotTo(HaveOccurred()) Expect(conf.Server.Port).To(Equal(3333)) }) }) - When("VCAP_SERVICES has service config", func() { + When("VCAP_SERVICES is empty", func() { + BeforeEach(func() { + vcapServicesJson = "{}" + }) + + It("should error with config service not found", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: metricsforwarder config service not found"))) + }) + }) + + When("VCAP_SERVICES has credentials for syslog client", func() { + var expectedTLSConfig models.TLSCerts + + BeforeEach(func() { + vcapServicesJson = `{ "user-provided": [ { "name": "config", "credentials": { "metricsforwarder": { } } }] }` + + expectedTLSConfig = models.TLSCerts{ + CertFile: "/tmp/client_cert.sslcert", + KeyFile: "/tmp/client_key.sslkey", + CACertFile: "/tmp/server_ca.sslrootcert", + } + + mockVCAPConfigurationReader.MaterializeTLSConfigFromServiceReturns(expectedTLSConfig, nil) + }) + + It("loads the syslog config from VCAP_SERVICES", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.SyslogConfig.TLS).To(Equal(expectedTLSConfig)) + }) + }) + + When("VCAP_SERVICES has relational db service bind to app for policy db", func() { + BeforeEach(func() { + vcapServicesJson = getVcapConfigWithCredImplementation("default") + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" + }) + + It("loads the db config from VCAP_SERVICES successfully", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Db[db.PolicyDb].URL).To(Equal(expectedDbUrl)) + Expect(mockVCAPConfigurationReader.MaterializeDBFromServiceCallCount()).To(Equal(1)) + actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(0) + Expect(actualDbName).To(Equal(db.PolicyDb)) + }) + }) + + When("storedProcedure_db service is provided and cred_helper_impl is stored_procedure", func() { + BeforeEach(func() { + vcapServicesJson = getVcapConfigWithCredImplementation("stored_procedure") + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" + }) + + It("reads the store procedure service from vcap", func() { + Expect(err).NotTo(HaveOccurred()) + _, storeProcedureFound := conf.Db[db.StoredProcedureDb] + Expect(storeProcedureFound).To(BeTrue()) + Expect(conf.Db[db.StoredProcedureDb].URL).To(Equal(expectedDbUrl)) + Expect(mockVCAPConfigurationReader.MaterializeDBFromServiceCallCount()).To(Equal(2)) + actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(1) + Expect(actualDbName).To(Equal(db.StoredProcedureDb)) + }) + }) + + When("storedProcedure_db service is provided and cred_helper_impl is default", func() { + BeforeEach(func() { + vcapServicesJson = getVcapConfigWithCredImplementation("default") + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" + }) + + It("ignores the service gracefully", func() { + Expect(err).NotTo(HaveOccurred()) + _, storeProcedureFound := conf.Db[db.StoredProcedureDb] + Expect(storeProcedureFound).To(BeFalse()) + }) + }) + + When("VCAP_SERVICES has metricsforwarder config", func() { BeforeEach(func() { - // VCAP_SERVICES={"user-provided":[ - //{"label":"user-provided", - // "name":"config", - // "tags":[], - // "instance_guid":"444c838e-17d9-429d-a1ea-660904db9841", - // "instance_name":"config", - // "binding_guid":"2cb523a1-773a-4fa4-ba05-3a76cc488ff7", - // "binding_name":null, - // "credentials":{ - // "db":null, - // "logging":{"level":"info"}, - // "policy_poller_interval":"60s", - // "rate_limit":{"max_amount":10,"valid_duration":"1s"}, - // "syslog":{ - // "port":6067, - // "server_address":"log-cache.service.cf.internal", - // "tls":{"ca_file":"/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt","cert_file":"/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt","key_file":"/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key"} - // }, - // } - //}, - // "syslog_drain_url":null, - // "volume_mounts":[]}]} - // vcapServicesJson = `{ "user-provided": [ { "label":"user-provided", "name": "config", "credentials": { - "metricsforwarder": { - "cache_cleanup_interval":"10h", - "cache_ttl":"90s", - "cred_helper_impl": "default", - "health":{"password":"health-password","username":"health-user"}, - "logging": { - "level": "debug" - }, - "loggregator": { - "metron_address": "metron-vcap-addrs:3457", - "tls": { - "ca_file": "../testcerts/ca.crt", - "cert_file": "../testcerts/client.crt", - "key_file": "../testcerts/client.key" + "metricsforwarder": { + "cache_cleanup_interval":"10h", + "cache_ttl":"90s", + "cred_helper_impl": "default", + "health":{"password":"health-password","username":"health-user"}, + "logging": { + "level": "debug" + }, + "loggregator": { + "metron_address": "metron-vcap-addrs:3457", + "tls": { + "ca_file": "../testcerts/ca.crt", + "cert_file": "../testcerts/client.crt", + "key_file": "../testcerts/client.key" + } } } } - } - }], - "autoscaler": [ { - "name": "policy_db", - "label": "postgres", - "credentials": { - "uri":"postgres://foo:bar@postgres.example.com:5432/policy_db" - }, - "syslog_drain_url": "", - "tags": ["postgres","postgresql","relational"] - } - ] - }` // #nosec G101 + } + ]}` // #nosec G101 }) It("loads the config from VCAP_SERVICES", func() { @@ -130,22 +184,13 @@ var _ = Describe("Config", func() { Expect(conf.LoggregatorConfig.MetronAddress).To(Equal("metron-vcap-addrs:3457")) Expect(conf.CacheTTL).To(Equal(90 * time.Second)) }) - - It("loads the db config from VCAP_SERVICES", func() { - expectedDbConfig := db.DatabaseConfig{ - URL: "postgres://foo:bar@postgres.example.com:5432/policy_db", - } - - Expect(err).NotTo(HaveOccurred()) - Expect(conf.Db[db.PolicyDb]).To(Equal(expectedDbConfig)) - }) }) }) When("config is read from file", func() { JustBeforeEach(func() { configFile = bytesToFile(configBytes) - conf, err = LoadConfig(configFile) + conf, err = LoadConfig(configFile, mockVCAPConfigurationReader) }) AfterEach(func() { @@ -390,3 +435,16 @@ health: }) }) }) + +func getVcapConfigWithCredImplementation(credHelperImplementation string) string { + return `{ + "user-provided": [ { + "name": "config", + "credentials": { + "metricsforwarder": { + "cred_helper_impl": "` + credHelperImplementation + `" + } + } + }] + }` // #nosec G101 +} diff --git a/src/autoscaler/metricsforwarder/default_config.json b/src/autoscaler/metricsforwarder/default_config.json new file mode 100644 index 0000000000..1493696415 --- /dev/null +++ b/src/autoscaler/metricsforwarder/default_config.json @@ -0,0 +1,34 @@ +{ + "metricsforwarder": { + "cache_cleanup_interval": "6h", + "cache_ttl": "900s", + "cred_helper_impl": "default", + "health": { + "username": "metricsforwarder" + }, + "logging": { + "level": "debug" + }, + "syslog": { + "server_address": "log-cache.service.cf.internal", + "port": 6067 + }, + "db": { + "policy_db": { + "max_open_connections": 100, + "max_idle_connections": 10, + "connection_max_lifetime": "60s" + }, + "storedprocedure_db": { + "max_open_connections": 20, + "max_idle_connections": 10, + "connection_max_lifetime": "60s" + } + }, + "policy_poller_interval": "60s", + "rate_limit": { + "valid_duration": "1s", + "max_amount": 10 + } + } +} diff --git a/src/autoscaler/mta.yaml b/src/autoscaler/mta.yaml index eff830b6c9..ab59e706d3 100644 --- a/src/autoscaler/mta.yaml +++ b/src/autoscaler/mta.yaml @@ -9,10 +9,32 @@ modules: - name: metricsforwarder type: binary path: build + requires: + - name: config + - name: policydb + - name: syslog-client parameters: memory: 1G disk-quota: 1G - instances: 1 + instances: 2 stack: cflinuxfs4 - command: ./metricsforwarder -c metricsforwarder.yml + command: ./metricsforwarder routes: + +resources: +- name: config + type: org.cloudfoundry.user-provided-service + parameters: + path: metricsforwarder/default_config.json +- name: policydb + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - policy_db + - relational +- name: syslog-client + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - syslog-client +