diff --git a/Makefile b/Makefile index 32c9f1e019..7d0ef2115b 100644 --- a/Makefile +++ b/Makefile @@ -453,5 +453,7 @@ go-get-u: $(addsuffix .go-get-u,$(go_modules)) go get -u ./... -mta-deploy: - @make --directory='./src/autoscaler' mta-deploy +deploy-apps: + echo " - deploying apps" + DEBUG="${DEBUG}" ${CI_DIR}/autoscaler/scripts/deploy-apps.sh + diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index 1c69188f10..2603a4d55f 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -6,6 +6,10 @@ script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/common.sh" source "${script_dir}/vars.source.sh" +pushd "${bbl_state_path}" > /dev/null + eval "$(bbl print-env)" +popd > /dev/null + function deploy() { log "Deploying autoscaler apps for bosh deployment '${deployment_name}' " pushd "${autoscaler_dir}/src/autoscaler" > /dev/null diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index e534c5912e..4236ab21db 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:jammy +FROM ubuntu:noble MAINTAINER autoscaler-team ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" @@ -36,7 +36,6 @@ RUN apt-get update && \ cf8-cli \ gnupg \ gnupg2 \ - netcat \ gh \ make \ mysql-client && \ diff --git a/ci/infrastructure/scripts/deploy-multiapps-controller.sh b/ci/infrastructure/scripts/deploy-multiapps-controller.sh index 3483a44c8a..56d471ca9e 100755 --- a/ci/infrastructure/scripts/deploy-multiapps-controller.sh +++ b/ci/infrastructure/scripts/deploy-multiapps-controller.sh @@ -23,7 +23,6 @@ function deploy_multiapps_controller() { mv multiapps-controller-web-war/*.war . pushd multiapps-controller-web-manifest cf push -f ./*.yml "${app_name}" - popd } diff --git a/devbox.json b/devbox.json index 86e61f6dcc..ca6c88335d 100644 --- a/devbox.json +++ b/devbox.json @@ -11,7 +11,7 @@ "bundix": "latest", "cloudfoundry-cli": "8.7.11", "coreutils": "latest", - "credhub-cli": "2.9.29", + "credhub-cli": "2.9.35", "delve": "latest", "direnv": "2.34.0", "fly": "7.10.0", diff --git a/devbox.lock b/devbox.lock index 87f6781cc7..047ab26bed 100644 --- a/devbox.lock +++ b/devbox.lock @@ -313,51 +313,51 @@ } } }, - "credhub-cli@2.9.29": { - "last_modified": "2024-05-29T10:04:41Z", - "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#credhub-cli", + "credhub-cli@2.9.35": { + "last_modified": "2024-08-14T11:41:26Z", + "resolved": "github:NixOS/nixpkgs/0cb2fd7c59fed0cd82ef858cbcbdb552b9a33465#credhub-cli", "source": "devbox-search", - "version": "2.9.29", + "version": "2.9.35", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/20xi78w44q6wn4847jrrlmxddyjycbrn-credhub-cli-2.9.29", + "path": "/nix/store/2xkv0vlafk8zgzjpz3yjagbgyv5kawkn-credhub-cli-2.9.35", "default": true } ], - "store_path": "/nix/store/20xi78w44q6wn4847jrrlmxddyjycbrn-credhub-cli-2.9.29" + "store_path": "/nix/store/2xkv0vlafk8zgzjpz3yjagbgyv5kawkn-credhub-cli-2.9.35" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/5hv8vj4dhy521m49sc4wwmaz2q319z9a-credhub-cli-2.9.29", + "path": "/nix/store/r0kkggv2kbdma8l2n177nl0sw4lwvqr5-credhub-cli-2.9.35", "default": true } ], - "store_path": "/nix/store/5hv8vj4dhy521m49sc4wwmaz2q319z9a-credhub-cli-2.9.29" + "store_path": "/nix/store/r0kkggv2kbdma8l2n177nl0sw4lwvqr5-credhub-cli-2.9.35" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/c5d2ggn5c819spywi74pzksbzds9ryym-credhub-cli-2.9.29", + "path": "/nix/store/wkgc414d175cv1pa3h257pm264ayk8if-credhub-cli-2.9.35", "default": true } ], - "store_path": "/nix/store/c5d2ggn5c819spywi74pzksbzds9ryym-credhub-cli-2.9.29" + "store_path": "/nix/store/wkgc414d175cv1pa3h257pm264ayk8if-credhub-cli-2.9.35" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/3y6r6lgrbaq8521hk3zbalf3gz3albcg-credhub-cli-2.9.29", + "path": "/nix/store/zkmy2g042p75p3i6qib6s1589qymrg3k-credhub-cli-2.9.35", "default": true } ], - "store_path": "/nix/store/3y6r6lgrbaq8521hk3zbalf3gz3albcg-credhub-cli-2.9.29" + "store_path": "/nix/store/zkmy2g042p75p3i6qib6s1589qymrg3k-credhub-cli-2.9.35" } } }, @@ -1666,6 +1666,7 @@ }, "ruby@latest": { "last_modified": "2024-08-14T11:41:26Z", + "plugin_version": "0.0.2", "resolved": "github:NixOS/nixpkgs/0cb2fd7c59fed0cd82ef858cbcbdb552b9a33465#ruby_3_3", "source": "devbox-search", "version": "3.3.4", diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml index c56d25d284..20b3829767 100644 --- a/operations/use-cf-services.yml +++ b/operations/use-cf-services.yml @@ -4,11 +4,6 @@ host: ((metricsforwarder_host)) mtls_host: ((metricsforwarder_host)) -# Set the same port for metricsforwarder and healthenpoint routes -- type: replace - path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health/port - value: 6201 - ## add router tcp route for postgres - type: replace path: /instance_groups/name=postgres/jobs/- @@ -53,3 +48,6 @@ - type: replace path: /variables/name=postgres_client/options/alternative_names/- value: ((deployment_name))-postgres.tcp.((system_domain)) + +- type: remove + path: /instance_groups/name=metricsforwarder diff --git a/packages/metricsforwarder/spec b/packages/metricsforwarder/spec index 03ce465a23..411094f8a3 100644 --- a/packages/metricsforwarder/spec +++ b/packages/metricsforwarder/spec @@ -8,6 +8,7 @@ files: - autoscaler/* - autoscaler/vendor/* - autoscaler/cf/* # gosub +- autoscaler/configutil/* # gosub - autoscaler/cred_helper/* # gosub - autoscaler/db/* # gosub - autoscaler/db/sqldb/* # gosub @@ -44,6 +45,7 @@ files: - autoscaler/vendor/github.com/andybalholm/brotli/matchfinder/* # gosub - autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub +- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub - autoscaler/vendor/github.com/felixge/httpsnoop/* # gosub - autoscaler/vendor/github.com/go-logr/logr/* # gosub - autoscaler/vendor/github.com/go-logr/logr/funcr/* # gosub @@ -81,6 +83,7 @@ files: - autoscaler/vendor/github.com/klauspost/compress/zlib/* # gosub - autoscaler/vendor/github.com/klauspost/compress/zstd/* # gosub - autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub +- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub - autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub diff --git a/src/acceptance/assets/app/go_app/Makefile b/src/acceptance/assets/app/go_app/Makefile index 429a7f16ce..e4194da52b 100644 --- a/src/acceptance/assets/app/go_app/Makefile +++ b/src/acceptance/assets/app/go_app/Makefile @@ -72,8 +72,6 @@ build: ./build/app ./build/manifest.yml CGO_ENABLED='${CGO_ENABLED}' GOOS='linux' GOARCH='amd64' go build -o './build/app' cp './app_manifest.yml' './build/manifest.yml' - - .PHONY: build_tests build_tests: $(addprefix build_test-,$(test_dirs)) diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index b29f66ddab..4e01642ea2 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -12,7 +12,7 @@ PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' DB_HOST ?= localhost DBURL ?= "postgres://postgres:postgres@${DB_HOST}/autoscaler?sslmode=disable" -METRICSFORWARDER_APPNAME ?= "metricsforwarder" +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) EXTENSION_FILE := $(shell mktemp) export GOWORK=off @@ -154,14 +154,12 @@ clean: .PHONY: mta-deploy mta-deploy: mta-build build-extension-file $(MAKE) -f metricsforwarder/Makefile set-security-group - $(MAKE) -f metricsforwarder/Makefile stop-metricsforwarder-vm @echo "Deploying with extension file: $(EXTENSION_FILE)" - @cf deploy mta_archives/*.mtar -f -e $(EXTENSION_FILE) + @cf deploy mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE) build-extension-file: - cp example.mtaext $(EXTENSION_FILE); - sed -i "s/APP_NAME/$(METRICSFORWARDER_APPNAME)/g" $(EXTENSION_FILE); - echo "EXTENSION_FILE: $(EXTENSION_FILE)" + echo "extension file at: $(EXTENSION_FILE)" + $(MAKEFILE_DIR)/build-extension-file.sh $(EXTENSION_FILE); mta-logs: rm -rf mta-* @@ -170,7 +168,6 @@ mta-logs: .PHONY: mta-build mta-build: mta-build-clean cf-build - $(MAKE) -f metricsforwarder/Makefile fetch-config mbt build mta-build-clean: diff --git a/src/autoscaler/build-extension-file.sh b/src/autoscaler/build-extension-file.sh new file mode 100755 index 0000000000..cebda974c2 --- /dev/null +++ b/src/autoscaler/build-extension-file.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2155,SC2034,SC2086 + +set -e + +if [ -z "$1" ]; then + echo "extension file path not provided" + exit 1 +else + extension_file_path=$1 +fi + +if [ -z "${DEPLOYMENT_NAME}" ]; then + echo "DEPLOYMENT_NAME is not set" + exit 1 +fi + +if [ -z "${PR_NUMBER}" ]; then + echo "PR_NUMBER is not set" + exit 1 +fi + +export SYSTEM_DOMAIN="autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" +export POSTGRES_ADDRESS="${DEPLOYMENT_NAME}-postgres.tcp.${SYSTEM_DOMAIN}" +export POSTGRES_EXTERNAL_PORT="${PR_NUMBER:-5432}" + +export METRICSFORWARDER_HEALTH_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_metricsforwarder_health_password --quiet)" +export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}" + +export POLICY_DB_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/database_password --quiet)" +export POLICY_DB_SERVER_CA="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key ca --quiet )" +export POLICY_DB_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key certificate --quiet)" +export POLICY_DB_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key private_key --quiet)" + +export SYSLOG_CLIENT_CA="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key ca --quiet)" +export SYSLOG_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)" +export SYSLOG_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)" + +cat < "${extension_file_path}" +ID: development +extends: com.github.cloudfoundry.app-autoscaler-release +version: 1.0.0 +_schema-version: 3.3.0 + +modules: + - name: metricsforwarder + parameters: + routes: + - route: ${METRICSFORWARDER_APPNAME}.\${default-domain} + +resources: +- name: config + parameters: + config: + metricsforwarder: + health: + password: "${METRICSFORWARDER_HEALTH_PASSWORD}" +- name: policydb + parameters: + config: + uri: "postgres://postgres:${POLICY_DB_PASSWORD}@${POSTGRES_ADDRESS}:${POSTGRES_EXTERNAL_PORT}/autoscaler?application_name=metricsforwarder&sslmode=verify-full" + client_cert: "${POLICY_DB_CLIENT_CERT//$'\n'/\\n}" + client_key: "${POLICY_DB_CLIENT_KEY//$'\n'/\\n}" + server_ca: "${POLICY_DB_SERVER_CA//$'\n'/\\n}" +- name: syslog-client + parameters: + config: + client_cert: "${SYSLOG_CLIENT_CERT//$'\n'/\\n}" + client_key: "${SYSLOG_CLIENT_KEY//$'\n'/\\n}" + server_ca: "${SYSLOG_CLIENT_CA//$'\n'/\\n}" +EOF diff --git a/src/autoscaler/configutil/cf.go b/src/autoscaler/configutil/cf.go new file mode 100644 index 0000000000..2671b00a1c --- /dev/null +++ b/src/autoscaler/configutil/cf.go @@ -0,0 +1,199 @@ +package configutil + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "github.com/cloud-gov/go-cfenv" +) + +var ( + ErrDbServiceNotFound = errors.New("failed to get service by name") + ErrMissingCredential = errors.New("failed to get required credential from service") +) + +type VCAPConfigurationReader interface { + MaterializeDBFromService(dbName string) (string, error) + MaterializeTLSConfigFromService(serviceTag string) (models.TLSCerts, error) + GetServiceCredentialContent(serviceTag string, credentialKey string) ([]byte, error) + GetPort() int + IsRunningOnCF() bool +} + +type VCAPConfiguration struct { + appEnv *cfenv.App +} + +func NewVCAPConfigurationReader() (*VCAPConfiguration, error) { + appEnv, err := cfenv.Current() + if err != nil { + fmt.Println("failed to read VCAP_APPLICATION environment variable") + } + return &VCAPConfiguration{appEnv: appEnv}, nil +} + +func (vc *VCAPConfiguration) GetPort() int { + return vc.appEnv.Port +} +func (vc *VCAPConfiguration) IsRunningOnCF() bool { + return cfenv.IsRunningOnCF() +} + +func (vc *VCAPConfiguration) GetServiceCredentialContent(serviceTag, credentialKey string) ([]byte, error) { + service, err := vc.getServiceByTag(serviceTag) + if err != nil { + return []byte(""), err + } + + content, ok := service.Credentials[credentialKey] + if !ok { + return []byte(""), fmt.Errorf("%w: %s", ErrMissingCredential, credentialKey) + } + + rawJSON, err := json.Marshal(content) + if err != nil { + return []byte(""), err + } + + return rawJSON, nil +} + +func (vc *VCAPConfiguration) MaterializeTLSConfigFromService(serviceTag string) (models.TLSCerts, error) { + service, err := vc.getServiceByTag(serviceTag) + if err != nil { + return models.TLSCerts{}, err + } + + tlsCerts, err := vc.buildTLSCerts(service, serviceTag) + if err != nil { + return models.TLSCerts{}, err + } + + return tlsCerts, nil +} + +func (vc *VCAPConfiguration) MaterializeDBFromService(dbName string) (string, error) { + service, err := vc.getServiceByTag(dbName) + if err != nil { + return "", err + } + + dbURL, err := vc.buildDatabaseURL(service, dbName) + if err != nil { + return "", err + } + + return dbURL.String(), nil +} + +func (vc *VCAPConfiguration) getServiceByTag(serviceTag string) (*cfenv.Service, error) { + services, err := vc.appEnv.Services.WithTag(serviceTag) + if err != nil || len(services) == 0 { + return nil, fmt.Errorf("%w: %s", ErrDbServiceNotFound, serviceTag) + } + return &services[0], nil +} + +func (vc *VCAPConfiguration) buildTLSCerts(service *cfenv.Service, serviceTag string) (models.TLSCerts, error) { + certs := models.TLSCerts{} + + if err := vc.createCertFile(service, "client_cert", "sslcert", serviceTag, &certs.CertFile); err != nil { + return models.TLSCerts{}, err + } + + if err := vc.createCertFile(service, "client_key", "sslkey", serviceTag, &certs.KeyFile); err != nil { + return models.TLSCerts{}, err + } + + if err := vc.createCertFile(service, "server_ca", "sslrootcert", serviceTag, &certs.CACertFile); err != nil { + return models.TLSCerts{}, err + } + + return certs, nil +} + +func (vc *VCAPConfiguration) createCertFile(service *cfenv.Service, credentialKey, fileSuffix, serviceTag string, certFile *string) error { + content, ok := service.CredentialString(credentialKey) + if !ok { + return fmt.Errorf("%w: %s", ErrMissingCredential, credentialKey) + } + fileName := fmt.Sprintf("%s.%s", credentialKey, fileSuffix) + createdFile, err := materializeServiceProperty(serviceTag, fileName, content) + if err != nil { + return err + } + *certFile = createdFile + return nil +} + +func (vc *VCAPConfiguration) buildDatabaseURL(service *cfenv.Service, dbName string) (*url.URL, error) { + dbURI, ok := service.CredentialString("uri") + if !ok { + return nil, fmt.Errorf("%w: uri", ErrMissingCredential) + } + + dbURL, err := url.Parse(dbURI) + if err != nil { + return nil, err + } + + parameters, err := url.ParseQuery(dbURL.RawQuery) + if err != nil { + return nil, err + } + + if err := vc.addConnectionParams(service, dbName, parameters); err != nil { + return nil, err + } + + dbURL.RawQuery = parameters.Encode() + return dbURL, nil +} + +func (vc *VCAPConfiguration) addConnectionParams(service *cfenv.Service, dbName string, parameters url.Values) error { + keys := []struct { + binding, connection string + }{ + {"client_cert", "sslcert"}, + {"client_key", "sslkey"}, + {"server_ca", "sslrootcert"}, + } + + for _, key := range keys { + if err := vc.addConnectionParam(service, dbName, key.binding, key.connection, parameters); err != nil { + return err + } + } + return nil +} + +func (vc *VCAPConfiguration) addConnectionParam(service *cfenv.Service, dbName, bindingKey, connectionParam string, parameters url.Values) error { + content, ok := service.CredentialString(bindingKey) + if ok { + fileName := fmt.Sprintf("%s.%s", bindingKey, connectionParam) + createdFile, err := materializeServiceProperty(dbName, fileName, content) + if err != nil { + return err + } + parameters.Set(connectionParam, createdFile) + } + return nil +} + +func materializeServiceProperty(serviceTag, fileName, content string) (string, error) { + dirPath := fmt.Sprintf("/tmp/%s", serviceTag) + if err := os.MkdirAll(dirPath, 0700); err != nil { + return "", err + } + + filePath := fmt.Sprintf("%s/%s", dirPath, fileName) + if err := os.WriteFile(filePath, []byte(content), 0600); err != nil { + return "", err + } + + return filePath, nil +} diff --git a/src/autoscaler/configutil/configutil_suite_test.go b/src/autoscaler/configutil/configutil_suite_test.go new file mode 100644 index 0000000000..839968d1f9 --- /dev/null +++ b/src/autoscaler/configutil/configutil_suite_test.go @@ -0,0 +1,13 @@ +package configutil_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfigutil(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Configutil Suite") +} diff --git a/src/autoscaler/configutil/configutil_test.go b/src/autoscaler/configutil/configutil_test.go new file mode 100644 index 0000000000..96a2eab05b --- /dev/null +++ b/src/autoscaler/configutil/configutil_test.go @@ -0,0 +1,173 @@ +package configutil_test + +import ( + "encoding/json" + "io" + "net/url" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" +) + +var _ = Describe("Configutil", func() { + Describe("VCAPConfiguration", func() { + var vcapConfiguration *VCAPConfiguration + var vcapApplicationJson string + var vcapServicesJson string + var err error + + JustBeforeEach(func() { + os.Setenv("VCAP_APPLICATION", vcapApplicationJson) + os.Setenv("VCAP_SERVICES", vcapServicesJson) + vcapConfiguration, err = NewVCAPConfigurationReader() + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + os.Unsetenv("VCAP_SERVICES") + os.Unsetenv("VCAP_APPLICATION") + }) + + Describe("IsRunningOnCF", func() { + When("VCAP_APPLICATION is not set", func() { + BeforeEach(func() { + vcapApplicationJson = "" + }) + + It("returns false when vcap", func() { + Expect(vcapConfiguration.IsRunningOnCF()).To(BeFalse()) + }) + }) + }) + + Describe("MaterializeTLSConfigFromService", func() { + BeforeEach(func() { + vcapApplicationJson = `{}` + }) + + When("service has tls ca, cert and key credentials", func() { + var expectedClientCertContent = "client-cert-content" + var expectedClientKeyContent = "client-key-content" + var expectedServerCAContent = "server-ca-content" + + BeforeEach(func() { + vcapServicesJson = getDbVcapServices(map[string]string{ + "client_cert": expectedClientCertContent, + "client_key": expectedClientKeyContent, + "server_ca": expectedServerCAContent, + }, "some-custom-tls-service") + }) + + It("returns a tls.Config with the expected values", func() { + actualTLSConfig, err := vcapConfiguration.MaterializeTLSConfigFromService("some-custom-tls-service") + Expect(err).NotTo(HaveOccurred()) + + expectedTLSConfig := models.TLSCerts{ + KeyFile: "/tmp/some-custom-tls-service/client_key.sslkey", + CertFile: "/tmp/some-custom-tls-service/client_cert.sslcert", + CACertFile: "/tmp/some-custom-tls-service/server_ca.sslrootcert", + } + + Expect(actualTLSConfig).To(Equal(expectedTLSConfig)) + + By("writing certs to /tmp and assigns them to the TLS config") + assertCertFile(actualTLSConfig.CertFile, expectedClientCertContent) + assertCertFile(actualTLSConfig.KeyFile, expectedClientKeyContent) + assertCertFile(actualTLSConfig.CACertFile, expectedServerCAContent) + }) + }) + }) + + Describe("MaterializeDBFromService", func() { + BeforeEach(func() { + vcapApplicationJson = "" + vcapApplicationJson = `{}` + }) + + When("VCAP_SERVICES has relational db service bind to app", func() { + When("when uri is wrong", func() { + BeforeEach(func() { + vcapServicesJson = getDbVcapServices(map[string]string{ + "uri": "http://example.com/path\x00with/invalid/character", + }, "some-db") + }) + + It("errors", func() { + _, err = vcapConfiguration.MaterializeDBFromService("some-db") + Expect(err).To(HaveOccurred()) + }) + }) + + When("service uri is present", func() { + var expectedClientCertContent = "client-cert-content" + var expectedClientKeyContent = "client-key-content" + var expectedServerCAContent = "server-ca-content" + + BeforeEach(func() { + vcapServicesJson = getDbVcapServices(map[string]string{ + "uri": "postgres://foo:bar@postgres.example.com:5432/some-db", + "client_cert": expectedClientCertContent, + "client_key": expectedClientKeyContent, + "server_ca": expectedServerCAContent, + }, "some-db") + }) + + It("loads the db config from VCAP_SERVICES for some-db", func() { + expectedDbUrl := "postgres://foo:bar@postgres.example.com:5432/some-db?sslcert=%2Ftmp%2Fsome-db%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fsome-db%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fsome-db%2Fserver_ca.sslrootcert" // #nosec G101 + dbUrl, err := vcapConfiguration.MaterializeDBFromService("some-db") + Expect(err).NotTo(HaveOccurred()) + Expect(dbUrl).To(Equal(expectedDbUrl)) + + By("writing certs to /tmp and assigns them to the DB config") + Expect(err).NotTo(HaveOccurred()) + parsedURL, err := url.Parse(dbUrl) + Expect(err).NotTo(HaveOccurred()) + queryParams := parsedURL.Query() + + actualSSLCertPath := queryParams.Get("sslcert") + actualSSLKeyPath := queryParams.Get("sslkey") + actualSSLRootCertPath := queryParams.Get("sslrootcert") + + assertCertFile(actualSSLCertPath, expectedClientCertContent) + assertCertFile(actualSSLKeyPath, expectedClientKeyContent) + assertCertFile(actualSSLRootCertPath, expectedServerCAContent) + }) + + AfterEach(func() { + os.Remove("/tmp/some-db/client_cert.sslcert") + os.Remove("/tmp/some-db/client_key.sslkey") + os.Remove("/tmp/some-db/server_ca.sslrootcert") + }) + }) + }) + }) + }) +}) + +func getDbVcapServices(creds map[string]string, serviceName string) string { + credentials, err := json.Marshal(creds) + Expect(err).NotTo(HaveOccurred()) + return `{ + "user-provided": [ { "name": "config", "credentials": { "metricsforwarder": { } }}], + "autoscaler": [ { + "name": "some-service", + "credentials": ` + string(credentials) + `, + "syslog_drain_url": "", + "tags": ["` + serviceName + `"] + } + ]}` // #nosec G101 +} + +func assertCertFile(actualCertPath, expectedContent string) { + Expect(actualCertPath).NotTo(BeEmpty()) + file, err := os.Open(actualCertPath) + Expect(err).NotTo(HaveOccurred()) + defer file.Close() + actualContent, err := io.ReadAll(file) + Expect(err).NotTo(HaveOccurred()) + Expect(string(actualContent)).To(Equal(expectedContent)) +} diff --git a/src/autoscaler/example.mtaext b/src/autoscaler/example.mtaext deleted file mode 100644 index 134038a347..0000000000 --- a/src/autoscaler/example.mtaext +++ /dev/null @@ -1,10 +0,0 @@ -_schema-version: 3.3.0 -ID: development -extends: com.github.cloudfoundry.app-autoscaler-release -version: 1.0.0 - -modules: - - name: metricsforwarder - parameters: - routes: - - route: APP_NAME.${default-domain} diff --git a/src/autoscaler/generate-fakes.go b/src/autoscaler/generate-fakes.go index b14bcd09f4..c1b5e36660 100644 --- a/src/autoscaler/generate-fakes.go +++ b/src/autoscaler/generate-fakes.go @@ -24,3 +24,4 @@ package fakes //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_fetcher_creator.go ./eventgenerator/metric LogCacheFetcherCreator //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_fetcher.go ./eventgenerator/metric Fetcher //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_client.go ./eventgenerator/metric LogCacheClient +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_vcap_configuration_reader.go ./configutil VCAPConfigurationReader diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index dfce661477..83d5e496f8 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -11,6 +11,7 @@ require ( code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240723222507-f3307e073100 code.cloudfoundry.org/tlsconfig v0.0.0-20240730181439-b476395a9e4e github.com/cenkalti/backoff/v4 v4.3.0 + github.com/cloud-gov/go-cfenv v1.19.1 github.com/go-chi/chi/v5 v5.1.0 github.com/go-faster/errors v0.7.1 github.com/go-faster/jx v1.1.0 @@ -77,6 +78,7 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openzipkin/zipkin-go v0.4.2 // indirect github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index ebb9e930c3..634fc4cdcb 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -652,6 +652,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloud-gov/go-cfenv v1.19.1 h1:SJtZqRmADChsFAMT1f1ftyFgzdsyGiVRIfbg3/cRass= +github.com/cloud-gov/go-cfenv v1.19.1/go.mod h1:nA0JsA4rVNcjMZPyv9yq390r84gpPUabuvXOg+QB88c= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -869,6 +871,8 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= @@ -908,6 +912,8 @@ github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisdd github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= diff --git a/src/autoscaler/metricsforwarder/.gitignore b/src/autoscaler/metricsforwarder/.gitignore index 7995a170f2..7e2f179b52 100644 --- a/src/autoscaler/metricsforwarder/.gitignore +++ b/src/autoscaler/metricsforwarder/.gitignore @@ -1,3 +1 @@ assets -metricsforwarder -metricsforwarder.yml diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 5b69047ef2..1d3b9f9d95 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -1,46 +1,4 @@ -PR_NUMBER ?= $(shell gh pr view --json number --jq '.number') -DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) -SYSTEM_DOMAIN ?=autoscaler.app-runtime-interfaces.ci.cloudfoundry.org -METIRCSFORWARDER_VM := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') -POSTGRES_ADDRESS := $(DEPLOYMENT_NAME)-postgres.tcp.$(SYSTEM_DOMAIN) -LOG_CACHE_IP := $(shell bosh -d cf vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("log-cache")) | .ips' ) MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) -POSTGRES_EXTERNAL_PORT := $(or $(PR_NUMBER),5432) - - -.PHONY: fetch-config -fetch-config: start-metricsforwarder-vm - # how to define variables in deployment name - mkdir -p build/assets/certs/policy_db build/assets/certs/storedprocedure_db build/assets/certs/syslog_client - - echo "POSTGRES ADDRESS: $(POSTGRES_ADDRESS)" - echo "LOG_CACHE IP: $(LOG_CACHE_IP)" - - @echo "Pulling metricforwarder config from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml build/assets/metricsforwarder.yml - - @echo "Pulling policy db certs from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt build/assets/certs/policy_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt build/assets/certs/policy_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key build/assets/certs/policy_db/. - - @echo "Pulling storeprocedure db certs from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt build/assets/certs/storedprocedure_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt build/assets/certs/storedprocedure_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key build/assets/certs/storedprocedure_db/. - - @echo "Pulling syslog-client certs from $(METIRCSFORWARDER_VM)..." - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt build/assets/certs/syslog_client/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt build/assets/certs/syslog_client/. - bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key build/assets/certs/syslog_client/. - - @echo "Build metricsforwarder config yaml" - cp build/assets/metricsforwarder.yml build/metricsforwarder.yml - - sed -i'' -e 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' build/metricsforwarder.yml - sed -i'' -e 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal:5432|$(POSTGRES_ADDRESS):$(POSTGRES_EXTERNAL_PORT)|g' build/metricsforwarder.yml - - PHONY: set-security-group set-security-group: @@ -49,13 +7,3 @@ set-security-group: cf create-security-group metricsforwarder $(MAKEFILE_DIR)/security-group.json cf bind-security-group metricsforwarder $(ORG) - -PHONY: start-metricsforwarder-vm -start-metricsforwarder-vm: - bosh -d $(DEPLOYMENT_NAME) -n start $(METIRCSFORWARDER_VM) - -PHONY: stop-metricsforwarder-vm -stop-metricsforwarder-vm: - bosh -d $(DEPLOYMENT_NAME) -n stop $(METIRCSFORWARDER_VM) - - diff --git a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go index 511fbe4ede..4b92530cd7 100644 --- a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go +++ b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cred_helper" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" @@ -27,25 +28,22 @@ import ( func main() { var path string + var err error + var conf *config.Config + flag.StringVar(&path, "c", "", "config file") flag.Parse() - if path == "" { - _, _ = fmt.Fprintln(os.Stderr, "missing config file") - os.Exit(1) - } - configFile, err := os.Open(path) + + vcapConfiguration, err := configutil.NewVCAPConfigurationReader() if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to open config file '%s' : %s\n", path, err.Error()) + _, _ = fmt.Fprintf(os.Stdout, "failed to read vcap configuration : %s\n", err.Error()) os.Exit(1) } - - var conf *config.Config - conf, err = config.LoadConfig(configFile) + conf, err = config.LoadConfig(path, vcapConfiguration) if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to read config file '%s' : %s\n", path, err.Error()) + _, _ = fmt.Fprintf(os.Stdout, "failed to load config : %s\n", err.Error()) os.Exit(1) } - _ = configFile.Close() err = conf.Validate() if err != nil { diff --git a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go index a72a650957..8e64152c53 100644 --- a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go +++ b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go @@ -28,20 +28,6 @@ var _ = Describe("Metricsforwarder", func() { }) Describe("MetricsForwarder configuration check", func() { - - Context("with a missing config file", func() { - BeforeEach(func() { - runner.startCheck = "" - runner.configPath = "bogus" - runner.Start() - }) - - It("fails with an error", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to open config file")) - }) - }) - Context("with an invalid config file", func() { BeforeEach(func() { runner.startCheck = "" diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index e8a964f4f0..743ff79b61 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -1,12 +1,12 @@ package config import ( + "errors" "fmt" - "io" "os" - "strconv" "time" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -14,8 +14,16 @@ import ( "gopkg.in/yaml.v3" ) -// ErrInvalidPort is returned when the PORT environment variable is not a valid port number -var ErrInvalidPort = fmt.Errorf("Invalid port number in PORT environment variable") +// There are 3 type of errors that this package can return: +// - ErrReadYaml +// - ErrReadEnvironment +// - ErrReadVCAPEnvironment + +var ( + ErrReadYaml = errors.New("failed to read config file") + ErrReadJson = errors.New("failed to read vcap_services json") + ErrMetricsforwarderConfigNotFound = errors.New("Configuration error: metricsforwarder config service not found") +) const ( DefaultMetronAddress = "127.0.0.1:3458" @@ -70,8 +78,31 @@ type SyslogConfig struct { TLS models.TLSCerts `yaml:"tls"` } -func LoadConfig(reader io.Reader) (*Config, error) { - conf := &Config{ +func decodeYamlFile(filepath string, c *Config) error { + r, err := os.Open(filepath) + + if err != nil { + _, _ = fmt.Fprintf(os.Stdout, "failed to open config file '%s' : %s\n", filepath, err.Error()) + return err + } + + dec := yaml.NewDecoder(r) + dec.KnownFields(true) + err = dec.Decode(c) + + if err != nil { + return fmt.Errorf("%w: %w", ErrReadYaml, err) + } + + defer r.Close() + return nil +} + +func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) { + var conf Config + var err error + + conf = Config{ Server: defaultServerConfig, Logging: defaultLoggingConfig, LoggregatorConfig: LoggregatorConfig{ @@ -87,23 +118,60 @@ func LoadConfig(reader io.Reader) (*Config, error) { }, } - dec := yaml.NewDecoder(reader) - dec.KnownFields(true) - err := dec.Decode(conf) - if err != nil { - return nil, err + if filepath != "" { + err = decodeYamlFile(filepath, &conf) + if err != nil { + return nil, err + } } - if os.Getenv("PORT") != "" { - port := os.Getenv("PORT") - portNumber, err := strconv.Atoi(port) + if vcapReader.IsRunningOnCF() { + conf.Server.Port = vcapReader.GetPort() + + data, err := vcapReader.GetServiceCredentialContent("config", "metricsforwarder") + if err != nil { + return &conf, fmt.Errorf("%w: %w", ErrMetricsforwarderConfigNotFound, err) + } + + err = yaml.Unmarshal(data, &conf) + if err != nil { + return &conf, fmt.Errorf("%w: %w", ErrReadJson, err) + } + + if conf.Db == nil { + conf.Db = make(map[string]db.DatabaseConfig) + } + + currentPolicyDb, ok := conf.Db[db.PolicyDb] + if !ok { + conf.Db[db.PolicyDb] = db.DatabaseConfig{} + } + + currentPolicyDb.URL, err = vcapReader.MaterializeDBFromService(db.PolicyDb) + if err != nil { + return &conf, err + } + conf.Db[db.PolicyDb] = currentPolicyDb + + if conf.CredHelperImpl == "stored_procedure" { + currentStoredProcedureDb, ok := conf.Db[db.StoredProcedureDb] + if !ok { + conf.Db[db.StoredProcedureDb] = db.DatabaseConfig{} + } + currentStoredProcedureDb.URL, err = vcapReader.MaterializeDBFromService(db.StoredProcedureDb) + if err != nil { + return &conf, err + } + conf.Db[db.StoredProcedureDb] = currentStoredProcedureDb + } + + conf.SyslogConfig.TLS, err = vcapReader.MaterializeTLSConfigFromService("syslog-client") if err != nil { - return nil, ErrInvalidPort + return &conf, err } - conf.Server.Port = portNumber } - return conf, nil + return &conf, nil } func (c *Config) UsingSyslog() bool { diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index fe02004a8b..01c00332a9 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -1,260 +1,207 @@ package config_test import ( - "bytes" + "fmt" "os" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" ) -var _ = Describe("Config", func() { +func bytesToFile(b []byte) string { + if len(b) == 0 { + return "" + } + + file, err := os.CreateTemp("", "") + Expect(err).NotTo(HaveOccurred()) + _, err = file.Write(b) + Expect(err).NotTo(HaveOccurred()) + return file.Name() +} +var _ = Describe("Config", func() { var ( - conf *Config - err error - configBytes []byte + conf *Config + err error + configBytes []byte + configFile string + mockVCAPConfigurationReader *fakes.FakeVCAPConfigurationReader ) + BeforeEach(func() { + mockVCAPConfigurationReader = &fakes.FakeVCAPConfigurationReader{} + }) Describe("LoadConfig", func() { - JustBeforeEach(func() { - conf, err = LoadConfig(bytes.NewReader(configBytes)) - }) - - Context("with invalid yaml", func() { - BeforeEach(func() { - configBytes = []byte(` - server: - port: 8081 - logging: - level: info + When("config is read from env", func() { + var expectedDbUrl string -loggregator - metron_address: 127.0.0.1:3457 - tls: - cert_file: "../testcerts/ca.crt" -`) + JustBeforeEach(func() { + mockVCAPConfigurationReader.IsRunningOnCFReturns(true) + mockVCAPConfigurationReader.MaterializeDBFromServiceReturns(expectedDbUrl, nil) + conf, err = LoadConfig("", mockVCAPConfigurationReader) }) - It("returns an error", func() { - Expect(err).To(MatchError(MatchRegexp("yaml: .*"))) - }) - }) + When("vcap PORT is set to a number ", func() { + BeforeEach(func() { + mockVCAPConfigurationReader.GetPortReturns(3333) + }) - Context("with valid yaml", func() { - BeforeEach(func() { - configBytes = []byte(` -server: - port: 8081 -logging: - level: debug -loggregator: - metron_address: 127.0.0.1:3457 - tls: - ca_file: "../testcerts/ca.crt" - cert_file: "../testcerts/client.crt" - key_file: "../testcerts/client.key" -db: - policy_db: - url: "postgres://pqgotest:password@localhost/pqgotest" - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -health: - port: 9999 -cred_helper_impl: default -`) + It("sets env variable over config file", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Server.Port).To(Equal(3333)) + }) }) - It("returns the config", func() { - Expect(conf.Server.Port).To(Equal(8081)) - Expect(conf.Logging.Level).To(Equal("debug")) - Expect(conf.Health.Port).To(Equal(9999)) - Expect(conf.LoggregatorConfig.MetronAddress).To(Equal("127.0.0.1:3457")) - Expect(conf.Db[db.PolicyDb]).To(Equal( - db.DatabaseConfig{ - URL: "postgres://pqgotest:password@localhost/pqgotest", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - })) - Expect(conf.CredHelperImpl).To(Equal("default")) - }) - }) + When("service is empty", func() { + var expectedErr error + BeforeEach(func() { + expectedErr = fmt.Errorf("Configuration error: metricsforwarder config service not found") + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(""), expectedErr) + }) - Context("with partial config", func() { - BeforeEach(func() { - configBytes = []byte(` -loggregator: - tls: - ca_file: "../testcerts/ca.crt" - cert_file: "../testcerts/client.crt" - key_file: "../testcerts/client.key" -db: - policy_db: - url: "postgres://pqgotest:password@localhost/pqgotest" - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -health: - port: 8081 -`) + It("should error with config service not found", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: metricsforwarder config service not found"))) + }) }) - It("returns default values", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(conf.Server.Port).To(Equal(6110)) - Expect(conf.Logging.Level).To(Equal("info")) - Expect(conf.LoggregatorConfig.MetronAddress).To(Equal(DefaultMetronAddress)) - Expect(conf.CacheTTL).To(Equal(DefaultCacheTTL)) - Expect(conf.CacheCleanupInterval).To(Equal(DefaultCacheCleanupInterval)) - Expect(conf.Health.Port).To(Equal(8081)) - }) + When("VCAP_SERVICES has credentials for syslog client", func() { + var expectedTLSConfig models.TLSCerts - When("PORT env variable is set", func() { + BeforeEach(func() { + expectedTLSConfig = models.TLSCerts{ + CertFile: "/tmp/client_cert.sslcert", + KeyFile: "/tmp/client_key.sslkey", + CACertFile: "/tmp/server_ca.sslrootcert", + } - AfterEach(func() { - os.Setenv("PORT", "") + mockVCAPConfigurationReader.MaterializeTLSConfigFromServiceReturns(expectedTLSConfig, nil) }) - When("PORT env is a number", func() { - BeforeEach(func() { - os.Setenv("PORT", "3333") - }) - It("prioritize env variable over config file", func() { - Expect(conf.Server.Port).NotTo(Equal(6110)) - Expect(conf.Server.Port).To(Equal(3333)) - }) + It("loads the syslog config from VCAP_SERVICES", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.SyslogConfig.TLS).To(Equal(expectedTLSConfig)) }) + }) - When("PORT env is not number", func() { - BeforeEach(func() { - os.Setenv("PORT", "NAN") - }) + When("VCAP_SERVICES has relational db service bind to app for policy db", func() { + BeforeEach(func() { + mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("default"), nil) + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 + }) - It("return invalid port error", func() { - Expect(err).To(MatchError(ErrInvalidPort)) - }) + It("loads the db config from VCAP_SERVICES successfully", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Db[db.PolicyDb].URL).To(Equal(expectedDbUrl)) + Expect(mockVCAPConfigurationReader.MaterializeDBFromServiceCallCount()).To(Equal(1)) + actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(0) + Expect(actualDbName).To(Equal(db.PolicyDb)) }) }) - }) - When("it gives a non integer port", func() { - BeforeEach(func() { - configBytes = []byte(` -server: - port: port -`) - }) + When("storedProcedure_db service is provided and cred_helper_impl is stored_procedure", func() { + BeforeEach(func() { + mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("stored_procedure"), nil) + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 + }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) + It("reads the store procedure service from vcap", func() { + Expect(err).NotTo(HaveOccurred()) + _, storeProcedureFound := conf.Db[db.StoredProcedureDb] + Expect(storeProcedureFound).To(BeTrue()) + Expect(conf.Db[db.StoredProcedureDb].URL).To(Equal(expectedDbUrl)) + Expect(mockVCAPConfigurationReader.MaterializeDBFromServiceCallCount()).To(Equal(2)) + actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(1) + Expect(actualDbName).To(Equal(db.StoredProcedureDb)) + }) }) - }) - When("it gives a non integer health server port", func() { - BeforeEach(func() { - configBytes = []byte(` -health: - port: port -`) + When("storedProcedure_db service is provided and cred_helper_impl is default", func() { + BeforeEach(func() { + mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("default"), nil) + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 + }) + + It("ignores the service gracefully", func() { + Expect(err).NotTo(HaveOccurred()) + _, storeProcedureFound := conf.Db[db.StoredProcedureDb] + Expect(storeProcedureFound).To(BeFalse()) + }) }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) + When("VCAP_SERVICES has metricsforwarder config", func() { + BeforeEach(func() { + + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(` { + "cache_cleanup_interval":"10h", + "cache_ttl":"90s", + "cred_helper_impl": "default", + "health":{"password":"health-password","username":"health-user"}, + "logging": { + "level": "debug" + }, + "loggregator": { + "metron_address": "metron-vcap-addrs:3457", + } + }`), nil) // #nosec G101 + }) + + It("loads the config from VCAP_SERVICES", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Logging.Level).To(Equal("debug")) + Expect(conf.LoggregatorConfig.MetronAddress).To(Equal("metron-vcap-addrs:3457")) + Expect(conf.CacheTTL).To(Equal(90 * time.Second)) + }) }) }) - When("it gives a non integer max_open_connections of policydb", func() { - BeforeEach(func() { - configBytes = []byte(` -loggregator: - metron_address: 127.0.0.1:3457 - tls: - ca_file: "../testcerts/ca.crt" - cert_file: "../testcerts/client.crt" - key_file: "../testcerts/client.key" -db: - policy_db: - url: postgres://pqgotest:password@localhost/pqgotest - max_open_connections: NOT-INTEGER-VALUE - max_idle_connections: 5 - connection_max_lifetime: 60s -health: - port: 8081 -`) + When("config is read from file", func() { + JustBeforeEach(func() { + configFile = bytesToFile(configBytes) + conf, err = LoadConfig(configFile, mockVCAPConfigurationReader) }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) + AfterEach(func() { + Expect(os.Remove(configFile)).To(Succeed()) }) - }) - When("it gives a non integer max_idle_connections of policydb", func() { BeforeEach(func() { - configBytes = []byte(` -loggregator: - metron_address: 127.0.0.1:3457 - tls: - ca_file: "../testcerts/ca.crt" - cert_file: "../testcerts/client.crt" - key_file: "../testcerts/client.key" -db: - policy_db: - url: postgres://pqgotest:password@localhost/pqgotest - max_open_connections: 10 - max_idle_connections: NOT-INTEGER-VALUE - connection_max_lifetime: 60s -health: - port: 8081 -`) + mockVCAPConfigurationReader.IsRunningOnCFReturns(false) }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - }) + Context("with invalid yaml", func() { + BeforeEach(func() { + configBytes = []byte(` + server: + port: 8081 + logging: + level: info - When("connection_max_lifetime of policydb is not a time duration", func() { - BeforeEach(func() { - configBytes = []byte(` -loggregator: +loggregator metron_address: 127.0.0.1:3457 tls: - ca_file: "../testcerts/ca.crt" - cert_file: "../testcerts/client.crt" - key_file: "../testcerts/client.key" -db: - policy_db: - url: postgres://pqgotest:password@localhost/pqgotest - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 6K -health: - port: 8081 + cert_file: "../testcerts/ca.crt" `) - }) + }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal .* into time.Duration"))) + It("returns an error", func() { + Expect(err).To(MatchError(MatchRegexp("yaml: .*"))) + }) }) - }) - - When("max_amount of rate_limit is not an interger", func() { - BeforeEach(func() { - configBytes = []byte(` + Context("with valid yaml", func() { + BeforeEach(func() { + configBytes = []byte(` +server: + port: 8081 +logging: + level: debug loggregator: metron_address: 127.0.0.1:3457 tls: @@ -263,52 +210,64 @@ loggregator: key_file: "../testcerts/client.key" db: policy_db: - url: postgres://pqgotest:password@localhost/pqgotest + url: "postgres://pqgotest:password@localhost/pqgotest" max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s health: - port: 8081 -rate_limit: - max_amount: NOT-INTEGER - valid_duration: 1s + port: 9999 +cred_helper_impl: default `) - }) + }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal .* into int"))) - }) - }) + It("returns the config", func() { + Expect(conf.Server.Port).To(Equal(8081)) + Expect(conf.Logging.Level).To(Equal("debug")) + Expect(conf.Health.Port).To(Equal(9999)) + Expect(conf.LoggregatorConfig.MetronAddress).To(Equal("127.0.0.1:3457")) + Expect(conf.Db[db.PolicyDb]).To(Equal( + db.DatabaseConfig{ + URL: "postgres://pqgotest:password@localhost/pqgotest", + MaxOpenConnections: 10, + MaxIdleConnections: 5, + ConnectionMaxLifetime: 60 * time.Second, + })) + Expect(conf.CredHelperImpl).To(Equal("default")) + }) - When("valid_duration of rate_limit is not a time duration", func() { - BeforeEach(func() { - configBytes = []byte(` + }) + Context("with partial config", func() { + BeforeEach(func() { + configBytes = []byte(` loggregator: - metron_address: 127.0.0.1:3457 tls: ca_file: "../testcerts/ca.crt" cert_file: "../testcerts/client.crt" key_file: "../testcerts/client.key" db: policy_db: - url: postgres://pqgotest:password@localhost/pqgotest + url: "postgres://pqgotest:password@localhost/pqgotest" max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s health: port: 8081 -rate_limit: - max_amount: 2 - valid_duration: NOT-TIME-DURATION `) - }) + }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal .* into time.Duration"))) + It("returns default values", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Server.Port).To(Equal(6110)) + Expect(conf.Logging.Level).To(Equal("info")) + Expect(conf.LoggregatorConfig.MetronAddress).To(Equal(DefaultMetronAddress)) + Expect(conf.CacheTTL).To(Equal(DefaultCacheTTL)) + Expect(conf.CacheCleanupInterval).To(Equal(DefaultCacheCleanupInterval)) + Expect(conf.Health.Port).To(Equal(8081)) + }) }) + }) + }) Describe("Validate", func() { @@ -437,8 +396,10 @@ rate_limit: BeforeEach(func() { conf.RateLimit.MaxAmount = 0 }) + It("should err", func() { Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.MaxAmount is equal or less than zero"))) + }) }) @@ -446,9 +407,14 @@ rate_limit: BeforeEach(func() { conf.RateLimit.ValidDuration = 0 * time.Nanosecond }) + It("should err", func() { Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.ValidDuration is equal or less than zero nanosecond"))) }) }) }) }) + +func getVcapConfigWithCredImplementation(credHelperImplementation string) []byte { + return []byte(`{ "cred_helper_impl": "` + credHelperImplementation + `" }`) // #nosec G101 +} diff --git a/src/autoscaler/metricsforwarder/default_config.json b/src/autoscaler/metricsforwarder/default_config.json new file mode 100644 index 0000000000..1493696415 --- /dev/null +++ b/src/autoscaler/metricsforwarder/default_config.json @@ -0,0 +1,34 @@ +{ + "metricsforwarder": { + "cache_cleanup_interval": "6h", + "cache_ttl": "900s", + "cred_helper_impl": "default", + "health": { + "username": "metricsforwarder" + }, + "logging": { + "level": "debug" + }, + "syslog": { + "server_address": "log-cache.service.cf.internal", + "port": 6067 + }, + "db": { + "policy_db": { + "max_open_connections": 100, + "max_idle_connections": 10, + "connection_max_lifetime": "60s" + }, + "storedprocedure_db": { + "max_open_connections": 20, + "max_idle_connections": 10, + "connection_max_lifetime": "60s" + } + }, + "policy_poller_interval": "60s", + "rate_limit": { + "valid_duration": "1s", + "max_amount": 10 + } + } +} diff --git a/src/autoscaler/mta.yaml b/src/autoscaler/mta.yaml index 3ed1cdaf48..3b14d94d3f 100644 --- a/src/autoscaler/mta.yaml +++ b/src/autoscaler/mta.yaml @@ -9,11 +9,34 @@ modules: - name: metricsforwarder type: binary path: build + requires: + - name: config + - name: policydb + - name: syslog-client parameters: memory: 1G disk-quota: 1G - instances: 1 + instances: 2 stack: cflinuxfs4 - command: ./metricsforwarder -c metricsforwarder.yml + command: ./metricsforwarder routes: +resources: +- name: config + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - config + path: metricsforwarder/default_config.json +- name: policydb + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - policy_db + - relational +- name: syslog-client + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - syslog-client +