feat(CustomMetrics): Allow Different App to Send Custom Metrics

jobs/metricsforwarder/spec

@@ -24,6 +24,10 @@ templates:
   policy_db.crt.erb: config/certs/policy_db/crt
   policy_db.key.erb: config/certs/policy_db/key
 
+  binding_db_ca.crt.erb: config/certs/binding_db/ca.crt
+  binding_db.crt.erb: config/certs/binding_db/crt
+  binding_db.key.erb: config/certs/binding_db/key
+
   storedprocedure_db_ca.crt.erb: config/certs/storedprocedure_db/ca.crt
   storedprocedure_db.crt.erb: config/certs/storedprocedure_db/crt
   storedprocedure_db.key.erb: config/certs/storedprocedure_db/key
@@ -126,6 +130,37 @@ properties:
   autoscaler.policy_db_connection_config.connection_max_lifetime:
     default: 60s
 
+  autoscaler.binding_db.address:
+    description: "IP address on which the bindingdb server will listen"
+    default: "autoscalerpostgres.service.cf.internal"
+  autoscaler.binding_db.databases:
+    description: "The list of databases used in bindingdb database including name"
+  autoscaler.binding_db.db_scheme:
+    description: "Database scheme to be used to access bindingdb"
+    default: postgres
+  autoscaler.binding_db.port:
+    description: "Port on which the bindingdb server will listen"
+  autoscaler.binding_db.roles:
+    description: "The list of database roles used in bindingdb database including name/password"
+  autoscaler.binding_db.tls.ca:
+    default: ''
+    description: 'PEM-encoded ca certificate for TLS database server'
+  autoscaler.binding_db.tls.certificate:
+    default: ''
+    description: 'PEM-encoded certificate for TLS database client'
+  autoscaler.binding_db.tls.private_key:
+    default: ''
+    description: 'PEM-encoded key for TLS database client'
+  autoscaler.binding_db.sslmode:
+    default: disable
+    description: "sslmode to connect to postgres server"
+  autoscaler.binding_db_connection_config.max_open_connections:
+    default: 20
+  autoscaler.binding_db_connection_config.max_idle_connections:
+    default: 10
+  autoscaler.binding_db_connection_config.connection_max_lifetime:
+    default: 60s
+
   autoscaler.storedprocedure_db.address:
     description: "IP address on which the storedproceduredb server will listen"
     default: ""

jobs/metricsforwarder/templates/binding_db.crt.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.binding_db.tls.certificate") do |value| %>
+<%= value %>
+<% end %>

jobs/metricsforwarder/templates/binding_db.key.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.binding_db.tls.private_key") do |value| %>
+<%= value %>
+<% end %>

jobs/metricsforwarder/templates/binding_db_ca.crt.erb

@@ -0,0 +1,3 @@
+<% if_p("autoscaler.binding_db.tls.ca") do |value| %>
+<%= value %>
+<% end %>

jobs/metricsforwarder/templates/metricsforwarder.yml.erb

@@ -38,6 +38,7 @@ end
 ###########################################
   job_name = 'metricsforwarder'
   policy_db_url = build_db_url('policy_db', job_name)
+  binding_db_url = build_db_url('binding_db', job_name)
   if p("autoscaler.storedprocedure_db.address") != ''
     storedprocedure_db_url = build_db_url('storedprocedure_db', job_name)
   end
@@ -80,6 +81,11 @@ db:
     max_open_connections: <%= p("autoscaler.policy_db_connection_config.max_open_connections") %>
     max_idle_connections: <%= p("autoscaler.policy_db_connection_config.max_idle_connections") %>
     connection_max_lifetime: <%= p("autoscaler.policy_db_connection_config.connection_max_lifetime") %>
+  binding_db:
+    url: <%= binding_db_url %>
+    max_open_connections: <%= p("autoscaler.binding_db_connection_config.max_open_connections") %>
+    max_idle_connections: <%= p("autoscaler.binding_db_connection_config.max_idle_connections") %>
+    connection_max_lifetime: <%= p("autoscaler.binding_db_connection_config.connection_max_lifetime") %>
 <% if p("autoscaler.storedprocedure_db.address") != '' %>
   storedprocedure_db:
     url: <%= storedprocedure_db_url %>

scripts/generate_test_certs.sh

@@ -102,7 +102,7 @@ if [[ "$OPENSSL_VERSION" == LibreSSL* ]]; then
 	echo "OpenSSL needs to be used rather than LibreSSL"
 	exit 1
 fi
-# valid certificate
+# valid client certificates
 echo "${depot_path}"
 openssl  req -new -newkey rsa:2048  -nodes -subj "/CN=sap.com/O=SAP SE/OU=organization:AB1234ORG/OU=app:an-app-id/OU=space:AB1234SPACE" -out "${depot_path}"/validmtls_client-1.csr
 openssl x509 -req -in "${depot_path}"/validmtls_client-1.csr -CA "${depot_path}"/valid-mtls-local-ca-1.crt -CAkey "${depot_path}"/valid-mtls-local-ca-1.key -CAcreateserial -out "${depot_path}"/validmtls_client-1.crt -days 365 -sha256

spec/fixtures/metricsforwarder.yml

@@ -14,6 +14,21 @@ autoscaler:
       ca: BEGIN---CA---END
       certificate: BEGIN---CERT---END
       private_key: BEGIN---KEY---END
+  binding_db:
+    address: 10.11.137.101
+    databases:
+      - name: foo
+        tag: default
+    db_scheme: postgres
+    port: 5432
+    roles:
+      - name: foo
+        password: default
+        tag: default
+    tls:
+      ca: BEGIN---CA---END
+      certificate: BEGIN---CERT---END
+      private_key: BEGIN---KEY---END
   cf:
     api: https://api.cf.domain
     auth_endpoint: https://login.cf.domain

spec/jobs/metricsforwarder/metricsforwarder_spec.rb

@@ -114,6 +114,20 @@
           end
         end
       end
+      context "binding_db" do
+        it "includes the ca, cert and key in url when configured" do
+          rendered_template["db"]["binding_db"]["url"].tap do |url|
+            check_if_certs_in_url(url, "binding_db")
+          end
+        end
+
+        it "does not include the ca, cert and key in url when not configured" do
+          properties["autoscaler"]["binding_db"]["tls"] = nil
+          rendered_template["db"]["binding_db"]["url"].tap do |url|
+            check_if_certs_not_in_url(url, "binding_db")
+          end
+        end
+      end
     end
   end
 end

src/acceptance/app/app_suite_test.go

@@ -29,8 +29,12 @@ var (
 	instanceName         string
 	initialInstanceCount int
 
-	appName string
-	appGUID string
+	appToScaleName string
+	appToScaleGUID string
+
+	metricProducerAppName string
+
+	metricProducerAppGUID string
 )
 
 const componentName = "Application Scale Suite"
@@ -59,10 +63,15 @@ func AppAfterEach() {
 	if os.Getenv("SKIP_TEARDOWN") == "true" {
 		fmt.Println("Skipping Teardown...")
 	} else {
-		DebugInfo(cfg, setup, appName)
-		if appName != "" {
-			DeleteService(cfg, instanceName, appName)
-			DeleteTestApp(appName, cfg.DefaultTimeoutDuration())
+		DebugInfo(cfg, setup, appToScaleName)
+		if appToScaleName != "" {
+			DeleteService(cfg, instanceName, appToScaleName)
+			DeleteTestApp(appToScaleName, cfg.DefaultTimeoutDuration())
+		}
+		if metricProducerAppName != "" {
+			DebugInfo(cfg, setup, metricProducerAppName)
+			DeleteService(cfg, instanceName, metricProducerAppName)
+			DeleteTestApp(metricProducerAppName, cfg.DefaultTimeoutDuration())
 		}
 	}
 }

src/acceptance/app/cf_metadata_test.go

@@ -15,19 +15,19 @@ var _ = Describe("AutoScaler CF metadata support", func() {
 	)
 	BeforeEach(func() {
 		policy = GenerateDynamicScaleOutAndInPolicy(1, 2, "test_metric", 500, 500)
-		appName = CreateTestApp(cfg, "labeled-go_app", 1)
-		appGUID, err = GetAppGuid(cfg, appName)
+		appToScaleName = CreateTestApp(cfg, "labeled-go_app", 1)
+		appToScaleGUID, err = GetAppGuid(cfg, appToScaleName)
 		Expect(err).NotTo(HaveOccurred())
-		instanceName = CreatePolicy(cfg, appName, appGUID, policy)
-		StartApp(appName, cfg.CfPushTimeoutDuration())
+		instanceName = CreatePolicy(cfg, appToScaleName, appToScaleGUID, policy)
+		StartApp(appToScaleName, cfg.CfPushTimeoutDuration())
 	})
 	AfterEach(AppAfterEach)
 
 	When("the label app-autoscaler.cloudfoundry.org/disable-autoscaling is set", func() {
 		It("should not scale out", func() {
 			By("Set the label app-autoscaler.cloudfoundry.org/disable-autoscaling to true")
-			SetLabel(cfg, appGUID, "app-autoscaler.cloudfoundry.org/disable-autoscaling", "true")
-			scaleOut := sendMetricToAutoscaler(cfg, appGUID, appName, 550, true)
+			SetLabel(cfg, appToScaleGUID, "app-autoscaler.cloudfoundry.org/disable-autoscaling", "true")
+			scaleOut := sendMetricToAutoscaler(cfg, appToScaleGUID, appToScaleName, 550, true)
 			Consistently(scaleOut).
 				WithTimeout(5 * time.Minute).
 				WithPolling(15 * time.Second).

src/acceptance/app/custom_metric_test.go

@@ -4,75 +4,131 @@ import (
 	"acceptance"
 	"acceptance/config"
 	. "acceptance/helpers"
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("AutoScaler custom metrics policy", func() {
+var _ = Describe("AutoScaler custom metrics", func() {
 	var (
 		policy string
 		err    error
 	)
 	BeforeEach(func() {
-		policy = GenerateDynamicScaleOutAndInPolicy(1, 2, "test_metric", 500, 500)
-		appName = CreateTestApp(cfg, "node-custom-metric", 1)
-		appGUID, err = GetAppGuid(cfg, appName)
+
+		appToScaleName = CreateTestApp(cfg, "go-custom-metric", 1)
+		appToScaleGUID, err = GetAppGuid(cfg, appToScaleName)
 		Expect(err).NotTo(HaveOccurred())
-		instanceName = CreatePolicy(cfg, appName, appGUID, policy)
-		StartApp(appName, cfg.CfPushTimeoutDuration())
+
 	})
 	AfterEach(AppAfterEach)
 
-	// This test will fail if credential-type is set to X509 in autoscaler broker.
-	// Therefore, only mtls connection will be supported for custom metrics in future
-	Context("when scaling by custom metrics", func() {
-		It("should scale out and scale in", Label(acceptance.LabelSmokeTests), func() {
-			By("Scale out to 2 instances")
-			scaleOut := sendMetricToAutoscaler(cfg, appGUID, appName, 550, false)
-			Eventually(scaleOut).
-				WithTimeout(5 * time.Minute).
-				WithPolling(15 * time.Second).
-				Should(Equal(2))
+	Describe("custom metrics policy for same app", func() {
+		BeforeEach(func() {
+			policy = GenerateDynamicScaleOutAndInPolicy(1, 2, "test_metric", 500, 500)
+			instanceName = CreatePolicy(cfg, appToScaleName, appToScaleGUID, policy)
+			StartApp(appToScaleName, cfg.CfPushTimeoutDuration())
+		})
+		// This test will fail if credential-type is set to X509 in autoscaler broker.
+		// Therefore, only mtls connection will be supported for custom metrics in future
+		Context("when scaling by custom metrics", func() {
+			It("should scale out and scale in", Label(acceptance.LabelSmokeTests), func() {
+				By("Scale out to 2 instances")
+				scaleOut := sendMetricToAutoscaler(cfg, appToScaleGUID, appToScaleName, 550, false)
+				Eventually(scaleOut).
+					WithTimeout(5 * time.Minute).
+					WithPolling(15 * time.Second).
+					Should(Equal(2))
 
-			By("Scale in to 1 instances")
-			scaleIn := sendMetricToAutoscaler(cfg, appGUID, appName, 100, false)
-			Eventually(scaleIn).
-				WithTimeout(5 * time.Minute).
-				WithPolling(15 * time.Second).
-				Should(Equal(1))
+				By("Scale in to 1 instances")
+				scaleIn := sendMetricToAutoscaler(cfg, appToScaleGUID, appToScaleName, 100, false)
+				Eventually(scaleIn).
+					WithTimeout(5 * time.Minute).
+					WithPolling(15 * time.Second).
+					Should(Equal(1))
 
+			})
+		})
+
+		Context("when scaling by custom metrics via mtls", func() {
+			It("should scale out and scale in", Label(acceptance.LabelSmokeTests), func() {
+				By("Scale out to 2 instances")
+				scaleOut := sendMetricToAutoscaler(cfg, appToScaleGUID, appToScaleName, 550, true)
+				Eventually(scaleOut).
+					WithTimeout(5 * time.Minute).
+					WithPolling(15 * time.Second).
+					Should(Equal(2))
+
+				By("Scale in to 1 instance")
+				scaleIn := sendMetricToAutoscaler(cfg, appToScaleGUID, appToScaleName, 100, true)
+				Eventually(scaleIn).
+					WithTimeout(5 * time.Minute).
+					WithPolling(15 * time.Second).
+					Should(Equal(1))
+			})
 		})
 	})
 
-	Context("when scaling by custom metrics via mtls", func() {
-		It("should scale out and scale in", Label(acceptance.LabelSmokeTests), func() {
-			By("Scale out to 2 instances")
-			scaleOut := sendMetricToAutoscaler(cfg, appGUID, appName, 550, true)
-			Eventually(scaleOut).
-				WithTimeout(5 * time.Minute).
-				WithPolling(15 * time.Second).
-				Should(Equal(2))
+	Describe("Custom metrics with producer app", func() {
+		BeforeEach(func() {
+			// attach policy to appToScale B
+			policy = GenerateBindingsWithScalingPolicy("bound_app", 1, 2, "test_metric", 100, 500)
+			instanceName = CreatePolicy(cfg, appToScaleName, appToScaleGUID, policy)
+			StartApp(appToScaleName, cfg.CfPushTimeoutDuration())
+
+			// push producer app without policy
+			metricProducerAppName = CreateTestApp(cfg, "go-custom_metric_producer-app", 1)
+			metricProducerAppGUID, err = GetAppGuid(cfg, metricProducerAppName)
+			Expect(err).NotTo(HaveOccurred())
+			err := BindServiceToAppWithPolicy(cfg, metricProducerAppName, instanceName, "")
+			Expect(err).NotTo(HaveOccurred())
+			StartApp(metricProducerAppName, cfg.CfPushTimeoutDuration())
 
-			By("Scale in to 1 instance")
-			scaleIn := sendMetricToAutoscaler(cfg, appGUID, appName, 100, true)
-			Eventually(scaleIn).
-				WithTimeout(5 * time.Minute).
-				WithPolling(15 * time.Second).
-				Should(Equal(1))
+		})
+		Context("producer app A sends custom metrics for appToScale B via mtls", func() {
+			When("policy is attached with the appToScale B with bound_app", func() {
+				BeforeEach(func() {
+					policy = GenerateBindingsWithScalingPolicy("bound_app", 1, 2, "test_metric", 100, 500)
+				})
+				It("should scale out and scale in app B", Label(acceptance.LabelSmokeTests), func() {
+					By(fmt.Sprintf("Scale out %s to 2 instance", appToScaleName))
+					scaleOut := sendMetricToAutoscaler(cfg, appToScaleGUID, metricProducerAppName, 550, true)
+					Eventually(scaleOut).
+						WithTimeout(5 * time.Minute).
+						WithPolling(15 * time.Second).
+						Should(Equal(2))
 
+					By(fmt.Sprintf("Scale in %s to 1 instance", appToScaleName))
+					scaleIn := sendMetricToAutoscaler(cfg, appToScaleGUID, metricProducerAppName, 80, true)
+					Eventually(scaleIn).
+						WithTimeout(5 * time.Minute).
+						WithPolling(15 * time.Second).
+						Should(Equal(1))
+				})
+			})
+		})
+		Context("appToScale B tries to send metrics for producer app A with strategy same_app", func() {
+			BeforeEach(func() {
+				policy = GenerateBindingsWithScalingPolicy("same_app", 1, 2, "test_metric", 100, 500)
+			})
+			It("should not scale producer app", func() {
+				By(fmt.Sprintf("Fail Scale %s ", metricProducerAppName))
+				sendMetricToAutoscaler(cfg, metricProducerAppGUID, appToScaleName, 550, true)
+				WaitForNInstancesRunning(metricProducerAppGUID, 1, 5*time.Second, "expected 1 instance running")
+			})
 		})
 	})
 })
 
-func sendMetricToAutoscaler(config *config.Config, appGUID string, appName string, metricThreshold int, mtls bool) func() (int, error) {
+func sendMetricToAutoscaler(config *config.Config, appToScaleGUID string, metricProducerAppName string, metricThreshold int, mtls bool) func() (int, error) {
 	return func() (int, error) {
 		if mtls {
-			SendMetricMTLS(config, appName, metricThreshold)
+			SendMetricMTLS(config, appToScaleGUID, metricProducerAppName, metricThreshold)
 		} else {
-			SendMetric(config, appName, metricThreshold)
+			SendMetric(config, metricProducerAppName, metricThreshold)
 		}
-		return RunningInstances(appGUID, 5*time.Second)
+		return RunningInstances(appToScaleGUID, 5*time.Second)
 	}
 }