go buildpack

Makefile

@@ -453,5 +453,7 @@ go-get-u: $(addsuffix .go-get-u,$(go_modules))
 	go get -u ./...
 
 
-mta-deploy:
-	@make --directory='./src/autoscaler' mta-deploy
+deploy-apps:
+	echo " - deploying apps"
+	DEBUG="${DEBUG}" ${CI_DIR}/autoscaler/scripts/deploy-apps.sh
+

ci/autoscaler/scripts/deploy-apps.sh

@@ -6,6 +6,10 @@ script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source "${script_dir}/common.sh"
 source "${script_dir}/vars.source.sh"
 
+pushd "${bbl_state_path}" > /dev/null
+  eval "$(bbl print-env)"
+popd > /dev/null
+
 function deploy() {
   log "Deploying autoscaler apps for bosh deployment '${deployment_name}' "
   pushd "${autoscaler_dir}/src/autoscaler" > /dev/null

ci/dockerfiles/autoscaler-tools/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:jammy
+FROM ubuntu:noble
 MAINTAINER autoscaler-team
 
 ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London"
@@ -36,7 +36,6 @@ RUN apt-get update && \
             cf8-cli \
             gnupg \
             gnupg2 \
-            netcat \
             gh  \
             make \
             mysql-client && \

ci/infrastructure/scripts/deploy-multiapps-controller.sh

@@ -23,7 +23,6 @@ function deploy_multiapps_controller() {
   mv multiapps-controller-web-war/*.war .
   pushd multiapps-controller-web-manifest
   cf push -f ./*.yml "${app_name}"
-
   popd
 }
 

devbox.json

@@ -11,7 +11,7 @@
     "bundix":                            "latest",
     "cloudfoundry-cli":                  "8.7.11",
     "coreutils":                         "latest",
-    "credhub-cli":                       "2.9.29",
+    "credhub-cli":                       "2.9.35",
     "delve":                             "latest",
     "direnv":                            "2.34.0",
     "fly":                               "7.10.0",

devbox.lock

@@ -313,51 +313,51 @@
         }
       }
     },
-    "[email protected].29": {
-      "last_modified": "2024-05-29T10:04:41Z",
-      "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#credhub-cli",
+    "[email protected].35": {
+      "last_modified": "2024-08-14T11:41:26Z",
+      "resolved": "github:NixOS/nixpkgs/0cb2fd7c59fed0cd82ef858cbcbdb552b9a33465#credhub-cli",
       "source": "devbox-search",
-      "version": "2.9.29",
+      "version": "2.9.35",
       "systems": {
         "aarch64-darwin": {
           "outputs": [
             {
               "name": "out",
-              "path": "/nix/store/20xi78w44q6wn4847jrrlmxddyjycbrn-credhub-cli-2.9.29",
+              "path": "/nix/store/2xkv0vlafk8zgzjpz3yjagbgyv5kawkn-credhub-cli-2.9.35",
               "default": true
             }
           ],
-          "store_path": "/nix/store/20xi78w44q6wn4847jrrlmxddyjycbrn-credhub-cli-2.9.29"
+          "store_path": "/nix/store/2xkv0vlafk8zgzjpz3yjagbgyv5kawkn-credhub-cli-2.9.35"
         },
         "aarch64-linux": {
           "outputs": [
             {
               "name": "out",
-              "path": "/nix/store/5hv8vj4dhy521m49sc4wwmaz2q319z9a-credhub-cli-2.9.29",
+              "path": "/nix/store/r0kkggv2kbdma8l2n177nl0sw4lwvqr5-credhub-cli-2.9.35",
               "default": true
             }
           ],
-          "store_path": "/nix/store/5hv8vj4dhy521m49sc4wwmaz2q319z9a-credhub-cli-2.9.29"
+          "store_path": "/nix/store/r0kkggv2kbdma8l2n177nl0sw4lwvqr5-credhub-cli-2.9.35"
         },
         "x86_64-darwin": {
           "outputs": [
             {
               "name": "out",
-              "path": "/nix/store/c5d2ggn5c819spywi74pzksbzds9ryym-credhub-cli-2.9.29",
+              "path": "/nix/store/wkgc414d175cv1pa3h257pm264ayk8if-credhub-cli-2.9.35",
               "default": true
             }
           ],
-          "store_path": "/nix/store/c5d2ggn5c819spywi74pzksbzds9ryym-credhub-cli-2.9.29"
+          "store_path": "/nix/store/wkgc414d175cv1pa3h257pm264ayk8if-credhub-cli-2.9.35"
         },
         "x86_64-linux": {
           "outputs": [
             {
               "name": "out",
-              "path": "/nix/store/3y6r6lgrbaq8521hk3zbalf3gz3albcg-credhub-cli-2.9.29",
+              "path": "/nix/store/zkmy2g042p75p3i6qib6s1589qymrg3k-credhub-cli-2.9.35",
               "default": true
             }
           ],
-          "store_path": "/nix/store/3y6r6lgrbaq8521hk3zbalf3gz3albcg-credhub-cli-2.9.29"
+          "store_path": "/nix/store/zkmy2g042p75p3i6qib6s1589qymrg3k-credhub-cli-2.9.35"
         }
       }
     },
@@ -1666,6 +1666,7 @@
     },
     "ruby@latest": {
       "last_modified": "2024-08-14T11:41:26Z",
+      "plugin_version": "0.0.2",
       "resolved": "github:NixOS/nixpkgs/0cb2fd7c59fed0cd82ef858cbcbdb552b9a33465#ruby_3_3",
       "source": "devbox-search",
       "version": "3.3.4",

operations/use-cf-services.yml

@@ -4,11 +4,6 @@
     host: ((metricsforwarder_host))
     mtls_host: ((metricsforwarder_host))
 
-# Set the same port for metricsforwarder and healthenpoint routes
-- type: replace
-  path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health/port
-  value: 6201
-
   ## add router tcp route for postgres
 - type: replace
   path: /instance_groups/name=postgres/jobs/-
@@ -53,3 +48,6 @@
 - type: replace
   path: /variables/name=postgres_client/options/alternative_names/-
   value: ((deployment_name))-postgres.tcp.((system_domain))
+
+- type: remove
+  path: /instance_groups/name=metricsforwarder

packages/metricsforwarder/spec

@@ -8,6 +8,7 @@ files:
 - autoscaler/*
 - autoscaler/vendor/*
 - autoscaler/cf/* # gosub
+- autoscaler/configutil/* # gosub
 - autoscaler/cred_helper/* # gosub
 - autoscaler/db/* # gosub
 - autoscaler/db/sqldb/* # gosub
@@ -44,6 +45,7 @@ files:
 - autoscaler/vendor/github.com/andybalholm/brotli/matchfinder/* # gosub
 - autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub
 - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub
+- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub
 - autoscaler/vendor/github.com/felixge/httpsnoop/* # gosub
 - autoscaler/vendor/github.com/go-logr/logr/* # gosub
 - autoscaler/vendor/github.com/go-logr/logr/funcr/* # gosub
@@ -81,6 +83,7 @@ files:
 - autoscaler/vendor/github.com/klauspost/compress/zlib/* # gosub
 - autoscaler/vendor/github.com/klauspost/compress/zstd/* # gosub
 - autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub
+- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub
 - autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub
 - autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub
 - autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub

src/acceptance/assets/app/go_app/Makefile

@@ -72,8 +72,6 @@ build: ./build/app ./build/manifest.yml
 	CGO_ENABLED='${CGO_ENABLED}' GOOS='linux' GOARCH='amd64' go build -o './build/app'
 	cp './app_manifest.yml' './build/manifest.yml'
 
-
-
 .PHONY: build_tests
 build_tests: $(addprefix build_test-,$(test_dirs))
 

src/autoscaler/Makefile

@@ -12,7 +12,7 @@ PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/'
 DB_HOST ?= localhost
 DBURL ?= "postgres://postgres:postgres@${DB_HOST}/autoscaler?sslmode=disable"
 
-METRICSFORWARDER_APPNAME ?= "metricsforwarder"
+MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
 EXTENSION_FILE := $(shell mktemp)
 
 export GOWORK=off
@@ -154,14 +154,12 @@ clean:
 .PHONY: mta-deploy
 mta-deploy: mta-build build-extension-file
 	$(MAKE) -f metricsforwarder/Makefile set-security-group
-	$(MAKE) -f metricsforwarder/Makefile stop-metricsforwarder-vm
 	@echo "Deploying with extension file: $(EXTENSION_FILE)"
-	@cf deploy mta_archives/*.mtar -f -e $(EXTENSION_FILE)
+	@cf deploy mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE)
 
 build-extension-file:
-	cp example.mtaext $(EXTENSION_FILE);
-	sed -i "s/APP_NAME/$(METRICSFORWARDER_APPNAME)/g" $(EXTENSION_FILE);
-	echo "EXTENSION_FILE: $(EXTENSION_FILE)"
+	echo "extension file at: $(EXTENSION_FILE)"
+	$(MAKEFILE_DIR)/build-extension-file.sh $(EXTENSION_FILE);
 
 mta-logs:
 	rm -rf mta-*
@@ -170,7 +168,6 @@ mta-logs:
 
 .PHONY: mta-build
 mta-build: mta-build-clean cf-build
-	$(MAKE) -f metricsforwarder/Makefile fetch-config
 	mbt build
 
 mta-build-clean:

src/autoscaler/build-extension-file.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2155,SC2034,SC2086
+
+set -e
+
+if [ -z "$1" ]; then
+  echo "extension file path not provided"
+  exit 1
+else
+  extension_file_path=$1
+fi
+
+if [ -z "${DEPLOYMENT_NAME}" ]; then
+  echo "DEPLOYMENT_NAME is not set"
+  exit 1
+fi
+
+if [ -z "${PR_NUMBER}" ]; then
+  echo "PR_NUMBER is not set"
+  exit 1
+fi
+
+export SYSTEM_DOMAIN="autoscaler.app-runtime-interfaces.ci.cloudfoundry.org"
+export POSTGRES_ADDRESS="${DEPLOYMENT_NAME}-postgres.tcp.${SYSTEM_DOMAIN}"
+export POSTGRES_EXTERNAL_PORT="${PR_NUMBER:-5432}"
+
+export METRICSFORWARDER_HEALTH_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_metricsforwarder_health_password --quiet)"
+export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}"
+
+export POLICY_DB_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/database_password --quiet)"
+export POLICY_DB_SERVER_CA="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key ca --quiet )"
+export POLICY_DB_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key certificate --quiet)"
+export POLICY_DB_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key private_key --quiet)"
+
+export SYSLOG_CLIENT_CA="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key ca --quiet)"
+export SYSLOG_CLIENT_CERT="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)"
+export SYSLOG_CLIENT_KEY="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)"
+
+cat <<EOF > "${extension_file_path}"
+ID: development
+extends: com.github.cloudfoundry.app-autoscaler-release
+version: 1.0.0
+_schema-version: 3.3.0
+
+modules:
+  - name: metricsforwarder
+    parameters:
+      routes:
+      - route: ${METRICSFORWARDER_APPNAME}.\${default-domain}
+
+resources:
+- name: config
+  parameters:
+    config:
+      metricsforwarder:
+        health:
+          password: "${METRICSFORWARDER_HEALTH_PASSWORD}"
+- name: policydb
+  parameters:
+    config:
+      uri: "postgres://postgres:${POLICY_DB_PASSWORD}@${POSTGRES_ADDRESS}:${POSTGRES_EXTERNAL_PORT}/autoscaler?application_name=metricsforwarder&sslmode=verify-full"
+      client_cert: "${POLICY_DB_CLIENT_CERT//$'\n'/\\n}"
+      client_key: "${POLICY_DB_CLIENT_KEY//$'\n'/\\n}"
+      server_ca: "${POLICY_DB_SERVER_CA//$'\n'/\\n}"
+- name: syslog-client
+  parameters:
+    config:
+      client_cert: "${SYSLOG_CLIENT_CERT//$'\n'/\\n}"
+      client_key: "${SYSLOG_CLIENT_KEY//$'\n'/\\n}"
+      server_ca: "${SYSLOG_CLIENT_CA//$'\n'/\\n}"
+EOF