Merge pull request #2983 from cloudfoundry/appautoscaler-703/make-log…

…cache-default

feat(metric-architecture)!: remove loggregator support and make log-cache the default metric source

.github/ISSUE_TEMPLATE/bug_report.md

@@ -14,7 +14,7 @@ Describe your issue here.
 * Version of app-autoscaler-release
 * Specify the database (postgres/mysql) along-with it's version
 * How did you deploy the release?
-  * Is your manifest based on [app-autoscaler-deployment.yml](https://github.com/cloudfoundry/app-autoscaler-release/blob/main/templates/app-autoscaler-deployment.yml)?
+  * Is your manifest based on [app-autoscaler.yml](https://github.com/cloudfoundry/app-autoscaler-release/blob/main/templates/app-autoscaler.yml)?
   * Which, if any, [ops-files](https://github.com/cloudfoundry/app-autoscaler-release/tree/main/example/operation) did you apply?
 * How did you run the acceptance-tests?
   * If you did not run them, please consider running them and attaching the output as well.

Makefile

@@ -251,7 +251,7 @@ spec-test:
 	bundle install
 	bundle exec rspec
 
-.PHONY: release
+.PHONY: bosh-release
 bosh-release: go-mod-tidy go-mod-vendor scheduler db build/autoscaler-test.tgz
 build/autoscaler-test.tgz:
 	@echo " - building bosh release into build/autoscaler-test.tgz"

ci/autoscaler/pipeline.yml

@@ -1,30 +1,14 @@
 ---
 anchors:
-  app-autoscaler-ops-files: &app-autoscaler-ops-files
-    OPS_FILES: |
-      operations/add-releases.yml
-      operations/instance-identity-cert-from-cf.yml
-      operations/add-postgres-variables.yml
-      operations/enable-nats-tls.yml
-      operations/loggregator-certs-from-cf.yml
-      operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml
-      operations/add-extra-plan.yml
-      operations/set-release-version.yml
-      operations/enable-scheduler-logging.yml
-
   app-autoscaler-ops-files-log-cache-metron: &app-autoscaler-ops-files-log-cache-metron
     OPS_FILES: |
       operations/add-releases.yml
       operations/instance-identity-cert-from-cf.yml
       operations/add-postgres-variables.yml
       operations/enable-nats-tls.yml
-      operations/loggregator-certs-from-cf.yml
       operations/add-extra-plan.yml
       operations/set-release-version.yml
-      operations/configure-log-cache-and-forward-metrics-via-mtls.yml
       operations/enable-metricsforwarder-via-metron-agent.yml
-      operations/remove-metricsserver.yml
-      operations/remove-metricsgateway.yml
       operations/enable-scheduler-logging.yml
 
   app-autoscaler-ops-files-log-cache-syslog: &app-autoscaler-ops-files-log-cache-syslog
@@ -33,13 +17,9 @@ anchors:
       operations/instance-identity-cert-from-cf.yml
       operations/add-postgres-variables.yml
       operations/enable-nats-tls.yml
-      operations/loggregator-certs-from-cf.yml
       operations/add-extra-plan.yml
       operations/set-release-version.yml
-      operations/configure-log-cache-and-forward-metrics-via-mtls.yml
       operations/enable-metricsforwarder-via-syslog-agent.yml
-      operations/remove-metricsserver.yml
-      operations/remove-metricsgateway.yml
       operations/enable-scheduler-logging.yml
 
   app-autoscaler-ops-files-log-cache-syslog-cf: &app-autoscaler-ops-files-log-cache-syslog-cf
@@ -48,13 +28,9 @@ anchors:
       operations/instance-identity-cert-from-cf.yml
       operations/add-postgres-variables.yml
       operations/enable-nats-tls.yml
-      operations/loggregator-certs-from-cf.yml
       operations/add-extra-plan.yml
       operations/set-release-version.yml
-      operations/configure-log-cache-and-forward-metrics-via-mtls.yml
       operations/enable-metricsforwarder-via-syslog-agent.yml
-      operations/remove-metricsserver.yml
-      operations/remove-metricsgateway.yml
       operations/enable-scheduler-logging.yml
       operations/disable-postgres-tls-config.yml
       operations/use-cf-services.yml
@@ -66,16 +42,13 @@ anchors:
       operations/instance-identity-cert-from-cf.yml
       operations/add-postgres-variables.yml
       operations/enable-nats-tls.yml
-      operations/loggregator-certs-from-cf.yml
-      operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml
       operations/postgres-persistent-disk.yml
       operations/add-extra-plan.yml
       operations/set-release-version.yml
 
 groups:
 - name: all
   jobs:
-  - acceptance
   - acceptance-log-cache-metron
   - acceptance-log-cache-syslog
   - acceptance-log-cache-syslog-cf
@@ -92,7 +65,6 @@ groups:
   - update-java
 - name: autoscaler-release
   jobs:
-  - acceptance
   - acceptance-log-cache-metron
   - acceptance-log-cache-syslog
   - draft
@@ -246,59 +218,6 @@ jobs:
     file: ci/ci/autoscaler/tasks/run-integration-tests.yml
     timeout: 45m
 
-- name: acceptance
-  public: true
-  build_logs_to_retain: 100
-  serial: true
-  on_success:
-    task: cleanup
-    file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml
-    params: &acceptance-params
-      DEPLOYMENT_NAME: ((acceptance_deployment_name))
-  plan:
-  - in_parallel:
-    - get: bbl-state
-    - get: app-autoscaler-release
-      passed: [unit-tests, integration-tests]
-      trigger: true
-    - get: ci
-  - task: make-prerelease
-    file: ci/ci/autoscaler/tasks/make/make.yaml
-    params:
-      # ⚠️ Here it is used that make officially guarantees to reach the goals in the provided order.
-      TARGETS: generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler
-    timeout: 15m
-  - task: deploy-autoscaler
-    file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml
-    params:
-      <<: *acceptance-params
-      <<: *app-autoscaler-ops-files
-    timeout: 30m
-  - task: register-broker
-    file: ci/ci/autoscaler/tasks/register-broker.yml
-    params:
-      <<: *acceptance-params
-    timeout: 5m
-  - in_parallel:
-    - task: autoscaler-acceptance-api
-      file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml
-      params:
-        <<: *acceptance-params
-        SUITES: api
-      timeout: 15m
-    - task: autoscaler-acceptance-app
-      file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml
-      params:
-        <<: *acceptance-params
-        SUITES: app
-      timeout: 45m
-    - task: autoscaler-acceptance-broker
-      file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml
-      params:
-        <<: *acceptance-params
-        SUITES: broker
-      timeout: 15m
-
 - name: acceptance-log-cache-metron
   public: true
   build_logs_to_retain: 100
@@ -490,7 +409,7 @@ jobs:
     file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml
     params:
       <<: *performance-env
-      <<: *app-autoscaler-ops-files
+      <<: *app-autoscaler-ops-files-log-cache-syslog
     timeout: 30m
   - task: register-broker
     file: ci/ci/autoscaler/tasks/register-broker.yml
@@ -522,7 +441,6 @@ jobs:
     - get: ci
     - get: app-autoscaler-release
       passed:
-      - acceptance
       - acceptance-log-cache-metron
       - acceptance-log-cache-syslog
       trigger: true

ci/autoscaler/scripts/deploy-autoscaler.sh

@@ -1,7 +1,6 @@
 #! /usr/bin/env bash
 # shellcheck disable=SC2086,SC2034,SC2155
 set -euo pipefail
-
 script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source "${script_dir}/vars.source.sh"
 
@@ -14,13 +13,8 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\
   ${autoscaler_dir}/operations/add-postgres-variables.yml\
   ${autoscaler_dir}/operations/connect_to_postgres_with_certs.yml\
   ${autoscaler_dir}/operations/enable-nats-tls.yml\
-  ${autoscaler_dir}/operations/loggregator-certs-from-cf.yml\
   ${autoscaler_dir}/operations/add-extra-plan.yml\
   ${autoscaler_dir}/operations/set-release-version.yml\
-  ${autoscaler_dir}/operations/configure-log-cache-and-forward-metrics-via-mtls.yml\
-  ${autoscaler_dir}/operations/remove-metricsserver.yml\
-  ${autoscaler_dir}/operations/remove-metricsgateway.yml\
-  ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\
   ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\
   ${autoscaler_dir}/operations/enable-scheduler-logging.yml"}
 
@@ -96,11 +90,13 @@ function create_manifest(){
       -v admin_password="$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q)" \
       -v cf_client_id=autoscaler_client_id \
       -v cf_client_secret=autoscaler_client_secret \
-      -v eventgenerator_uaa_client_id=firehose_exporter \
-      -v eventgenerator_uaa_client_secret="$(credhub get -n /bosh-autoscaler/cf/uaa_clients_firehose_exporter_secret --quiet)"\
-      -v eventgenerator_uaa_skip_ssl_validation=true \
-      -v metricsforwarder_host="${metricsforwarder_host}" \
-    -v skip_ssl_validation=true \
+      -v log_cache_syslog_tls_ca="$(credhub get -n /bosh-autoscaler/cf/log_cache_syslog_tls --key ca --quiet)"\
+      -v syslog_agent_log_cache_tls_certificate="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)"\
+      -v syslog_agent_log_cache_tls_key="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)"\
+      -v metricscollector_ca_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key ca --quiet)"\
+      -v metricscollector_client_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key certificate --quiet)"\
+      -v metricscollector_client_key="$(credhub get -n /bosh-autoscaler/cf/log_cache --key private_key --quiet)"\
+      -v skip_ssl_validation=true \
       > "${tmp_manifest_file}"
 
     # shellcheck disable=SC2064

ci/autoscaler/set-pipeline.sh

@@ -77,7 +77,10 @@ function main(){
       export PIPELINE_NAME="app-autoscaler-release"
       set_pipeline $PIPELINE_NAME
     else
-      export PIPELINE_NAME="app-autoscaler-release-${CURRENT_BRANCH}"
+      local current_branch_without_slashes
+      current_branch_without_slashes="$(echo "${CURRENT_BRANCH}" | sed 's/\//-/g')" # Concourse can't handle slashes in pipeline names
+
+      export PIPELINE_NAME="app-autoscaler-release-${current_branch_without_slashes}"
       set_pipeline "$PIPELINE_NAME"
       pause_jobs "$PIPELINE_NAME"
       unpause_job "$PIPELINE_NAME/set-pipeline"

ci/autoscaler/tasks/deploy-previous-autoscaler.yml

@@ -14,7 +14,6 @@ params:
   # Optional: relative to the root of the release input. The files will be
   # checked to see if they exist, if not they will be excluded from the list
   OPS_FILES: |
-    example/operation/loggregator-certs-from-cf.yml
     example/operation/postgres-persistent-disk.yml
 
 

devbox.lock

@@ -1037,6 +1037,7 @@
     },
     "ruby@latest": {
       "last_modified": "2024-05-22T06:18:38Z",
+      "plugin_version": "0.0.2",
       "resolved": "github:NixOS/nixpkgs/3f316d2a50699a78afe5e77ca486ad553169061e#ruby_3_3",
       "source": "devbox-search",
       "version": "3.3.1",

docs/developer_guide.md

@@ -22,19 +22,3 @@
 - Evaluates app policies rules and generates scaling events based on metrics cache.
 - Manages coolDown threshold for scaling events.
 
-### MetricsServer (To be Deprecated)
-
-![Alt text](./images/metrics_server.svg)
-
-**Responsabilities:**
-
-- For Timer metrics it caches and compiles httpStartStop events to collect a average response time and throughput metric for a configured interval, by default 60 Seconds.
-- Keeps track of current metrics sharded by node.
-- if persistence is enabled, it stores metrics in DB.
-- Provides HTTPServer GET endpoint to retrieve metrics_history by appid/metrictype.
-- Transforms GAUGE envelopes into autoscaler compatible metrics (memoryutil, )
-
-### MetricsGateway (To be Deprecated)
-
-![Alt text](./images/metrics_gateway.svg)
-