diff --git a/.final_builds/packages/golang-1-linux/index.yml b/.final_builds/packages/golang-1-linux/index.yml index 36e31e3962..0eef0e2acf 100644 --- a/.final_builds/packages/golang-1-linux/index.yml +++ b/.final_builds/packages/golang-1-linux/index.yml @@ -7,6 +7,10 @@ builds: version: 126fca7399cba5bc8c1b02e392441f1688daf1586d57d1b3614e314b893a6cc4 blobstore_id: bcfa9608-1490-49d9-5488-b83dc9318735 sha1: sha256:70c3918c28cd74a0a9be561b5eac8b85e8e7de4a72bc66cd321288974937e0be + 15120b8865dac54af5d5a7d1d1bc4d35acd37c114c2398f9edb16323184de634: + version: 15120b8865dac54af5d5a7d1d1bc4d35acd37c114c2398f9edb16323184de634 + blobstore_id: fc0da6d7-04b8-4879-7370-0429a97de826 + sha1: sha256:ba5980b79910c449ce872b087411918e39d4bd55294153c449ded491141f7d72 1bcec40078343f776c33b8ab9bd319709ff1114479ed3e450fade285678a7dc3: version: 1bcec40078343f776c33b8ab9bd319709ff1114479ed3e450fade285678a7dc3 blobstore_id: 6bd50e18-21a8-443f-67d9-5e25a66e4ed9 diff --git a/.github/workflows/acceptance_tests_reusable.yaml b/.github/workflows/acceptance_tests_reusable.yaml index 06945330d0..cbf1e0fa3a 100644 --- a/.github/workflows/acceptance_tests_reusable.yaml +++ b/.github/workflows/acceptance_tests_reusable.yaml @@ -136,6 +136,7 @@ jobs: deployment_cleanup: needs: [ acceptance_tests_verify ] + if: "!contains(github.event.pull_request.labels.*.name, 'skip-cleanup')" name: Deployment cleanup runs-on: ubuntu-latest container: diff --git a/.gitignore b/.gitignore index 57fe57b76f..699df721f9 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ keys/ # easy sharing, see: # !.vscode/extensions.json + +.devbox diff --git a/.tool-versions b/.tool-versions index 41f14f3e17..828f6f16de 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,12 +1,13 @@ act 0.2.62 actionlint 1.6.27 bosh 7.3.1 +credhub 2.9.29 cf 8.7.10 concourse 7.10.0 direnv 2.34.0 gcloud 473.0.0 ginkgo 2.17.3 -golang 1.21.5 +golang 1.22.3 golangci-lint 1.57.2 java temurin-21.0.3+9.0.LTS make 4.4 @@ -14,3 +15,4 @@ maven 3.8.6 ruby 3.3.1 shellcheck 0.10.0 yq 4.43.1 +gum 0.13.0 diff --git a/Makefile b/Makefile index a2948fcd01..aefd25358e 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ clean-scheduler: @rm -rf src/scheduler/src/test/resources/certs clean-certs: @echo " - cleaning test certs" - @rm -f testcerts/* + @rm -f test-certs/* clean-bosh-release: @echo " - cleaning bosh dev releases" @rm -rf dev_releases @@ -231,7 +231,7 @@ lint: $(addprefix lint_,$(go_modules)) rubocop rubocop: @echo " - ruby scripts" @bundle install - @bundle exec rubocop ./spec ./packages + @bundle exec rubocop ${RUBOCOP_OPTS} ./spec ./packages .PHONY: markdownlint markdownlint: markdownlint-cli @@ -305,7 +305,7 @@ mod-download: .PHONY: acceptance.go-mod-vendor autoscaler.go-mod-vendor changelog.go-mod-vendor \ changeloglockcleander.go-mod-vendor -go-mod-vendor: acceptance.go-mod-vendor autoscaler.go-mod-vendor changelog.go-mod-vendor \ +go-mod-vendor: clean-vendor acceptance.go-mod-vendor autoscaler.go-mod-vendor changelog.go-mod-vendor \ changeloglockcleander.go-mod-vendor acceptance.go-mod-vendor: make --directory='${go-acceptance-dir}' go-mod-vendor diff --git a/ci/Makefile b/ci/Makefile new file mode 100644 index 0000000000..3a5c795c41 --- /dev/null +++ b/ci/Makefile @@ -0,0 +1,11 @@ + +.PHONY: set-autoscaler-pipeline + +set-autoscaler-pipeline: + @./autoscaler/set-pipeline.sh + +.PHONY: unpause-pipeline + +unpause-pipeline: + @./scripts/unpause-pipeline.sh + diff --git a/ci/README.md b/ci/README.md index d20f3ca376..1f56db8493 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,14 +1,16 @@ # app-autoscaler-ci This repository provides all public scripts and pipeline deployments used -by the app autoscaler team. The public pipeline is hosted at: . + +By the app autoscaler team. The public pipeline is hosted at: . To reproduce this pipeline, you can use your own private configuration files for the `pipeline.yml` files as described below. ## Autoscaler -This directory contains the concourse `pipeline.yml` for the autoscaler [pipeline](https://bosh.ci.cloudfoundry.org/pipelines/app-autoscaler) +This directory contains the concourse `pipeline.yml` for the autoscaler [pipeline](https://concourse.app-runtime-interfaces.ci.cloudfoundry.org/teams/app-autoscaler/pipelines/app-autoscaler-release) and all of the associated scripts. To use this manifest, you need to provide a private configuration file + for all of the template parameters. NOTE: If you are recreating this pipeline, for personal use and do not have authority to update @@ -28,10 +30,17 @@ This directory contains the terragrunt managed stacks of resouces in account app __Setup__ ``` -fly --target autoscaler login --team-name app-autoscaler --concourse-url https://bosh.ci.cloudfoundry.org/ -push autoscaler -./set-pipeline.sh -popd +fly --target app-autoscaler-release login --team-name app-autoscaler --concourse-url https://concourse.app-runtime-interfaces.ci.cloudfoundry.org +make set-autoscaler-pipeline +``` + +## Unpause all/some jobs for any set pipeline + +_Unpause_ + +``` +# You will be prompted to select the specific jobs you want to unpause. +make unpause-pipeline ``` ## Prometheus diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index d7bb03d2df..d93fc3973b 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -12,7 +12,7 @@ anchors: operations/set-release-version.yml operations/enable-scheduler-logging.yml - app-autoscaler-ops-files-log-cache: &app-autoscaler-ops-files-log-cache + app-autoscaler-ops-files-log-cache-metron: &app-autoscaler-ops-files-log-cache-metron OPS_FILES: | operations/add-releases.yml operations/instance-identity-cert-from-cf.yml @@ -22,6 +22,22 @@ anchors: operations/add-extra-plan.yml operations/set-release-version.yml operations/configure-log-cache-and-forward-metrics-via-mtls.yml + operations/enable-metricsforwarder-via-metron-agent.yml + operations/remove-metricsserver.yml + operations/remove-metricsgateway.yml + operations/enable-scheduler-logging.yml + + app-autoscaler-ops-files-log-cache-syslog: &app-autoscaler-ops-files-log-cache-syslog + OPS_FILES: | + operations/add-releases.yml + operations/instance-identity-cert-from-cf.yml + operations/add-postgres-variables.yml + operations/enable-nats-tls.yml + operations/loggregator-certs-from-cf.yml + operations/add-extra-plan.yml + operations/set-release-version.yml + operations/configure-log-cache-and-forward-metrics-via-mtls.yml + operations/enable-metricsforwarder-via-syslog-agent.yml operations/remove-metricsserver.yml operations/remove-metricsgateway.yml operations/enable-scheduler-logging.yml @@ -42,7 +58,8 @@ groups: - name: all jobs: - acceptance - - acceptance-log-cache + - acceptance-log-cache-metron + - acceptance-log-cache-syslog - cleanup-autoscaler-deployments - fetch-latest-stemcell - draft @@ -57,7 +74,8 @@ groups: - name: autoscaler-release jobs: - acceptance - - acceptance-log-cache + - acceptance-log-cache-metron + - acceptance-log-cache-syslog - draft - integration-tests - release @@ -177,7 +195,8 @@ jobs: vars: branch_name: ((branch_name)) acceptance_deployment_name: ((acceptance_deployment_name)) - logcache_acceptance_deployment_name: ((logcache_acceptance_deployment_name)) + acceptance_deployment_name_logcache_metron: ((acceptance_deployment_name_logcache_metron)) + acceptance_deployment_name_logcache_syslog: ((acceptance_deployment_name_logcache_syslog)) - name: unit-tests public: true @@ -260,7 +279,60 @@ jobs: SUITES: broker timeout: 15m -- name: acceptance-log-cache +- name: acceptance-log-cache-metron + public: true + build_logs_to_retain: 100 + serial: true + on_success: + task: cleanup + file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml + params: &acceptance-log-cache-params + DEPLOYMENT_NAME: ((acceptance_deployment_name_logcache_metron)) + plan: + - in_parallel: + - get: bbl-state + - get: app-autoscaler-release + passed: [unit-tests, integration-tests] + trigger: true + - get: ci + - task: make-prerelease + file: ci/ci/autoscaler/tasks/make/make.yaml + params: + # ⚠️ Here it is used that make officially guarantees to reach the goals in the provided order. + TARGETS: generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler + timeout: 15m + - task: deploy-autoscaler + file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml + params: + <<: *acceptance-log-cache-params + <<: *app-autoscaler-ops-files-log-cache-metron + timeout: 30m + - task: register-broker + file: ci/ci/autoscaler/tasks/register-broker.yml + params: + <<: *acceptance-log-cache-params + timeout: 5m + - in_parallel: + - task: autoscaler-acceptance-api + file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml + params: + <<: *acceptance-log-cache-params + SUITES: api + timeout: 15m + - task: autoscaler-acceptance-app + file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml + params: + <<: *acceptance-log-cache-params + SUITES: app + timeout: 45m + - task: autoscaler-acceptance-broker + file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml + params: + <<: *acceptance-log-cache-params + SUITES: broker + timeout: 15m + +- name: acceptance-log-cache-syslog public: true build_logs_to_retain: 100 serial: true @@ -268,7 +340,7 @@ jobs: task: cleanup file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml params: &acceptance-log-cache-params - DEPLOYMENT_NAME: ((logcache_acceptance_deployment_name)) + DEPLOYMENT_NAME: ((acceptance_deployment_name_logcache_syslog)) plan: - in_parallel: - get: bbl-state @@ -286,7 +358,7 @@ jobs: file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml params: <<: *acceptance-log-cache-params - <<: *app-autoscaler-ops-files-log-cache + <<: *app-autoscaler-ops-files-log-cache-syslog timeout: 30m - task: register-broker file: ci/ci/autoscaler/tasks/register-broker.yml @@ -374,7 +446,8 @@ jobs: - get: app-autoscaler-release passed: - acceptance - - acceptance-log-cache + - acceptance-log-cache-metron + - acceptance-log-cache-syslog trigger: true - get: previous-stable-release - task: deploy-previous-stable-release diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index d8db51bc3f..d8e0351339 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -20,6 +20,7 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/remove-metricsserver.yml\ ${autoscaler_dir}/operations/remove-metricsgateway.yml\ ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ + ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} diff --git a/ci/autoscaler/set-pipeline.sh b/ci/autoscaler/set-pipeline.sh index d8d7f63086..039f921631 100755 --- a/ci/autoscaler/set-pipeline.sh +++ b/ci/autoscaler/set-pipeline.sh @@ -11,7 +11,7 @@ if "$(which gh)" > /dev/null; then echo "gh cli found" gh --version else - echo "no gh cli found!" + echo "no gh cli found!" exit 1 fi @@ -31,10 +31,12 @@ function set_pipeline(){ add_var branch_name "${CURRENT_BRANCH}" if [[ -z $PR_NUMBER ]]; then add_var acceptance_deployment_name "acceptance" - add_var logcache_acceptance_deployment_name "acceptance-lc" + add_var acceptance_deployment_name_logcache_metron "acceptance-lc" + add_var acceptance_deployment_name_logcache_syslog "acceptance-lc-syslog" else add_var acceptance_deployment_name "${PR_NUMBER}-acceptance" - add_var logcache_acceptance_deployment_name "${PR_NUMBER}-acceptance-lc" + add_var acceptance_deployment_name_logcache_metron "${PR_NUMBER}-acceptance-lc" + add_var acceptance_deployment_name_logcache_syslog "${PR_NUMBER}-acceptance-lc-sl" fi # shellcheck disable=SC2086 diff --git a/ci/scripts/unpause-pipeline.sh b/ci/scripts/unpause-pipeline.sh new file mode 100755 index 0000000000..2332ef7459 --- /dev/null +++ b/ci/scripts/unpause-pipeline.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2086 +# + +target="app-autoscaler-release" + +function unpause-job(){ + pipeline="$1" + jobs=$(fly -t "$target" jobs -p "$pipeline" --json | jq ".[] | select(.paused==true) | .name" -r) + selected_job=$(gum choose --no-limit $jobs --header "Select jobs to unpause from pipeline $pipeline") + + + if [[ -z "$selected_job" ]]; then + echo "No job selected to unpause" + return + fi + + for j in $selected_job; do + fly -t "$target" unpause-job -j "$pipeline/$j" + done +} + +function unpause-pipeline(){ + payload=$(fly -t "$target" pipelines --json) + + pipelines=$(echo "$payload" | jq ".[] |.name" -r | sort) + # ignore shellcheck warning + pipeline=$(gum choose $pipelines "all") + + if [[ "$pipeline" == "all" ]]; then + for p in $pipelines; do + fly -t "$target" unpause-pipeline -p "$p" + unpause-job "$p" + done + else + fly -t "$target" unpause-pipeline -p "$pipeline" + unpause-job "$pipeline" + fi + +} + +function check-login(){ + if ! fly -t "$target" status; then + echo + echo "fly -t $target login" + echo + exit 1 + fi +} + +check-login +unpause-pipeline "${@:-}" diff --git a/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod b/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod index 697d476ebc..d7146d419e 100644 --- a/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod +++ b/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod @@ -1,3 +1,3 @@ module apitester -go 1.21.5 +go 1.22.3 diff --git a/devbox.json b/devbox.json index b228997ceb..6bd1043743 100644 --- a/devbox.json +++ b/devbox.json @@ -4,7 +4,6 @@ "path:local-flake#app-autoscaler-cli-plugin", "path:local-flake#log-cache-cli-plugin", "path:local-flake#uaac", - "credhub-cli@latest", "delve@latest", "gh@latest", "go-tools@latest", @@ -29,13 +28,15 @@ "cloudfoundry-cli@8.7.10", "shellcheck@0.10.0", "act@0.2.62", - "go@1.21.5", "ruby@3.3.1", "ginkgo@2.17.3", - "google-cloud-sdk@latest", - "temurin-bin-17@latest", "bundix@latest", - "oha@latest" + "oha@latest", + "credhub-cli@2.9.29", + "gum@0.13.0", + "google-cloud-sdk@latest", + "go@1.22.3", + "temurin-bin-17@latest" ], "shell": { "init_hook": [ diff --git a/devbox.lock b/devbox.lock index fc598d1a4e..cc15f844b0 100644 --- a/devbox.lock +++ b/devbox.lock @@ -213,23 +213,51 @@ } } }, - "credhub-cli@latest": { - "last_modified": "2024-02-24T23:06:34Z", - "resolved": "github:NixOS/nixpkgs/9a9dae8f6319600fa9aebde37f340975cab4b8c0#credhub-cli", + "credhub-cli@2.9.29": { + "last_modified": "2024-05-12T16:19:40Z", + "resolved": "github:NixOS/nixpkgs/3281bec7174f679eabf584591e75979a258d8c40#credhub-cli", "source": "devbox-search", - "version": "2.9.27", + "version": "2.9.29", "systems": { "aarch64-darwin": { - "store_path": "/nix/store/7yp4g9gxairkvf2jdx0a22iz6assfnvc-credhub-cli-2.9.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/nmp6m8m0yf3fhm64c0w0mb5k0gwp329w-credhub-cli-2.9.29", + "default": true + } + ], + "store_path": "/nix/store/nmp6m8m0yf3fhm64c0w0mb5k0gwp329w-credhub-cli-2.9.29" }, "aarch64-linux": { - "store_path": "/nix/store/2aaxi9nxnkr7d3mzqadb8yab2d7p4q3s-credhub-cli-2.9.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/8ppsf1ghqdlpqjhsnm7sp1s6j8lfyavf-credhub-cli-2.9.29", + "default": true + } + ], + "store_path": "/nix/store/8ppsf1ghqdlpqjhsnm7sp1s6j8lfyavf-credhub-cli-2.9.29" }, "x86_64-darwin": { - "store_path": "/nix/store/jb12m850bbkv2w2hfgxranyw8zdzvy0w-credhub-cli-2.9.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/sb7qgq8c1xdcnfizzc6z89h695ddc672-credhub-cli-2.9.29", + "default": true + } + ], + "store_path": "/nix/store/sb7qgq8c1xdcnfizzc6z89h695ddc672-credhub-cli-2.9.29" }, "x86_64-linux": { - "store_path": "/nix/store/gcqisc7987hwfxhqgrx5lqdprhdpj1xw-credhub-cli-2.9.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/s7k85s4x2c3n5asvccbrjr56449mdlw8-credhub-cli-2.9.29", + "default": true + } + ], + "store_path": "/nix/store/s7k85s4x2c3n5asvccbrjr56449mdlw8-credhub-cli-2.9.29" } } }, @@ -429,51 +457,51 @@ } } }, - "go@1.21.5": { - "last_modified": "2024-01-14T03:55:27Z", - "resolved": "github:NixOS/nixpkgs/dd5621df6dcb90122b50da5ec31c411a0de3e538#go", + "go@1.22.3": { + "last_modified": "2024-05-22T06:18:38Z", + "resolved": "github:NixOS/nixpkgs/3f316d2a50699a78afe5e77ca486ad553169061e#go", "source": "devbox-search", - "version": "1.21.5", + "version": "1.22.3", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5", + "path": "/nix/store/i04a1a6qgxhjw6c0ld2b3x1v815sbxjc-go-1.22.3", "default": true } ], - "store_path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5" + "store_path": "/nix/store/i04a1a6qgxhjw6c0ld2b3x1v815sbxjc-go-1.22.3" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5", + "path": "/nix/store/d68f2iblysnl0r4qcfdacmdpmvvy86kf-go-1.22.3", "default": true } ], - "store_path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5" + "store_path": "/nix/store/d68f2iblysnl0r4qcfdacmdpmvvy86kf-go-1.22.3" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5", + "path": "/nix/store/1p6vr83cgyfwm8517jhfmf6lypzhy3q2-go-1.22.3", "default": true } ], - "store_path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5" + "store_path": "/nix/store/1p6vr83cgyfwm8517jhfmf6lypzhy3q2-go-1.22.3" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5", + "path": "/nix/store/00mg4vlhzmm7gi9bd5v5ydjlgrywpc3n-go-1.22.3", "default": true } ], - "store_path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5" + "store_path": "/nix/store/00mg4vlhzmm7gi9bd5v5ydjlgrywpc3n-go-1.22.3" } } }, @@ -593,6 +621,54 @@ } } }, + "gum@0.13.0": { + "last_modified": "2024-04-19T21:36:04Z", + "resolved": "github:NixOS/nixpkgs/92d295f588631b0db2da509f381b4fb1e74173c5#gum", + "source": "devbox-search", + "version": "0.13.0", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/rv7y82ylid613qkg947a1icajp41jhr1-gum-0.13.0", + "default": true + } + ], + "store_path": "/nix/store/rv7y82ylid613qkg947a1icajp41jhr1-gum-0.13.0" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/mcbsnqvcz9q3j6mnbpwy5brid0zgzll6-gum-0.13.0", + "default": true + } + ], + "store_path": "/nix/store/mcbsnqvcz9q3j6mnbpwy5brid0zgzll6-gum-0.13.0" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/vf21qsvvsmfs9diw5bk9b32iavp9pdfx-gum-0.13.0", + "default": true + } + ], + "store_path": "/nix/store/vf21qsvvsmfs9diw5bk9b32iavp9pdfx-gum-0.13.0" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/lmc0z0s5zlq5vvyv5y0cxhi727dzvq02-gum-0.13.0", + "default": true + } + ], + "store_path": "/nix/store/lmc0z0s5zlq5vvyv5y0cxhi727dzvq02-gum-0.13.0" + } + } + }, "jq@latest": { "last_modified": "2024-04-21T11:44:28-04:00", "resolved": "github:NixOS/nixpkgs/69ee1d82f1fa4c70a3dc9a64111e7eef3b8e4527#jq", @@ -1150,11 +1226,11 @@ "outputs": [ { "name": "out", - "path": "/nix/store/5wm01i1h0i2c4sknbh5mwkiv97k75f5n-temurin-bin-17.0.9", + "path": "/nix/store/191lc0lvhk6d34yn5l35s0bphd3bsf7n-temurin-bin-17.0.9", "default": true } ], - "store_path": "/nix/store/5wm01i1h0i2c4sknbh5mwkiv97k75f5n-temurin-bin-17.0.9" + "store_path": "/nix/store/191lc0lvhk6d34yn5l35s0bphd3bsf7n-temurin-bin-17.0.9" }, "aarch64-linux": { "outputs": [ @@ -1170,11 +1246,11 @@ "outputs": [ { "name": "out", - "path": "/nix/store/nb6883hq2789nyb68j3d1m3rmbh4j0qf-temurin-bin-17.0.9", + "path": "/nix/store/96ijp2sqyx9bm1sra9f6qdrfw4gg94qx-temurin-bin-17.0.9", "default": true } ], - "store_path": "/nix/store/nb6883hq2789nyb68j3d1m3rmbh4j0qf-temurin-bin-17.0.9" + "store_path": "/nix/store/96ijp2sqyx9bm1sra9f6qdrfw4gg94qx-temurin-bin-17.0.9" }, "x86_64-linux": { "outputs": [ diff --git a/go.work b/go.work index 6dd2e0c8ba..1fd73397d3 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.21.5 +go 1.22.3 use ( ./src/acceptance diff --git a/jobs/metricsforwarder/spec b/jobs/metricsforwarder/spec index 716824f0a9..d8feaf1be3 100644 --- a/jobs/metricsforwarder/spec +++ b/jobs/metricsforwarder/spec @@ -16,6 +16,10 @@ templates: metron_client.crt.erb: config/certs/metron_client/client.crt metron_client.key.erb: config/certs/metron_client/client.key + syslog_client_ca.crt.erb: config/certs/syslog_client/ca.crt + syslog_client.crt.erb: config/certs/syslog_client/client.crt + syslog_client.key.erb: config/certs/syslog_client/client.key + policy_db_ca.crt.erb: config/certs/policy_db/ca.crt policy_db.crt.erb: config/certs/policy_db/crt policy_db.key.erb: config/certs/policy_db/key @@ -43,6 +47,20 @@ properties: autoscaler.metricsforwarder.server.server_key: description: "PEM-encoded server key for the metricsforwarder server" + autoscaler.metricsforwarder.syslog.server_address: + description: "host where the syslog server is running" + + autoscaler.metricsforwarder.syslog.port: + description: "port where the syslog server is running" + default: "6067" + + autoscaler.metricsforwarder.syslog.tls.cert: + description: "PEM-encoded tls client certificate to connect to syslog server" + autoscaler.metricsforwarder.syslog.tls.key: + description: "PEM-encoded tls client key to connect to syslog server" + autoscaler.metricsforwarder.syslog.tls.ca_cert: + description: "PEM-encoded ca certificate of syslog server" + autoscaler.metricsforwarder.loggregator.metron_address: description: "IP address and port where the metron agent is running" default: "127.0.0.1:3458" diff --git a/jobs/metricsforwarder/templates/metricsforwarder.yml.erb b/jobs/metricsforwarder/templates/metricsforwarder.yml.erb index 945dbd76ca..e518676955 100644 --- a/jobs/metricsforwarder/templates/metricsforwarder.yml.erb +++ b/jobs/metricsforwarder/templates/metricsforwarder.yml.erb @@ -55,12 +55,25 @@ server: logging: level: <%= p("autoscaler.metricsforwarder.logging.level") %> + + +<% if_p("autoscaler.metricsforwarder.syslog.server_address") do |prop| %> +syslog: + server_address: <%= p("autoscaler.metricsforwarder.syslog.server_address") %> + port: <%= p("autoscaler.metricsforwarder.syslog.port") %> + tls: + key_file: /var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key + cert_file: /var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt + ca_file: /var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt +<% end.else do %> loggregator: metron_address: <%= p("autoscaler.metricsforwarder.loggregator.metron_address") %> tls: key_file: /var/vcap/jobs/metricsforwarder/config/certs/metron_client/client.key cert_file: /var/vcap/jobs/metricsforwarder/config/certs/metron_client/client.crt ca_file: /var/vcap/jobs/metricsforwarder/config/certs/metron_client/ca.crt +<% end %> + db: policy_db: url: <%= policy_db_url %> diff --git a/jobs/metricsforwarder/templates/syslog_client.crt.erb b/jobs/metricsforwarder/templates/syslog_client.crt.erb new file mode 100644 index 0000000000..b7440691a9 --- /dev/null +++ b/jobs/metricsforwarder/templates/syslog_client.crt.erb @@ -0,0 +1,4 @@ +<% if_p("autoscaler.metricsforwarder.syslog.tls.cert") do |value| %> +<%= value %> +<% end %> + diff --git a/jobs/metricsforwarder/templates/syslog_client.key.erb b/jobs/metricsforwarder/templates/syslog_client.key.erb new file mode 100644 index 0000000000..4afc5d70ba --- /dev/null +++ b/jobs/metricsforwarder/templates/syslog_client.key.erb @@ -0,0 +1,3 @@ +<% if_p("autoscaler.metricsforwarder.syslog.tls.key") do |value| %> +<%= value %> +<% end %> diff --git a/jobs/metricsforwarder/templates/syslog_client_ca.crt.erb b/jobs/metricsforwarder/templates/syslog_client_ca.crt.erb new file mode 100644 index 0000000000..17ba5df743 --- /dev/null +++ b/jobs/metricsforwarder/templates/syslog_client_ca.crt.erb @@ -0,0 +1,4 @@ +<% if_p("autoscaler.metricsforwarder.syslog.tls.ca_cert") do |value| %> +<%= value %> +<% end %> + diff --git a/operations/enable-metricsforwarder-via-metron-agent.yml b/operations/enable-metricsforwarder-via-metron-agent.yml new file mode 100644 index 0000000000..96811277f5 --- /dev/null +++ b/operations/enable-metricsforwarder-via-metron-agent.yml @@ -0,0 +1,7 @@ +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/loggregator + value: + tls: # connection to syslog-agent + ca_cert: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.ca)) + cert: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.certificate)) + key: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.private_key)) diff --git a/operations/enable-metricsforwarder-via-syslog-agent.yml b/operations/enable-metricsforwarder-via-syslog-agent.yml new file mode 100644 index 0000000000..5ca1cb4b30 --- /dev/null +++ b/operations/enable-metricsforwarder-via-syslog-agent.yml @@ -0,0 +1,9 @@ +#t configure metricsforwarder +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/syslog? + value: + server_address: log-cache.service.cf.internal + tls: + ca_cert: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.ca)) + cert: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.certificate)) + key: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.private_key)) diff --git a/packages/golang-1-linux/spec.lock b/packages/golang-1-linux/spec.lock index 6a23a78647..4dc39f56c9 100644 --- a/packages/golang-1-linux/spec.lock +++ b/packages/golang-1-linux/spec.lock @@ -1,2 +1,2 @@ name: golang-1-linux -fingerprint: a53d60ed79d278964faa9bc17836ed3bc5f084c0f3d41110c8a6e0b2de5f2dc0 +fingerprint: 15120b8865dac54af5d5a7d1d1bc4d35acd37c114c2398f9edb16323184de634 diff --git a/packages/golang-1-linux/vendored-commit b/packages/golang-1-linux/vendored-commit index 71e8079258..2ef63f4d2d 100644 --- a/packages/golang-1-linux/vendored-commit +++ b/packages/golang-1-linux/vendored-commit @@ -1 +1 @@ -5dcfbfc3789c847c524993c9c4fbeb085c0f87ec +c35d64e785152d04e7aea49b2ee4b4988c33958d diff --git a/packages/golang-1-linux/version b/packages/golang-1-linux/version index d6a49eb373..98f7da137c 100644 --- a/packages/golang-1-linux/version +++ b/packages/golang-1-linux/version @@ -1 +1 @@ -1.21.5 \ No newline at end of file +1.22.3 \ No newline at end of file diff --git a/packages/metricsforwarder/spec b/packages/metricsforwarder/spec index 87d48e1347..17665c5216 100644 --- a/packages/metricsforwarder/spec +++ b/packages/metricsforwarder/spec @@ -29,12 +29,19 @@ files: - autoscaler/vendor/code.cloudfoundry.org/clock/* # gosub - autoscaler/vendor/code.cloudfoundry.org/go-diodes/* # gosub - autoscaler/vendor/code.cloudfoundry.org/go-loggregator/v9/* # gosub +- autoscaler/vendor/code.cloudfoundry.org/go-loggregator/v9/rfc5424/* # gosub - autoscaler/vendor/code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2/* # gosub +- autoscaler/vendor/code.cloudfoundry.org/go-metric-registry/* # gosub - autoscaler/vendor/code.cloudfoundry.org/lager/v3/* # gosub - autoscaler/vendor/code.cloudfoundry.org/lager/v3/internal/truncate/* # gosub +- autoscaler/vendor/code.cloudfoundry.org/loggregator-agent-release/src/pkg/diodes/* # gosub +- autoscaler/vendor/code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/* # gosub +- autoscaler/vendor/code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/syslog/* # gosub - autoscaler/vendor/code.cloudfoundry.org/tlsconfig/* # gosub - autoscaler/vendor/filippo.io/edwards25519/* # gosub - autoscaler/vendor/filippo.io/edwards25519/field/* # gosub +- autoscaler/vendor/github.com/andybalholm/brotli/* # gosub +- autoscaler/vendor/github.com/andybalholm/brotli/matchfinder/* # gosub - autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub - autoscaler/vendor/github.com/felixge/httpsnoop/* # gosub @@ -64,6 +71,16 @@ files: - autoscaler/vendor/github.com/jackc/puddle/v2/internal/genstack/* # gosub - autoscaler/vendor/github.com/jmoiron/sqlx/* # gosub - autoscaler/vendor/github.com/jmoiron/sqlx/reflectx/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/flate/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/fse/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/gzip/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/huff0/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/internal/cpuinfo/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/internal/snapref/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/zlib/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/zstd/* # gosub +- autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub - autoscaler/vendor/github.com/nu7hatch/gouuid/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub @@ -85,6 +102,10 @@ files: - autoscaler/vendor/github.com/tedsuo/ifrit/sigmon/* # gosub - autoscaler/vendor/github.com/uptrace/opentelemetry-go-extra/otelsql/* # gosub - autoscaler/vendor/github.com/uptrace/opentelemetry-go-extra/otelsqlx/* # gosub +- autoscaler/vendor/github.com/valyala/bytebufferpool/* # gosub +- autoscaler/vendor/github.com/valyala/fasthttp/* # gosub +- autoscaler/vendor/github.com/valyala/fasthttp/fasthttputil/* # gosub +- autoscaler/vendor/github.com/valyala/fasthttp/stackless/* # gosub - autoscaler/vendor/go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux/* # gosub - autoscaler/vendor/go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux/internal/semconvutil/* # gosub - autoscaler/vendor/go.opentelemetry.io/otel/* # gosub diff --git a/scripts/asdf2devbox.py b/scripts/asdf2devbox.py index 84604416e8..c8fec97260 100755 --- a/scripts/asdf2devbox.py +++ b/scripts/asdf2devbox.py @@ -27,6 +27,8 @@ def get_installed_version(package): "bosh": "bosh-cli", "cf": "cloudfoundry-cli", "concourse": "fly", + "concourse": "fly", + "credhub": "credhub-cli", "gcloud": "google-cloud-sdk", "golang": "go", "java": "temurin-bin-17", diff --git a/scripts/generate_test_certs.sh b/scripts/generate_test_certs.sh index 9397172a43..029cc3cc16 100755 --- a/scripts/generate_test_certs.sh +++ b/scripts/generate_test_certs.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e + script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # Place keys and certificates here @@ -20,11 +21,17 @@ ${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name log mv -f "${depot_path}"/loggregatorCA.crt "${depot_path}"/loggregator-ca.crt mv -f "${depot_path}"/loggregatorCA.key "${depot_path}"/loggregator-ca.key +# CA to distribute to dummy syslog emitter certs +${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name LogCacheSyslogServerCA --years "20" +mv -f "${depot_path}"/LogCacheSyslogServerCA.crt "${depot_path}"/log-cache-syslog-server-ca.crt +mv -f "${depot_path}"/LogCacheSyslogServerCA.key "${depot_path}"/log-cache-syslog-server-ca.key + # CA for local testing mTLS certs ${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name validMTLSLocalCA --years "20" mv -f "${depot_path}"/validMTLSLocalCA.crt "${depot_path}"/valid-mtls-local-ca-1.crt mv -f "${depot_path}"/validMTLSLocalCA.key "${depot_path}"/valid-mtls-local-ca-1.key rm -f "${depot_path}"/validMTLSLocalCA.crl + ${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name validMTLSLocalCA --years "20" mv -f "${depot_path}"/validMTLSLocalCA.crt "${depot_path}"/valid-mtls-local-ca-2.crt mv -f "${depot_path}"/validMTLSLocalCA.key "${depot_path}"/valid-mtls-local-ca-2.key @@ -88,6 +95,11 @@ ${CERTSTRAP} --depot-path "${depot_path}" sign metricserver_client --CA autoscal ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metron ${CERTSTRAP} --depot-path "${depot_path}" sign metron --CA loggregator-ca --years "20" + +# metricsforwarder certificate for log-cache-syslog-server +${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain cf-app +${CERTSTRAP} --depot-path "${depot_path}" sign cf-app --CA log-cache-syslog-server-ca --years "20" + # mTLS client certificate for local testing ## certstrap with multiple OU not working at the moment. Pull request is created in the upstream. Therefore, using openssl at the moment ## https://github.com/square/certstrap/pull/120 diff --git a/scripts/sync-package-specs b/scripts/sync-package-specs index c5bfb9089f..349ad13d98 100755 --- a/scripts/sync-package-specs +++ b/scripts/sync-package-specs @@ -16,10 +16,10 @@ function sync_package() { cat "${spec_dir}/spec" | grep --invert-match '# gosub' go run github.com/loggregator/gosub@2819a49a12756cab02592b5ee9f17f02148b0617 list "$@" \ | grep 'autoscaler' \ - | sed --expression='s|code.cloudfoundry.org/app-autoscaler/src/\(.*\)|- \1/* # gosub|g' + | sed -e 's|code.cloudfoundry.org/app-autoscaler/src/\(.*\)|- \1/* # gosub|g' go run github.com/loggregator/gosub@2819a49a12756cab02592b5ee9f17f02148b0617 list "$@" \ | grep --invert-match 'autoscaler' \ - | sed --expression='s|\(.*\)|- autoscaler/vendor/\1/* # gosub|g' + | sed -e 's|\(.*\)|- autoscaler/vendor/\1/* # gosub|g' } > "${spec_dir}/spec.new" mv "${spec_dir}/spec.new" "${spec_dir}/spec" diff --git a/spec/fixtures/metricsforwarder.yml b/spec/fixtures/metricsforwarder.yml index e47d23e453..434d1bbfcb 100644 --- a/spec/fixtures/metricsforwarder.yml +++ b/spec/fixtures/metricsforwarder.yml @@ -20,4 +20,4 @@ autoscaler: client_id: client_id secret: uaa_secret uaa_api: https://login.cf.domain/uaa - grant_type: ALLOW_ALL \ No newline at end of file + grant_type: ALLOW_ALL diff --git a/spec/jobs/metricsforwarder/metricsforwarder_spec.rb b/spec/jobs/metricsforwarder/metricsforwarder_spec.rb index 6d7e24ef69..4c64b52f61 100644 --- a/spec/jobs/metricsforwarder/metricsforwarder_spec.rb +++ b/spec/jobs/metricsforwarder/metricsforwarder_spec.rb @@ -12,6 +12,28 @@ let(:rendered_template) { YAML.safe_load(template.render(properties)) } context "config/metricsforwarder.yml" do + it "supports syslog forwarding" do + properties["autoscaler"]["metricsforwarder"] = { + "syslog" => { + "server_address" => "syslog-server" + } + } + + expect(rendered_template).to include( + { + "syslog" => { + "server_address" => "syslog-server", + "port" => 6067, + "tls" => { + "key_file" => "/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key", + "cert_file" => "/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt", + "ca_file" => "/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt" + } + } + } + ) + end + it "does not set username nor password if not configured" do properties["autoscaler"]["metricsforwarder"] = { "health" => { @@ -43,11 +65,9 @@ end it "has a cred helper impl by default" do - expect(rendered_template).to include( - { - "cred_helper_impl" => "default" - } - ) + expect(rendered_template).to include({ + "cred_helper_impl" => "default" + }) end it "has a cred helper impl configured for stored procedures" do diff --git a/src/acceptance/Makefile b/src/acceptance/Makefile index 3ae2486a00..0b10972421 100644 --- a/src/acceptance/Makefile +++ b/src/acceptance/Makefile @@ -5,7 +5,7 @@ MAKEFLAGS := -s GO := GO111MODULE=on GO15VENDOREXPERIMENT=1 go GO_NOMOD := GO111MODULE=off go -GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_VERSION = $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') GO_DEPENDENCIES = $(shell find . -type f -name '*.go') PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \ | grep --invert-match --regexp='e2e') @@ -66,7 +66,4 @@ test: bin/test $(filter-out $@,$(MAKECMDGOALS)) lint: - @go run github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} run - -lint-fix: - go run github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} run --fix + make -C ../.. lint_acceptance OPTS=${OPTS} diff --git a/src/acceptance/assets/app/go_app/Makefile b/src/acceptance/assets/app/go_app/Makefile index db492e23eb..429a7f16ce 100644 --- a/src/acceptance/assets/app/go_app/Makefile +++ b/src/acceptance/assets/app/go_app/Makefile @@ -7,7 +7,7 @@ aes_terminal_reset := \e[0m # TODO: Do we need the next line? MAKEFLAGS = -GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_VERSION = $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') GO_DEPENDENCIES = $(shell find . -type f -name '*.go') PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \ | grep --invert-match --regexp='e2e') diff --git a/src/acceptance/assets/app/go_app/go.mod b/src/acceptance/assets/app/go_app/go.mod index 50d5e15c29..401f975575 100644 --- a/src/acceptance/assets/app/go_app/go.mod +++ b/src/acceptance/assets/app/go_app/go.mod @@ -1,6 +1,6 @@ module code.cloudfoundry.org/app-autoscaler-release/src/acceptance/assets/app/go_app -go 1.21.5 +go 1.22.3 require ( github.com/cloudfoundry-community/go-cfenv v1.18.0 diff --git a/src/acceptance/go.mod b/src/acceptance/go.mod index 3271890977..cdd46370b3 100644 --- a/src/acceptance/go.mod +++ b/src/acceptance/go.mod @@ -1,6 +1,6 @@ module acceptance -go 1.21.5 +go 1.22.3 require ( github.com/cloudfoundry/cf-test-helpers/v2 v2.9.0 diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index 6b86e168e7..abc9e08758 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -4,14 +4,11 @@ MAKEFLAGS := -s aes_terminal_font_yellow := \e[38;2;255;255;0m aes_terminal_reset := \e[0m -GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_VERSION = $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') GO_DEPENDENCIES = $(shell find . -type f -name '*.go') PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \ | grep --invert-match --regexp='e2e') -# `CGO_ENABLED := 1` is required to enforce dynamic linking which is a requirement of dynatrace. -CGO_ENABLED := 1 -BUILDTAGS := export GOWORK=off BUILDFLAGS := -ldflags '-linkmode=external' @@ -86,11 +83,10 @@ go-mod-vendor: ${go-vendoring-folder} ${go-vendored-files} ${go-vendoring-folder} ${go-vendored-files} &: ${app-fakes-dir} ${app-fakes-files} go mod vendor - - +# CGO_ENABLED := 1 is required to enforce dynamic linking which is a requirement of dynatrace. build-%: ${openapi-generated-clients-and-servers-dir} ${openapi-generated-clients-and-servers-files} @echo "# building $*" - @CGO_ENABLED=$(CGO_ENABLED) go build $(BUILDTAGS) $(BUILDFLAGS) -o build/$* $*/cmd/$*/main.go + @CGO_ENABLED=1 go build $(BUILDTAGS) $(BUILDFLAGS) -o build/$* $*/cmd/$*/main.go build: $(addprefix build-,$(binaries)) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index d2fe977c87..8672734b23 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -1,6 +1,6 @@ module code.cloudfoundry.org/app-autoscaler/src/autoscaler -go 1.21.5 +go 1.22.3 require ( code.cloudfoundry.org/cfhttp/v2 v2.0.1 @@ -8,13 +8,14 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5 code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 github.com/go-faster/errors v0.7.1 github.com/go-faster/jx v1.1.0 - github.com/go-logr/logr v1.4.1 + github.com/go-logr/logr v1.4.2 github.com/go-sql-driver/mysql v1.8.1 github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.1 @@ -25,12 +26,12 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d github.com/ogen-go/ogen v1.1.0 - github.com/onsi/ginkgo/v2 v2.17.3 + github.com/onsi/ginkgo/v2 v2.18.0 github.com/onsi/gomega v1.33.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pivotal-cf/brokerapi/v11 v11.0.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/rubyist/circuitbreaker v2.2.1+incompatible github.com/steinfletcher/apitest v1.5.15 github.com/stretchr/testify v1.9.0 @@ -47,13 +48,15 @@ require ( golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.25.0 golang.org/x/time v0.5.0 - google.golang.org/grpc v1.63.2 + google.golang.org/grpc v1.64.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - code.cloudfoundry.org/go-diodes v0.0.0-20240419195010-376885f5f3d4 // indirect + code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 // indirect + code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563 // indirect filippo.io/edwards25519 v1.1.0 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect @@ -69,14 +72,15 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/pprof v0.0.0-20240521024322-9665fa269a30 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.2.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/openzipkin/zipkin-go v0.4.2 // indirect @@ -85,8 +89,10 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.53.0 // indirect - github.com/prometheus/procfs v0.14.0 // indirect + github.com/prometheus/procfs v0.15.0 // indirect github.com/segmentio/asm v1.2.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.53.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -96,8 +102,8 @@ require ( golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 05b53129a2..5b9dc0658e 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -596,14 +596,18 @@ code.cloudfoundry.org/cfhttp/v2 v2.0.1 h1:xCbkvDVUDuFCquDo66QnvM9hG6vxM2cvTMjj/A code.cloudfoundry.org/cfhttp/v2 v2.0.1/go.mod h1:yD0n22D4KP4xamgTxbz82KsWzK5kcfUrmJNrrmd4YBg= code.cloudfoundry.org/clock v1.1.0 h1:XLzC6W3Ah/Y7ht1rmZ6+QfPdt1iGWEAAtIZXgiaj57c= code.cloudfoundry.org/clock v1.1.0/go.mod h1:yA3fxddT9RINQL2XHS7PS+OXxKCGhfrZmlNUCIM6AKo= -code.cloudfoundry.org/go-diodes v0.0.0-20240419195010-376885f5f3d4 h1:exUBTQS/m2MhcsRJspBU9hClxrAPNihvQz9SDQBwE+4= -code.cloudfoundry.org/go-diodes v0.0.0-20240419195010-376885f5f3d4/go.mod h1:mndfNpz8bKGHkWW+mTJpXW0N8rT4cZuBjo0QFWn2EIA= +code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 h1:6wazSuHaJjatGy8pvchSy6L+4M67WPfjgK9yh7cJLMs= +code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718/go.mod h1:eVHabU/rcpC5ocoIAXvnaySkdH6+PgORRVlY5l3SDys= code.cloudfoundry.org/go-log-cache/v2 v2.0.7 h1:yR/JjQ/RscO1n4xVAT9HDYcpx5ET/3Cq2/RhpJml6ZU= code.cloudfoundry.org/go-log-cache/v2 v2.0.7/go.mod h1:6KQe2FeeaqRheD5vCvpyTa80YoJojB/r21E54mT97Mc= code.cloudfoundry.org/go-loggregator/v9 v9.2.1 h1:S6Lgg5UJbhh2bt2TGQxs6R00CF8PrUA3GFPYDxy56Fk= code.cloudfoundry.org/go-loggregator/v9 v9.2.1/go.mod h1:FTFFruqGeOhVCDFvyLgl8EV8YW63NNwRzLhxJcporu8= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563 h1:mkCsrZ0h4ZF9jcVTEtLiSRJ0Mu2AIQANEWVDcXrOSVY= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563/go.mod h1:zYhHvL+6LLy/lGiIe/T5Ma6ae0Tv5Qxm6TNWOvsQVrQ= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5 h1:5ae8G6wQss4vqKkBQQ8fNQoXEhdV5vEkKStGlPQ68Rc= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5/go.mod h1:2cFY9sx7jyRwEglCfSLMV57cWE0opfIBQhgXHaHIx+0= code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a h1:yWneME1ZPxsWkZT1R9e0n6CymukD5pU/ceEYo4ZVMak= code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a/go.mod h1:TI9y0bK+FJ9OKSLaJmBo5Z2tr+/VULZ2jiQwSg1+Z5c= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -622,6 +626,8 @@ github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= @@ -724,8 +730,8 @@ github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpx github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= @@ -816,8 +822,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240521024322-9665fa269a30 h1:r6YdmbD41tGHeCWDyHF691LWtL7D1iSTyJaKejTWwVU= +github.com/google/pprof v0.0.0-20240521024322-9665fa269a30/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -849,8 +855,8 @@ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZH github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= @@ -882,6 +888,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -938,8 +946,8 @@ github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxm github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= -github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU= -github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.18.0 h1:W9Y7IWXxPUpAit9ieMOLI7PJZGaW22DTKgiVAuhDTLc= +github.com/onsi/ginkgo/v2 v2.18.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -978,8 +986,8 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= @@ -987,8 +995,8 @@ github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= -github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= -github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= +github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= +github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1033,6 +1041,10 @@ github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.4 h1:x3omFAG2XkvWFg1hvXRi github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.4/go.mod h1:qMKJr5fTnY0p7hqCQMNrAk62bCARWR5rAbTrGUFRuh4= github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4 h1:Pt/+CUTRusJb471SBXwkRCz+9pbOjNr80M6LlwqV07w= github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4/go.mod h1:kQgNoghy4K/wguxbOd/u0OJw/Y0maNPc7PF4JpEGeUc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.53.0 h1:lW/+SUkOxCx2vlIu0iaImv4JLrVRnbbkpCoaawvA4zc= +github.com/valyala/fasthttp v1.53.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1672,10 +1684,10 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1718,8 +1730,8 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20230512210959-5dcfb37c0b43/go.mod h1:irORyHPQXotoshbRTZVFvPDcfTfFHL23efQeop+H45M= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 60c6e6d164..ded7468bb8 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -25,6 +25,7 @@ type Config struct { Logging helpers.LoggingConfig `yaml:"logging"` Server helpers.ServerConfig `yaml:"server"` LoggregatorConfig LoggregatorConfig `yaml:"loggregator"` + SyslogConfig SyslogConfig `yaml:"syslog"` Db map[string]db.DatabaseConfig `yaml:"db"` CacheTTL time.Duration `yaml:"cache_ttl"` CacheCleanupInterval time.Duration `yaml:"cache_cleanup_interval"` @@ -58,6 +59,12 @@ type LoggregatorConfig struct { TLS models.TLSCerts `yaml:"tls"` } +type SyslogConfig struct { + ServerAddress string `yaml:"server_address"` + Port int `yaml:"port"` + TLS models.TLSCerts `yaml:"tls"` +} + func LoadConfig(reader io.Reader) (*Config, error) { conf := &Config{ Server: defaultServerConfig, @@ -86,19 +93,36 @@ func LoadConfig(reader io.Reader) (*Config, error) { return conf, nil } +func (c *Config) UsingSyslog() bool { + return c.SyslogConfig.ServerAddress != "" && c.SyslogConfig.Port != 0 +} + func (c *Config) Validate() error { if c.Db[db.PolicyDb].URL == "" { return fmt.Errorf("Configuration error: Policy DB url is empty") } - if c.LoggregatorConfig.TLS.CACertFile == "" { - return fmt.Errorf("Configuration error: Loggregator CACert is empty") - } - if c.LoggregatorConfig.TLS.CertFile == "" { - return fmt.Errorf("Configuration error: Loggregator ClientCert is empty") - } - if c.LoggregatorConfig.TLS.KeyFile == "" { - return fmt.Errorf("Configuration error: Loggregator ClientKey is empty") + if c.UsingSyslog() { + if c.SyslogConfig.TLS.CACertFile == "" { + return fmt.Errorf("Configuration error: SyslogServer Loggregator CACert is empty") + } + if c.SyslogConfig.TLS.CertFile == "" { + return fmt.Errorf("Configuration error: SyslogServer ClientCert is empty") + } + if c.SyslogConfig.TLS.KeyFile == "" { + return fmt.Errorf("Configuration error: SyslogServer ClientKey is empty") + } + } else { + if c.LoggregatorConfig.TLS.CACertFile == "" { + return fmt.Errorf("Configuration error: Loggregator CACert is empty") + } + if c.LoggregatorConfig.TLS.CertFile == "" { + return fmt.Errorf("Configuration error: Loggregator ClientCert is empty") + } + if c.LoggregatorConfig.TLS.KeyFile == "" { + return fmt.Errorf("Configuration error: Loggregator ClientKey is empty") + } } + if c.RateLimit.MaxAmount <= 0 { return fmt.Errorf("Configuration error: RateLimit.MaxAmount is equal or less than zero") } diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index 7f5ee97b3a..34e1518cd8 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -6,6 +6,7 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -309,6 +310,55 @@ rate_limit: err = conf.Validate() }) + When("syslog is available", func() { + BeforeEach(func() { + conf.SyslogConfig = SyslogConfig{ + ServerAddress: "localhost", + Port: 514, + TLS: models.TLSCerts{ + CACertFile: "../testcerts/ca.crt", + CertFile: "../testcerts/client.crt", + KeyFile: "../testcerts/client.crt", + }, + } + conf.LoggregatorConfig = LoggregatorConfig{} + }) + + It("should not error", func() { + Expect(err).NotTo(HaveOccurred()) + }) + + When("SyslogServer CACert is not set", func() { + BeforeEach(func() { + conf.SyslogConfig.TLS.CACertFile = "" + }) + + It("should error", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer Loggregator CACert is empty"))) + }) + }) + + Context("when SyslogServer CertFile is not set", func() { + BeforeEach(func() { + conf.SyslogConfig.TLS.KeyFile = "" + }) + + It("should error", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer ClientKey is empty"))) + }) + }) + + Context("when SyslogServer ClientCert is not set", func() { + BeforeEach(func() { + conf.SyslogConfig.TLS.CertFile = "" + }) + + It("should error", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer ClientCert is empty"))) + }) + }) + }) + Context("when all the configs are valid", func() { It("should not error", func() { Expect(err).NotTo(HaveOccurred()) diff --git a/src/autoscaler/metricsforwarder/forwarder/forwarder_test.go b/src/autoscaler/metricsforwarder/forwarder/forwarder_test.go index 2f967a4d99..7ea677c065 100644 --- a/src/autoscaler/metricsforwarder/forwarder/forwarder_test.go +++ b/src/autoscaler/metricsforwarder/forwarder/forwarder_test.go @@ -1,119 +1,71 @@ package forwarder_test import ( - "time" + "path/filepath" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/forwarder" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/testhelpers" - - "errors" - "path/filepath" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" "code.cloudfoundry.org/lager/v3" ) var _ = Describe("MetricForwarder", func() { - var ( - metricForwarder forwarder.MetricForwarder - metrics *models.CustomMetric - grpcIngressTestServer *testhelpers.TestIngressServer - err error + metricForwarder forwarder.MetricForwarder + loggregatorConfig config.LoggregatorConfig + syslogConfig config.SyslogConfig + err error + conf *config.Config ) - BeforeEach(func() { - testCertDir := "../../../../test-certs" - - grpcIngressTestServer, err = testhelpers.NewTestIngressServer( - filepath.Join(testCertDir, "metron.crt"), - filepath.Join(testCertDir, "metron.key"), - filepath.Join(testCertDir, "loggregator-ca.crt"), - ) - - Expect(err).ToNot(HaveOccurred()) - err = grpcIngressTestServer.Start() - Expect(err).NotTo(HaveOccurred()) - loggregatorConfig := config.LoggregatorConfig{ - MetronAddress: grpcIngressTestServer.GetAddr(), - TLS: models.TLSCerts{ - KeyFile: filepath.Join(testCertDir, "metron.key"), - CertFile: filepath.Join(testCertDir, "metron.crt"), - CACertFile: filepath.Join(testCertDir, "loggregator-ca.crt"), - }, - } - serverConfig := helpers.ServerConfig{ - Port: 10000 + GinkgoParallelProcess(), - } - - loggerConfig := helpers.LoggingConfig{ - Level: "debug", - } - - conf := &config.Config{ - Server: serverConfig, - Logging: loggerConfig, + JustBeforeEach(func() { + conf = &config.Config{ LoggregatorConfig: loggregatorConfig, + SyslogConfig: syslogConfig, } logger := lager.NewLogger("metricsforwarder-test") - metricForwarder, err = forwarder.NewMetricForwarder(logger, conf) Expect(err).ToNot(HaveOccurred()) - }) - Describe("EmitMetrics", func() { - - Context("when a request to emit custom metrics comes", func() { + Describe("NewMetricForwarder", func() { + When("syslog it is present it creates a SyslogEmitter", func() { BeforeEach(func() { - metrics = &models.CustomMetric{Name: "queuelength", Value: 12.5, Unit: "unit", InstanceIndex: 123, AppGUID: "dummy-guid"} - metricForwarder.EmitMetric(metrics) + // Loggregator config in spec has this default value + loggregatorConfig.MetronAddress = "127.0.0.1:3458" + syslogConfig = config.SyslogConfig{ + ServerAddress: "syslog://some-server-address", + } + }) + It("should create a SyslogClient", func() { + Expect(metricForwarder).To(BeAssignableToTypeOf(&forwarder.SyslogEmitter{})) }) + }) - It("Should emit gauge metrics", func() { - env, err := getEnvelopeAt(grpcIngressTestServer.Receivers, 0) - Expect(err).NotTo(HaveOccurred()) - ts := time.Unix(0, env.Timestamp) - Expect(ts).Should(BeTemporally("~", time.Now(), time.Second)) - metrics := env.GetGauge() - Expect(metrics).NotTo(BeNil()) - Expect(metrics.GetMetrics()).To(HaveLen(1)) - Expect(metrics.GetMetrics()["queuelength"].Value).To(Equal(12.5)) - Expect(env.Tags["origin"]).To(Equal("autoscaler_metrics_forwarder")) + When("loggregatorConfig is present creates a metronForwarder", func() { + BeforeEach(func() { + testCertDir := "../../../../test-certs" + syslogConfig = config.SyslogConfig{} + loggregatorConfig = config.LoggregatorConfig{ + MetronAddress: "some-address", + TLS: models.TLSCerts{ + KeyFile: filepath.Join(testCertDir, "metron.key"), + CertFile: filepath.Join(testCertDir, "metron.crt"), + CACertFile: filepath.Join(testCertDir, "loggregator-ca.crt"), + }, + } }) + It("should create a metronClient", func() { + Expect(metricForwarder).To(BeAssignableToTypeOf(&forwarder.MetronEmitter{})) + }) }) }) - - AfterEach(func() { - grpcIngressTestServer.Stop() - }) - }) - -func getEnvelopeAt(receivers chan loggregator_v2.Ingress_BatchSenderServer, idx int) (*loggregator_v2.Envelope, error) { - var recv loggregator_v2.Ingress_BatchSenderServer - Eventually(receivers, 10).Should(Receive(&recv)) - - envBatch, err := recv.Recv() - if err != nil { - return nil, err - } - - if len(envBatch.Batch) < 1 { - return nil, errors.New("no envelopes") - } - - return envBatch.Batch[idx], nil -} diff --git a/src/autoscaler/metricsforwarder/forwarder/metric_forwarder.go b/src/autoscaler/metricsforwarder/forwarder/metric_forwarder.go index 89473b9772..34242d60dc 100644 --- a/src/autoscaler/metricsforwarder/forwarder/metric_forwarder.go +++ b/src/autoscaler/metricsforwarder/forwarder/metric_forwarder.go @@ -1,10 +1,10 @@ package forwarder import ( - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" "code.cloudfoundry.org/go-loggregator/v9" + "code.cloudfoundry.org/lager/v3" ) @@ -12,48 +12,23 @@ type MetricForwarder interface { EmitMetric(*models.CustomMetric) } -type metricForwarder struct { +type MetronEmitter struct { client *loggregator.IngressClient logger lager.Logger } const METRICS_FORWARDER_ORIGIN = "autoscaler_metrics_forwarder" -func NewMetricForwarder(logger lager.Logger, conf *config.Config) (MetricForwarder, error) { - tlsConfig, err := loggregator.NewIngressTLSConfig( - conf.LoggregatorConfig.TLS.CACertFile, - conf.LoggregatorConfig.TLS.CertFile, - conf.LoggregatorConfig.TLS.KeyFile, - ) - if err != nil { - logger.Error("could-not-create-TLS-config", err, lager.Data{"config": conf}) - return &metricForwarder{}, err - } - - client, err := loggregator.NewIngressClient( - tlsConfig, - loggregator.WithAddr(conf.LoggregatorConfig.MetronAddress), - loggregator.WithTag("origin", METRICS_FORWARDER_ORIGIN), - loggregator.WithLogger(helpers.NewLoggregatorGRPCLogger(logger.Session("metric_forwarder"))), - ) - - if err != nil { - logger.Error("could-not-create-loggregator-client", err, lager.Data{"config": conf}) - return &metricForwarder{}, err - } - - return &metricForwarder{ - client: client, - logger: logger, - }, nil +func hasSyslogConfig(conf *config.Config) bool { + return conf.SyslogConfig.ServerAddress != "" } -func (mf *metricForwarder) EmitMetric(metric *models.CustomMetric) { - mf.logger.Debug("custom-metric-emit-request-received", lager.Data{"metric": metric}) - - options := []loggregator.EmitGaugeOption{ - loggregator.WithGaugeAppInfo(metric.AppGUID, int(metric.InstanceIndex)), - loggregator.WithGaugeValue(metric.Name, metric.Value, metric.Unit), +func NewMetricForwarder(logger lager.Logger, conf *config.Config) (MetricForwarder, error) { + if hasSyslogConfig(conf) { + logger.Info("using-syslog-emitter") + return NewSyslogEmitter(logger, conf) + } else { + logger.Info("Using metron-emitter") + return NewMetronEmitter(logger, conf) } - mf.client.EmitGauge(options...) } diff --git a/src/autoscaler/metricsforwarder/forwarder/metron_emitter.go b/src/autoscaler/metricsforwarder/forwarder/metron_emitter.go new file mode 100644 index 0000000000..dc37d0fb5d --- /dev/null +++ b/src/autoscaler/metricsforwarder/forwarder/metron_emitter.go @@ -0,0 +1,48 @@ +package forwarder + +import ( + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/go-loggregator/v9" + "code.cloudfoundry.org/lager/v3" +) + +func NewMetronEmitter(logger lager.Logger, conf *config.Config) (MetricForwarder, error) { + tlsConfig, err := loggregator.NewIngressTLSConfig( + conf.LoggregatorConfig.TLS.CACertFile, + conf.LoggregatorConfig.TLS.CertFile, + conf.LoggregatorConfig.TLS.KeyFile, + ) + + if err != nil { + logger.Error("could-not-create-TLS-config", err, lager.Data{"config": conf}) + return &MetronEmitter{}, err + } + + client, err := loggregator.NewIngressClient( + tlsConfig, + loggregator.WithAddr(conf.LoggregatorConfig.MetronAddress), + loggregator.WithTag("origin", METRICS_FORWARDER_ORIGIN), + loggregator.WithLogger(helpers.NewLoggregatorGRPCLogger(logger.Session("metric_forwarder"))), + ) + if err != nil { + logger.Error("could-not-create-loggregator-client", err, lager.Data{"config": conf}) + return &MetronEmitter{}, err + } + + return &MetronEmitter{ + client: client, + logger: logger, + }, nil +} + +func (mf *MetronEmitter) EmitMetric(metric *models.CustomMetric) { + mf.logger.Debug("custom-metric-emit-request-received", lager.Data{"metric": metric}) + + options := []loggregator.EmitGaugeOption{ + loggregator.WithGaugeAppInfo(metric.AppGUID, int(metric.InstanceIndex)), + loggregator.WithGaugeValue(metric.Name, metric.Value, metric.Unit), + } + mf.client.EmitGauge(options...) +} diff --git a/src/autoscaler/metricsforwarder/forwarder/metron_emitter_test.go b/src/autoscaler/metricsforwarder/forwarder/metron_emitter_test.go new file mode 100644 index 0000000000..c35b0d5a78 --- /dev/null +++ b/src/autoscaler/metricsforwarder/forwarder/metron_emitter_test.go @@ -0,0 +1,107 @@ +package forwarder_test + +import ( + "errors" + "path/filepath" + "time" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/forwarder" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/testhelpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" + "code.cloudfoundry.org/lager/v3" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("MetricForwarder", func() { + + var ( + grpcIngressTestServer *testhelpers.TestIngressServer + metrics *models.CustomMetric + err error + serverConfig helpers.ServerConfig + loggerConfig helpers.LoggingConfig + emitter forwarder.MetricForwarder + conf *config.Config + ) + + BeforeEach(func() { + testCertDir := "../../../../test-certs" + + grpcIngressTestServer, err = testhelpers.NewTestIngressServer( + filepath.Join(testCertDir, "metron.crt"), + filepath.Join(testCertDir, "metron.key"), + filepath.Join(testCertDir, "loggregator-ca.crt"), + ) + + Expect(err).ToNot(HaveOccurred()) + err = grpcIngressTestServer.Start() + Expect(err).NotTo(HaveOccurred()) + + loggregatorConfig := config.LoggregatorConfig{ + MetronAddress: grpcIngressTestServer.GetAddr(), + TLS: models.TLSCerts{ + KeyFile: filepath.Join(testCertDir, "metron.key"), + CertFile: filepath.Join(testCertDir, "metron.crt"), + CACertFile: filepath.Join(testCertDir, "loggregator-ca.crt"), + }, + } + + serverConfig = helpers.ServerConfig{ + Port: 10000 + GinkgoParallelProcess(), + } + + loggerConfig = helpers.LoggingConfig{ + Level: "debug", + } + + conf = &config.Config{ + Server: serverConfig, + Logging: loggerConfig, + LoggregatorConfig: loggregatorConfig, + } + logger := lager.NewLogger("metricsforwarder-test") + emitter, err = forwarder.NewMetronEmitter(logger, conf) + Expect(err).NotTo(HaveOccurred()) + + }) + + Describe("EmitMetrics", func() { + It("Should emit gauge metrics", func() { + metrics = &models.CustomMetric{Name: "queuelength", Value: 12.5, Unit: "unit", InstanceIndex: 123, AppGUID: "dummy-guid"} + emitter.EmitMetric(metrics) + env, err := getEnvelopeAt(grpcIngressTestServer.Receivers, 0) + Expect(err).NotTo(HaveOccurred()) + ts := time.Unix(0, env.Timestamp) + Expect(ts).Should(BeTemporally("~", time.Now(), time.Second)) + metrics := env.GetGauge() + Expect(metrics).NotTo(BeNil()) + Expect(metrics.GetMetrics()).To(HaveLen(1)) + Expect(metrics.GetMetrics()["queuelength"].Value).To(Equal(12.5)) + Expect(env.Tags["origin"]).To(Equal("autoscaler_metrics_forwarder")) + }) + }) + + AfterEach(func() { + grpcIngressTestServer.Stop() + }) +}) + +func getEnvelopeAt(receivers chan loggregator_v2.Ingress_BatchSenderServer, idx int) (*loggregator_v2.Envelope, error) { + var recv loggregator_v2.Ingress_BatchSenderServer + Eventually(receivers, 10).Should(Receive(&recv)) + + envBatch, err := recv.Recv() + if err != nil { + return nil, err + } + + if len(envBatch.Batch) < 1 { + return nil, errors.New("no envelopes") + } + + return envBatch.Batch[idx], nil +} diff --git a/src/autoscaler/metricsforwarder/forwarder/syslog_emitter.go b/src/autoscaler/metricsforwarder/forwarder/syslog_emitter.go new file mode 100644 index 0000000000..84ff7e0a90 --- /dev/null +++ b/src/autoscaler/metricsforwarder/forwarder/syslog_emitter.go @@ -0,0 +1,102 @@ +package forwarder + +import ( + "fmt" + "net/url" + "os" + "time" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" + "code.cloudfoundry.org/lager/v3" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/syslog" +) + +type SyslogEmitter struct { + logger lager.Logger + Writer egress.WriteCloser +} + +type Counter struct{} + +func (c *Counter) Add(delta float64) { +} +func (c *Counter) Set(delta float64) { +} + +func NewSyslogEmitter(logger lager.Logger, conf *config.Config) (MetricForwarder, error) { + var writer egress.WriteCloser + var protocol string + + tlsConfig, _ := conf.SyslogConfig.TLS.CreateClientConfig() + + netConf := syslog.NetworkTimeoutConfig{ + WriteTimeout: time.Second, + DialTimeout: 100 * time.Millisecond, + } + + if conf.SyslogConfig.TLS.CACertFile != "" { + protocol = "syslog-tls" + } else { + protocol = "syslog" + } + + syslogUrl, _ := url.Parse(fmt.Sprintf("%s://%s:%d", protocol, conf.SyslogConfig.ServerAddress, conf.SyslogConfig.Port)) + + logger.Info("using-syslog-url", lager.Data{"url": syslogUrl}) + + hostname, _ := os.Hostname() + + binding := &syslog.URLBinding{ + URL: syslogUrl, + Hostname: hostname, + } + + switch binding.URL.Scheme { + case "syslog": + writer = syslog.NewTCPWriter( + binding, + netConf, + &Counter{}, + syslog.NewConverter(), + ) + case "syslog-tls": + writer = syslog.NewTLSWriter( + binding, + netConf, + tlsConfig, + &Counter{}, + syslog.NewConverter(), + ) + } + + return &SyslogEmitter{ + Writer: writer, + logger: logger, + }, nil +} + +func (mf *SyslogEmitter) EmitMetric(metric *models.CustomMetric) { + e := &loggregator_v2.Envelope{ + InstanceId: fmt.Sprintf("%d", metric.InstanceIndex), + Timestamp: time.Now().UnixNano(), + SourceId: metric.AppGUID, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + metric.Name: { + Unit: metric.Unit, + Value: metric.Value, + }, + }, + }, + }, + } + + err := mf.Writer.Write(e) + if err != nil { + mf.logger.Error("failed-to-write-metric-to-syslog", err) + } +} diff --git a/src/autoscaler/metricsforwarder/forwarder/syslog_emitter_test.go b/src/autoscaler/metricsforwarder/forwarder/syslog_emitter_test.go new file mode 100644 index 0000000000..01a0c1dc98 --- /dev/null +++ b/src/autoscaler/metricsforwarder/forwarder/syslog_emitter_test.go @@ -0,0 +1,116 @@ +package forwarder_test + +import ( + "bufio" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/forwarder" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/lager/v3" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/syslog" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SyslogEmitter", func() { + var ( + listener net.Listener + err error + port int + conf *config.Config + tlsCerts models.TLSCerts + emitter forwarder.MetricForwarder + ) + + BeforeEach(func() { + port = 10000 + GinkgoParallelProcess() + listener, err = net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + Expect(err).ToNot(HaveOccurred()) + tlsCerts = models.TLSCerts{} + Expect(err).ToNot(HaveOccurred()) + + }) + + JustBeforeEach(func() { + host, port, err := net.SplitHostPort(listener.Addr().String()) + Expect(err).ToNot(HaveOccurred()) + fmt.Println(host, port) + + portNumber, err := strconv.Atoi(port) + Expect(err).ToNot(HaveOccurred()) + + conf = &config.Config{ + SyslogConfig: config.SyslogConfig{ + ServerAddress: host, + Port: portNumber, + TLS: tlsCerts, + }, + } + + logger := lager.NewLogger("metricsforwarder-test") + emitter, err = forwarder.NewSyslogEmitter(logger, conf) + Expect(err).ToNot(HaveOccurred()) + + }) + + AfterEach(func() { + listener.Close() + }) + + Describe("NewSyslogEmitter", func() { + When("tls config is provided", func() { + BeforeEach(func() { + testCertDir := "../../../../test-certs" + tlsCerts = models.TLSCerts{ + KeyFile: filepath.Join(testCertDir, "cf-app.key"), + CertFile: filepath.Join(testCertDir, "cf-app.crt"), + CACertFile: filepath.Join(testCertDir, "log-cache-syslog-server-ca.crt"), + } + }) + + It("Writer should be TLS", func() { + // cast emitter to syslogEmitter to access writer + Expect(emitter.(*forwarder.SyslogEmitter).Writer).To(BeAssignableToTypeOf(&syslog.TLSWriter{})) + }) + }) + + When("tls config is not provided", func() { + JustBeforeEach(func() { + conf.SyslogConfig.TLS = models.TLSCerts{} + }) + + It("Writer should be TCP", func() { + Expect(emitter.(*forwarder.SyslogEmitter).Writer).To(BeAssignableToTypeOf(&syslog.TCPWriter{})) + }) + }) + }) + + Describe("EmitMetric", func() { + It("should send message to syslog server", func() { + metric := &models.CustomMetric{Name: "queuelength", Value: 12, Unit: "bytes", InstanceIndex: 123, AppGUID: "dummy-guid"} + + expectedHostname, err := os.Hostname() + Expect(err).ToNot(HaveOccurred()) + + emitter.EmitMetric(metric) + + conn, err := listener.Accept() + Expect(err).ToNot(HaveOccurred()) + + buf := bufio.NewReader(conn) + + actual, err := buf.ReadString('\n') + Expect(err).ToNot(HaveOccurred()) + + priorityAndVersion := actual[:4] + expectedMsglen := len(actual) - len(priorityAndVersion) // 4 is the length of the syslog priority and version + expected := fmt.Sprintf(`%d <14>1 \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2} %s %s \[%d\] - \[gauge@47450 name="%s" value="%.0f" unit="%s"\]`, expectedMsglen, expectedHostname, metric.AppGUID, metric.InstanceIndex, metric.Name, metric.Value, metric.Unit) + Expect(actual).To(MatchRegexp(expected)) + }) + }) +}) diff --git a/src/changelog/Makefile b/src/changelog/Makefile index 6ecb51efa3..3a9fff0655 100644 --- a/src/changelog/Makefile +++ b/src/changelog/Makefile @@ -5,7 +5,7 @@ MAKEFLAGS := -s GO := GO111MODULE=on GO15VENDOREXPERIMENT=1 go GO_NOMOD := GO111MODULE=off go -GO_VERSION := $(shell ${GO} version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_VERSION := $(shell ${GO} version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') GO_DEPENDENCIES = $(shell find . -type f -name '*.go') PACKAGE_DIRS = $(shell ${GO} list './...' | grep --invert-match --regexp='/vendor/' \ | grep --invert-match --regexp='e2e') diff --git a/src/changelog/go.mod b/src/changelog/go.mod index e234623981..d1ddf0b826 100644 --- a/src/changelog/go.mod +++ b/src/changelog/go.mod @@ -1,6 +1,6 @@ module changelog -go 1.21.5 +go 1.22.3 require ( github.com/Masterminds/semver/v3 v3.2.1 diff --git a/src/changeloglockcleaner/go.mod b/src/changeloglockcleaner/go.mod index 93aff1bbf6..9a8ec5b4c5 100644 --- a/src/changeloglockcleaner/go.mod +++ b/src/changeloglockcleaner/go.mod @@ -1,6 +1,6 @@ module changeloglockcleaner -go 1.21.5 +go 1.22.3 require ( github.com/go-sql-driver/mysql v1.8.1