From dc1cb995822480a4ef5d26b38383653fe8fbb7b9 Mon Sep 17 00:00:00 2001 From: rashmichandrashekar Date: Fri, 15 Nov 2024 11:53:14 -0800 Subject: [PATCH] Adding support for global settings (#1003) [comment]: # (Note that your PR title should follow the conventional commit format: https://conventionalcommits.org/en/v1.0.0/#summary) # PR Description Adding support for global settings [comment]: # (The below checklist is for PRs adding new features. If a box is not checked, add a reason why it's not needed.) # New Feature Checklist - [ ] List telemetry added about the feature. - [ ] Link to the one-pager about the feature. - [ ] List any tasks necessary for release (3P docs, AKS RP chart changes, etc.) after merging the PR. - [ ] Attach results of scale and perf testing. [comment]: # (The below checklist is for code changes. Not all boxes necessarily need to be checked. Build, doc, and template changes do not need to fill out the checklist.) # Tests Checklist - [ ] Have end-to-end Ginkgo tests been run on your cluster and passed? To bootstrap your cluster to run the tests, follow [these instructions](/otelcollector/test/README.md#bootstrap-a-dev-cluster-to-run-ginkgo-tests). - Labels used when running the tests on your cluster: - [ ] `operator` - [ ] `windows` - [ ] `arm64` - [ ] `arc-extension` - [ ] `fips` - [ ] Have new tests been added? For features, have tests been added for this feature? For fixes, is there a test that could have caught this issue and could validate that the fix works? - [ ] Is a new scrape job needed? - [ ] The scrape job was added to the folder [test-cluster-yamls](/otelcollector/test/test-cluster-yamls/) in the correct configmap or as a CR. - [ ] Was a new test label added? - [ ] A string constant for the label was added to [constants.go](/otelcollector/test/utils/constants.go). - [ ] The label and description was added to the [test README](/otelcollector/test/README.md). - [ ] The label was added to this [PR checklist](/.github/pull_request_template). - [ ] The label was added as needed to [testkube-test-crs.yaml](/otelcollector/test/testkube/testkube-test-crs.yaml). - [ ] Are additional API server permissions needed for the new tests? - [ ] These permissions have been added to [api-server-permissions.yaml](/otelcollector/test/testkube/api-server-permissions.yaml). - [ ] Was a new test suite (a new folder under `/tests`) added? - [ ] The new test suite is included in [testkube-test-crs.yaml](/otelcollector/test/testkube/testkube-test-crs.yaml). --- otelcollector/main/main.go | 1 + .../prom-config-validator-builder/main.go | 30 ++++++- .../shared/configmap/mp/configmapparser.go | 4 +- .../mp/tomlparser-set-global-config.go | 87 +++++++++++++++++++ .../telegraf-prometheus-collector-ds.conf | 2 + ...egraf-prometheus-collector-ta-enabled.conf | 2 + ...telegraf-prometheus-collector-windows.conf | 3 + 7 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 otelcollector/shared/configmap/mp/tomlparser-set-global-config.go diff --git a/otelcollector/main/main.go b/otelcollector/main/main.go index c4c8d817d..401f7dc8a 100644 --- a/otelcollector/main/main.go +++ b/otelcollector/main/main.go @@ -147,6 +147,7 @@ func main() { collectorConfig = "/opt/microsoft/otelcollector/ccp-collector-config-replicaset.yml" } else { collectorConfig = "/opt/microsoft/otelcollector/collector-config-replicaset.yml" + configmapsettings.SetGlobalSettingsInCollectorConfig() } } else if azmonUseDefaultPrometheusConfig == "true" { fmt.Println("Starting otelcollector with only default scrape configs enabled") diff --git a/otelcollector/prom-config-validator-builder/main.go b/otelcollector/prom-config-validator-builder/main.go index a311217db..a35a2cdc3 100644 --- a/otelcollector/prom-config-validator-builder/main.go +++ b/otelcollector/prom-config-validator-builder/main.go @@ -71,15 +71,15 @@ func setFatalErrorMessageAsEnvVar(message string) { truncatedMessage = re.ReplaceAllString(truncatedMessage, "") // Write env var to a file so it can be used by other processes - file, err := os.Create("/opt/microsoft/prom_config_validator_env_var") + file, err := os.OpenFile("/opt/microsoft/prom_config_validator_env_var", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - log.Println("prom-config-validator::Unable to create file for prom_config_validator_env_var") + log.Println("prom-config-validator::Unable to open file - prom_config_validator_env_var") } setEnvVarString := fmt.Sprintf("export INVALID_CONFIG_FATAL_ERROR=\"%s\"\n", truncatedMessage) if os.Getenv("OS_TYPE") != "linux" { setEnvVarString = fmt.Sprintf("INVALID_CONFIG_FATAL_ERROR=%s\n", truncatedMessage) } - _, err = file.WriteString(setEnvVarString) + _, err = file.Write([]byte(setEnvVarString)) if err != nil { log.Println("prom-config-validator::Unable to write to the file prom_config_validator_env_var") } @@ -215,6 +215,30 @@ func generateOtelConfig(promFilePath string, outputFilePath string, otelConfigTe return fmt.Errorf("unsupported features:\n\t%s", strings.Join(unsupportedFeatures, "\n\t")) } + globalSettingsFromMergedOtelConfig := prometheusConfig["global"] + + if globalSettingsFromMergedOtelConfig != nil { + globalSettings := globalSettingsFromMergedOtelConfig.(map[interface{}]interface{}) + scrapeInterval := globalSettings["scrape_interval"] + if (len(globalSettings) > 1) || (len(globalSettings) == 1 && scrapeInterval != "15s") { + setEnvVarString := fmt.Sprintf("AZMON_GLOBAL_SETTINGS_CONFIGURED=true\n") + file, err := os.OpenFile("/opt/microsoft/prom_config_validator_env_var", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Println("prom-config-validator::Unable to open file - prom_config_validator_env_var") + } + _, err = file.Write([]byte(setEnvVarString)) + if err != nil { + log.Println("prom-config-validator::Unable to write to the file prom_config_validator_env_var") + } + file.Close() + if err != nil { + log.Println("prom-config-validator::Unable to close file prom_config_validator_env_var", err) + } else { + log.Printf("prom-config-validator::Successfully set env variables for global config in file prom_config_validator_env_var\n") + } + } + } + otelConfig.Receivers.Prometheus.Config = prometheusConfig if os.Getenv("DEBUG_MODE_ENABLED") == "true" { diff --git a/otelcollector/shared/configmap/mp/configmapparser.go b/otelcollector/shared/configmap/mp/configmapparser.go index fea3485c2..419f4b214 100644 --- a/otelcollector/shared/configmap/mp/configmapparser.go +++ b/otelcollector/shared/configmap/mp/configmapparser.go @@ -183,6 +183,8 @@ func Configmapparser() { } } shared.SetEnvAndSourceBashrcOrPowershell("AZMON_USE_DEFAULT_PROMETHEUS_CONFIG", "true", true) + } else { + shared.SetEnvAndSourceBashrcOrPowershell("AZMON_SET_GLOBAL_SETTINGS", "true", true) } } } else if _, err := os.Stat(mergedDefaultConfigPath); err == nil { @@ -225,7 +227,7 @@ func Configmapparser() { if len(parts) == 2 { key := parts[0] value := parts[1] - os.Setenv(key, value) + shared.SetEnvAndSourceBashrcOrPowershell(key, value, true) // Write to envvars.env fmt.Fprintf(envFile, "%s=%s\n", key, value) diff --git a/otelcollector/shared/configmap/mp/tomlparser-set-global-config.go b/otelcollector/shared/configmap/mp/tomlparser-set-global-config.go new file mode 100644 index 000000000..2b3962723 --- /dev/null +++ b/otelcollector/shared/configmap/mp/tomlparser-set-global-config.go @@ -0,0 +1,87 @@ +package configmapsettings + +import ( + "fmt" + "log" + "os" + + yaml "gopkg.in/yaml.v2" +) + +type OtelConfig struct { + Exporters interface{} `yaml:"exporters"` + Processors interface{} `yaml:"processors"` + Extensions interface{} `yaml:"extensions"` + Receivers struct { + Prometheus struct { + Config map[string]interface{} `yaml:"config"` + TargetAllocator interface{} `yaml:"target_allocator"` + } `yaml:"prometheus"` + } `yaml:"receivers"` + Service struct { + Extensions interface{} `yaml:"extensions"` + Pipelines struct { + Metrics struct { + Exporters interface{} `yaml:"exporters"` + Processors interface{} `yaml:"processors"` + Receivers interface{} `yaml:"receivers"` + } `yaml:"metrics"` + } `yaml:"pipelines"` + Telemetry struct { + Logs struct { + Level interface{} `yaml:"level"` + Encoding interface{} `yaml:"encoding"` + } `yaml:"logs"` + } `yaml:"telemetry"` + } `yaml:"service"` +} + +func SetGlobalSettingsInCollectorConfig() { + azmonSetGlobalSettings := os.Getenv("AZMON_SET_GLOBAL_SETTINGS") + + if azmonSetGlobalSettings == "true" { + mergedCollectorConfigPath := "/opt/microsoft/otelcollector/collector-config.yml" + mergedCollectorConfigFileContents, err := os.ReadFile(mergedCollectorConfigPath) + if err != nil { + fmt.Printf("Unable to read file contents from: %s - %v\n", mergedCollectorConfigPath, err) + return + } + var promScrapeConfig map[string]interface{} + var otelConfig OtelConfig + err = yaml.Unmarshal([]byte(mergedCollectorConfigFileContents), &otelConfig) + if err != nil { + fmt.Printf("Unable to unmarshal merged otel configuration from: %s - %v\n", mergedCollectorConfigFileContents, err) + return + } + + promScrapeConfig = otelConfig.Receivers.Prometheus.Config + globalSettingsFromMergedOtelConfig := promScrapeConfig["global"] + + if globalSettingsFromMergedOtelConfig != nil { + fmt.Println("Found global settings in merged otel config, triyng to replace replicaset collector config") + collectorConfigReplicasetPath := "/opt/microsoft/otelcollector/collector-config-replicaset.yml" + replicasetCollectorConfigFileContents, err := os.ReadFile(collectorConfigReplicasetPath) + if err != nil { + fmt.Printf("Unable to read file contents from: %s - %v\n", replicasetCollectorConfigFileContents, err) + return + } + var otelConfigReplicaset OtelConfig + err = yaml.Unmarshal([]byte(replicasetCollectorConfigFileContents), &otelConfigReplicaset) + if err != nil { + fmt.Printf("Unable to unmarshal merged otel configuration from: %s - %v\n", replicasetCollectorConfigFileContents, err) + return + } + otelConfigReplicaset.Receivers.Prometheus.Config = map[string]interface{}{"global": ""} + otelConfigReplicaset.Receivers.Prometheus.Config["global"] = globalSettingsFromMergedOtelConfig + otelReplacedConfigYaml, _ := yaml.Marshal(otelConfigReplicaset) + if err := os.WriteFile(collectorConfigReplicasetPath, otelReplacedConfigYaml, 0644); err != nil { + fmt.Printf("Unable to write to: %s - %v\n", collectorConfigReplicasetPath, err) + return + } + + log.Println("Updated file with global settings", collectorConfigReplicasetPath) + return + } + fmt.Println("Global settings are empty in custom config map, making no replacement") + } +} diff --git a/otelcollector/telegraf/telegraf-prometheus-collector-ds.conf b/otelcollector/telegraf/telegraf-prometheus-collector-ds.conf index 260dfab05..417343f4c 100644 --- a/otelcollector/telegraf/telegraf-prometheus-collector-ds.conf +++ b/otelcollector/telegraf/telegraf-prometheus-collector-ds.conf @@ -33,6 +33,8 @@ podname = "$POD_NAME" ostype = "$OS_TYPE" mip="$MINIMAL_INGESTION_PROFILE" + setGlobalSettings = "$AZMON_SET_GLOBAL_SETTINGS" + globalSettingsConfigured = "$AZMON_GLOBAL_SETTINGS_CONFIGURED" # Configuration for telegraf agent [agent] diff --git a/otelcollector/telegraf/telegraf-prometheus-collector-ta-enabled.conf b/otelcollector/telegraf/telegraf-prometheus-collector-ta-enabled.conf index 11547a903..815bd44ad 100644 --- a/otelcollector/telegraf/telegraf-prometheus-collector-ta-enabled.conf +++ b/otelcollector/telegraf/telegraf-prometheus-collector-ta-enabled.conf @@ -211,6 +211,8 @@ httpproxyenabled = "$HTTP_PROXY_ENABLED" tadapterh="$tokenadapterHealthyAfterSecs" tadapterf="$tokenadapterUnhealthyAfterSecs" + setGlobalSettings="$AZMON_SET_GLOBAL_SETTINGS" + globalSettingsConfigured="$AZMON_GLOBAL_SETTINGS_CONFIGURED" [[inputs.procstat]] exe = "MetricsExtension" diff --git a/otelcollector/telegraf/telegraf-prometheus-collector-windows.conf b/otelcollector/telegraf/telegraf-prometheus-collector-windows.conf index 7ecd99e69..14c406fdc 100644 --- a/otelcollector/telegraf/telegraf-prometheus-collector-windows.conf +++ b/otelcollector/telegraf/telegraf-prometheus-collector-windows.conf @@ -35,6 +35,9 @@ ostype = "$OS_TYPE" debugmodeenabled = "$DEBUG_MODE_ENABLED" windowsVersion = "$windowsVersion" + setGlobalSettings = "$AZMON_SET_GLOBAL_SETTINGS" + globalSettingsConfigured = "$AZMON_GLOBAL_SETTINGS_CONFIGURED" + # Configuration for telegraf agent [agent]