Skip to content

Commit

Permalink
retina
Browse files Browse the repository at this point in the history
  • Loading branch information
matmerr committed Oct 19, 2023
1 parent 6eea3f3 commit f6f2b2c
Show file tree
Hide file tree
Showing 7 changed files with 47 additions and 47 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
scrape_configs:
- job_name: "hubble-metrics"
- job_name: "retina-metrics"
scrape_interval: $$SCRAPE_INTERVAL$$
label_limit: 63
label_name_length_limit: 511
Expand All @@ -11,7 +11,7 @@ scrape_configs:
- source_labels:
[__meta_kubernetes_namespace, __meta_kubernetes_service_name]
action: keep
regex: kube-system;hubble-peer
regex: kube-system;retina-peer
- source_labels: [__address__]
target_label: __address__
replacement: "$$NODE_IP$$:9965"
Expand Down
22 changes: 11 additions & 11 deletions otelcollector/configmapparser/prometheus-config-merger.rb
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
@podannotationsDefaultFile = @defaultPromConfigPathPrefix + "podannotationsDefault.yml"
@windowskubeproxyDefaultRsAdvancedFile = @defaultPromConfigPathPrefix + "windowskubeproxyDefaultRsAdvanced.yml"
@kappiebasicDefaultFileDs = @defaultPromConfigPathPrefix + "kappieBasicDefaultDs.yml"
@hubbleDefaultFileDs = @defaultPromConfigPathPrefix + "hubbleDefaultDs.yml"
@retinaDefaultFileDs = @defaultPromConfigPathPrefix + "retinaDefaultDs.yml"

def parseConfigMap
begin
Expand Down Expand Up @@ -336,22 +336,22 @@ def populateDefaultPrometheusConfig
end
end

if !ENV["AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED"].downcase == "true"
hubbleMetricsKeepListRegex = @regexHash["HUBBLE_METRICS_KEEP_LIST_REGEX"]
hubblebasicScrapeInterval = @intervalHash["HUBBLE_SCRAPE_INTERVAL"]
if !ENV["AZMON_PROMETHEUS_RETINA_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_RETINA_SCRAPING_ENABLED"].downcase == "true"
retinaMetricsKeepListRegex = @regexHash["RETINA_METRICS_KEEP_LIST_REGEX"]
retinabasicScrapeInterval = @intervalHash["RETINA_SCRAPE_INTERVAL"]
if currentControllerType == @replicasetControllerType
#do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
else #hubble scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
else #retina scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
if advancedMode == true && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" #&& ENV["OS_TYPE"].downcase == "linux"
UpdateScrapeIntervalConfig(@hubbleDefaultFileDs, hubblebasicScrapeInterval)
if !hubbleMetricsKeepListRegex.nil? && !hubbleMetricsKeepListRegex.empty?
AppendMetricRelabelConfig(@hubbleDefaultFileDs, hubbleMetricsKeepListRegex)
UpdateScrapeIntervalConfig(@retinaDefaultFileDs, retinabasicScrapeInterval)
if !retinaMetricsKeepListRegex.nil? && !retinaMetricsKeepListRegex.empty?
AppendMetricRelabelConfig(@retinaDefaultFileDs, retinaMetricsKeepListRegex)
end
contents = File.read(@hubbleDefaultFileDs)
contents = File.read(@retinaDefaultFileDs)
contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
File.open(@hubbleDefaultFileDs, "w") { |file| file.puts contents }
defaultConfigs.push(@hubbleDefaultFileDs)
File.open(@retinaDefaultFileDs, "w") { |file| file.puts contents }
defaultConfigs.push(@retinaDefaultFileDs)
end
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
@windowsexporterEnabled = false
@windowskubeproxyEnabled = false
@kappiebasicEnabled = true
@hubbleEnabled = false
@retinaEnabled = false
@noDefaultsEnabled = false
@sendDSUpMetric = false

Expand Down Expand Up @@ -94,9 +94,9 @@ def populateSettingValuesFromConfigMap(parsedConfig)
@kappiebasicEnabled = parsedConfig[:kappiebasic]
puts "config::Using configmap scrape settings for kappiebasic: #{@kappiebasicEnabled}"
end
if !parsedConfig[:hubble].nil?
@hubbleEnabled = parsedConfig[:hubble]
puts "config::Using configmap scrape settings for hubble: #{@hubbleEnabled}"
if !parsedConfig[:retina].nil?
@retinaEnabled = parsedConfig[:retina]
puts "config::Using configmap scrape settings for retina: #{@retinaEnabled}"
end

windowsDaemonset = false
Expand Down Expand Up @@ -133,7 +133,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
ConfigParseErrorLogger.logSection(LOGGING_PREFIX, "Start default-scrape-settings Processing")
# set default targets for MAC mode
if !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true"
ConfigParseErrorLogger.logWarning(LOGGING_PREFIX, "MAC mode is enabled. Only enabling targets kubestate,cadvisor,kubelet,kappiebasic,hubble & nodeexporter for linux before config map processing....")
ConfigParseErrorLogger.logWarning(LOGGING_PREFIX, "MAC mode is enabled. Only enabling targets kubestate,cadvisor,kubelet,kappiebasic,retina & nodeexporter for linux before config map processing....")

@corednsEnabled = false
@kubeproxyEnabled = false
Expand Down Expand Up @@ -173,7 +173,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
file.write($export + "AZMON_PROMETHEUS_WINDOWSEXPORTER_SCRAPING_ENABLED=#{@windowsexporterEnabled}\n")
file.write($export + "AZMON_PROMETHEUS_WINDOWSKUBEPROXY_SCRAPING_ENABLED=#{@windowskubeproxyEnabled}\n")
file.write($export + "AZMON_PROMETHEUS_KAPPIEBASIC_SCRAPING_ENABLED=#{@kappiebasicEnabled}\n")
file.write($export + "AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED=#{@hubbleEnabled}\n")
file.write($export + "AZMON_PROMETHEUS_RETINA_SCRAPING_ENABLED=#{@retinaEnabled}\n")
file.write($export + "AZMON_PROMETHEUS_POD_ANNOTATION_SCRAPING_ENABLED=#{@podannotationEnabled}\n")
# Close file after writing all metric collection setting environment variables
file.close
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
@windowskubeproxyRegex = ""
@podannotationRegex = ""
@kappiebasicRegex = ""
@hubbleRegex = ""
@retinaRegex = ""

#This will always be string "true" as we set the string value in the chart for both MAC and non MAC modes
@minimalIngestionProfile = ENV["MINIMAL_INGESTION_PROFILE"]
Expand All @@ -45,7 +45,7 @@
@kubestateRegex_minimal = "kube_horizontalpodautoscaler_spec_min_replicas|kube_horizontalpodautoscaler_status_desired_replicas|kube_job_status_active|kube_node_status_capacity|kube_job_status_succeeded|kube_job_spec_completions|kube_daemonset_status_number_misscheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_number_ready|kube_deployment_spec_replicas|kube_deployment_status_replicas_ready|kube_pod_container_status_last_terminated_reason|kube_node_status_condition|kube_pod_container_status_restarts_total|kube_pod_container_resource_requests|kube_pod_status_phase|kube_pod_container_resource_limits|kube_node_status_allocatable|kube_pod_info|kube_pod_owner|kube_resourcequota|kube_statefulset_replicas|kube_statefulset_status_replicas|kube_statefulset_status_replicas_ready|kube_statefulset_status_replicas_current|kube_statefulset_status_replicas_updated|kube_namespace_status_phase|kube_node_info|kube_statefulset_metadata_generation|kube_pod_labels|kube_pod_annotations|kube_horizontalpodautoscaler_status_current_replicas|kube_horizontalpodautoscaler_spec_max_replicas|kube_node_spec_taint|kube_pod_container_status_waiting_reason|kube_job_failed|kube_job_status_start_time|kube_deployment_status_replicas_available|kube_deployment_status_replicas_updated|kube_replicaset_owner|kubernetes_build_info|kube_pod_container_info"
@nodeexporterRegex_minimal = "node_filesystem_readonly|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_Slab_bytes|node_memory_MemTotal_bytes|node_netstat_Tcp_RetransSegs|node_netstat_Tcp_OutSegs|node_netstat_TcpExt_TCPSynRetrans|node_load1|node_load5|node_load15|node_disk_read_bytes_total|node_disk_written_bytes_total|node_disk_io_time_seconds_total|node_filesystem_size_bytes|node_filesystem_avail_bytes|node_network_receive_bytes_total|node_network_transmit_bytes_total|node_vmstat_pgmajfault|node_network_receive_drop_total|node_network_transmit_drop_total|node_disk_io_time_weighted_seconds_total|node_exporter_build_info|node_time_seconds|node_uname_info|kubernetes_build_info"
@kappiebasicRegex_minimal = "kappie.*"
@hubbleRegex_minimal = "hubble.*"
@retinaRegex_minimal = "retina.*"
@windowsexporterRegex_minimal = "windows_system_system_up_time|windows_cpu_time_total|windows_memory_available_bytes|windows_os_visible_memory_bytes|windows_memory_cache_bytes|windows_memory_modified_page_list_bytes|windows_memory_standby_cache_core_bytes|windows_memory_standby_cache_normal_priority_bytes|windows_memory_standby_cache_reserve_bytes|windows_memory_swap_page_operations_total|windows_logical_disk_read_seconds_total|windows_logical_disk_write_seconds_total|windows_logical_disk_size_bytes|windows_logical_disk_free_bytes|windows_net_bytes_total|windows_net_packets_received_discarded_total|windows_net_packets_outbound_discarded_total|windows_container_available|windows_container_cpu_usage_seconds_total|windows_container_memory_usage_commit_bytes|windows_container_memory_usage_private_working_set_bytes|windows_container_network_receive_bytes_total|windows_container_network_transmit_bytes_total"
@windowskubeproxyRegex_minimal = "kubeproxy_sync_proxy_rules_duration_seconds|kubeproxy_sync_proxy_rules_duration_seconds_bucket|kubeproxy_sync_proxy_rules_duration_seconds_sum|kubeproxy_sync_proxy_rules_duration_seconds_count|rest_client_requests_total|rest_client_request_duration_seconds|rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines"

Expand All @@ -58,7 +58,7 @@
@kubestateRegex_minimal_mac = "kube_job_status_succeeded|kube_job_spec_completions|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_ready|kube_deployment_status_replicas_ready|kube_pod_container_status_last_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_node_status_allocatable|kube_pod_owner|kube_pod_container_resource_requests|kube_pod_status_phase|kube_pod_container_resource_limits|kube_replicaset_owner|kube_resourcequota|kube_namespace_status_phase|kube_node_status_capacity|kube_node_info|kube_pod_info|kube_deployment_spec_replicas|kube_deployment_status_replicas_available|kube_deployment_status_replicas_updated|kube_statefulset_status_replicas_ready|kube_statefulset_status_replicas|kube_statefulset_status_replicas_updated|kube_job_status_start_time|kube_job_status_active|kube_job_failed|kube_horizontalpodautoscaler_status_desired_replicas|kube_horizontalpodautoscaler_status_current_replicas|kube_horizontalpodautoscaler_spec_min_replicas|kube_horizontalpodautoscaler_spec_max_replicas|kubernetes_build_info|kube_node_status_condition|kube_node_spec_taint|kube_pod_container_info|kube_.*_labels|kube_.*_annotations"
@nodeexporterRegex_minimal_mac = "node_filesystem_readonly|node_memory_MemTotal_bytes|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_Slab_bytes|node_filesystem_avail_bytes|node_filesystem_size_bytes|node_time_seconds|node_exporter_build_info|node_load1|node_vmstat_pgmajfault|node_network_receive_bytes_total|node_network_transmit_bytes_total|node_network_receive_drop_total|node_network_transmit_drop_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_load5|node_load15|node_disk_read_bytes_total|node_disk_written_bytes_total|node_uname_info|kubernetes_build_info"
@kappiebasicRegex_minimal_mac = "kappie.*"
@hubbleRegex_minimal_mac = "hubble.*"
@retinaRegex_minimal_mac = "retina.*"
@windowsexporterRegex_minimal_mac = "windows_system_system_up_time|windows_cpu_time_total|windows_memory_available_bytes|windows_os_visible_memory_bytes|windows_memory_cache_bytes|windows_memory_modified_page_list_bytes|windows_memory_standby_cache_core_bytes|windows_memory_standby_cache_normal_priority_bytes|windows_memory_standby_cache_reserve_bytes|windows_memory_swap_page_operations_total|windows_logical_disk_read_seconds_total|windows_logical_disk_write_seconds_total|windows_logical_disk_size_bytes|windows_logical_disk_free_bytes|windows_net_bytes_total|windows_net_packets_received_discarded_total|windows_net_packets_outbound_discarded_total|windows_container_available|windows_container_cpu_usage_seconds_total|windows_container_memory_usage_commit_bytes|windows_container_memory_usage_private_working_set_bytes|windows_container_network_receive_bytes_total|windows_container_network_transmit_bytes_total"
@windowskubeproxyRegex_minimal_mac = "kubeproxy_sync_proxy_rules_duration_seconds|kubeproxy_sync_proxy_rules_duration_seconds_bucket|kubeproxy_sync_proxy_rules_duration_seconds_sum|kubeproxy_sync_proxy_rules_duration_seconds_count|rest_client_requests_total|rest_client_request_duration_seconds|rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines"

Expand Down Expand Up @@ -194,18 +194,18 @@ def populateSettingValuesFromConfigMap(parsedConfig)
ConfigParseErrorLogger.logError(LOGGING_PREFIX, "kappiebasicRegex either not specified or not of type string")
end

hubbleRegex = parsedConfig[:hubble]
if !hubbleRegex.nil? && hubbleRegex.kind_of?(String)
if !hubbleRegex.empty?
if isValidRegex(hubbleRegex) == true
@hubbleRegex = hubbleRegex
ConfigParseErrorLogger.log(LOGGING_PREFIX, "Using configmap metrics keep list regex for hubble")
retinaRegex = parsedConfig[:retina]
if !retinaRegex.nil? && retinaRegex.kind_of?(String)
if !retinaRegex.empty?
if isValidRegex(retinaRegex) == true
@retinaRegex = retinaRegex
ConfigParseErrorLogger.log(LOGGING_PREFIX, "Using configmap metrics keep list regex for retina")
else
ConfigParseErrorLogger.logError(LOGGING_PREFIX, "Invalid keep list regex for hubble")
ConfigParseErrorLogger.logError(LOGGING_PREFIX, "Invalid keep list regex for retina")
end
end
else
ConfigParseErrorLogger.logError(LOGGING_PREFIX, "hubbleRegex either not specified or not of type string")
ConfigParseErrorLogger.logError(LOGGING_PREFIX, "retinaRegex either not specified or not of type string")
end

windowsexporterRegex = parsedConfig[:windowsexporter]
Expand Down Expand Up @@ -281,7 +281,7 @@ def populateRegexValuesWithMinimalIngestionProfile
@kubestateRegex = @kubestateRegex + "|" + @kubestateRegex_minimal_mac
@nodeexporterRegex = @nodeexporterRegex + "|" + @nodeexporterRegex_minimal_mac
@kappiebasicRegex = @kappiebasicRegex + "|" + @kappiebasicRegex_minimal_mac
@hubbleRegex = @hubbleRegex + "|" + @hubbleRegex_minimal_mac
@retinaRegex = @retinaRegex + "|" + @retinaRegex_minimal_mac
@windowsexporterRegex = @windowsexporterRegex + "|" + @windowsexporterRegex_minimal_mac
@windowskubeproxyRegex = @windowskubeproxyRegex + "|" + @windowskubeproxyRegex_minimal_mac
else
Expand All @@ -293,7 +293,7 @@ def populateRegexValuesWithMinimalIngestionProfile
@apiserverRegex = @apiserverRegex + "|" + @apiserverRegex_minimal
@kubestateRegex = @kubestateRegex + "|" + @kubestateRegex_minimal
@nodeexporterRegex = @nodeexporterRegex + "|" + @nodeexporterRegex_minimal
@hubbleRegex = @hubbleRegex + "|" + @hubbleRegex_minimal
@retinaRegex = @retinaRegex + "|" + @retinaRegex_minimal
@windowsexporterRegex = @windowsexporterRegex + "|" + @windowsexporterRegex_minimal
@windowskubeproxyRegex = @windowskubeproxyRegex + "|" + @windowskubeproxyRegex_minimal
end
Expand Down Expand Up @@ -336,7 +336,7 @@ def populateRegexValuesWithMinimalIngestionProfile
regexHash["WINDOWSKUBEPROXY_METRICS_KEEP_LIST_REGEX"] = @windowskubeproxyRegex
regexHash["POD_ANNOTATION_METRICS_KEEP_LIST_REGEX"] = @podannotationRegex
regexHash["KAPPIEBASIC_METRICS_KEEP_LIST_REGEX"] = @kappiebasicRegex
regexHash["HUBBLE_METRICS_KEEP_LIST_REGEX"] = @hubbleRegex
regexHash["RETINA_METRICS_KEEP_LIST_REGEX"] = @retinaRegex

if !file.nil?
# Close file after writing regex keep list hash
Expand Down
20 changes: 10 additions & 10 deletions otelcollector/configmapparser/tomlparser-scrape-interval.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
@prometheusCollectorHealthInterval = "30s"
@podannotationScrapeInterval = "30s"
@kappiebasicScrapeInterval = "30s"
@hubbleScrapeInterval = "30s"
@retinaScrapeInterval = "30s"

# Use parser to parse the configmap toml file to a ruby structure
def parseConfigMap
Expand Down Expand Up @@ -209,20 +209,20 @@ def populateSettingValuesFromConfigMap(parsedConfig)
ConfigParseErrorLogger.log(LOGGING_PREFIX, "kappiebasicScrapeInterval override not specified in configmap")
end

hubbleScrapeInterval = parsedConfig[:hubble]
if !hubbleScrapeInterval.nil?
matched = MATCHER.match(hubbleScrapeInterval)
retinaScrapeInterval = parsedConfig[:retina]
if !retinaScrapeInterval.nil?
matched = MATCHER.match(retinaScrapeInterval)
if !matched
# set default scrape interval to 30s if its not in the proper format
hubbleScrapeInterval = "30s"
@hubbleScrapeInterval = hubbleScrapeInterval
ConfigParseErrorLogger.log(LOGGING_PREFIX, "Incorrect regex pattern for duration, set default scrape interval to 30s for hubble")
retinaScrapeInterval = "30s"
@retinaScrapeInterval = retinaScrapeInterval
ConfigParseErrorLogger.log(LOGGING_PREFIX, "Incorrect regex pattern for duration, set default scrape interval to 30s for retina")
else
@hubbleScrapeInterval = hubbleScrapeInterval
ConfigParseErrorLogger.log(LOGGING_PREFIX, "Using configmap scrape settings for hubbleScrapeInterval")
@retinaScrapeInterval = retinaScrapeInterval
ConfigParseErrorLogger.log(LOGGING_PREFIX, "Using configmap scrape settings for retinaScrapeInterval")
end
else
ConfigParseErrorLogger.log(LOGGING_PREFIX, "hubbleScrapeInterval override not specified in configmap")
ConfigParseErrorLogger.log(LOGGING_PREFIX, "retinaScrapeInterval override not specified in configmap")
end

prometheusCollectorHealthInterval = parsedConfig[:prometheuscollectorhealth]
Expand Down
6 changes: 3 additions & 3 deletions otelcollector/configmaps/ama-metrics-settings-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ data:
windowsexporter = false
windowskubeproxy = false
kappiebasic = true
hubble = false
retina = false
prometheuscollectorhealth = false
# Regex for which namespaces to scrape through pod annotation based scraping.
# This is none by default. Use '.*' to scrape all namespaces of annotated pods.
Expand All @@ -38,7 +38,7 @@ data:
windowskubeproxy = ""
podannotations = ""
kappiebasic = ""
hubble = ""
retina = ""
minimalingestionprofile = true
default-targets-scrape-interval-settings: |-
kubelet = "30s"
Expand All @@ -51,7 +51,7 @@ data:
windowsexporter = "30s"
windowskubeproxy = "30s"
kappiebasic = "30s"
hubble = "30s"
retina = "30s"
prometheuscollectorhealth = "30s"
podannotations = "30s"
debug-mode: |-
Expand Down
2 changes: 1 addition & 1 deletion otelcollector/telegraf/telegraf-prometheus-collector.conf
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@
defaultscrapepodannotations = "$AZMON_PROMETHEUS_POD_ANNOTATION_SCRAPING_ENABLED"
podannotationns = "$AZMON_PROMETHEUS_POD_ANNOTATION_NAMESPACES_REGEX"
defaultscrapekappiebasic = "$AZMON_PROMETHEUS_KAPPIEBASIC_SCRAPING_ENABLED"
defaultscrapehubblebasic = "$AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED"
defaultscraperetinabasic = "$AZMON_PROMETHEUS_RETINA_SCRAPING_ENABLED"
nodeexportertargetport= "$NODE_EXPORTER_TARGETPORT"
nodeexportername = "$NODE_EXPORTER_NAME"
kubestatename = "$KUBE_STATE_NAME"
Expand Down

0 comments on commit f6f2b2c

Please sign in to comment.