add hubble scrape config

otelcollector/configmapparser/default-prom-configs/hubbleDefaultDs.yml

@@ -0,0 +1,21 @@
+scrape_configs:
+  - job_name: "hubble-metrics"
+    scrape_interval: $$SCRAPE_INTERVAL$$
+    label_limit: 63
+    label_name_length_limit: 511
+    label_value_length_limit: 1023
+    kubernetes_sd_configs:
+      - role: service
+    scheme: http
+    relabel_configs:
+      - source_labels:
+          [__meta_kubernetes_namespace, __meta_kubernetes_service_name]
+        action: keep
+        regex: kube-system;hubble-peer
+      - source_labels: [__address__]
+        target_label: __address__
+        replacement: "$$NODE_IP$$:9965"
+        action: replace
+      - source_labels: [__address__]
+        replacement: "$$NODE_NAME$$"
+        target_label: instance

otelcollector/configmapparser/prometheus-config-merger.rb

@@ -42,6 +42,7 @@
 @podannotationsDefaultFile = @defaultPromConfigPathPrefix + "podannotationsDefault.yml"
 @windowskubeproxyDefaultRsAdvancedFile = @defaultPromConfigPathPrefix + "windowskubeproxyDefaultRsAdvanced.yml"
 @kappiebasicDefaultFileDs = @defaultPromConfigPathPrefix + "kappieBasicDefaultDs.yml"
+@hubbleDefaultFileDs = @defaultPromConfigPathPrefix + "hubbleDefaultDs.yml"
 
 def parseConfigMap
   begin
@@ -335,6 +336,26 @@ def populateDefaultPrometheusConfig
       end
     end
 
+    if !ENV["AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED"].downcase == "true"
+      hubbleMetricsKeepListRegex = @regexHash["HUBBLE_METRICS_KEEP_LIST_REGEX"]
+      hubblebasicScrapeInterval = @intervalHash["HUBBLE_SCRAPE_INTERVAL"]
+      if currentControllerType == @replicasetControllerType
+        #do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
+      else #hubble scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
+        if advancedMode == true  && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" #&& ENV["OS_TYPE"].downcase == "linux"
+          UpdateScrapeIntervalConfig(@hubbleDefaultFileDs, hubblebasicScrapeInterval)
+          if !hubbleMetricsKeepListRegex.nil? && !hubbleMetricsKeepListRegex.empty?
+            AppendMetricRelabelConfig(@hubbleDefaultFileDs, hubbleMetricsKeepListRegex)
+          end
+          contents = File.read(@hubbleDefaultFileDs)
+          contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
+          contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
+          File.open(@hubbleDefaultFileDs, "w") { |file| file.puts contents }
+          defaultConfigs.push(@hubbleDefaultFileDs)
+        end
+      end
+    end
+
     # Collector health config should be enabled or disabled for both replicaset and daemonset
     if !ENV["AZMON_PROMETHEUS_COLLECTOR_HEALTH_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_COLLECTOR_HEALTH_SCRAPING_ENABLED"].downcase == "true"
       prometheusCollectorHealthInterval = @intervalHash["PROMETHEUS_COLLECTOR_HEALTH_SCRAPE_INTERVAL"]

otelcollector/configmapparser/tomlparser-default-scrape-settings.rb

@@ -22,6 +22,7 @@
 @windowsexporterEnabled = false
 @windowskubeproxyEnabled = false
 @kappiebasicEnabled = true
+@hubbleEnabled = false
 @noDefaultsEnabled = false
 @sendDSUpMetric = false
 
@@ -93,6 +94,10 @@ def populateSettingValuesFromConfigMap(parsedConfig)
       @kappiebasicEnabled = parsedConfig[:kappiebasic]
       puts "config::Using configmap scrape settings for kappiebasic: #{@kappiebasicEnabled}"
     end
+    if !parsedConfig[:hubble].nil?
+      @hubbleEnabled = parsedConfig[:hubble]
+      puts "config::Using configmap scrape settings for hubble: #{@hubbleEnabled}"
+    end
 
     windowsDaemonset = false
     if ENV["WINMODE"].nil? && ENV["WINMODE"].strip.downcase == "advanced"
@@ -128,7 +133,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
 ConfigParseErrorLogger.logSection(LOGGING_PREFIX, "Start default-scrape-settings Processing")
 # set default targets for MAC mode
 if !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true"
-  ConfigParseErrorLogger.logWarning(LOGGING_PREFIX, "MAC mode is enabled. Only enabling targets kubestate,cadvisor,kubelet,kappiebasic & nodeexporter for linux before config map processing....")
+  ConfigParseErrorLogger.logWarning(LOGGING_PREFIX, "MAC mode is enabled. Only enabling targets kubestate,cadvisor,kubelet,kappiebasic,hubble & nodeexporter for linux before config map processing....")
 
   @corednsEnabled = false
   @kubeproxyEnabled = false
@@ -168,10 +173,11 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write($export + "AZMON_PROMETHEUS_WINDOWSEXPORTER_SCRAPING_ENABLED=#{@windowsexporterEnabled}\n")
   file.write($export + "AZMON_PROMETHEUS_WINDOWSKUBEPROXY_SCRAPING_ENABLED=#{@windowskubeproxyEnabled}\n")
   file.write($export + "AZMON_PROMETHEUS_KAPPIEBASIC_SCRAPING_ENABLED=#{@kappiebasicEnabled}\n")
+  file.write($export + "AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED=#{@hubbleEnabled}\n")
   file.write($export + "AZMON_PROMETHEUS_POD_ANNOTATION_SCRAPING_ENABLED=#{@podannotationEnabled}\n")
   # Close file after writing all metric collection setting environment variables
   file.close
 else
   ConfigParseErrorLogger.logError(LOGGING_PREFIX, "Exception while opening file for writing default-scrape-settings config environment variables")
 end
-ConfigParseErrorLogger.logSection(LOGGING_PREFIX, "End default-scrape-settings Processing")
+ConfigParseErrorLogger.logSection(LOGGING_PREFIX, "End default-scrape-settings Processing")

otelcollector/configmapparser/tomlparser-default-targets-metrics-keep-list.rb

@@ -26,6 +26,7 @@
 @windowskubeproxyRegex = ""
 @podannotationRegex = ""
 @kappiebasicRegex = ""
+@hubbleRegex = ""
 
 #This will always be string "true" as we set the string value in the chart for both MAC and non MAC modes
 @minimalIngestionProfile = ENV["MINIMAL_INGESTION_PROFILE"]
@@ -44,6 +45,7 @@
 @kubestateRegex_minimal = "kube_horizontalpodautoscaler_spec_min_replicas|kube_horizontalpodautoscaler_status_desired_replicas|kube_job_status_active|kube_node_status_capacity|kube_job_status_succeeded|kube_job_spec_completions|kube_daemonset_status_number_misscheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_number_ready|kube_deployment_spec_replicas|kube_deployment_status_replicas_ready|kube_pod_container_status_last_terminated_reason|kube_node_status_condition|kube_pod_container_status_restarts_total|kube_pod_container_resource_requests|kube_pod_status_phase|kube_pod_container_resource_limits|kube_node_status_allocatable|kube_pod_info|kube_pod_owner|kube_resourcequota|kube_statefulset_replicas|kube_statefulset_status_replicas|kube_statefulset_status_replicas_ready|kube_statefulset_status_replicas_current|kube_statefulset_status_replicas_updated|kube_namespace_status_phase|kube_node_info|kube_statefulset_metadata_generation|kube_pod_labels|kube_pod_annotations|kube_horizontalpodautoscaler_status_current_replicas|kube_horizontalpodautoscaler_spec_max_replicas|kube_node_spec_taint|kube_pod_container_status_waiting_reason|kube_job_failed|kube_job_status_start_time|kube_deployment_status_replicas_available|kube_deployment_status_replicas_updated|kube_replicaset_owner|kubernetes_build_info|kube_pod_container_info"
 @nodeexporterRegex_minimal = "node_filesystem_readonly|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_Slab_bytes|node_memory_MemTotal_bytes|node_netstat_Tcp_RetransSegs|node_netstat_Tcp_OutSegs|node_netstat_TcpExt_TCPSynRetrans|node_load1|node_load5|node_load15|node_disk_read_bytes_total|node_disk_written_bytes_total|node_disk_io_time_seconds_total|node_filesystem_size_bytes|node_filesystem_avail_bytes|node_network_receive_bytes_total|node_network_transmit_bytes_total|node_vmstat_pgmajfault|node_network_receive_drop_total|node_network_transmit_drop_total|node_disk_io_time_weighted_seconds_total|node_exporter_build_info|node_time_seconds|node_uname_info|kubernetes_build_info"
 @kappiebasicRegex_minimal = "kappie.*"
+@hubbleRegex_minimal = "hubble.*"
 @windowsexporterRegex_minimal = "windows_system_system_up_time|windows_cpu_time_total|windows_memory_available_bytes|windows_os_visible_memory_bytes|windows_memory_cache_bytes|windows_memory_modified_page_list_bytes|windows_memory_standby_cache_core_bytes|windows_memory_standby_cache_normal_priority_bytes|windows_memory_standby_cache_reserve_bytes|windows_memory_swap_page_operations_total|windows_logical_disk_read_seconds_total|windows_logical_disk_write_seconds_total|windows_logical_disk_size_bytes|windows_logical_disk_free_bytes|windows_net_bytes_total|windows_net_packets_received_discarded_total|windows_net_packets_outbound_discarded_total|windows_container_available|windows_container_cpu_usage_seconds_total|windows_container_memory_usage_commit_bytes|windows_container_memory_usage_private_working_set_bytes|windows_container_network_receive_bytes_total|windows_container_network_transmit_bytes_total"
 @windowskubeproxyRegex_minimal = "kubeproxy_sync_proxy_rules_duration_seconds|kubeproxy_sync_proxy_rules_duration_seconds_bucket|kubeproxy_sync_proxy_rules_duration_seconds_sum|kubeproxy_sync_proxy_rules_duration_seconds_count|rest_client_requests_total|rest_client_request_duration_seconds|rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines"
 
@@ -56,6 +58,7 @@
 @kubestateRegex_minimal_mac = "kube_job_status_succeeded|kube_job_spec_completions|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_ready|kube_deployment_status_replicas_ready|kube_pod_container_status_last_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_node_status_allocatable|kube_pod_owner|kube_pod_container_resource_requests|kube_pod_status_phase|kube_pod_container_resource_limits|kube_replicaset_owner|kube_resourcequota|kube_namespace_status_phase|kube_node_status_capacity|kube_node_info|kube_pod_info|kube_deployment_spec_replicas|kube_deployment_status_replicas_available|kube_deployment_status_replicas_updated|kube_statefulset_status_replicas_ready|kube_statefulset_status_replicas|kube_statefulset_status_replicas_updated|kube_job_status_start_time|kube_job_status_active|kube_job_failed|kube_horizontalpodautoscaler_status_desired_replicas|kube_horizontalpodautoscaler_status_current_replicas|kube_horizontalpodautoscaler_spec_min_replicas|kube_horizontalpodautoscaler_spec_max_replicas|kubernetes_build_info|kube_node_status_condition|kube_node_spec_taint|kube_pod_container_info|kube_.*_labels|kube_.*_annotations"
 @nodeexporterRegex_minimal_mac = "node_filesystem_readonly|node_memory_MemTotal_bytes|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_Slab_bytes|node_filesystem_avail_bytes|node_filesystem_size_bytes|node_time_seconds|node_exporter_build_info|node_load1|node_vmstat_pgmajfault|node_network_receive_bytes_total|node_network_transmit_bytes_total|node_network_receive_drop_total|node_network_transmit_drop_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_load5|node_load15|node_disk_read_bytes_total|node_disk_written_bytes_total|node_uname_info|kubernetes_build_info"
 @kappiebasicRegex_minimal_mac = "kappie.*"
+@hubbleRegex_minimal_mac = "hubble.*"
 @windowsexporterRegex_minimal_mac = "windows_system_system_up_time|windows_cpu_time_total|windows_memory_available_bytes|windows_os_visible_memory_bytes|windows_memory_cache_bytes|windows_memory_modified_page_list_bytes|windows_memory_standby_cache_core_bytes|windows_memory_standby_cache_normal_priority_bytes|windows_memory_standby_cache_reserve_bytes|windows_memory_swap_page_operations_total|windows_logical_disk_read_seconds_total|windows_logical_disk_write_seconds_total|windows_logical_disk_size_bytes|windows_logical_disk_free_bytes|windows_net_bytes_total|windows_net_packets_received_discarded_total|windows_net_packets_outbound_discarded_total|windows_container_available|windows_container_cpu_usage_seconds_total|windows_container_memory_usage_commit_bytes|windows_container_memory_usage_private_working_set_bytes|windows_container_network_receive_bytes_total|windows_container_network_transmit_bytes_total"
 @windowskubeproxyRegex_minimal_mac = "kubeproxy_sync_proxy_rules_duration_seconds|kubeproxy_sync_proxy_rules_duration_seconds_bucket|kubeproxy_sync_proxy_rules_duration_seconds_sum|kubeproxy_sync_proxy_rules_duration_seconds_count|rest_client_requests_total|rest_client_request_duration_seconds|rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count|process_resident_memory_bytes|process_cpu_seconds_total|go_goroutines"
 
@@ -191,6 +194,20 @@ def populateSettingValuesFromConfigMap(parsedConfig)
       ConfigParseErrorLogger.logError(LOGGING_PREFIX, "kappiebasicRegex either not specified or not of type string")
     end
 
+    hubbleRegex = parsedConfig[:hubble]
+    if !hubbleRegex.nil? && hubbleRegex.kind_of?(String)
+      if !hubbleRegex.empty?
+        if isValidRegex(hubbleRegex) == true
+          @hubbleRegex = hubbleRegex
+          ConfigParseErrorLogger.log(LOGGING_PREFIX, "Using configmap metrics keep list regex for hubble")
+        else
+          ConfigParseErrorLogger.logError(LOGGING_PREFIX, "Invalid keep list regex for hubble")
+        end
+      end
+    else
+      ConfigParseErrorLogger.logError(LOGGING_PREFIX, "hubbleRegex either not specified or not of type string")
+    end
+
     windowsexporterRegex = parsedConfig[:windowsexporter]
     if !windowsexporterRegex.nil? && windowsexporterRegex.kind_of?(String)
       if !windowsexporterRegex.empty?
@@ -264,6 +281,7 @@ def populateRegexValuesWithMinimalIngestionProfile
         @kubestateRegex = @kubestateRegex + "|" + @kubestateRegex_minimal_mac
         @nodeexporterRegex = @nodeexporterRegex + "|" + @nodeexporterRegex_minimal_mac
         @kappiebasicRegex = @kappiebasicRegex + "|" + @kappiebasicRegex_minimal_mac
+        @hubbleRegex = @hubbleRegex + "|" + @hubbleRegex_minimal_mac
         @windowsexporterRegex = @windowsexporterRegex + "|" + @windowsexporterRegex_minimal_mac
         @windowskubeproxyRegex = @windowskubeproxyRegex + "|" + @windowskubeproxyRegex_minimal_mac
       else
@@ -275,7 +293,7 @@ def populateRegexValuesWithMinimalIngestionProfile
         @apiserverRegex = @apiserverRegex + "|" + @apiserverRegex_minimal
         @kubestateRegex = @kubestateRegex + "|" + @kubestateRegex_minimal
         @nodeexporterRegex = @nodeexporterRegex + "|" + @nodeexporterRegex_minimal
-        @kappiebasicRegex = @kappiebasicRegex + "|" + @kappiebasicRegex_minimal
+        @hubbleRegex = @hubbleRegex + "|" + @hubbleRegex_minimal
         @windowsexporterRegex = @windowsexporterRegex + "|" + @windowsexporterRegex_minimal
         @windowskubeproxyRegex = @windowskubeproxyRegex + "|" + @windowskubeproxyRegex_minimal
       end
@@ -318,6 +336,7 @@ def populateRegexValuesWithMinimalIngestionProfile
 regexHash["WINDOWSKUBEPROXY_METRICS_KEEP_LIST_REGEX"] = @windowskubeproxyRegex
 regexHash["POD_ANNOTATION_METRICS_KEEP_LIST_REGEX"] = @podannotationRegex
 regexHash["KAPPIEBASIC_METRICS_KEEP_LIST_REGEX"] = @kappiebasicRegex
+regexHash["HUBBLE_METRICS_KEEP_LIST_REGEX"] = @hubbleRegex
 
 if !file.nil?
   # Close file after writing regex keep list hash

otelcollector/configmapparser/tomlparser-scrape-interval.rb

@@ -27,6 +27,7 @@
 @prometheusCollectorHealthInterval = "30s"
 @podannotationScrapeInterval = "30s"
 @kappiebasicScrapeInterval = "30s"
+@hubbleScrapeInterval = "30s"
 
 # Use parser to parse the configmap toml file to a ruby structure
 def parseConfigMap
@@ -208,6 +209,22 @@ def populateSettingValuesFromConfigMap(parsedConfig)
       ConfigParseErrorLogger.log(LOGGING_PREFIX, "kappiebasicScrapeInterval override not specified in configmap")
     end
 
+    hubbleScrapeInterval = parsedConfig[:hubble]
+    if !hubbleScrapeInterval.nil?
+      matched = MATCHER.match(hubbleScrapeInterval)
+      if !matched
+        # set default scrape interval to 30s if its not in the proper format
+        hubbleScrapeInterval = "30s"
+        @hubbleScrapeInterval = hubbleScrapeInterval
+        ConfigParseErrorLogger.log(LOGGING_PREFIX, "Incorrect regex pattern for duration, set default scrape interval to 30s for hubble")
+      else
+        @hubbleScrapeInterval = hubbleScrapeInterval
+        ConfigParseErrorLogger.log(LOGGING_PREFIX, "Using configmap scrape settings for hubbleScrapeInterval")
+      end
+    else
+      ConfigParseErrorLogger.log(LOGGING_PREFIX, "hubbleScrapeInterval override not specified in configmap")
+    end
+
     prometheusCollectorHealthInterval = parsedConfig[:prometheuscollectorhealth]
     if !prometheusCollectorHealthInterval.nil?
       matched = MATCHER.match(prometheusCollectorHealthInterval)

otelcollector/configmaps/ama-metrics-settings-configmap.yaml

@@ -20,6 +20,7 @@ data:
     windowsexporter = false
     windowskubeproxy = false
     kappiebasic = true
+    hubble = false
     prometheuscollectorhealth = false
   # Regex for which namespaces to scrape through pod annotation based scraping.
   # This is none by default. Use '.*' to scrape all namespaces of annotated pods.
@@ -37,6 +38,7 @@ data:
     windowskubeproxy = ""
     podannotations = ""
     kappiebasic = ""
+    hubble = ""
     minimalingestionprofile = true
   default-targets-scrape-interval-settings: |-
     kubelet = "30s"
@@ -49,6 +51,7 @@ data:
     windowsexporter = "30s"
     windowskubeproxy = "30s"
     kappiebasic = "30s"
+    hubble = "30s"
     prometheuscollectorhealth = "30s"
     podannotations = "30s"
   debug-mode: |-

otelcollector/telegraf/telegraf-prometheus-collector.conf

@@ -170,6 +170,7 @@
     defaultscrapepodannotations = "$AZMON_PROMETHEUS_POD_ANNOTATION_SCRAPING_ENABLED"
     podannotationns = "$AZMON_PROMETHEUS_POD_ANNOTATION_NAMESPACES_REGEX"
     defaultscrapekappiebasic = "$AZMON_PROMETHEUS_KAPPIEBASIC_SCRAPING_ENABLED"
+    defaultscrapehubblebasic = "$AZMON_PROMETHEUS_HUBBLE_SCRAPING_ENABLED"
     nodeexportertargetport= "$NODE_EXPORTER_TARGETPORT"
     nodeexportername = "$NODE_EXPORTER_NAME"
     kubestatename = "$KUBE_STATE_NAME"