networkobservability-metrics

.pipelines/azure-pipeline-build.yml

@@ -2,13 +2,15 @@ trigger:
   branches:
     include:
     - main
+    - netobsmetrics
 
 pr:
   autoCancel: true
   branches:
     include:
     - main
-
+    - netobsmetrics
+
 variables:
   HELM_CHART_NAME: 'prometheus-collector'
   ARC_HELM_CHART_NAME: 'ama-metrics-arc'

otelcollector/configmapparser/default-prom-configs/networkobservabilityCiliumDefaultDs.yml

@@ -0,0 +1,30 @@
+scrape_configs:
+  - job_name: "networkobservability-cilium"
+    scrape_interval: $$SCRAPE_INTERVAL$$
+    kubernetes_sd_configs:
+      - role: service
+    scheme: http
+    relabel_configs:
+      - source_labels:
+          [
+            __meta_kubernetes_namespace,
+            __meta_kubernetes_service_name,
+            __meta_kubernetes_service_port_name,
+          ]
+        action: keep
+        regex: kube-system;network-observability;cilium
+
+      - source_labels: [__address__]
+        target_label: __address__
+        replacement: $$NODE_IP$$
+        action: replace
+
+      - source_labels: [__address__, __meta_kubernetes_service_port_number]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+
+      - source_labels: [__address__]
+        replacement: "$$NODE_NAME$$"
+        target_label: instance

otelcollector/configmapparser/default-prom-configs/networkobservabilityHubbleDefaultDs.yml

@@ -0,0 +1,30 @@
+scrape_configs:
+  - job_name: "networkobservability-hubble"
+    scrape_interval: $$SCRAPE_INTERVAL$$
+    kubernetes_sd_configs:
+      - role: service
+    scheme: http
+    relabel_configs:
+      - source_labels:
+          [
+            __meta_kubernetes_namespace,
+            __meta_kubernetes_service_name,
+            __meta_kubernetes_service_port_name,
+          ]
+        action: keep
+        regex: kube-system;network-observability;hubble
+
+      - source_labels: [__address__]
+        target_label: __address__
+        replacement: $$NODE_IP$$
+        action: replace
+
+      - source_labels: [__address__, __meta_kubernetes_service_port_number]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+
+      - source_labels: [__address__]
+        replacement: "$$NODE_NAME$$"
+        target_label: instance

otelcollector/configmapparser/default-prom-configs/networkobservabilityRetinaDefaultDs.yml

@@ -0,0 +1,30 @@
+scrape_configs:
+  - job_name: "networkobservability-retina"
+    scrape_interval: $$SCRAPE_INTERVAL$$
+    kubernetes_sd_configs:
+      - role: service
+    scheme: http
+    relabel_configs:
+      - source_labels:
+          [
+            __meta_kubernetes_namespace,
+            __meta_kubernetes_service_name,
+            __meta_kubernetes_service_port_name,
+          ]
+        action: keep
+        regex: kube-system;network-observability;retina
+
+      - source_labels: [__address__]
+        target_label: __address__
+        replacement: $$NODE_IP$$
+        action: replace
+
+      - source_labels: [__address__, __meta_kubernetes_service_port_number]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+
+      - source_labels: [__address__]
+        replacement: "$$NODE_NAME$$"
+        target_label: instance

otelcollector/configmapparser/prometheus-config-merger-with-operator.rb

@@ -44,6 +44,11 @@
 @windowskubeproxyDefaultRsAdvancedFile = @defaultPromConfigPathPrefix + "windowskubeproxyDefaultRsAdvanced.yml"
 @kappiebasicDefaultFileDs = @defaultPromConfigPathPrefix + "kappieBasicDefaultDs.yml"
 
+@networkobservabilityRetinaDefaultFileDs = @defaultPromConfigPathPrefix + "networkobservabilityRetinaDefaultDs.yml"
+@networkobservabilityHubbleDefaultFileDs = @defaultPromConfigPathPrefix + "networkobservabilityHubbleDefaultDs.yml"
+@networkobservabilityCiliumDefaultFileDs = @defaultPromConfigPathPrefix + "networkobservabilityCiliumDefaultDs.yml"
+
+
 def parseConfigMap
   begin
     # Check to see if config map is created
@@ -352,6 +357,68 @@ def populateDefaultPrometheusConfig
       end
     end
 
+    if !ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYRETINA_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYRETINA_SCRAPING_ENABLED"].downcase == "true"
+      networkobservabilityRetinaMetricsKeepListRegex = @regexHash["NETWORKOBSERVABILITYRETINA_METRICS_KEEP_LIST_REGEX"]
+      networkobservabilityRetinaScrapeInterval = @intervalHash["NETWORKOBSERVABILITYRETINA_SCRAPE_INTERVAL"]
+      if currentControllerType == @replicasetControllerType
+        #do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
+      else #networkobservabilityRetina scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
+        if advancedMode == true  && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" #&& ENV["OS_TYPE"].downcase == "linux"
+          UpdateScrapeIntervalConfig(@networkobservabilityRetinaDefaultFileDs, networkobservabilityRetinaScrapeInterval)
+          if !networkobservabilityRetinaMetricsKeepListRegex.nil? && !networkobservabilityRetinaMetricsKeepListRegex.empty?
+            AppendMetricRelabelConfig(@networkobservabilityRetinaDefaultFileDs, networkobservabilityRetinaMetricsKeepListRegex)
+          end
+          contents = File.read(@networkobservabilityRetinaDefaultFileDs)
+          contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
+          contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
+          File.open(@networkobservabilityRetinaDefaultFileDs, "w") { |file| file.puts contents }
+          defaultConfigs.push(@networkobservabilityRetinaDefaultFileDs)
+        end
+      end
+    end
+
+    if !ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYHUBBLE_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYHUBBLE_SCRAPING_ENABLED"].downcase == "true"
+      networkobservabilityHubbleMetricsKeepListRegex = @regexHash["NETWORKOBSERVABILITYHUBBLE_METRICS_KEEP_LIST_REGEX"]
+      networkobservabilityHubbleScrapeInterval = @intervalHash["NETWORKOBSERVABILITYHUBBLE_SCRAPE_INTERVAL"]
+      if currentControllerType == @replicasetControllerType
+        #do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
+      else #networkobservabilityHubble scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
+        if advancedMode == true  && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" && ENV["OS_TYPE"].downcase == "linux"
+          UpdateScrapeIntervalConfig(@networkobservabilityHubbleDefaultFileDs, networkobservabilityHubbleScrapeInterval)
+          if !networkobservabilityHubbleMetricsKeepListRegex.nil? && !networkobservabilityHubbleMetricsKeepListRegex.empty?
+            AppendMetricRelabelConfig(@networkobservabilityHubbleDefaultFileDs, networkobservabilityHubbleMetricsKeepListRegex)
+          end
+          contents = File.read(@networkobservabilityHubbleDefaultFileDs)
+          contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
+          contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
+          File.open(@networkobservabilityHubbleDefaultFileDs, "w") { |file| file.puts contents }
+          defaultConfigs.push(@networkobservabilityHubbleDefaultFileDs)
+        end
+      end
+    end
+
+    if !ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYCILIUM_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYCILIUM_SCRAPING_ENABLED"].downcase == "true"
+      networkobservabilityCiliumMetricsKeepListRegex = @regexHash["NETWORKOBSERVABILITYCILIUM_METRICS_KEEP_LIST_REGEX"]
+      networkobservabilityCiliumScrapeInterval = @intervalHash["NETWORKOBSERVABILITYCILIUM_SCRAPE_INTERVAL"]
+      if currentControllerType == @replicasetControllerType
+        #do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
+      else #networkobservabilityCilium scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
+        if advancedMode == true  && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" && ENV["OS_TYPE"].downcase == "linux"
+          UpdateScrapeIntervalConfig(@networkobservabilityCiliumDefaultFileDs, networkobservabilityCiliumScrapeInterval)
+          if !networkobservabilityCiliumMetricsKeepListRegex.nil? && !networkobservabilityCiliumMetricsKeepListRegex.empty?
+            AppendMetricRelabelConfig(@networkobservabilityCiliumDefaultFileDs, networkobservabilityCiliumMetricsKeepListRegex)
+          end
+          contents = File.read(@networkobservabilityCiliumDefaultFileDs)
+          contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
+          contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
+          File.open(@networkobservabilityCiliumDefaultFileDs, "w") { |file| file.puts contents }
+          defaultConfigs.push(@networkobservabilityCiliumDefaultFileDs)
+        end
+      end
+    end
+
+
+
     # Collector health config should be enabled or disabled for both replicaset and daemonset
     if !ENV["AZMON_PROMETHEUS_COLLECTOR_HEALTH_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_COLLECTOR_HEALTH_SCRAPING_ENABLED"].downcase == "true"
       prometheusCollectorHealthInterval = @intervalHash["PROMETHEUS_COLLECTOR_HEALTH_SCRAPE_INTERVAL"]

otelcollector/configmapparser/prometheus-config-merger.rb

@@ -42,6 +42,9 @@
 @podannotationsDefaultFile = @defaultPromConfigPathPrefix + "podannotationsDefault.yml"
 @windowskubeproxyDefaultRsAdvancedFile = @defaultPromConfigPathPrefix + "windowskubeproxyDefaultRsAdvanced.yml"
 @kappiebasicDefaultFileDs = @defaultPromConfigPathPrefix + "kappieBasicDefaultDs.yml"
+@networkobservabilityRetinaDefaultFileDs = @defaultPromConfigPathPrefix + "networkobservabilityRetinaDefaultDs.yml"
+@networkobservabilityHubbleDefaultFileDs = @defaultPromConfigPathPrefix + "networkobservabilityHubbleDefaultDs.yml"
+@networkobservabilityCiliumDefaultFileDs = @defaultPromConfigPathPrefix + "networkobservabilityCiliumDefaultDs.yml"
 
 def parseConfigMap
   begin
@@ -335,6 +338,67 @@ def populateDefaultPrometheusConfig
       end
     end
 
+    if !ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYRETINA_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYRETINA_SCRAPING_ENABLED"].downcase == "true"
+      networkobservabilityRetinaMetricsKeepListRegex = @regexHash["NETWORKOBSERVABILITYRETINA_METRICS_KEEP_LIST_REGEX"]
+      networkobservabilityRetinaScrapeInterval = @intervalHash["NETWORKOBSERVABILITYRETINA_SCRAPE_INTERVAL"]
+      if currentControllerType == @replicasetControllerType
+        #do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
+      else #networkobservabilityRetina scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
+        if advancedMode == true  && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" #&& ENV["OS_TYPE"].downcase == "linux"
+          UpdateScrapeIntervalConfig(@networkobservabilityRetinaDefaultFileDs, networkobservabilityRetinaScrapeInterval)
+          if !networkobservabilityRetinaMetricsKeepListRegex.nil? && !networkobservabilityRetinaMetricsKeepListRegex.empty?
+            AppendMetricRelabelConfig(@networkobservabilityRetinaDefaultFileDs, networkobservabilityRetinaMetricsKeepListRegex)
+          end
+          contents = File.read(@networkobservabilityRetinaDefaultFileDs)
+          contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
+          contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
+          File.open(@networkobservabilityRetinaDefaultFileDs, "w") { |file| file.puts contents }
+          defaultConfigs.push(@networkobservabilityRetinaDefaultFileDs)
+        end
+      end
+    end
+
+    if !ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYHUBBLE_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYHUBBLE_SCRAPING_ENABLED"].downcase == "true"
+      networkobservabilityHubbleMetricsKeepListRegex = @regexHash["NETWORKOBSERVABILITYHUBBLE_METRICS_KEEP_LIST_REGEX"]
+      networkobservabilityHubbleScrapeInterval = @intervalHash["NETWORKOBSERVABILITYHUBBLE_SCRAPE_INTERVAL"]
+      if currentControllerType == @replicasetControllerType
+        #do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
+      else #networkobservabilityHubble scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
+        if advancedMode == true  && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" && ENV["OS_TYPE"].downcase == "linux"
+          UpdateScrapeIntervalConfig(@networkobservabilityHubbleDefaultFileDs, networkobservabilityHubbleScrapeInterval)
+          if !networkobservabilityHubbleMetricsKeepListRegex.nil? && !networkobservabilityHubbleMetricsKeepListRegex.empty?
+            AppendMetricRelabelConfig(@networkobservabilityHubbleDefaultFileDs, networkobservabilityHubbleMetricsKeepListRegex)
+          end
+          contents = File.read(@networkobservabilityHubbleDefaultFileDs)
+          contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
+          contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
+          File.open(@networkobservabilityHubbleDefaultFileDs, "w") { |file| file.puts contents }
+          defaultConfigs.push(@networkobservabilityHubbleDefaultFileDs)
+        end
+      end
+    end
+
+    if !ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYCILIUM_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_NETWORKOBSERVABILITYCILIUM_SCRAPING_ENABLED"].downcase == "true"
+      networkobservabilityCiliumMetricsKeepListRegex = @regexHash["NETWORKOBSERVABILITYCILIUM_METRICS_KEEP_LIST_REGEX"]
+      networkobservabilityCiliumScrapeInterval = @intervalHash["NETWORKOBSERVABILITYCILIUM_SCRAPE_INTERVAL"]
+      if currentControllerType == @replicasetControllerType
+        #do nothing -- kappie is not supported to be scrapped automatically outside ds. if needed, customer can disable this ds target, and enable rs scraping thru custom config map
+      else #networkobservabilityCilium scraping will be turned ON by default only when in MAC/addon mode (for both windows & linux)
+        if advancedMode == true  && !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true" && ENV["OS_TYPE"].downcase == "linux"
+          UpdateScrapeIntervalConfig(@networkobservabilityCiliumDefaultFileDs, networkobservabilityCiliumScrapeInterval)
+          if !networkobservabilityCiliumMetricsKeepListRegex.nil? && !networkobservabilityCiliumMetricsKeepListRegex.empty?
+            AppendMetricRelabelConfig(@networkobservabilityCiliumDefaultFileDs, networkobservabilityCiliumMetricsKeepListRegex)
+          end
+          contents = File.read(@networkobservabilityCiliumDefaultFileDs)
+          contents = contents.gsub("$$NODE_IP$$", ENV["NODE_IP"])
+          contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"])
+          File.open(@networkobservabilityCiliumDefaultFileDs, "w") { |file| file.puts contents }
+          defaultConfigs.push(@networkobservabilityCiliumDefaultFileDs)
+        end
+      end
+    end
+
+
     # Collector health config should be enabled or disabled for both replicaset and daemonset
     if !ENV["AZMON_PROMETHEUS_COLLECTOR_HEALTH_SCRAPING_ENABLED"].nil? && ENV["AZMON_PROMETHEUS_COLLECTOR_HEALTH_SCRAPING_ENABLED"].downcase == "true"
       prometheusCollectorHealthInterval = @intervalHash["PROMETHEUS_COLLECTOR_HEALTH_SCRAPE_INTERVAL"]

otelcollector/configmapparser/tomlparser-default-scrape-settings.rb

@@ -22,6 +22,9 @@
 @windowsexporterEnabled = false
 @windowskubeproxyEnabled = false
 @kappiebasicEnabled = true
+@networkobservabilityRetinaEnabled = true
+@networkobservabilityHubbleEnabled = true
+@networkobservabilityCiliumEnabled = true
 @noDefaultsEnabled = false
 @sendDSUpMetric = false
 
@@ -93,6 +96,18 @@ def populateSettingValuesFromConfigMap(parsedConfig)
       @kappiebasicEnabled = parsedConfig[:kappiebasic]
       puts "config::Using configmap scrape settings for kappiebasic: #{@kappiebasicEnabled}"
     end
+    if !parsedConfig[:networkobservabilityRetina].nil?
+      @networkobservabilityRetinaEnabled = parsedConfig[:networkobservabilityRetina]
+      puts "config::Using configmap scrape settings for networkobservabilityRetina: #{@networkobservabilityRetinaEnabled}"
+    end
+    if !parsedConfig[:networkobservabilityHubble].nil?
+      @networkobservabilityHubbleEnabled = parsedConfig[:networkobservabilityHubble]
+      puts "config::Using configmap scrape settings for networkobservabilityHubble: #{@networkobservabilityHubbleEnabled}"
+    end
+    if !parsedConfig[:networkobservabilityCilium].nil?
+      @networkobservabilityCiliumEnabled = parsedConfig[:networkobservabilityCilium]
+      puts "config::Using configmap scrape settings for networkobservabilityCilium: #{@networkobservabilityCiliumEnabled}"
+    end
 
     windowsDaemonset = false
     if ENV["WINMODE"].nil? && ENV["WINMODE"].strip.downcase == "advanced"
@@ -128,7 +143,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
 ConfigParseErrorLogger.logSection(LOGGING_PREFIX, "Start default-scrape-settings Processing")
 # set default targets for MAC mode
 if !ENV['MAC'].nil? && !ENV['MAC'].empty? && ENV['MAC'].strip.downcase == "true"
-  ConfigParseErrorLogger.logWarning(LOGGING_PREFIX, "MAC mode is enabled. Only enabling targets kubestate,cadvisor,kubelet,kappiebasic & nodeexporter for linux before config map processing....")
+  ConfigParseErrorLogger.logWarning(LOGGING_PREFIX, "MAC mode is enabled. Only enabling targets kubestate,cadvisor,kubelet,kappiebasic,networkobservabilityRetina,networkobservabilityHubble,networkobservabilityCilium & nodeexporter for linux before config map processing....")
 
   @corednsEnabled = false
   @kubeproxyEnabled = false
@@ -168,10 +183,13 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write($export + "AZMON_PROMETHEUS_WINDOWSEXPORTER_SCRAPING_ENABLED=#{@windowsexporterEnabled}\n")
   file.write($export + "AZMON_PROMETHEUS_WINDOWSKUBEPROXY_SCRAPING_ENABLED=#{@windowskubeproxyEnabled}\n")
   file.write($export + "AZMON_PROMETHEUS_KAPPIEBASIC_SCRAPING_ENABLED=#{@kappiebasicEnabled}\n")
+  file.write($export + "AZMON_PROMETHEUS_NETWORKOBSERVABILITYRETINA_SCRAPING_ENABLED=#{@networkobservabilityRetinaEnabled}\n")
+  file.write($export + "AZMON_PROMETHEUS_NETWORKOBSERVABILITYHUBBLE_SCRAPING_ENABLED=#{@networkobservabilityHubbleEnabled}\n")
+  file.write($export + "AZMON_PROMETHEUS_NETWORKOBSERVABILITYCILIUM_SCRAPING_ENABLED=#{@networkobservabilityCiliumEnabled}\n")
   file.write($export + "AZMON_PROMETHEUS_POD_ANNOTATION_SCRAPING_ENABLED=#{@podannotationEnabled}\n")
   # Close file after writing all metric collection setting environment variables
   file.close
 else
   ConfigParseErrorLogger.logError(LOGGING_PREFIX, "Exception while opening file for writing default-scrape-settings config environment variables")
 end
-ConfigParseErrorLogger.logSection(LOGGING_PREFIX, "End default-scrape-settings Processing")
+ConfigParseErrorLogger.logSection(LOGGING_PREFIX, "End default-scrape-settings Processing")