Merge branch 'main' into grace/testkube

Azure · Apr 11, 2024 · 55bdd8a · 55bdd8a
2 parents 07e178f + f315c8a
commit 55bdd8a
Show file tree

Hide file tree

Showing 17 changed files with 658 additions and 723 deletions.
diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml
@@ -24,7 +24,6 @@ variables:
   IS_PR: $[eq(variables['Build.Reason'], 'PullRequest')]
   IS_MAIN_BRANCH: $[eq(variables['Build.SourceBranchName'], 'main')]
 
-
 stages:
 - stage: Build
   jobs:
@@ -1330,4 +1329,4 @@ stages:
                 exit 1
               fi
             workingDirectory: $(Build.SourcesDirectory)
-            displayName: "Run tests"
+            displayName: "Run tests"
diff --git a/.trivyignore b/.trivyignore
@@ -6,15 +6,10 @@
 
 # =========== HIGH ================
 # HIGH - otelcollector
-# HIGH - telegraf
 GHSA-fr2g-9hjm-wr23
 # HIGH - promconfigvalidator
-# HIGH - telegraf
+# HIGH - go vulnerabilities
 CVE-2023-39325
-GHSA-m425-mq94-257g
-CVE-2023-47090
-CVE-2023-46129
-CVE-2024-21626
 
 # =========== MEDIUM ================
 # MEDIUM - otelcollector
@@ -25,22 +20,9 @@ CVE-2023-48795
 CVE-2024-24786
 CVE-2024-24557
 CVE-2024-27281
-# MEDIUM - telegraf
-GHSA-jq35-85cj-fj4p
-GHSA-7ww5-4wqc-m92c
-GHSA-mhpq-9638-x6pw
-CVE-2024-27304
-GHSA-7jwh-3vrq-q3m8
-CVE-2023-50658
-CVE-2023-48795
+# MEDIUM - go vulnerabilities
 CVE-2023-3978
 CVE-2023-44487
-CVE-2023-50658
-CVE-2024-28110
-CVE-2024-27289
-CVE-2024-24557
-CVE-2024-29018
-# MEDIUM - go vulnerabilities
 # MEDIUM - mariner
 CVE-2023-5678
 # MEDIUM - ruby

diff --git a/RELEASENOTES.md b/RELEASENOTES.md
@@ -2,10 +2,10 @@
 
 ## Release 04-08-2024
 
-* Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:<tbd>`
-* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:<tbd>`
-* TA image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:<tbd>`
-* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:<tbd>`
+* Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97`
+* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97-win`
+* TA image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97-targetallocator`
+* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97-cfg`
 * Change log -
   * fix: Fix: For Arc, use a default value CloudEnvironment that customers can change for other clouds- https://github.com/Azure/prometheus-collector/pull/753
   * Upgrade: fluent-bit from 2.0.9 to 2.1.10 - https://github.com/Azure/prometheus-collector/pull/809
@@ -25,9 +25,9 @@
 ## Release 03-08-2024
 
 * Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb`
-* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-cfg`
+* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-win`
 * TA image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-targetallocator`
-* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-win`
+* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-cfg`
 * Change log -
   * fix: Fix for yaml unmarshal bug for keepequal/dropequal - https://github.com/Azure/prometheus-collector/pull/753
   * fix: dollar fix for TA - https://github.com/Azure/prometheus-collector/pull/769

diff --git a/otelcollector/build/linux/Dockerfile b/otelcollector/build/linux/Dockerfile
@@ -72,7 +72,6 @@ COPY --from=prom-config-validator-builder /src/prom-config-validator-builder/pro
 
 COPY ./scripts/*.sh $tmpdir/
 COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config ./metricextension/me_ds_internal.config /usr/sbin/
-COPY ./telegraf/ $tmpdir/telegraf/
 COPY ./fluent-bit/fluent-bit.conf ./fluent-bit/fluent-bit-daemonset.conf ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/
 COPY --from=fluent-bit-builder /src/out_appinsights.so $tmpdir/fluent-bit/bin/
 COPY ./react /static/react
@@ -131,7 +130,6 @@ COPY --from=builder /usr/bin/inotifywait /usr/bin/inotifywait
 COPY --from=builder /usr/bin/bash /usr/bin/bash
 COPY --from=builder /usr/sbin/busybox /usr/sbin/busybox
 COPY --from=builder /usr/bin/fluent-bit /usr/bin/fluent-bit
-COPY --from=builder /usr/bin/telegraf /usr/bin/telegraf
 COPY --from=builder /usr/sbin/crond /usr/sbin/crond
 COPY --from=builder /usr/bin/vim /usr/bin/vim
 COPY --from=builder /usr/share/vim /usr/share/vim
@@ -164,8 +162,6 @@ COPY --from=builder /lib64/libuuid.so.1 /lib64
 # fluent-bit dependencies
 # libssl.so.1.1 & libcrypto.so.1.1 are already available with openssl in distroless and copying them over causes FIPS HMAC verification failures
 COPY --from=builder /lib/libyaml-0.so.2 /lib/libsystemd.so.0 /lib/libcurl.so.4 /lib/libm.so.6 /lib/libz.so.1 /lib/libzstd.so.1 /lib/libsasl2.so.3 /lib/libgcc_s.so.1 /lib/libc.so.6 /lib/liblzma.so.5 /lib/liblz4.so.1  /lib/libcap.so.2 /lib/libgcrypt.so.20 /lib/libnghttp2.so.14 /lib/libssh2.so.1 /lib/libgssapi_krb5.so.2 /lib/libresolv.so.2 /lib/libgpg-error.so.0 /usr/lib/libkrb5.so.3 /usr/lib/libk5crypto.so.3 /usr/lib/libcom_err.so.2 /usr/lib/libkrb5support.so.0 /lib/
-# telegraf dependencies
-COPY --from=builder /lib/libc.so.6 /lib/
 # mdsd dependencies
 COPY --from=builder /usr/lib/libdl.so.2 /usr/lib/librt.so.1 /usr/lib/libpthread.so.0 /usr/lib/libm.so.6 /usr/lib/libstdc++.so.6 /usr/lib/libgcc_s.so.1 /usr/lib/
 # logrotate dependencies

diff --git a/otelcollector/fluent-bit/fluent-bit-daemonset.conf b/otelcollector/fluent-bit/fluent-bit-daemonset.conf
@@ -130,6 +130,49 @@
     Skip_Long_Lines On
     Ignore_Older 2m
 
+[INPUT]
+    Name          proc
+    Proc_Name     otelcollector
+    Interval_Sec  10
+    Tag           proc.otelcollector.memvmrss.scrape
+
+[INPUT]
+    Name          proc
+    Proc_Name     metricsextension
+    Interval_Sec  10
+    Tag           proc.metricsextension.memvmrss.scrape
+
+[INPUT]
+    name prometheus_scrape
+    host localhost
+    port 8888
+    tag          promscrape.prometheus.8888
+    scrape_interval 10s
+
+[INPUT]
+    name prometheus_scrape
+    host localhost
+    port 9090
+    tag           promscrape.prometheus.9090
+    scrape_interval 10s
+
+[INPUT]
+    name prometheus_scrape
+    host ama-metrics-operator-targets.kube-system.svc.cluster.local
+    port 80
+    tag           promscrape.prometheus.operator
+    scrape_interval 10s
+
+[INPUT]
+    Name cpu
+    Tag  cpu.otelcollector
+    PID  ${OTEL_PID}
+
+[INPUT]
+    Name cpu
+    Tag  cpu.me
+    PID  ${ME_PID}
+
 # Send log lines that contain the telemetry we want to a different tag
 # to then send to customMetrics table
 [FILTER]
@@ -201,3 +244,98 @@
     Format                          json_lines
     json_date_key                   time
     Match                           prometheus.log.noconfiguration
+
+[OUTPUT]
+    Name file
+    Match proc.*
+    Path /opt/fluent-bit
+    File proc.log
+
+[OUTPUT]
+    Name file
+    Match promscrape.*
+    Path /opt/fluent-bit
+    File promscrape.log
+
+[OUTPUT]
+    Name file
+    Match cpu.otelcollector
+    Path /opt/fluent-bit
+    File otelcpu.log
+
+[OUTPUT]
+    Name file
+    Match cpu.me
+    Path /opt/fluent-bit
+    File mecpu.log
+
+[INPUT]
+    Name tail
+    Path /opt/fluent-bit/proc.log
+    Read_from_Head true
+    Tag proc.scrape
+    Parser proc-parser
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
+[INPUT]
+    Name tail
+    Path /opt/fluent-bit/promscrape.log
+    Read_from_Head true
+    Tag promscrape.scrape
+    Parser promscrape-parser
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
+
+[INPUT]
+    Name tail
+    Path /opt/fluent-bit/otelcpu.log
+    Read_from_Head true
+    Tag cpu.otel
+    Parser cpu-parser
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
+[INPUT]
+    Name tail
+    Path /opt/fluent-bit/mecpu.log
+    Read_from_Head true
+    Tag cpu.metricsextension
+    Parser cpu-parser
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
+[FILTER]
+    Name   rewrite_tag
+    Match  proc.scrape
+    Rule   $message .*otelcollector.* procai.otelcollector.memvmrss.scrape false
+
+[FILTER]
+    Name   rewrite_tag
+    Match  proc.scrape
+    Rule   $message .*metricsextension.* procai.metricsextension.memvmrss.scrape false
+
+[OUTPUT]
+    Name                            appinsights
+    Match                           procai.*
+
+[OUTPUT]
+    Name                            appinsights
+    Match                           cpu.otel
+
+[OUTPUT]
+    Name                            appinsights
+    Match                           cpu.metricsextension
+
+[OUTPUT]
+    Name                            appinsights
+    Match                           promscrape.scrape
diff --git a/otelcollector/fluent-bit/fluent-bit-parsers.conf b/otelcollector/fluent-bit/fluent-bit-parsers.conf
@@ -34,3 +34,27 @@
     Time_Key time
     Time_Format %Y-%m-%dT%H:%M:%S
     Time_Keep On
+
+[PARSER]
+    Name proc-parser
+    Format regex
+    Regex ^[^\[]*\[\s*(?<time>[^\s,]+),\s*(?<message>.*)\]$
+    Time_Key time
+    Time_Format %s.%L
+    Time_Keep On
+
+[PARSER]
+    Name promscrape-parser
+    Format regex
+    Regex ^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+(?<message>.*)$
+    Time_Key time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L
+    Time_Keep On
+
+[PARSER]
+    Name cpu-parser
+    Format regex
+    Regex ^[^:]+: \[(?<time>[^\s,]+), \{"cpu_p":(?<message>[^,]+),.*
+    Time_Key time
+    Time_Format %s.%L
+    Time_Keep On