diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml index 19a566814..19669174a 100644 --- a/.pipelines/azure-pipeline-build.yml +++ b/.pipelines/azure-pipeline-build.yml @@ -24,7 +24,6 @@ variables: IS_PR: $[eq(variables['Build.Reason'], 'PullRequest')] IS_MAIN_BRANCH: $[eq(variables['Build.SourceBranchName'], 'main')] - stages: - stage: Build jobs: @@ -1330,4 +1329,4 @@ stages: exit 1 fi workingDirectory: $(Build.SourcesDirectory) - displayName: "Run tests" \ No newline at end of file + displayName: "Run tests" diff --git a/.trivyignore b/.trivyignore index c69968542..cd719bbad 100644 --- a/.trivyignore +++ b/.trivyignore @@ -6,15 +6,10 @@ # =========== HIGH ================ # HIGH - otelcollector -# HIGH - telegraf GHSA-fr2g-9hjm-wr23 # HIGH - promconfigvalidator -# HIGH - telegraf +# HIGH - go vulnerabilities CVE-2023-39325 -GHSA-m425-mq94-257g -CVE-2023-47090 -CVE-2023-46129 -CVE-2024-21626 # =========== MEDIUM ================ # MEDIUM - otelcollector @@ -25,22 +20,9 @@ CVE-2023-48795 CVE-2024-24786 CVE-2024-24557 CVE-2024-27281 -# MEDIUM - telegraf -GHSA-jq35-85cj-fj4p -GHSA-7ww5-4wqc-m92c -GHSA-mhpq-9638-x6pw -CVE-2024-27304 -GHSA-7jwh-3vrq-q3m8 -CVE-2023-50658 -CVE-2023-48795 +# MEDIUM - go vulnerabilities CVE-2023-3978 CVE-2023-44487 -CVE-2023-50658 -CVE-2024-28110 -CVE-2024-27289 -CVE-2024-24557 -CVE-2024-29018 -# MEDIUM - go vulnerabilities # MEDIUM - mariner CVE-2023-5678 # MEDIUM - ruby diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 60bcd3341..59a6faaa4 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,10 +2,10 @@ ## Release 04-08-2024 -* Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:` -* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:` -* TA image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:` -* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:` +* Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97` +* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97-win` +* TA image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97-targetallocator` +* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.7-main-04-09-2024-82adbf97-cfg` * Change log - * fix: Fix: For Arc, use a default value CloudEnvironment that customers can change for other clouds- https://github.com/Azure/prometheus-collector/pull/753 * Upgrade: fluent-bit from 2.0.9 to 2.1.10 - https://github.com/Azure/prometheus-collector/pull/809 @@ -25,9 +25,9 @@ ## Release 03-08-2024 * Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb` -* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-cfg` +* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-win` * TA image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-targetallocator` -* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-win` +* cfg sidecar image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.8.6-main-03-08-2024-fd4f13cb-cfg` * Change log - * fix: Fix for yaml unmarshal bug for keepequal/dropequal - https://github.com/Azure/prometheus-collector/pull/753 * fix: dollar fix for TA - https://github.com/Azure/prometheus-collector/pull/769 diff --git a/otelcollector/build/linux/Dockerfile b/otelcollector/build/linux/Dockerfile index 932d97f16..2f8508cef 100644 --- a/otelcollector/build/linux/Dockerfile +++ b/otelcollector/build/linux/Dockerfile @@ -72,7 +72,6 @@ COPY --from=prom-config-validator-builder /src/prom-config-validator-builder/pro COPY ./scripts/*.sh $tmpdir/ COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config ./metricextension/me_ds_internal.config /usr/sbin/ -COPY ./telegraf/ $tmpdir/telegraf/ COPY ./fluent-bit/fluent-bit.conf ./fluent-bit/fluent-bit-daemonset.conf ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/ COPY --from=fluent-bit-builder /src/out_appinsights.so $tmpdir/fluent-bit/bin/ COPY ./react /static/react @@ -131,7 +130,6 @@ COPY --from=builder /usr/bin/inotifywait /usr/bin/inotifywait COPY --from=builder /usr/bin/bash /usr/bin/bash COPY --from=builder /usr/sbin/busybox /usr/sbin/busybox COPY --from=builder /usr/bin/fluent-bit /usr/bin/fluent-bit -COPY --from=builder /usr/bin/telegraf /usr/bin/telegraf COPY --from=builder /usr/sbin/crond /usr/sbin/crond COPY --from=builder /usr/bin/vim /usr/bin/vim COPY --from=builder /usr/share/vim /usr/share/vim @@ -164,8 +162,6 @@ COPY --from=builder /lib64/libuuid.so.1 /lib64 # fluent-bit dependencies # libssl.so.1.1 & libcrypto.so.1.1 are already available with openssl in distroless and copying them over causes FIPS HMAC verification failures COPY --from=builder /lib/libyaml-0.so.2 /lib/libsystemd.so.0 /lib/libcurl.so.4 /lib/libm.so.6 /lib/libz.so.1 /lib/libzstd.so.1 /lib/libsasl2.so.3 /lib/libgcc_s.so.1 /lib/libc.so.6 /lib/liblzma.so.5 /lib/liblz4.so.1 /lib/libcap.so.2 /lib/libgcrypt.so.20 /lib/libnghttp2.so.14 /lib/libssh2.so.1 /lib/libgssapi_krb5.so.2 /lib/libresolv.so.2 /lib/libgpg-error.so.0 /usr/lib/libkrb5.so.3 /usr/lib/libk5crypto.so.3 /usr/lib/libcom_err.so.2 /usr/lib/libkrb5support.so.0 /lib/ -# telegraf dependencies -COPY --from=builder /lib/libc.so.6 /lib/ # mdsd dependencies COPY --from=builder /usr/lib/libdl.so.2 /usr/lib/librt.so.1 /usr/lib/libpthread.so.0 /usr/lib/libm.so.6 /usr/lib/libstdc++.so.6 /usr/lib/libgcc_s.so.1 /usr/lib/ # logrotate dependencies diff --git a/otelcollector/fluent-bit/fluent-bit-daemonset.conf b/otelcollector/fluent-bit/fluent-bit-daemonset.conf index 5540619f1..7b750ff2a 100644 --- a/otelcollector/fluent-bit/fluent-bit-daemonset.conf +++ b/otelcollector/fluent-bit/fluent-bit-daemonset.conf @@ -130,6 +130,49 @@ Skip_Long_Lines On Ignore_Older 2m +[INPUT] + Name proc + Proc_Name otelcollector + Interval_Sec 10 + Tag proc.otelcollector.memvmrss.scrape + +[INPUT] + Name proc + Proc_Name metricsextension + Interval_Sec 10 + Tag proc.metricsextension.memvmrss.scrape + +[INPUT] + name prometheus_scrape + host localhost + port 8888 + tag promscrape.prometheus.8888 + scrape_interval 10s + +[INPUT] + name prometheus_scrape + host localhost + port 9090 + tag promscrape.prometheus.9090 + scrape_interval 10s + +[INPUT] + name prometheus_scrape + host ama-metrics-operator-targets.kube-system.svc.cluster.local + port 80 + tag promscrape.prometheus.operator + scrape_interval 10s + +[INPUT] + Name cpu + Tag cpu.otelcollector + PID ${OTEL_PID} + +[INPUT] + Name cpu + Tag cpu.me + PID ${ME_PID} + # Send log lines that contain the telemetry we want to a different tag # to then send to customMetrics table [FILTER] @@ -201,3 +244,98 @@ Format json_lines json_date_key time Match prometheus.log.noconfiguration + +[OUTPUT] + Name file + Match proc.* + Path /opt/fluent-bit + File proc.log + +[OUTPUT] + Name file + Match promscrape.* + Path /opt/fluent-bit + File promscrape.log + +[OUTPUT] + Name file + Match cpu.otelcollector + Path /opt/fluent-bit + File otelcpu.log + +[OUTPUT] + Name file + Match cpu.me + Path /opt/fluent-bit + File mecpu.log + +[INPUT] + Name tail + Path /opt/fluent-bit/proc.log + Read_from_Head true + Tag proc.scrape + Parser proc-parser + Mem_Buf_Limit 1m + Path_Key filepath + Skip_Long_Lines On + Ignore_Older 2m + +[INPUT] + Name tail + Path /opt/fluent-bit/promscrape.log + Read_from_Head true + Tag promscrape.scrape + Parser promscrape-parser + Mem_Buf_Limit 1m + Path_Key filepath + Skip_Long_Lines On + Ignore_Older 2m + + +[INPUT] + Name tail + Path /opt/fluent-bit/otelcpu.log + Read_from_Head true + Tag cpu.otel + Parser cpu-parser + Mem_Buf_Limit 1m + Path_Key filepath + Skip_Long_Lines On + Ignore_Older 2m + +[INPUT] + Name tail + Path /opt/fluent-bit/mecpu.log + Read_from_Head true + Tag cpu.metricsextension + Parser cpu-parser + Mem_Buf_Limit 1m + Path_Key filepath + Skip_Long_Lines On + Ignore_Older 2m + +[FILTER] + Name rewrite_tag + Match proc.scrape + Rule $message .*otelcollector.* procai.otelcollector.memvmrss.scrape false + +[FILTER] + Name rewrite_tag + Match proc.scrape + Rule $message .*metricsextension.* procai.metricsextension.memvmrss.scrape false + +[OUTPUT] + Name appinsights + Match procai.* + +[OUTPUT] + Name appinsights + Match cpu.otel + +[OUTPUT] + Name appinsights + Match cpu.metricsextension + +[OUTPUT] + Name appinsights + Match promscrape.scrape diff --git a/otelcollector/fluent-bit/fluent-bit-parsers.conf b/otelcollector/fluent-bit/fluent-bit-parsers.conf index 2f4db187e..94ec2b694 100644 --- a/otelcollector/fluent-bit/fluent-bit-parsers.conf +++ b/otelcollector/fluent-bit/fluent-bit-parsers.conf @@ -34,3 +34,27 @@ Time_Key time Time_Format %Y-%m-%dT%H:%M:%S Time_Keep On + +[PARSER] + Name proc-parser + Format regex + Regex ^[^\[]*\[\s*(?