Operator changes to support CRD

.pipelines/deployment/ServiceGroupRoot/Parameters/PrometheusCollector.ConfigReader.Parameters.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "http://schema.express.azure.com/schemas/2015-01-01-alpha/RolloutParameters.json",
+  "contentVersion": "1.0.0.0",
+  "wait": [
+    {
+      "name": "waitSdpBakeTime",
+      "properties": {
+        "duration": "PT24H"
+      }
+    }
+  ],
+  "shellExtensions": [
+    {
+      "name": "PushAgentToACR",
+      "type": "ShellExtensionType",
+      "properties": {
+        "maxexecutiontime": "PT1H"
+      },
+      "package": {
+        "reference": {
+          "path": "artifacts.tar.gz"
+        }
+      },
+      "launch": {
+        "command": [
+          "/bin/bash",
+          "pushAgentToAcr.sh"
+        ],
+        "environmentVariables": [
+          {
+            "name": "ACR_REGISTRY",
+            "value": "__ACR_REGISTRY__"
+          },
+          {
+            "name": "PROD_ACR_REPOSITORY",
+            "value": "__PROD_ACR_AGENT_REPOSITORY__"
+          },
+          {
+            "name": "MCR_REGISTRY",
+            "value": "__MCR_REGISTRY__"
+          },
+          {
+            "name": "PROD_MCR_REPOSITORY",
+            "value": "__PROD_MCR_AGENT_REPOSITORY__"
+          },
+          {
+            "name": "DEV_MCR_REPOSITORY",
+            "value": "__DEV_MCR_AGENT_REPOSITORY__"
+          },
+          {
+            "name": "IMAGE_TAG",
+            "value": "__CONFIGREADER_TAG__"
+          }
+        ], 
+        "identity": {
+          "type": "userAssigned",
+          "userAssignedIdentities": [
+            "__MANAGED_IDENTITY__"
+          ]
+        }
+      }
+    }
+  ]
+}

.pipelines/deployment/ServiceGroupRoot/Parameters/PrometheusCollector.TargetAllocator.Parameters.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "http://schema.express.azure.com/schemas/2015-01-01-alpha/RolloutParameters.json",
+  "contentVersion": "1.0.0.0",
+  "wait": [
+    {
+      "name": "waitSdpBakeTime",
+      "properties": {
+        "duration": "PT24H"
+      }
+    }
+  ],
+  "shellExtensions": [
+    {
+      "name": "PushAgentToACR",
+      "type": "ShellExtensionType",
+      "properties": {
+        "maxexecutiontime": "PT1H"
+      },
+      "package": {
+        "reference": {
+          "path": "artifacts.tar.gz"
+        }
+      },
+      "launch": {
+        "command": [
+          "/bin/bash",
+          "pushAgentToAcr.sh"
+        ],
+        "environmentVariables": [
+          {
+            "name": "ACR_REGISTRY",
+            "value": "__ACR_REGISTRY__"
+          },
+          {
+            "name": "PROD_ACR_REPOSITORY",
+            "value": "__PROD_ACR_AGENT_REPOSITORY__"
+          },
+          {
+            "name": "MCR_REGISTRY",
+            "value": "__MCR_REGISTRY__"
+          },
+          {
+            "name": "PROD_MCR_REPOSITORY",
+            "value": "__PROD_MCR_AGENT_REPOSITORY__"
+          },
+          {
+            "name": "DEV_MCR_REPOSITORY",
+            "value": "__DEV_MCR_AGENT_REPOSITORY__"
+          },
+          {
+            "name": "IMAGE_TAG",
+            "value": "__TARGETALLOCATOR_TAG__"
+          }
+        ], 
+        "identity": {
+          "type": "userAssigned",
+          "userAssignedIdentities": [
+            "__MANAGED_IDENTITY__"
+          ]
+        }
+      }
+    }
+  ]
+}

.pipelines/deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json

@@ -32,6 +32,20 @@
       "actions": [ "Shell/PushAgentToACR" ],
       "dependsOn": [ ]
     },
+    {
+      "name": "PushTargetAllocator",
+      "targetType": "ServiceResource",
+      "targetName": "PushTargetAllocator",
+      "actions": [ "Shell/PushAgentToACR" ],
+      "dependsOn": [ ]
+    },
+    {
+      "name": "PushConfigReader",
+      "targetType": "ServiceResource",
+      "targetName": "PushConfigReader",
+      "actions": [ "Shell/PushAgentToACR" ],
+      "dependsOn": [ ]
+    },
     {
       "name": "PushKSMChart",
       "targetType": "ServiceResource",

.pipelines/deployment/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json

@@ -53,6 +53,14 @@
                   "find": "__WINDOWS_TAG__",
                   "replaceWith": "$(WindowsTag)"
               },
+              {
+                "find": "__TARGETALLOCATOR_TAG__",
+                "replaceWith": "$(TargetAllocatorTag)"
+              },
+              {
+                "find": "__TARGETALLOCATOR_TAG__",
+                "replaceWith": "$(ConfigReaderTag)"
+              },
               {
                 "find": "__PROD_MCR_AGENT_REPOSITORY__",
                 "replaceWith": "$(ProdMCRAgentRepository)"

.pipelines/deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json

@@ -50,6 +50,16 @@
           "InstanceOf": "ShellExtension",
           "RolloutParametersPath": "Parameters\\PrometheusCollector.Windows.Parameters.json"
         },
+        {
+          "Name": "PushTargetAllocator",
+          "InstanceOf": "ShellExtension",
+          "RolloutParametersPath": "Parameters\\PrometheusCollector.TargetAllocator.Parameters.json"
+        },
+        {
+          "Name": "PushConfigReader",
+          "InstanceOf": "ShellExtension",
+          "RolloutParametersPath": "Parameters\\PrometheusCollector.ConfigReader.Parameters.json"
+        },
         {
           "Name": "Push1PHelmChart",
           "InstanceOf": "ShellExtension",

.trivyignore

@@ -13,6 +13,8 @@ CVE-2022-27191
 CVE-2022-23471
 CVE-2023-25153
 CVE-2023-25173
+# target-allocator
+CVE-2023-2253
 
 # MEDIUM
 # opt/telegraf/telegraf
@@ -27,3 +29,6 @@ CVE-2022-41723
 CVE-2022-41717
 CVE-2022-46146
 CVE-2022-41721
+# target-allocator
+CVE-2023-29401
+CVE-2023-40577

internal/referenceapp/prometheus-reference-app.yaml

@@ -39,6 +39,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: prometheus-reference-service
+  labels:
+    app: prometheus-reference-app
 spec:
   selector:
     app: prometheus-reference-app

otelcollector/build/linux/Dockerfile

@@ -54,12 +54,13 @@ COPY ./logrotate/crontab /etc/crontab
 COPY ./scripts/livenessprobe.sh $tmpdir/microsoft/liveness/livenessprobe.sh
 COPY ./configmapparser/*.rb $tmpdir/microsoft/configmapparser/
 COPY ./configmapparser/default-prom-configs/*.yml $tmpdir/microsoft/otelcollector/default-prom-configs/
-COPY ./opentelemetry-collector-builder/collector-config-default.yml ./opentelemetry-collector-builder/collector-config-template.yml ./opentelemetry-collector-builder/PROMETHEUS_VERSION $tmpdir/microsoft/otelcollector/
+COPY ./opentelemetry-collector-builder/collector-config-default.yml ./opentelemetry-collector-builder/collector-config-template.yml ./opentelemetry-collector-builder/collector-config-replicaset.yml ./opentelemetry-collector-builder/PROMETHEUS_VERSION $tmpdir/microsoft/otelcollector/
 COPY --from=otelcollector-builder /src/opentelemetry-collector-builder/otelcollector $tmpdir/microsoft/otelcollector/
 COPY --from=otelcollector-builder /src/opentelemetry-collector-builder/otelcollector $tmpdir/microsoft/otelcollector/
 COPY --from=otelcollector-builder /src/goversion.txt $tmpdir/goversion.txt
 COPY --from=prom-config-validator-builder /src/prom-config-validator-builder/promconfigvalidator $tmpdir/
-COPY ./scripts/setup.sh ./scripts/main.sh $tmpdir/
+
+COPY ./scripts/*.sh $tmpdir/
 COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config ./metricextension/me_ds_internal.config /usr/sbin/
 COPY ./telegraf/telegraf-prometheus-collector.conf $tmpdir/telegraf/
 COPY ./fluent-bit/fluent-bit.conf ./fluent-bit/fluent-bit-daemonset.conf ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/
@@ -73,7 +74,7 @@ COPY ./build/linux/rpm-repos/ /etc/yum.repos.d/
 ARG TARGETARCH
 RUN tdnf clean all
 RUN tdnf repolist --refresh
-RUN tdnf update
+RUN tdnf update -y
 RUN tdnf install -y wget sudo net-tools cronie vim ruby-devel logrotate procps-ng busybox diffutils curl
 RUN mkdir /busybin && busybox --install /busybin
 RUN chmod 775 /etc/cron.daily/logrotate

otelcollector/build/linux/configuration-reader/Dockerfile

@@ -0,0 +1,108 @@
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang:1.19 as prom-config-validator-builder
+WORKDIR /src
+RUN apt-get update && apt-get install gcc-aarch64-linux-gnu -y
+COPY ./prom-config-validator-builder/go.mod ./prom-config-validator-builder/go.sum ./prom-config-validator-builder/
+COPY ./prometheusreceiver/go.mod ./prometheusreceiver/go.sum ./prometheusreceiver/
+WORKDIR /src/prometheusreceiver
+RUN go version
+RUN go mod download
+WORKDIR /src/prom-config-validator-builder
+RUN go mod download
+COPY ./prom-config-validator-builder /src/prom-config-validator-builder
+COPY ./prometheusreceiver /src/prometheusreceiver
+ARG TARGETOS TARGETARCH
+RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o promconfigvalidator . ; else CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o promconfigvalidator . ; fi
+
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang:1.19 as configuration-reader-builder
+WORKDIR /src
+RUN apt-get update && apt-get install gcc-aarch64-linux-gnu -y
+COPY ./configuration-reader-builder/go.mod ./configuration-reader-builder/go.sum ./configuration-reader-builder/
+RUN go version > goversion.txt
+WORKDIR /src/configuration-reader-builder
+RUN go mod download
+COPY ./configuration-reader-builder /src/configuration-reader-builder
+ARG TARGETOS TARGETARCH
+RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o configurationreader . ; else CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o configurationreader . ; fi
+
+FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as builder
+LABEL description="Azure Monitor Prometheus metrics collector - configuration reader sidecar"
+LABEL maintainer="[email protected]"
+ENV OS_TYPE "linux"
+ENV tmpdir /opt
+COPY ./logrotate/logrotate /etc/cron.daily/logrotate
+COPY ./logrotate/crontab /etc/crontab
+COPY ./scripts/livenessprobe-configreader.sh $tmpdir/microsoft/liveness/livenessprobe-configreader.sh
+COPY ./configmapparser/*.rb $tmpdir/microsoft/configmapparser/
+COPY ./configmapparser/default-prom-configs/*.yml $tmpdir/microsoft/otelcollector/default-prom-configs/
+COPY ./opentelemetry-collector-builder/collector-config-default.yml ./opentelemetry-collector-builder/collector-config-template.yml ./opentelemetry-collector-builder/PROMETHEUS_VERSION $tmpdir/microsoft/otelcollector/
+COPY --from=configuration-reader-builder /src/goversion.txt $tmpdir/goversion.txt
+COPY --from=prom-config-validator-builder /src/prom-config-validator-builder/promconfigvalidator $tmpdir/
+COPY --from=configuration-reader-builder /src/configuration-reader-builder/configurationreader $tmpdir/
+
+COPY ./scripts/*.sh $tmpdir/
+COPY ./LICENSE $tmpdir/microsoft
+COPY ./NOTICE $tmpdir/microsoft
+COPY ./build/linux/rpm-repos/ /etc/yum.repos.d/
+
+ARG TARGETARCH
+RUN tdnf clean all
+RUN tdnf repolist --refresh
+RUN tdnf update -y
+RUN tdnf install -y wget sudo net-tools cronie vim ruby-devel logrotate procps-ng busybox diffutils curl
+RUN mkdir /busybin && busybox --install /busybin
+RUN chmod 775 /etc/cron.daily/logrotate
+RUN chmod 775 $tmpdir/*.sh;
+RUN sync;
+RUN $tmpdir/setup-configreader.sh ${TARGETARCH}
+
+FROM mcr.microsoft.com/cbl-mariner/distroless/base:2.0
+ENV PATH="/busybin:${PATH}"
+ENV OS_TYPE "linux"
+
+# files
+COPY --from=builder /opt /opt
+COPY --from=builder /etc /etc
+COPY --from=builder /busybin /busybin
+COPY --from=builder /var/lib/logrotate /var/lib/logrotate
+COPY --from=builder /var/spool/cron /var/spool/cron
+
+# executables
+COPY --from=builder /usr/bin/ruby /usr/bin/ruby
+COPY --from=builder /usr/lib/ruby /usr/lib/ruby
+COPY --from=builder /usr/bin/inotifywait /usr/bin/inotifywait
+COPY --from=builder /usr/bin/bash /usr/bin/bash
+COPY --from=builder /usr/sbin/busybox /usr/sbin/busybox
+COPY --from=builder /usr/sbin/crond /usr/sbin/crond
+COPY --from=builder /usr/bin/vim /usr/bin/vim
+COPY --from=builder /usr/share/vim /usr/share/vim
+COPY --from=builder /usr/sbin/logrotate /usr/sbin/logrotate
+COPY --from=builder /usr/bin/gzip /usr/bin/
+COPY --from=builder /usr/bin/curl /usr/bin/
+COPY --from=builder /bin/sh /bin/sh
+
+# bash dependencies
+COPY --from=builder /lib/libreadline.so.8 /lib/
+COPY --from=builder /usr/lib/libncursesw.so.6 /usr/lib/libtinfo.so.6 /usr/lib/
+# inotifywait dependencies
+COPY --from=builder /lib/libinotifytools.so.0 /lib/
+# crond dependencies
+COPY --from=builder /lib/libselinux.so.1 /lib/libpam.so.0 /lib/libc.so.6 /lib/libpcre.so.1 /lib/libaudit.so.1 /lib/libcap-ng.so.0/ /lib/
+# vim dependencies
+COPY --from=builder /lib/libm.so.6 /lib/libtinfo.so.6 /lib/
+# ruby dependencies
+COPY --from=builder /usr/lib/libruby.so.3.1 /usr/lib/libz.so.1 /usr/lib/libgmp.so.10 /usr/lib/libcrypt.so.1 /usr/lib/libm.so.6 /usr/lib/
+# ruby re2 dependencies
+COPY --from=builder /usr/lib/libre2.so.0a /usr/lib/libstdc++.so.6 /usr/lib/libgcc_s.so.1 /usr/lib/libz.so.1 /usr/lib/libgmp.so.10 /usr/lib/libcrypt.so.1 /usr/lib/libm.so.6 /usr/lib/
+# logrotate dependencies
+COPY --from=builder /lib/libselinux.so.1 /lib/libpopt.so.0 /lib/libpcre.so.1 /lib/
+# curl dependencies
+COPY --from=builder /lib/libcurl.so.4 /lib/libz.so.1 /lib/libc.so.6 /lib/libnghttp2.so.14 /lib/libssh2.so.1 /lib/libssl.so.1.1   /lib/libcrypto.so.1.1  /lib/libgssapi_krb5.so.2 /lib/libzstd.so.1 /lib/
+COPY --from=builder /usr/lib/libkrb5.so.3 /usr/lib/libk5crypto.so.3 /usr/lib/libcom_err.so.2 /usr/lib/libkrb5support.so.0 /usr/lib/libresolv.so.2 /usr/lib/
+# sh dependencies
+COPY --from=builder /lib/libreadline.so.8 /lib/libc.so.6 /usr/lib/libncursesw.so.6 /usr/lib/libtinfo.so.6 /lib/
+
+RUN [ "/bin/bash", "-c", "chmod 644 /etc/crontab" ]
+RUN [ "/bin/bash", "-c", "chown root.root /etc/crontab" ]
+RUN [ "/bin/bash", "-c", "chmod 755 /etc/cron.daily/logrotate" ]
+ENTRYPOINT [ "/bin/bash" ]
+CMD [ "/opt/main-configreader.sh" ]