Build: pipeline feedback and fixes (#803)

.pipelines/azure-pipeline-build.yml

@@ -95,6 +95,7 @@ jobs:
         echo "##vso[task.setvariable variable=SEMVER;isOutput=true]$SEMVER"
         echo "##vso[task.setvariable variable=LINUX_FULL_IMAGE_NAME;isOutput=true]$LINUX_FULL_IMAGE_NAME"
         echo "##vso[task.setvariable variable=TARGET_ALLOCATOR_IMAGE_TAG;isOutput=true]$TARGET_ALLOCATOR_IMAGE_TAG"
+        echo "##vso[task.setvariable variable=LINUX_CONFIG_READER_IMAGE_TAG;isOutput=true]$LINUX_CONFIG_READER_IMAGE_TAG"
         echo "##vso[task.setvariable variable=TARGET_ALLOCATOR_FULL_IMAGE_NAME;isOutput=true]$TARGET_ALLOCATOR_FULL_IMAGE_NAME"
         echo "##vso[task.setvariable variable=LINUX_CONFIG_READER_FULL_IMAGE_NAME;isOutput=true]$LINUX_CONFIG_READER_FULL_IMAGE_NAME"
         echo "##vso[task.setvariable variable=LINUX_CCP_FULL_IMAGE_NAME;isOutput=true]$LINUX_CCP_FULL_IMAGE_NAME"
@@ -171,7 +172,7 @@ jobs:
     LINUX_REF_APP_GOLANG_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.LINUX_REF_APP_GOLANG_FULL_IMAGE_NAME'] ]
   # This is necessary because of: https://github.com/moby/moby/issues/37965
     DOCKER_BUILDKIT: 1
-  condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+  condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true)))
   steps:
     - checkout: self
       persistCredentials: true
@@ -196,7 +197,7 @@ jobs:
     LINUX_REF_APP_PYTHON_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.LINUX_REF_APP_PYTHON_FULL_IMAGE_NAME'] ]
   # This is necessary because of: https://github.com/moby/moby/issues/37965
     DOCKER_BUILDKIT: 1
-  condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+  condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true)))
   steps:
     - checkout: self
       persistCredentials: true
@@ -219,7 +220,7 @@ jobs:
   variables:
     WINDOWS_REF_APP_GOLANG_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_REF_APP_GOLANG_FULL_IMAGE_NAME'] ]
     skipComponentGovernanceDetection: true
-  condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+  condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true)))
   steps:
     - powershell: |
         docker build . --isolation=hyperv --file windows/Dockerfile -t $(WINDOWS_REF_APP_GOLANG_FULL_IMAGE_NAME)
@@ -236,7 +237,7 @@ jobs:
   variables:
     WINDOWS_REF_APP_PYTHON_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_REF_APP_PYTHON_FULL_IMAGE_NAME'] ]
     skipComponentGovernanceDetection: true
-  condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+  condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true)))
   steps:
     - powershell: |
         docker build . --isolation=hyperv --file windows/Dockerfile -t $(WINDOWS_REF_APP_PYTHON_FULL_IMAGE_NAME)
@@ -363,7 +364,7 @@ jobs:
         EOF
       workingDirectory: $(Build.SourcesDirectory)/otelcollector/
       displayName: "Build: Set values in payload.json for signing"
-      condition: eq(variables.IS_MAIN_BRANCH, true)
+      condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true))
 
     - task: EsrpCodeSigning@3
       displayName: "ESRP CodeSigning for Prometheus"
@@ -401,7 +402,7 @@ jobs:
           -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]"
       workingDirectory: $(Build.ArtifactStagingDirectory)/linux/
       displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linux/"
-      condition: eq(variables.IS_MAIN_BRANCH, true)
+      condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true))
 
     - bash: |
         curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
@@ -435,7 +436,7 @@ jobs:
 
     - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
       displayName: "Ev2: Generate image artifacts"
-      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
       inputs:
         BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux'
         DockerImagesToScan: '$(LINUX_FULL_IMAGE_NAME)'
@@ -462,7 +463,7 @@ jobs:
 
     - task: PublishBuildArtifacts@1
       displayName: "Ev2: Publish image artifacts"
-      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
       inputs:
         pathToPublish: '$(Build.ArtifactStagingDirectory)'
         artifactName: drop
@@ -527,7 +528,7 @@ jobs:
         EOF
       workingDirectory: $(Build.SourcesDirectory)/otelcollector/
       displayName: "Build: Set values in payload.json for signing"
-      condition: eq(variables.IS_MAIN_BRANCH, true)
+      condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true))
 
     - task: EsrpCodeSigning@3
       displayName: "ESRP CodeSigning for Prometheus"
@@ -565,7 +566,7 @@ jobs:
           -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]"
       workingDirectory: $(Build.ArtifactStagingDirectory)/linuxccp/
       displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linuxccp/"
-      condition: eq(variables.IS_MAIN_BRANCH, true)
+      condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true))
 
     - bash: |
         curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
@@ -591,7 +592,7 @@ jobs:
 
     - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
       displayName: "Ev2: Generate image artifacts"
-      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
       inputs:
         BuildDropPath: '$(Build.ArtifactStagingDirectory)/linuxccp'
         DockerImagesToScan: '$(LINUX_CCP_FULL_IMAGE_NAME)'
@@ -618,7 +619,7 @@ jobs:
 
     - task: PublishBuildArtifacts@1
       displayName: "Ev2: Publish image artifacts"
-      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
       inputs:
         pathToPublish: '$(Build.ArtifactStagingDirectory)'
         artifactName: drop
@@ -654,6 +655,8 @@ jobs:
         sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static
         docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
+        docker system prune --all -f
+
         docker buildx create --name dockerbuilder
         docker buildx use dockerbuilder
         docker login containerinsightsprod.azurecr.io -u $(ACR_USERNAME) -p $(ACR_PASSWORD)
@@ -712,7 +715,7 @@ jobs:
           -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]"
       workingDirectory: $(Build.ArtifactStagingDirectory)/targetallocator/
       displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/targetallocator/"
-      condition: eq(variables.IS_MAIN_BRANCH, true)
+      condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true))
 
 - job: Linux_Config_Reader
   displayName: "Build: config reader image"
@@ -731,6 +734,7 @@ jobs:
         # Necessary due to necessary due to https://stackoverflow.com/questions/60080264/docker-cannot-build-multi-platform-images-with-docker-buildx
         sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static
         docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+        docker system prune --all -f
 
         docker buildx create --name dockerbuilder
         docker buildx use dockerbuilder
@@ -788,7 +792,7 @@ jobs:
           -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]"
       workingDirectory: $(Build.ArtifactStagingDirectory)/linuxcfgreader/
       displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linuxcfgreader/"
-      condition: eq(variables.IS_MAIN_BRANCH, true)
+      condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true))
 
 - job: Windows2019_Prometheus_Collector
   displayName: "Build: windows 2019 prometheus-collector image"
@@ -943,7 +947,7 @@ jobs:
       displayName: "Build: Windows multi-arch manifest"
 
     - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
-      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
       displayName: "Ev2: generate image artifacts"
       inputs:
         BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows'
@@ -1000,10 +1004,10 @@ jobs:
         oras attach $(WINDOWS_FULL_IMAGE_NAME) --artifact-type application/vnd.cncf.notary.signature ./payload.json:application/cose -a io.cncf.notary.x509chain.thumbprint#S256=[\""79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\""]
       workingDirectory: $(Build.ArtifactStagingDirectory)/windows
       displayName: "Download, install Oras and run oras attach"
-      condition: eq(variables.IS_MAIN_BRANCH, true)
+      condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true))
 
     - task: PublishBuildArtifacts@1
-      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
       displayName: "Ev2: publish image artifacts"
       inputs:
         pathToPublish: '$(Build.ArtifactStagingDirectory)'
@@ -1055,14 +1059,14 @@ jobs:
 
   - task: PublishBuildArtifacts@1
     displayName: "Ev2: publish helm chart artifacts"
-    condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+    condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
     inputs:
       pathToPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: drop
 
 - job: Deploy_Chart_ARC
   displayName: "Deploy: Arc dev cluster"
-  condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+  condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
   pool:
     name: Azure-Pipelines-CI-Test-EO
   dependsOn:
@@ -1173,17 +1177,20 @@ jobs:
   displayName: "Deploy: AKS dev cluster"
   pool:
     name: Azure-Pipelines-CI-Test-EO
-  condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+  condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
   dependsOn:
   - Image_Tags_and_Ev2_Artifacts
   - Linux_Prometheus_Collector
+  - Linux_Config_Reader
+  - Linux_Target_Allocator
   - WindowsMultiArch_Prometheus_Collector
   variables:
     HELM_CHART_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.HELM_CHART_NAME'] ]
     HELM_SEMVER: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.SEMVER'] ]
     IMAGE_TAG: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.SEMVER'] ]
     IMAGE_TAG_WINDOWS: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_IMAGE_TAG'] ]
-    HELM_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.HELM_FULL_IMAGE_NAME'] ]
+    IMAGE_TAG_TARGET_ALLOCATOR: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.TARGET_ALLOCATOR_IMAGE_TAG'] ]
+    IMAGE_TAG_CONFIG_READER: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.CONFIG_READER_IMAGE_TAG'] ]
     skipComponentGovernanceDetection: true
   steps:
   - checkout: self
@@ -1205,10 +1212,9 @@ jobs:
           do
             sleep 30
             echo $(MCR_REGISTRY)$(MCR_REPOSITORY):$(IMAGE_TAG_WINDOWS)
-            echo $(MCR_REGISTRY)$(MCR_REPOSITORY_HELM):$(IMAGE_TAG)
 
             output=$(curl -s https://$(MCR_REGISTRY)/v2$(MCR_REPOSITORY)/tags/list)
-            if (echo $output | grep $(IMAGE_TAG_WINDOWS)) && (echo $output | grep $(IMAGE_TAG))
+            if (echo $output | grep $(IMAGE_TAG_WINDOWS)) && (echo $output | grep $(IMAGE_TAG) && (echo $output | grep $(IMAGE_TAG_TARGET_ALLOCATOR) && (echo $output | grep $(IMAGE_TAG_CONFIG_READER))
             then
               echo "Images are published to mcr"
               exit 0

otelcollector/otel-allocator/Dockerfile

@@ -32,7 +32,7 @@ COPY . .
 
 ARG TARGETOS TARGETARCH
 # Build the Go app
-RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -installsuffix -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; else CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -installsuffix -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; fi
+RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; else CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; fi
 
 ######## Start a new stage from scratch #######
 FROM mcr.microsoft.com/cbl-mariner/distroless/debug:2.0