diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml index 4313231f3..34eaf2b1c 100644 --- a/.pipelines/azure-pipeline-build.yml +++ b/.pipelines/azure-pipeline-build.yml @@ -95,6 +95,7 @@ jobs: echo "##vso[task.setvariable variable=SEMVER;isOutput=true]$SEMVER" echo "##vso[task.setvariable variable=LINUX_FULL_IMAGE_NAME;isOutput=true]$LINUX_FULL_IMAGE_NAME" echo "##vso[task.setvariable variable=TARGET_ALLOCATOR_IMAGE_TAG;isOutput=true]$TARGET_ALLOCATOR_IMAGE_TAG" + echo "##vso[task.setvariable variable=LINUX_CONFIG_READER_IMAGE_TAG;isOutput=true]$LINUX_CONFIG_READER_IMAGE_TAG" echo "##vso[task.setvariable variable=TARGET_ALLOCATOR_FULL_IMAGE_NAME;isOutput=true]$TARGET_ALLOCATOR_FULL_IMAGE_NAME" echo "##vso[task.setvariable variable=LINUX_CONFIG_READER_FULL_IMAGE_NAME;isOutput=true]$LINUX_CONFIG_READER_FULL_IMAGE_NAME" echo "##vso[task.setvariable variable=LINUX_CCP_FULL_IMAGE_NAME;isOutput=true]$LINUX_CCP_FULL_IMAGE_NAME" @@ -171,7 +172,7 @@ jobs: LINUX_REF_APP_GOLANG_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.LINUX_REF_APP_GOLANG_FULL_IMAGE_NAME'] ] # This is necessary because of: https://github.com/moby/moby/issues/37965 DOCKER_BUILDKIT: 1 - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true))) steps: - checkout: self persistCredentials: true @@ -196,7 +197,7 @@ jobs: LINUX_REF_APP_PYTHON_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.LINUX_REF_APP_PYTHON_FULL_IMAGE_NAME'] ] # This is necessary because of: https://github.com/moby/moby/issues/37965 DOCKER_BUILDKIT: 1 - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true))) steps: - checkout: self persistCredentials: true @@ -219,7 +220,7 @@ jobs: variables: WINDOWS_REF_APP_GOLANG_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_REF_APP_GOLANG_FULL_IMAGE_NAME'] ] skipComponentGovernanceDetection: true - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true))) steps: - powershell: | docker build . --isolation=hyperv --file windows/Dockerfile -t $(WINDOWS_REF_APP_GOLANG_FULL_IMAGE_NAME) @@ -236,7 +237,7 @@ jobs: variables: WINDOWS_REF_APP_PYTHON_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_REF_APP_PYTHON_FULL_IMAGE_NAME'] ] skipComponentGovernanceDetection: true - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), or(eq(variables.IS_PR, true), eq(variables.IS_MAIN_BRANCH, true))) steps: - powershell: | docker build . --isolation=hyperv --file windows/Dockerfile -t $(WINDOWS_REF_APP_PYTHON_FULL_IMAGE_NAME) @@ -363,7 +364,7 @@ jobs: EOF workingDirectory: $(Build.SourcesDirectory)/otelcollector/ displayName: "Build: Set values in payload.json for signing" - condition: eq(variables.IS_MAIN_BRANCH, true) + condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true)) - task: EsrpCodeSigning@3 displayName: "ESRP CodeSigning for Prometheus" @@ -401,7 +402,7 @@ jobs: -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]" workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linux/" - condition: eq(variables.IS_MAIN_BRANCH, true) + condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true)) - bash: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin @@ -435,7 +436,7 @@ jobs: - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: "Ev2: Generate image artifacts" - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) inputs: BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux' DockerImagesToScan: '$(LINUX_FULL_IMAGE_NAME)' @@ -462,7 +463,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: "Ev2: Publish image artifacts" - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) inputs: pathToPublish: '$(Build.ArtifactStagingDirectory)' artifactName: drop @@ -527,7 +528,7 @@ jobs: EOF workingDirectory: $(Build.SourcesDirectory)/otelcollector/ displayName: "Build: Set values in payload.json for signing" - condition: eq(variables.IS_MAIN_BRANCH, true) + condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true)) - task: EsrpCodeSigning@3 displayName: "ESRP CodeSigning for Prometheus" @@ -565,7 +566,7 @@ jobs: -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]" workingDirectory: $(Build.ArtifactStagingDirectory)/linuxccp/ displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linuxccp/" - condition: eq(variables.IS_MAIN_BRANCH, true) + condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true)) - bash: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin @@ -591,7 +592,7 @@ jobs: - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: "Ev2: Generate image artifacts" - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) inputs: BuildDropPath: '$(Build.ArtifactStagingDirectory)/linuxccp' DockerImagesToScan: '$(LINUX_CCP_FULL_IMAGE_NAME)' @@ -618,7 +619,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: "Ev2: Publish image artifacts" - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) inputs: pathToPublish: '$(Build.ArtifactStagingDirectory)' artifactName: drop @@ -654,6 +655,8 @@ jobs: sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker system prune --all -f + docker buildx create --name dockerbuilder docker buildx use dockerbuilder docker login containerinsightsprod.azurecr.io -u $(ACR_USERNAME) -p $(ACR_PASSWORD) @@ -712,7 +715,7 @@ jobs: -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]" workingDirectory: $(Build.ArtifactStagingDirectory)/targetallocator/ displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/targetallocator/" - condition: eq(variables.IS_MAIN_BRANCH, true) + condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true)) - job: Linux_Config_Reader displayName: "Build: config reader image" @@ -731,6 +734,7 @@ jobs: # Necessary due to necessary due to https://stackoverflow.com/questions/60080264/docker-cannot-build-multi-platform-images-with-docker-buildx sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker system prune --all -f docker buildx create --name dockerbuilder docker buildx use dockerbuilder @@ -788,7 +792,7 @@ jobs: -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]" workingDirectory: $(Build.ArtifactStagingDirectory)/linuxcfgreader/ displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linuxcfgreader/" - condition: eq(variables.IS_MAIN_BRANCH, true) + condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true)) - job: Windows2019_Prometheus_Collector displayName: "Build: windows 2019 prometheus-collector image" @@ -943,7 +947,7 @@ jobs: displayName: "Build: Windows multi-arch manifest" - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) displayName: "Ev2: generate image artifacts" inputs: BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' @@ -1000,10 +1004,10 @@ jobs: oras attach $(WINDOWS_FULL_IMAGE_NAME) --artifact-type application/vnd.cncf.notary.signature ./payload.json:application/cose -a io.cncf.notary.x509chain.thumbprint#S256=[\""79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\""] workingDirectory: $(Build.ArtifactStagingDirectory)/windows displayName: "Download, install Oras and run oras attach" - condition: eq(variables.IS_MAIN_BRANCH, true) + condition: and(succeeded(), eq(variables.IS_MAIN_BRANCH, true)) - task: PublishBuildArtifacts@1 - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) displayName: "Ev2: publish image artifacts" inputs: pathToPublish: '$(Build.ArtifactStagingDirectory)' @@ -1055,14 +1059,14 @@ jobs: - task: PublishBuildArtifacts@1 displayName: "Ev2: publish helm chart artifacts" - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) inputs: pathToPublish: '$(Build.ArtifactStagingDirectory)' artifactName: drop - job: Deploy_Chart_ARC displayName: "Deploy: Arc dev cluster" - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) pool: name: Azure-Pipelines-CI-Test-EO dependsOn: @@ -1173,17 +1177,20 @@ jobs: displayName: "Deploy: AKS dev cluster" pool: name: Azure-Pipelines-CI-Test-EO - condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)) + condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))) dependsOn: - Image_Tags_and_Ev2_Artifacts - Linux_Prometheus_Collector + - Linux_Config_Reader + - Linux_Target_Allocator - WindowsMultiArch_Prometheus_Collector variables: HELM_CHART_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.HELM_CHART_NAME'] ] HELM_SEMVER: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.SEMVER'] ] IMAGE_TAG: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.SEMVER'] ] IMAGE_TAG_WINDOWS: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_IMAGE_TAG'] ] - HELM_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.HELM_FULL_IMAGE_NAME'] ] + IMAGE_TAG_TARGET_ALLOCATOR: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.TARGET_ALLOCATOR_IMAGE_TAG'] ] + IMAGE_TAG_CONFIG_READER: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.CONFIG_READER_IMAGE_TAG'] ] skipComponentGovernanceDetection: true steps: - checkout: self @@ -1205,10 +1212,9 @@ jobs: do sleep 30 echo $(MCR_REGISTRY)$(MCR_REPOSITORY):$(IMAGE_TAG_WINDOWS) - echo $(MCR_REGISTRY)$(MCR_REPOSITORY_HELM):$(IMAGE_TAG) output=$(curl -s https://$(MCR_REGISTRY)/v2$(MCR_REPOSITORY)/tags/list) - if (echo $output | grep $(IMAGE_TAG_WINDOWS)) && (echo $output | grep $(IMAGE_TAG)) + if (echo $output | grep $(IMAGE_TAG_WINDOWS)) && (echo $output | grep $(IMAGE_TAG) && (echo $output | grep $(IMAGE_TAG_TARGET_ALLOCATOR) && (echo $output | grep $(IMAGE_TAG_CONFIG_READER)) then echo "Images are published to mcr" exit 0 diff --git a/otelcollector/otel-allocator/Dockerfile b/otelcollector/otel-allocator/Dockerfile index 8b76d4dfa..50ae2bca0 100644 --- a/otelcollector/otel-allocator/Dockerfile +++ b/otelcollector/otel-allocator/Dockerfile @@ -32,7 +32,7 @@ COPY . . ARG TARGETOS TARGETARCH # Build the Go app -RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -installsuffix -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; else CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -installsuffix -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; fi +RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; else CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=pie -ldflags '-linkmode external -extldflags=-Wl,-z,now' -o main . ; fi ######## Start a new stage from scratch ####### FROM mcr.microsoft.com/cbl-mariner/distroless/debug:2.0