Step 0 : Merge CCP changes to main with a separate image

.pipelines/azure-pipeline-build.yml

@@ -50,10 +50,11 @@ jobs:
         LINUX_IMAGE_TAG=$SEMVER
         # Truncating to 128 characters as it is required by docker
         LINUX_IMAGE_TAG=$(echo "${LINUX_IMAGE_TAG}" | cut -c1-128)
-
+        
         #Truncating this to 124 to add the cfg suffix
         LINUX_IMAGE_TAG_PREFIX=$(echo "${LINUX_IMAGE_TAG}" | cut -c1-124)
         LINUX_CONFIG_READER_IMAGE_TAG=$LINUX_IMAGE_TAG_PREFIX-cfg
+        LINUX_CCP_IMAGE_TAG=$LINUX_IMAGE_TAG_PREFIX-ccp
 
         #Truncating this to 113 to add the ref app suffices
         LINUX_REF_APP_IMAGE_TAG_PREFIX=$(echo "${LINUX_IMAGE_TAG}" | cut -c1-113)
@@ -81,6 +82,7 @@ jobs:
         LINUX_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$LINUX_IMAGE_TAG
         TARGET_ALLOCATOR_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$TARGET_ALLOCATOR_IMAGE_TAG
         LINUX_CONFIG_READER_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$LINUX_CONFIG_READER_IMAGE_TAG
+        LINUX_CCP_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$LINUX_CCP_IMAGE_TAG
         WINDOWS_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY:$WINDOWS_IMAGE_TAG
         HELM_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY_HELM/$HELM_CHART_NAME:$SEMVER
         ARC_HELM_FULL_IMAGE_NAME=$ACR_REGISTRY$ACR_REPOSITORY_HELM/$ARC_HELM_CHART_NAME:$SEMVER
@@ -95,6 +97,7 @@ jobs:
         echo "##vso[task.setvariable variable=TARGET_ALLOCATOR_IMAGE_TAG;isOutput=true]$TARGET_ALLOCATOR_IMAGE_TAG"
         echo "##vso[task.setvariable variable=TARGET_ALLOCATOR_FULL_IMAGE_NAME;isOutput=true]$TARGET_ALLOCATOR_FULL_IMAGE_NAME"
         echo "##vso[task.setvariable variable=LINUX_CONFIG_READER_FULL_IMAGE_NAME;isOutput=true]$LINUX_CONFIG_READER_FULL_IMAGE_NAME"
+        echo "##vso[task.setvariable variable=LINUX_CCP_FULL_IMAGE_NAME;isOutput=true]$LINUX_CCP_FULL_IMAGE_NAME"
         echo "##vso[task.setvariable variable=WINDOWS_FULL_IMAGE_NAME;isOutput=true]$WINDOWS_FULL_IMAGE_NAME"
         echo "##vso[task.setvariable variable=LINUX_REF_APP_GOLANG_FULL_IMAGE_NAME;isOutput=true]$LINUX_REF_APP_GOLANG_FULL_IMAGE_NAME"
         echo "##vso[task.setvariable variable=LINUX_REF_APP_PYTHON_FULL_IMAGE_NAME;isOutput=true]$LINUX_REF_APP_PYTHON_FULL_IMAGE_NAME"
@@ -258,7 +261,6 @@ jobs:
         targetType: 'F'
         targetArgument: '$(Build.SourcesDirectory)'
 
-
 - job: SDL_Binary_Scan
   displayName: "SDL: linux binary scanning"
   pool:
@@ -339,15 +341,16 @@ jobs:
 
         # Necessary due to necessary due to https://stackoverflow.com/questions/60080264/docker-cannot-build-multi-platform-images-with-docker-buildx
         sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static
-        #docker system prune --volumes -y
         docker system prune --all -f
+        docker images -q --filter "dangling=true" | xargs docker rmi
         docker login containerinsightsprod.azurecr.io -u $(ACR_USERNAME) -p $(ACR_PASSWORD)
         docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
         docker buildx create --name dockerbuilder
         docker buildx use dockerbuilder
         docker buildx build . --platform=linux/amd64,linux/arm64 --file ./build/linux/Dockerfile -t $(LINUX_FULL_IMAGE_NAME) --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --push # --cache-to type=registry,ref=$(ACR_REGISTRY)$(ACR_REPOSITORY)/cache:prometheuscollector,mode=max --cache-from type=registry,ref=$(ACR_REGISTRY)$(ACR_REPOSITORY)/cache:prometheuscollector
         docker pull $(LINUX_FULL_IMAGE_NAME)
+        docker system prune --all -f
       workingDirectory: $(Build.SourcesDirectory)/otelcollector/
       displayName: "Build: build and push image to dev ACR"
 
@@ -475,6 +478,162 @@ jobs:
         GdnBreakGdnToolSemmle: true
         GdnBreakGdnToolSemmleSeverity: 'Warning'
 
+- job: Linux_CCP_Prometheus_Collector
+  displayName: "Build: linux CCP prometheus-collector image"
+  pool:
+    name: Azure-Pipelines-CI-Test-EO
+  dependsOn: Image_Tags_and_Ev2_Artifacts
+  variables:
+    LINUX_CCP_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.LINUX_CCP_FULL_IMAGE_NAME'] ]
+  # This is necessary because of: https://github.com/moby/moby/issues/37965
+    DOCKER_BUILDKIT: 1
+  steps:
+    - checkout: self
+      submodules: true
+
+    - task: CodeQL3000Init@0
+      displayName: 'SDL: init codeql'
+
+    - task: GoTool@0
+      displayName: "Build: specify golang version"
+      inputs:
+        version: '1.20'
+
+    - bash: |
+        mkdir -p $(Build.ArtifactStagingDirectory)/linuxccp
+
+        # Necessary due to necessary due to https://stackoverflow.com/questions/60080264/docker-cannot-build-multi-platform-images-with-docker-buildx
+        sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static
+        docker system prune --volumes -y
+        docker system prune --all -f
+        docker images -q --filter "dangling=true" | xargs docker rmi
+        docker login containerinsightsprod.azurecr.io -u $(ACR_USERNAME) -p $(ACR_PASSWORD)
+        docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+        docker buildx create --name dockerbuilder
+        docker buildx use dockerbuilder
+        docker buildx build . --platform=linux/amd64 --file ./build/linux/ccp/Dockerfile -t $(LINUX_CCP_FULL_IMAGE_NAME) --metadata-file $(Build.ArtifactStagingDirectory)/linuxccp/metadata.json --push # --cache-to type=registry,ref=$(ACR_REGISTRY)$(ACR_REPOSITORY)/cache:prometheuscollectorccp,mode=max --cache-from type=registry,ref=$(ACR_REGISTRY)$(ACR_REPOSITORY)/cache:prometheuscollectorccp
+        docker pull $(LINUX_CCP_FULL_IMAGE_NAME)
+        docker system prune --all -f
+      workingDirectory: $(Build.SourcesDirectory)/otelcollector/
+      displayName: "Build: build and push CCP image to dev ACR"
+
+    - bash: |
+        MEDIA_TYPE=$(docker manifest inspect -v $(LINUX_CCP_FULL_IMAGE_NAME) | jq '.Descriptor.mediaType')
+        DIGEST=$(docker manifest inspect -v $(LINUX_CCP_FULL_IMAGE_NAME) | jq '.Descriptor.digest')
+        SIZE=$(docker manifest inspect -v $(LINUX_CCP_FULL_IMAGE_NAME) | jq '.Descriptor.size')
+        cat <<EOF >>$(Build.ArtifactStagingDirectory)/linuxccp/payload.json
+        {"targetArtifact":{"mediaType":$MEDIA_TYPE,"digest":$DIGEST,"size":$SIZE}}
+        EOF
+      workingDirectory: $(Build.SourcesDirectory)/otelcollector/
+      displayName: "Build: Set values in payload.json for signing"
+      condition: eq(variables.IS_MAIN_BRANCH, true)
+
+    - task: EsrpCodeSigning@3
+      displayName: "ESRP CodeSigning for Prometheus"
+      inputs:
+        ConnectedServiceName: "ESRPServiceConnectionForPrometheusImages"
+        FolderPath: $(Build.ArtifactStagingDirectory)/linuxccp/
+        Pattern: "*.json"
+        signConfigType: inlineSignParams
+        inlineOperation: |
+          [
+            {
+                "keyCode": "CP-469451",
+                "operationSetCode": "NotaryCoseSign",
+                "parameters": [
+                  {
+                    "parameterName": "CoseFlags",
+                    "parameterValue": "chainunprotected"
+                  }
+                ],
+                "toolName": "sign",
+                "toolVersion": "1.0"
+            }
+          ]
+
+    - bash: |
+        set -euxo pipefail
+        curl -LO "https://github.com/oras-project/oras/releases/download/v1.0.0/oras_1.0.0_linux_amd64.tar.gz"
+        mkdir -p oras-install/
+        tar -zxf oras_1.0.0_*.tar.gz -C oras-install/
+        sudo mv oras-install/oras /usr/local/bin/
+        rm -rf oras_1.0.0_*.tar.gz oras-install/
+        oras attach $(LINUX_CCP_FULL_IMAGE_NAME) \
+          --artifact-type 'application/vnd.cncf.notary.signature' \
+          ./payload.json:application/cose \
+          -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]"
+      workingDirectory: $(Build.ArtifactStagingDirectory)/linuxccp/
+      displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linuxccp/"
+      condition: eq(variables.IS_MAIN_BRANCH, true)
+
+    - bash: |
+        curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
+        trivy image --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM --exit-code 1 $(LINUX_CCP_FULL_IMAGE_NAME)
+        if [ $? -ne 0 ]; then
+          exit 1
+        fi
+      workingDirectory: $(Build.SourcesDirectory)
+      displayName: "Build: run trivy scan"
+
+    - task: CodeQL3000Finalize@0
+      displayName: 'SDL: run codeql'
+
+    - task: ComponentGovernanceComponentDetection@0
+      displayName: "SDL: run component governance"
+      inputs:
+        scanType: 'Register'
+        verbosity: 'Verbose'
+        dockerImagesToScan: '$(LINUX_CCP_FULL_IMAGE_NAME)'
+        alertWarningLevel: 'High'
+        sourceScanPath: '$(Build.SourcesDirectory)/otelcollector'
+        ignoreDirectories: '$(Build.SourcesDirectory)/mixins,$(Build.SourcesDirectory)/tools,$(Build.SourcesDirectory)/otelcollector/react'
+
+    - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+      displayName: "Ev2: Generate image artifacts"
+      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      inputs:
+        BuildDropPath: '$(Build.ArtifactStagingDirectory)/linuxccp'
+        DockerImagesToScan: '$(LINUX_CCP_FULL_IMAGE_NAME)'
+
+    - task: SdtReport@2
+      displayName: 'SDL: generate report'
+      inputs:
+        GdnExportAllTools: false
+        GdnExportGdnToolBinSkim: true
+        GdnExportGdnToolBinSkimSeverity: 'Note'
+        GdnExportGdnToolGosec: true
+        GdnExportGdnToolGosecSeverity: 'Note'
+        GdnExportGdnToolSemmle: true
+        GdnExportGdnToolSemmleSeverity: 'Note'
+
+    - task: PublishSecurityAnalysisLogs@3
+      displayName: 'SDL: publish report'
+      inputs:
+        ArtifactName: 'CodeAnalysisLogs'
+        ArtifactType: 'Container'
+        PublishProcessedResults: true
+        AllTools: true
+        ToolLogsNotFoundAction: 'Standard'
+
+    - task: PublishBuildArtifacts@1
+      displayName: "Ev2: Publish image artifacts"
+      condition: and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true))
+      inputs:
+        pathToPublish: '$(Build.ArtifactStagingDirectory)'
+        artifactName: drop
+
+    - task: PostAnalysis@2
+      displayName: 'SDL: Post-Build Analysis'
+      inputs:
+        GdnBreakAllTools: false
+        GdnBreakGdnToolBinSkim: true
+        GdnBreakGdnToolBinSkimSeverity: 'Warning'
+        GdnBreakGdnToolGosec: true
+        GdnBreakGdnToolGosecSeverity: 'Warning'
+        GdnBreakGdnToolSemmle: true
+        GdnBreakGdnToolSemmleSeverity: 'Warning'
+
 - job: Linux_Target_Allocator
   displayName: "Build: target allocator image"
   pool: