Arc: new release image, enable target-allocator, add CI/CD proxy clus…

…ter (#977) [comment]: # (Note that your PR title should follow the conventional commit format: https://conventionalcommits.org/en/v1.0.0/#summary) # PR Description ### Chart: - Enable the target-allocator for Arc - Disable the HPA chart setting for Arc ### Pipeline: - Add deploying to an Arc CI/CD proxy cluster - Add running testkube after every merge and nightly for the Arc CI/CD cluster - Add BUILD_WINDOWS pipeline variable for faster building of the chart when just testing linux ### Tests: - README clarifications - Ignore proxy for some localhost curl calls in the tests
Azure · Sep 25, 2024 · 74a753d · 74a753d
1 parent d9b3ed9
commit 74a753d
Show file tree

Hide file tree

Showing 8 changed files with 144 additions and 32 deletions.
diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml
@@ -2,6 +2,7 @@ trigger:
   branches:
     include:
     - main
+
 pr:
   autoCancel: true
   branches:
@@ -22,6 +23,7 @@ variables:
   NODE_EXPORTER_IMAGE: 'mcr.microsoft.com/oss/prometheus/node-exporter:v1.6.0'
   IS_PR: $[eq(variables['Build.Reason'], 'PullRequest')]
   IS_MAIN_BRANCH: $[eq(variables['Build.SourceBranchName'], 'main')]
+  BUILD_WINDOWS: true
   Codeql.Enabled: true
 
 stages:
@@ -1126,6 +1128,7 @@ stages:
       WINDOWS_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_FULL_IMAGE_NAME'] ]
       WINDOWS_2019_BASE_IMAGE_VERSION: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_2019_BASE_IMAGE_VERSION'] ]
       skipComponentGovernanceDetection: true
+    condition: and(succeeded(), eq(variables.BUILD_WINDOWS, true))
     steps:
       - task: GoTool@0
         displayName: "Build: specify golang version"
@@ -1159,6 +1162,7 @@ stages:
       WINDOWS_FULL_IMAGE_NAME: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_FULL_IMAGE_NAME'] ]
       WINDOWS_2022_BASE_IMAGE_VERSION: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_2022_BASE_IMAGE_VERSION'] ]
       skipComponentGovernanceDetection: true
+    condition: and(succeeded(), eq(variables.BUILD_WINDOWS, true))
     steps:
       - task: GoTool@0
         displayName: "Build: specify golang version"
@@ -1196,6 +1200,7 @@ stages:
       WINDOWS_2019_BASE_IMAGE_VERSION: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_2019_BASE_IMAGE_VERSION'] ]
       WINDOWS_2022_BASE_IMAGE_VERSION: $[ dependencies.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_2022_BASE_IMAGE_VERSION'] ]
       skipComponentGovernanceDetection: true
+    condition: and(succeeded(), eq(variables.BUILD_WINDOWS, true))
     steps:
       - task: GoTool@0
         displayName: "Build: specify golang version"
@@ -1510,6 +1515,112 @@ stages:
                 az config set extension.use_dynamic_install=yes_without_prompt
                 az k8s-extension update --name azuremonitor-metrics --resource-group ci-dev-arc-wcus --cluster-name ci-dev-arc-wcus --cluster-type connectedClusters --version $HELM_SEMVER --release-train pipeline
 
+          - task: AzureCLI@2
+            displayName: "Deploy: ci-dev-arc-proxy cluster"
+            inputs:
+              azureSubscription: 'ContainerInsights_Build_Subscription(9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb)'
+              scriptType: 'bash'
+              scriptLocation: 'inlineScript'
+              inlineScript: |
+                az config set extension.use_dynamic_install=yes_without_prompt
+                az k8s-extension update --name azuremonitor-metrics --resource-group ci-dev-arc-proxy --cluster-name ci-dev-arc-proxy --cluster-type connectedClusters --version $HELM_SEMVER --release-train pipeline
+
+  - deployment: Testkube_ARC
+    displayName: "Test: Arc testkube tests"
+    environment: Prometheus-Collector
+    dependsOn: Deploy_Chart_ARC
+    pool:
+      name: Azure-Pipelines-CI-Test-EO
+    condition: and(succeeded(), and(eq(variables.IS_PR, false), eq(variables.IS_MAIN_BRANCH, true)))
+    variables:
+      HELM_CHART_NAME: $[ stageDependencies.Build.Image_Tags_and_Ev2_Artifacts.outputs['setup.HELM_CHART_NAME'] ]
+      HELM_SEMVER: $[ stageDependencies.Build.Image_Tags_and_Ev2_Artifacts.outputs['setup.SEMVER'] ]
+      IMAGE_TAG: $[ stageDependencies.Build.Image_Tags_and_Ev2_Artifacts.outputs['setup.SEMVER'] ]
+      IMAGE_TAG_WINDOWS: $[ stageDependencies.Build.Image_Tags_and_Ev2_Artifacts.outputs['setup.WINDOWS_IMAGE_TAG'] ]
+      HELM_FULL_IMAGE_NAME: $[ stageDependencies.Build.Image_Tags_and_Ev2_Artifacts.outputs['setup.HELM_FULL_IMAGE_NAME'] ]
+      skipComponentGovernanceDetection: true
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - checkout: self
+            persistCredentials: true
+
+          - bash: |
+              wget -qO - https://repo.testkube.io/key.pub | sudo apt-key add -
+              echo "deb https://repo.testkube.io/linux linux main" | sudo tee -a /etc/apt/sources.list
+              sudo apt-get update
+              sudo apt-get install -y testkube
+
+              exit 0
+            workingDirectory: $(Build.SourcesDirectory)
+            displayName: "Install testkube CLI"
+
+          - task: AzureCLI@1
+            displayName: Get kubeconfig
+            inputs:
+              azureSubscription: 'ContainerInsights_Build_Subscription(9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb)'
+              scriptLocation: 'inlineScript'
+              inlineScript: 'az aks get-credentials -g ci-dev-arc-wcus -n ci-dev-arc-wcus'
+
+          - bash: |
+              export AMW_QUERY_ENDPOINT="https://ci-dev-arc-amw-p3eu.eastus.prometheus.monitor.azure.com"
+              export AZURE_CLIENT_ID="5f13547e-a4e2-4efd-85fe-a2b03d5b8661"
+
+              envsubst < ./testkube/testkube-test-crs-arc.yaml > ./testkube/testkube-test-crs-ci-dev-arc-wcus.yaml
+              kubectl apply -f ./testkube/api-server-permissions.yaml
+              kubectl apply -f ./testkube/testkube-test-crs-ci-dev-arc-wcus.yaml
+              kubectl apply -f ./test-cluster-yamls/configmaps
+              kubectl apply -f ./test-cluster-yamls/prometheus-reference-app.yaml
+
+              exit 0
+            workingDirectory: $(Build.SourcesDirectory)/otelcollector/test/
+            displayName: "Apply TestKube CRs, scrape configs and pod/service monitors"
+
+          - bash: |
+              sleep 360
+
+              exit 0
+            displayName: "Wait for cluster to be ready"
+
+          - bash: |
+              # Run the full test suite
+              kubectl testkube run testsuite e2e-tests-merge --verbose
+
+              # Get the current id of the test suite now running
+              execution_id=$(kubectl testkube get testsuiteexecutions --test-suite e2e-tests-merge --limit 1 | grep e2e-tests | awk '{print $1}')
+
+              # Watch until the all the tests in the test suite finish
+              kubectl testkube watch testsuiteexecution $execution_id
+
+              # Get the results as a formatted json file
+              kubectl testkube get testsuiteexecution $execution_id --output json > testkube-results.json
+
+              # For any test that has failed, print out the Ginkgo logs
+              if [[ $(jq -r '.status' testkube-results.json) == "failed" ]]; then
+
+                # Get each test name and id that failed
+                jq -r '.executeStepResults[].execute[] | select(.execution.executionResult.status=="failed") | "\(.execution.testName) \(.execution.id)"' testkube-results.json | while read line; do
+                  testName=$(echo $line | cut -d ' ' -f 1)
+                  id=$(echo $line | cut -d ' ' -f 2)
+                  echo "Test $testName failed. Test ID: $id"
+
+                  # Get the Ginkgo logs of the test
+                  kubectl testkube get execution $id > out 2>error.log
+
+                  # Remove superfluous logs of everything before the last occurence of 'go downloading'.
+                  # The actual errors can be viewed from the ADO run, instead of needing to view the testkube dashboard.
+                  cat error.log | tac | awk '/go: downloading/ {exit} 1' | tac
+                done
+
+                # Explicitly fail the ADO task since at least one test failed
+                exit 0
+              fi
+
+              exit 0
+            workingDirectory: $(Build.SourcesDirectory)
+            displayName: "Run tests"
+
   - deployment: Deploy_AKS_Chart
     displayName: "Deploy: AKS dev cluster"
     environment: Prometheus-Collector

diff --git a/...r/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-collector-hpa.yaml b/...r/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-collector-hpa.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.AzureMonitorMetrics.CollectorHPAEnabled}}
+{{- if and (ne .Values.AzureMonitorMetrics.ArcExtension true) (.Values.AzureMonitorMetrics.CollectorHPAEnabled)}}
 {{- $amaMetricsHpa := include "ama-metrics-merge-custom-hpa" . | fromYaml }}
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler

diff --git a/...ctor/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-deployment.yaml b/...ctor/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-deployment.yaml
@@ -9,7 +9,7 @@ metadata:
 spec:
 # New TargetAllocator Enabled scenario
 {{- if .Values.AzureMonitorMetrics.TargetAllocatorEnabled }}
-  {{- if .Values.AzureMonitorMetrics.CollectorHPAEnabled }} # If HPA toggle is enabled, allow HPA to modify the deployment spec
+  {{- if and (ne .Values.AzureMonitorMetrics.ArcExtension true) (.Values.AzureMonitorMetrics.CollectorHPAEnabled) }} # If HPA toggle is enabled, allow HPA to modify the deployment spec
   {{- $currentSpec := (lookup "apps/v1" "Deployment" "kube-system" "ama-metrics").spec }}
     {{- if $currentSpec }}
     # in if check
@@ -83,7 +83,7 @@ spec:
               value: "false"
             {{- end }}
             - name: AZMON_COLLECTOR_HPA_ENABLED
-            {{- if eq .Values.AzureMonitorMetrics.CollectorHPAEnabled true }}
+            {{- if and (ne .Values.AzureMonitorMetrics.ArcExtension true) (eq .Values.AzureMonitorMetrics.CollectorHPAEnabled true) }}
               value: "true"
             {{- else }}
               value: "false"

diff --git a/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/values-template.yaml b/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/values-template.yaml
@@ -74,7 +74,7 @@ AzureMonitorMetrics:
     ImageRepositoryWin: "/aks/hcp/addon-token-adapter"
     ImageTagWin: "master.240102.1"
   ArcExtension: ${ARC_EXTENSION}
-  ArcEnableOperator: false
+  ArcEnableOperator: true
   # Do not change the below settings. They are reserved for Arc Autonomous
   isArcACluster: false
   arcAutonomousSettings:

diff --git a/otelcollector/test/README.md b/otelcollector/test/README.md
@@ -307,13 +307,13 @@ Some highlights are that:
     wget -qO - https://repo.testkube.io/key.pub | sudo apt-key add -
     echo "deb https://repo.testkube.io/linux linux main" | sudo tee -a /etc/apt/sources.list
     sudo apt-get update
-    sudo apt-get install -y testkube
+    sudo apt-get install -y testkube=1.14.2
   ```
   Other OS installation instructions are [here](https://docs.testkube.io/articles/install-cli/).
 - Install the [helm chart](https://docs.testkube.io/articles/helm-chart/) on your cluster:
   ```bash
   helm repo add kubeshop https://kubeshop.github.io/helm-charts
-  helm install --create-namespace testkube kubeshop/testkube
+  helm install --create-namespace testkube kubeshop/testkube --version 1.15.17 -n testkube
   ```
 - The helm chart will install in the namespace `testkube`.
 - Run `testkube dashboard` to port-forward the dashboard.

diff --git a/otelcollector/test/ginkgo-e2e/prometheusui/prometheus_ui_test.go b/otelcollector/test/ginkgo-e2e/prometheusui/prometheus_ui_test.go
@@ -180,7 +180,7 @@ var _ = DescribeTable("The Prometheus UI should return the /metrics data",
 			// Execute the command and capture the output
 			var command []string
 			if isLinux {
-				command = []string{"sh", "-c", "curl \"http://localhost:9090/metrics\""}
+				command = []string{"sh", "-c", "curl --noproxy \"*\" \"http://localhost:9090/metrics\""}
 			} else {
 				command = []string{"powershell", "-c", "(curl \"http://localhost:9090/metrics\" -UseBasicParsing).Content"}
 			}
@@ -213,7 +213,7 @@ var _ = DescribeTable("The Prometheus UI should return a 200 for its UI pages",
 			for _, uiPath := range uiPaths {
 				var command []string
 				if isLinux {
-					command = []string{"sh", "-c", fmt.Sprintf("curl \"http://localhost:9090%s\"", uiPath)}
+					command = []string{"sh", "-c", fmt.Sprintf("curl --noproxy \"*\" \"http://localhost:9090%s\"", uiPath)}
 				} else {
 					command = []string{"powershell", "-c", fmt.Sprintf("(curl \"http://localhost:9090/%s\" -UseBasicParsing).Content", uiPath)}
 				}

diff --git a/otelcollector/test/ginkgo-e2e/utils/constants.go b/otelcollector/test/ginkgo-e2e/utils/constants.go
@@ -1,27 +1,28 @@
 package utils
 
-
-var(
-  // Slices can't be constants
-  LogLineErrorsToExclude = [...]string{
-    // Metrics Extension
-    "\"filepath\":\"/MetricsExtensionConsoleDebugLog.log\"",
-    // Arc token adapter
-    "create or renew cluster identity error",
-    "get token from status error",
-    // KSM
-    "ended with: an error on the server",
-    "Objects listed",
-    // Target allocator
-    "client connection lost",
-  }
+var (
+	// Slices can't be constants
+	LogLineErrorsToExclude = [...]string{
+		// Metrics Extension
+		"\"filepath\":\"/MetricsExtensionConsoleDebugLog.log\"",
+		// Arc token adapter
+		"create or renew cluster identity error",
+		"get token from status error",
+		// Arc node-exporter
+		"Failed to open directory, disabling udev device properties",
+		// KSM
+		"ended with: an error on the server",
+		"Objects listed",
+		// Target allocator
+		"client connection lost",
+	}
 )
 
 const (
-  OperatorLabel              = "operator"
-  ArcExtensionLabel          = "arc-extension"
-  WindowsLabel               = "windows"
-  ARM64Label                 = "arm64"
-  FIPSLabel                  = "fips"
-  LinuxDaemonsetCustomConfig = "linux-daemonset-custom-config"
+	OperatorLabel              = "operator"
+	ArcExtensionLabel          = "arc-extension"
+	WindowsLabel               = "windows"
+	ARM64Label                 = "arm64"
+	FIPSLabel                  = "fips"
+	LinuxDaemonsetCustomConfig = "linux-daemonset-custom-config"
 )
diff --git a/otelcollector/test/testkube/testkube-test-crs.yaml b/otelcollector/test/testkube/testkube-test-crs.yaml
@@ -18,7 +18,7 @@ spec:
   executionRequest:
     args:
       - "--label-filter"
-      - "!(arc-extension,linux-daemonset-custom-config)"
+      - "!(arc-extension,linux-daemonset-custom-config,arm64)"
       - "./containerstatus"
     executePostRunScriptBeforeScraping: false
 ---
@@ -42,7 +42,7 @@ spec:
   executionRequest:
     args:
       - "--label-filter"
-      - "!(arc-extension,linux-daemonset-custom-config)"
+      - "!(arc-extension,linux-daemonset-custom-config,arm64)"
       - "./livenessprobe"
     executePostRunScriptBeforeScraping: false
 ---
@@ -66,7 +66,7 @@ spec:
   executionRequest:
     args:
       - "--label-filter"
-      - "!(arc-extension,linux-daemonset-custom-config)"
+      - "!(arc-extension,linux-daemonset-custom-config,arm64)"
       - "./prometheusui"
     executePostRunScriptBeforeScraping: false
 ---