Run BK integration tests on PRs and release branches

.buildkite/bk.integration.pipeline.yml

@@ -6,21 +6,10 @@ env:
   ASDF_MAGE_VERSION: 1.14.0
 
 steps:
-  - label: "Integration tests: packaging"
-    key: "package-it"
-    command: ".buildkite/scripts/steps/integration-package.sh"
-    artifact_paths:
-      - build/distributions/**
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
   - label: Start ESS stack for integration tests
     key: integration-ess
     env:
       ASDF_TERRAFORM_VERSION: 1.9.2
-    depends_on:
-      - package-it
     command: |
       #!/usr/bin/env bash
       set -euo pipefail
@@ -36,11 +25,10 @@ steps:
     key: integration-tests-win
     depends_on:
       - integration-ess
-      - package-it
     steps:
       - label: "Win2022:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
@@ -62,7 +50,7 @@ steps:
 
       - label: "Win2022:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
@@ -83,11 +71,10 @@ steps:
     key: integration-tests-ubuntu
     depends_on:
       - integration-ess
-      - package-it
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -106,7 +93,7 @@ steps:
 
       - label: "x86_64:sudo: {{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -133,8 +120,9 @@ steps:
           - fleet-airgapped-privileged
 
       - label: "arm:sudo: {{matrix}}"
+        skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -161,8 +149,9 @@ steps:
           - fleet-airgapped-privileged
 
       - label: "arm:non-sudo: {{matrix}}"
+        skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -182,12 +171,11 @@ steps:
   - group: "Stateful(Sudo):RHEL8"
     key: integration-tests-rhel8
     depends_on:
-      - package-it
       - integration-ess
     steps:
       - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

.buildkite/integration.pipeline.yml

@@ -57,32 +57,13 @@ steps:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
 
-  - label: "Integration tests"
-    key: "integration-tests"
+  - label: "Triggering Integration tests"
     depends_on:
       - package-it
-    concurrency_group: elastic-agent-extended-testing/integration
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh stateful
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Integration tests"
-    plugins:
-      - test-collector#v1.10.1:
-          files: "build/TEST-*.xml"
-          format: "junit"
-          branches: "main"
-          debug: true
+    trigger: "elastic-agent-extended-testing-bk"
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
 
   - label: "Serverless Beats Tests"
     depends_on:

.buildkite/pipeline.yml

@@ -261,7 +261,7 @@ steps:
                   - GH_VERSION=2.4.0
 
   # Trigger for pull requests
-  - label: "Trigger Integration tests for Pull request"
+  - label: "Trigger Extended tests for Pull request"
     if: |
       (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) ||
       build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/
@@ -301,7 +301,7 @@ steps:
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
 
   # Trigger for branches
-  - label: "Triggering Integration tests for branches"
+  - label: "Triggering Extended tests for branches"
     if: build.pull_request.id == null
     trigger: "elastic-agent-extended-testing"
     build:

docs/test-framework-dev-guide.md

@@ -317,6 +317,39 @@ are defined the more instances will be provisioned to complete all tests. A bala
 groups is better than a ton of groups each executing a small set of tests, as the time to set up an instance can
 out weight the benefits of creating another group.
 
+#### Creating a new test group and Buildkite integration tests
+
+  When creating a new test group, it is important to add the new group to the job in the `.buildkite/bk.integration.pipeline.yml` file. This will ensure that the new group is executed in the CI pipeline. 
+
+  Add the new group to the `matrix` in the corresponding steps. The matrix is a list of all the test groups that are executed in the teh step. 
+  Example: 
+
+  ```yaml
+  - label: "x86_64:sudo: {{matrix}}"
+        command: |
+          ...
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+        matrix:
+          - default
+          - container
+          - fleet-upgrade-to-pr-build
+          - upgrade
+          - fleet
+  ```
+
+  This requirement is temporary and will be removed once the Buildkite pipeline is updated to automatically detect new test groups.
+
 ### Test namespaces
 
 Every test has access to its own unique namespace (a string value). This namespace can