[CI] Buildkite integration tests: Windows ess retry script (#6030) (#…

…6294)

* ess.ps1

* Cleanup

* Cleanup

* cleanup

---------

Co-authored-by: Geoff Rowland <[email protected]>
(cherry picked from commit 0c4e7a9)

# Conflicts:
#	.buildkite/scripts/integration-tests.ps1

Co-authored-by: Pavel Zorin <[email protected]>
Co-authored-by: Michal Pristas <[email protected]>

.buildkite/scripts/integration-tests.ps1

@@ -0,0 +1,41 @@
+param (
+    [string]$GROUP_NAME
+)
+
+echo "~~~ Preparing environment"
+
+$PSVersionTable.PSVersion
+
+. "$PWD\.buildkite\scripts\steps\ess.ps1"
+
+go install gotest.tools/gotestsum
+gotestsum --version
+
+# Read package version from .package-version file
+$PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue
+if ($PACKAGE_VERSION) {
+    $PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT"
+}
+$env:TEST_BINARY_NAME = "elastic-agent"
+$env:AGENT_VERSION = $PACKAGE_VERSION
+$env:SNAPSHOT = $true
+
+echo "~~~ Building test binaries"
+mage build:testBinaries
+
+try {
+    Get-Ess-Stack -StackVersion $PACKAGE_VERSION    
+    Write-Output "~~~ Running integration test group: $GROUP_NAME as user: $env:USERNAME"
+    gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=true"    
+} finally {
+    ess_down
+    # Generate HTML report if XML output exists
+    $outputXML = "build/${GROUP_NAME}.integration.xml"
+    if (Test-Path $outputXML) {
+        # Install junit2html if not installed
+        go install github.com/alexec/junit2html@latest
+        Get-Content $outputXML | junit2html > "build/TEST-report.html"
+    } else {
+        Write-Output "Cannot generate HTML test report: $outputXML not found"
+    }
+}

.buildkite/scripts/steps/ess.ps1

@@ -0,0 +1,131 @@
+function ess_up {
+  param (
+      [string]$StackVersion,
+      [string]$EssRegion = "gcp-us-west2"
+  )
+
+  Write-Output "~~~ Starting ESS Stack"
+
+  $Workspace = & git rev-parse --show-toplevel
+  $TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/"
+
+  if (-not $StackVersion) {
+      Write-Error "Error: Specify stack version: ess_up [stack_version]"
+      return 1
+  }
+
+  $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
+    vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
+  }
+
+  if (-not $Env:EC_API_KEY) {
+      Write-Error "Error: Failed to get EC API key from vault"
+      exit 1
+  }
+
+  $BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
+  $BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
+  $BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
+
+  Push-Location -Path $TfDir
+  & terraform init
+  & terraform apply -auto-approve `
+      -var="stack_version=$StackVersion" `
+      -var="ess_region=$EssRegion" `
+      -var="creator=$BuildkiteBuildCreator" `
+      -var="buildkite_id=$BuildkiteBuildNumber" `
+      -var="pipeline=$BuildkitePipelineSlug"
+
+  $Env:ELASTICSEARCH_HOST = & terraform output -raw es_host
+  $Env:ELASTICSEARCH_USERNAME = & terraform output -raw es_username
+  $Env:ELASTICSEARCH_PASSWORD = & terraform output -raw es_password
+  $Env:KIBANA_HOST = & terraform output -raw kibana_endpoint
+  $Env:KIBANA_USERNAME = $Env:ELASTICSEARCH_USERNAME
+  $Env:KIBANA_PASSWORD = $Env:ELASTICSEARCH_PASSWORD
+  Pop-Location
+}
+
+function ess_down {  
+  $Workspace = & git rev-parse --show-toplevel
+  $TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/"
+  $stateFilePath = Join-Path -Path $TfDir -ChildPath "terraform.tfstate"
+
+  if (-not (Test-Path -Path $stateFilePath)) {
+    Write-Output "Terraform state file not found. Skipping ESS destroy."
+    return 0
+  }
+  Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
+  try {  
+    $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
+      vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
+    }
+    Push-Location -Path $TfDir
+    & terraform init
+    & terraform destroy -auto-approve
+    Pop-Location
+  } catch {
+    Write-Output "Error: Failed to destroy ESS stack(it will be auto-deleted later): $_"
+  }
+}
+
+function get_git_user_email {
+  if (!(git rev-parse --is-inside-work-tree *>&1)) {
+      return "unknown"
+  }
+
+  $email = & git config --get user.email
+
+  if (-not $email) {
+      return "unknown"
+  } else {
+      return $email
+  }
+}
+
+function Retry-Command {
+  param (
+      [scriptblock]$ScriptBlock,
+      [int]$MaxRetries = 3,
+      [int]$DelaySeconds = 5
+  )
+
+  $lastError = $null
+
+  for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) {
+      try {          
+        $result = & $ScriptBlock        
+        return $result
+      }
+      catch {          
+          $lastError = $_
+          Write-Warning "Attempt $attempt failed: $($_.Exception.Message)"
+          Write-Warning "Retrying in $DelaySeconds seconds..."
+          Start-Sleep -Seconds $DelaySeconds
+      }
+  }
+
+  Write-Error "All $MaxRetries attempts failed. Original error: $($lastError.Exception.Message)"
+  throw $lastError.Exception
+}
+
+function Get-Ess-Stack {
+  param (
+      [string]$StackVersion
+  )
+
+  if ($Env:BUILDKITE_RETRY_COUNT -gt 0) {
+      Write-Output "The step is retried, starting the ESS stack again"        
+      ess_up $StackVersion
+      Write-Output "ESS stack is up. ES_HOST: $Env:ELASTICSEARCH_HOST"
+  } else {
+      # For the first run, we retrieve ESS stack metadata
+      Write-Output "~~~ Receiving ESS stack metadata"
+      $Env:ELASTICSEARCH_HOST = & buildkite-agent meta-data get "es.host"
+      $Env:ELASTICSEARCH_USERNAME = & buildkite-agent meta-data get "es.username"
+      $Env:ELASTICSEARCH_PASSWORD = & buildkite-agent meta-data get "es.pwd"
+      $Env:KIBANA_HOST = & buildkite-agent meta-data get "kibana.host"
+      $Env:KIBANA_USERNAME = & buildkite-agent meta-data get "kibana.username"
+      $Env:KIBANA_PASSWORD = & buildkite-agent meta-data get "kibana.pwd"
+      Write-Output "Received ESS stack data from previous step. ES_HOST: $Env:ELASTICSEARCH_HOST"
+  }
+}