Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Trigger external apps test #19065

Open
wants to merge 38 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 30 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
354d585
ci: enable on-demand and release-triggered Cypress tests across DHIS2…
adeldhis2 Feb 26, 2024
b38b5ff
chore: add GitHub Actions trigger for Cypress tests in stable and dev…
adeldhis2 Jul 30, 2024
997ef44
chore: comment out non-essential stages for testing
adeldhis2 Jul 31, 2024
d26cc9e
chore: comment out the failure notifications
adeldhis2 Jul 31, 2024
4213a63
chore: remove the cron and pollSCM triggers
adeldhis2 Aug 1, 2024
c95b06a
Add pipeline to trigger external tests post core deployment
adeldhis2 Aug 12, 2024
9d7b524
rename event_type to apps-e2e-tests-trigger
adeldhis2 Aug 19, 2024
5a164d2
chore: improve parameter validation and error handling
adeldhis2 Sep 9, 2024
629feda
chore: increase instance creation timeout to 20 minutes
adeldhis2 Sep 11, 2024
12e594c
chore: add GitHub dispatch response logging to Jenkins pipeline
adeldhis2 Sep 11, 2024
e3d5abe
chore: fix GitHub dispatch event for e2e tests
adeldhis2 Sep 11, 2024
bffc610
chore: switch to workflow_dispatch for triggering tests with branch t…
adeldhis2 Sep 12, 2024
08c5615
chore: trigger data-entry and data-viz
adeldhis2 Sep 19, 2024
529620f
chore: add verbose logging for debugging
adeldhis2 Sep 24, 2024
cd7aef3
chore: switch to dhis2-im-bot credentials
adeldhis2 Sep 24, 2024
7fddc30
chore: trigger more apps tests
adeldhis2 Sep 27, 2024
05d05ab
chore: debug
adeldhis2 Sep 30, 2024
77a578b
chore: debug token verification
adeldhis2 Sep 30, 2024
28c2a6e
chore: debug
adeldhis2 Sep 30, 2024
5c550c7
chore: remove GitHub token access verification code from pipeline
adeldhis2 Sep 30, 2024
69f0ce4
chore: debug
adeldhis2 Oct 9, 2024
947fd95
chore: use custom db
adeldhis2 Oct 10, 2024
116513b
chore: add steps to install App Hub apps and generate analytics tables
adeldhis2 Oct 14, 2024
474006c
chore: debug
adeldhis2 Oct 14, 2024
93c4701
debug: istall apps and run analytics
adeldhis2 Oct 16, 2024
72fc67d
fix script permission
adeldhis2 Oct 16, 2024
3d526c7
fix install app script
adeldhis2 Oct 16, 2024
3cc31c2
cleanup
adeldhis2 Oct 16, 2024
7face0d
chore: fix db path
adeldhis2 Oct 17, 2024
66de41d
cleanup
adeldhis2 Nov 5, 2024
842f8b6
debug: use fixed test instance URL
adeldhis2 Dec 3, 2024
da98959
debug
adeldhis2 Dec 3, 2024
3634433
debug
adeldhis2 Dec 3, 2024
ac9d1d4
debug
adeldhis2 Dec 9, 2024
3016b9d
debug: comment instance deployment
adeldhis2 Dec 9, 2024
5160b03
debug
adeldhis2 Dec 9, 2024
3c53dbf
debug
adeldhis2 Dec 16, 2024
3f6c053
debug: deploy instances
adeldhis2 Dec 20, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@ env:
# This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
on:
workflow_dispatch:
push:
branches:
- master
- trigger-external-apps-test
pull_request:
release:
types: [created]
concurrency:
group: ${{ github.workflow}}-${{ github.ref }}
cancel-in-progress: true
Expand Down
170 changes: 170 additions & 0 deletions jenkinsfiles/trigger-external-tests
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
#!/usr/bin/env groovy

@Library('pipeline-library') _

pipeline {
agent {
label 'ec2-jdk17'
}

options {
buildDiscarder(logRotator(daysToKeepStr: '5'))
timeout(time: 60)
disableConcurrentBuilds()
}

parameters {
string(name: 'DHIS2_VERSION', defaultValue: '', description: 'DHIS2 core version to use for deployment. If left empty, it will be derived from the Git branch.')
string(name: 'DB_VERSION', defaultValue: '', description: 'Database version to use. If left empty, it will be derived from the DHIS2 version.')
}

environment {
HTTP = "http --check-status"
MAVEN_OPTS = '-Xms1024m -Xmx4096m -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125'
DOCKER_IMAGE_TAG = "${params.DHIS2_VERSION ? params.DHIS2_VERSION.replace('SNAPSHOT', 'rc') : env.GIT_BRANCH}"
DB_VERSION_TAG = "${params.DB_VERSION ? params.DB_VERSION : DOCKER_IMAGE_TAG}"
IMAGE_TAG = "${DOCKER_IMAGE_TAG}"
IMAGE_REPOSITORY = 'core'
IM_REPO_URL = 'https://github.com/dhis2-sre/im-manager'
IM_ENVIRONMENT = 'im.dhis2.org'
IM_HOST = "https://api.$IM_ENVIRONMENT"
INSTANCE_GROUP_NAME = 'qa'
DATABASE_GROUP_NAME = 'test-dbs'
INSTANCE_NAME = "core-${env.GIT_BRANCH.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER"
INSTANCE_DOMAIN = "https://${INSTANCE_GROUP_NAME}.$IM_ENVIRONMENT"
INSTANCE_URL = "$INSTANCE_DOMAIN/$INSTANCE_NAME"
LIVENESS_PROBE_TIMEOUT_SECONDS = 3
READINESS_PROBE_TIMEOUT_SECONDS = 3
STARTUP_PROBE_FAILURE_THRESHOLD = 50
CORE_RESOURCES_REQUESTS_CPU = '900m'
DB_RESOURCES_REQUESTS_CPU = '900m'
CORE_RESOURCES_REQUESTS_MEMORY = '2500Mi'
DB_RESOURCES_REQUESTS_MEMORY = '500Mi'
DHIS2_CREDENTIALS = credentials('dhis2-default')
}

stages {
stage('Validate Parameters') {
steps {
script {
if (!params.DHIS2_VERSION) {
echo "DHIS2_VERSION is empty, using Git branch version: ${env.GIT_BRANCH}"
} else {
echo "Using DHIS2_VERSION: ${params.DHIS2_VERSION}"
}

if (!params.DB_VERSION) {
echo "DB_VERSION is empty, using DHIS2_VERSION: ${params.DHIS2_VERSION ? params.DHIS2_VERSION : env.GIT_BRANCH}"
} else {
echo "Using DB_VERSION: ${params.DB_VERSION}"
}

if (!env.GIT_BRANCH) {
error 'GIT_BRANCH is not defined. Make sure you are running this job from a Git branch.'
}
}
}
}

stage('Deploy DHIS2 Instance') {
steps {
script {
withCredentials([usernamePassword(credentialsId: 'dhis2-im-bot', passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) {
dir('im-manager') {
gitHelper.sparseCheckout(IM_REPO_URL, "${gitHelper.getLatestTag(IM_REPO_URL)}", '/scripts')

dir('scripts/databases') {
env.DATABASE_ID = sh(
returnStdout: true,
script: "./list.sh | jq -r '.[] | select(.name == \"$DATABASE_GROUP_NAME\") .databases[] | select(.name == \"sierra-leone/${DB_VERSION_TAG}.sql.gz\") .id'"
).trim()

if (!env.DATABASE_ID) {
echo "Couldn't find database for $DB_VERSION_TAG"

try {
env.DATABASE_ID = sh(
returnStdout: true,
script: "./upload.sh $DATABASE_GROUP_NAME \"sierra-leone/${DB_VERSION_TAG}.sql.gz\" ${DB_VERSION_TAG}.sql.gz | jq -r '.id'"
).trim()
} catch (err) {
echo "Couldn't download or upload database for ${DB_VERSION_TAG}: ${err}"

DHIS2_SHORT_VERSION = DB_VERSION_TAG.split('\\.').take(2).join('.')
env.DATABASE_ID = sh(
returnStdout: true,
script: "./list.sh | jq -r '.[] | select(.name == \"$DATABASE_GROUP_NAME\") .databases[] | select(.name == \"sierra-leone/${DHIS2_SHORT_VERSION}.sql.gz\") .id'"
).trim()
}
}

if (!env.DATABASE_ID) {
error "Failed to find or upload database for ${DB_VERSION_TAG}. Stopping build."
}

echo "DATABASE_ID is $DATABASE_ID"
}

dir('scripts/instances') {
description = "DHIS 2 instance for ${env.GIT_BRANCH}"
sh """
./findByName.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME | jq --exit-status 'has("id")' && \
./restart.sh \$(./findByName.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME | jq '.instances[] | select(.stackName=="dhis2-core") | .id') || \
IMAGE_TAG=${DOCKER_IMAGE_TAG} IMAGE_REPOSITORY=${IMAGE_REPOSITORY} ./deploy-dhis2.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME $description
"""

timeout(time: 20, unit: 'MINUTES') {
waitFor.statusOk("$INSTANCE_URL/$INSTANCE_NAME")
}
echo 'Generating analytics ...'
def NOTIFIER_ENDPOINT = dhis2.generateAnalytics("$INSTANCE_URL", '$DHIS2_CREDENTIALS')
timeout(time: 20, unit: 'MINUTES') {
waitFor.analyticsCompleted("${INSTANCE_URL}${NOTIFIER_ENDPOINT}", '$DHIS2_CREDENTIALS')
}
sh "chmod +x $WORKSPACE/scripts/install_apps_from_app_hub.sh"
sh "credentials=\$DHIS2_CREDENTIALS url=$INSTANCE_URL $WORKSPACE/scripts/install_apps_from_app_hub.sh"
}
}
}
}
}
}

stage('Trigger Cypress Tests') {
steps {
script {
// def repos = ['dhis2/line-listing-app', 'dhis2/data-visualizer-app', 'dhis2/aggregate-data-entry-app', 'dhis2/dashboard-app', 'dhis2/capture-app', 'dhis2/maps-app', 'dhis2/user-app']
def repos = ['dhis2/line-listing-app', 'dhis2/aggregate-data-entry-app']
def payload = [
dhis2_version: env.DOCKER_IMAGE_TAG,
instance_url: "$INSTANCE_URL"
]

repos.each { repo ->
withCredentials([string(credentialsId: 'github-token', variable: 'GITHUB_TOKEN')]) {
def response = sh (
script: """
curl -v -X POST \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: token \$GITHUB_TOKEN" \
https://api.github.com/repos/${repo}/actions/workflows/e2e-test-dispatch.yml/dispatches \
-d '{ "ref": "workflow-dispatch-cypress", "inputs": {"dhis2_version":"${env.DOCKER_IMAGE_TAG}", "instance_url":"$INSTANCE_URL"} }'
""",
returnStdout: true
).trim()
echo "GitHub Dispatch Response for ${repo}: ${response}"
}
}
}
}
}
}

post {
always {
script {
gitHelper.setCommitStatus("${env.DHIS2_COMMIT_SHA}", "${env.DHIS2_REPO_URL}")
}
}
}
}
63 changes: 63 additions & 0 deletions scripts/install_apps_from_app_hub.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'

url=$url
credentials=$credentials

APP_IDS=('92b75fd0-34cc-451c-942f-3dd0f283bcbd' 'a4cd3827-e717-4e09-965d-ab05df2591e5')

getLatestAvailableAppVersion() {
app_hub_id=$1
core_version=$(curl -u $credentials $url/api/system/info | jq -r '.version' | grep -o -E '2.[0-9]+')
latest_compatible_version_response=$(curl -u $credentials $url/api/appHub/v2/apps/$app_hub_id/versions?minDhisVersion=lte:$core_version | jq -r '.result[0] // empty')
echo "$latest_compatible_version_response"
}

installOrUpdate() {
app_hub_id=$1
app_response=$(curl -u $credentials $url/api/apps | jq --arg app_hub_id $app_hub_id -c '.[] | select(.app_hub_id==$app_hub_id)')
app_version=$(echo "$app_response" | jq -r .version)

latest_compatible_version_response="$(getLatestAvailableAppVersion $app_hub_id)"
latest_compatible_version=$(echo $latest_compatible_version_response | jq -r '.version // empty')
app_name=$(echo $latest_compatible_version_response | jq -r .slug)

if [[ -z "$latest_compatible_version" ]]; then
echo "App $app_hub_id is not compatible with this instance. Skipping install."
return
fi

if [ -z "$app_version" ] || [[ $app_version != $latest_compatible_version ]];then
echo "Installing $app_name app version $latest_compatible_version"
download_url=$(echo $latest_compatible_version_response | jq -r .downloadUrl )
download_name="$app_name.$latest_compatible_version.zip"
downloadApp $download_url $download_name
importApp $download_name

else
echo "$app_name app is up-to-date"
fi
}

downloadApp () {
download_name=$2
download_url=$1

curl -s -L $download_url --output $download_name

}
importApp() {
download_name=$1
status=$(curl -o /dev/null -u $credentials -F file=@$download_name $url/api/apps -w "%{http_code}")

if [ $status != 204 ];then
echo 'App install failed!'
fi
}


for i in "${APP_IDS[@]}";
do
installOrUpdate $i
done
Loading