diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 105cf42facc2..bde9ed4009d6 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -3,10 +3,14 @@ env: # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499 MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125 on: + workflow_dispatch: push: branches: - master + - trigger-external-apps-test pull_request: + release: + types: [created] concurrency: group: ${{ github.workflow}}-${{ github.ref }} cancel-in-progress: true diff --git a/jenkinsfiles/trigger-external-tests b/jenkinsfiles/trigger-external-tests new file mode 100644 index 000000000000..b31587d32ce8 --- /dev/null +++ b/jenkinsfiles/trigger-external-tests @@ -0,0 +1,170 @@ +#!/usr/bin/env groovy + +@Library('pipeline-library') _ + +pipeline { + agent { + label 'ec2-jdk17' + } + + options { + buildDiscarder(logRotator(daysToKeepStr: '5')) + timeout(time: 60) + disableConcurrentBuilds() + } + + parameters { + string(name: 'DHIS2_VERSION', defaultValue: '', description: 'DHIS2 core version to use for deployment. If left empty, it will be derived from the Git branch.') + string(name: 'DB_VERSION', defaultValue: '', description: 'Database version to use. If left empty, it will be derived from the DHIS2 version.') + } + + environment { + HTTP = "http --check-status" + MAVEN_OPTS = '-Xms1024m -Xmx4096m -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125' + DOCKER_IMAGE_TAG = "${params.DHIS2_VERSION ? params.DHIS2_VERSION.replace('SNAPSHOT', 'rc') : env.GIT_BRANCH}" + DB_VERSION_TAG = "${params.DB_VERSION ? params.DB_VERSION : DOCKER_IMAGE_TAG}" + IMAGE_TAG = "${DOCKER_IMAGE_TAG}" + IMAGE_REPOSITORY = 'core' + IM_REPO_URL = 'https://github.com/dhis2-sre/im-manager' + IM_ENVIRONMENT = 'im.dhis2.org' + IM_HOST = "https://api.$IM_ENVIRONMENT" + INSTANCE_GROUP_NAME = 'qa' + DATABASE_GROUP_NAME = 'test-dbs' + INSTANCE_NAME = "core-${env.GIT_BRANCH.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER" + INSTANCE_DOMAIN = "https://${INSTANCE_GROUP_NAME}.$IM_ENVIRONMENT" + INSTANCE_URL = "$INSTANCE_DOMAIN/$INSTANCE_NAME" + LIVENESS_PROBE_TIMEOUT_SECONDS = 3 + READINESS_PROBE_TIMEOUT_SECONDS = 3 + STARTUP_PROBE_FAILURE_THRESHOLD = 50 + CORE_RESOURCES_REQUESTS_CPU = '900m' + DB_RESOURCES_REQUESTS_CPU = '900m' + CORE_RESOURCES_REQUESTS_MEMORY = '2500Mi' + DB_RESOURCES_REQUESTS_MEMORY = '500Mi' + DHIS2_CREDENTIALS = credentials('dhis2-default') + } + + stages { + stage('Validate Parameters') { + steps { + script { + if (!params.DHIS2_VERSION) { + echo "DHIS2_VERSION is empty, using Git branch version: ${env.GIT_BRANCH}" + } else { + echo "Using DHIS2_VERSION: ${params.DHIS2_VERSION}" + } + + if (!params.DB_VERSION) { + echo "DB_VERSION is empty, using DHIS2_VERSION: ${params.DHIS2_VERSION ? params.DHIS2_VERSION : env.GIT_BRANCH}" + } else { + echo "Using DB_VERSION: ${params.DB_VERSION}" + } + + if (!env.GIT_BRANCH) { + error 'GIT_BRANCH is not defined. Make sure you are running this job from a Git branch.' + } + } + } + } + + stage('Deploy DHIS2 Instance') { + steps { + script { + withCredentials([usernamePassword(credentialsId: 'dhis2-im-bot', passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager') { + gitHelper.sparseCheckout(IM_REPO_URL, "${gitHelper.getLatestTag(IM_REPO_URL)}", '/scripts') + + dir('scripts/databases') { + env.DATABASE_ID = sh( + returnStdout: true, + script: "./list.sh | jq -r '.[] | select(.name == \"$DATABASE_GROUP_NAME\") .databases[] | select(.name == \"sierra-leone/${DB_VERSION_TAG}.sql.gz\") .id'" + ).trim() + + if (!env.DATABASE_ID) { + echo "Couldn't find database for $DB_VERSION_TAG" + + try { + env.DATABASE_ID = sh( + returnStdout: true, + script: "./upload.sh $DATABASE_GROUP_NAME \"sierra-leone/${DB_VERSION_TAG}.sql.gz\" ${DB_VERSION_TAG}.sql.gz | jq -r '.id'" + ).trim() + } catch (err) { + echo "Couldn't download or upload database for ${DB_VERSION_TAG}: ${err}" + + DHIS2_SHORT_VERSION = DB_VERSION_TAG.split('\\.').take(2).join('.') + env.DATABASE_ID = sh( + returnStdout: true, + script: "./list.sh | jq -r '.[] | select(.name == \"$DATABASE_GROUP_NAME\") .databases[] | select(.name == \"sierra-leone/${DHIS2_SHORT_VERSION}.sql.gz\") .id'" + ).trim() + } + } + + if (!env.DATABASE_ID) { + error "Failed to find or upload database for ${DB_VERSION_TAG}. Stopping build." + } + + echo "DATABASE_ID is $DATABASE_ID" + } + + dir('scripts/instances') { + description = "DHIS 2 instance for ${env.GIT_BRANCH}" + sh """ + ./findByName.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME | jq --exit-status 'has("id")' && \ + ./restart.sh \$(./findByName.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME | jq '.instances[] | select(.stackName=="dhis2-core") | .id') || \ + IMAGE_TAG=${DOCKER_IMAGE_TAG} IMAGE_REPOSITORY=${IMAGE_REPOSITORY} ./deploy-dhis2.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME $description + """ + + timeout(time: 20, unit: 'MINUTES') { + waitFor.statusOk("$INSTANCE_URL/$INSTANCE_NAME") + } + echo 'Generating analytics ...' + def NOTIFIER_ENDPOINT = dhis2.generateAnalytics("$INSTANCE_URL", '$DHIS2_CREDENTIALS') + timeout(time: 20, unit: 'MINUTES') { + waitFor.analyticsCompleted("${INSTANCE_URL}${NOTIFIER_ENDPOINT}", '$DHIS2_CREDENTIALS') + } + sh "chmod +x $WORKSPACE/scripts/install_apps_from_app_hub.sh" + sh "credentials=\$DHIS2_CREDENTIALS url=$INSTANCE_URL $WORKSPACE/scripts/install_apps_from_app_hub.sh" + } + } + } + } + } + } + + stage('Trigger Cypress Tests') { + steps { + script { + def repos = ['dhis2/line-listing-app', 'dhis2/data-visualizer-app', 'dhis2/aggregate-data-entry-app', 'dhis2/dashboard-app', 'dhis2/capture-app', 'dhis2/maps-app', 'dhis2/user-app'] + + def payload = [ + dhis2_version: env.DOCKER_IMAGE_TAG, + instance_url: "$INSTANCE_URL" + ] + + repos.each { repo -> + withCredentials([string(credentialsId: 'github-token', variable: 'GITHUB_TOKEN')]) { + def response = sh ( + script: """ + curl -v -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token \$GITHUB_TOKEN" \ + https://api.github.com/repos/${repo}/actions/workflows/e2e-test-dispatch.yml/dispatches \ + -d '{ "ref": "workflow-dispatch-cypress", "inputs": {"dhis2_version":"${env.DOCKER_IMAGE_TAG}", "instance_url":"$INSTANCE_URL"} }' + """, + returnStdout: true + ).trim() + echo "GitHub Dispatch Response for ${repo}: ${response}" + } + } + } + } + } + } + + post { + always { + script { + gitHelper.setCommitStatus("${env.DHIS2_COMMIT_SHA}", "${env.DHIS2_REPO_URL}") + } + } + } +} diff --git a/scripts/install_apps_from_app_hub.sh b/scripts/install_apps_from_app_hub.sh new file mode 100644 index 000000000000..12c57e2d9ab8 --- /dev/null +++ b/scripts/install_apps_from_app_hub.sh @@ -0,0 +1,63 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +url=$url +credentials=$credentials + +APP_IDS=('92b75fd0-34cc-451c-942f-3dd0f283bcbd' 'a4cd3827-e717-4e09-965d-ab05df2591e5') + +getLatestAvailableAppVersion() { + app_hub_id=$1 + core_version=$(curl -u $credentials $url/api/system/info | jq -r '.version' | grep -o -E '2.[0-9]+') + latest_compatible_version_response=$(curl -u $credentials $url/api/appHub/v2/apps/$app_hub_id/versions?minDhisVersion=lte:$core_version | jq -r '.result[0] // empty') + echo "$latest_compatible_version_response" +} + +installOrUpdate() { + app_hub_id=$1 + app_response=$(curl -u $credentials $url/api/apps | jq --arg app_hub_id $app_hub_id -c '.[] | select(.app_hub_id==$app_hub_id)') + app_version=$(echo "$app_response" | jq -r .version) + + latest_compatible_version_response="$(getLatestAvailableAppVersion $app_hub_id)" + latest_compatible_version=$(echo $latest_compatible_version_response | jq -r '.version // empty') + app_name=$(echo $latest_compatible_version_response | jq -r .slug) + + if [[ -z "$latest_compatible_version" ]]; then + echo "App $app_hub_id is not compatible with this instance. Skipping install." + return + fi + + if [ -z "$app_version" ] || [[ $app_version != $latest_compatible_version ]];then + echo "Installing $app_name app version $latest_compatible_version" + download_url=$(echo $latest_compatible_version_response | jq -r .downloadUrl ) + download_name="$app_name.$latest_compatible_version.zip" + downloadApp $download_url $download_name + importApp $download_name + + else + echo "$app_name app is up-to-date" + fi +} + +downloadApp () { + download_name=$2 + download_url=$1 + + curl -s -L $download_url --output $download_name + +} +importApp() { + download_name=$1 + status=$(curl -o /dev/null -u $credentials -F file=@$download_name $url/api/apps -w "%{http_code}") + + if [ $status != 204 ];then + echo 'App install failed!' + fi +} + + +for i in "${APP_IDS[@]}"; +do + installOrUpdate $i +done