From 929065c159267bcf8faea0546968c4b8aef9872b Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 21 Feb 2023 21:47:03 -0500 Subject: [PATCH] build!: remove Jenkins CI scripts These files were used to run tests on build.testeng.edx.org, a Jenkins server. All tests have sinced been moved over to GitHub Actions, and build.testeng.edx.org is decomissioned. --- scripts/Jenkinsfiles/python | 272 ------------------------- scripts/Jenkinsfiles/quality | 228 --------------------- scripts/jenkins-common.sh | 65 ------ scripts/jenkins-quality-report.sh | 13 -- scripts/jenkins-report.sh | 34 ---- scripts/xdist/get_worker_test_list.py | 56 ----- scripts/xdist/prepare_xdist_nodes.sh | 26 --- scripts/xdist/pytest_worker_manager.py | 219 -------------------- scripts/xdist/setup_worker.sh | 26 --- scripts/xdist/terminate_xdist_nodes.sh | 10 - 10 files changed, 949 deletions(-) delete mode 100644 scripts/Jenkinsfiles/python delete mode 100644 scripts/Jenkinsfiles/quality delete mode 100644 scripts/jenkins-common.sh delete mode 100755 scripts/jenkins-quality-report.sh delete mode 100755 scripts/jenkins-report.sh delete mode 100644 scripts/xdist/get_worker_test_list.py delete mode 100644 scripts/xdist/prepare_xdist_nodes.sh delete mode 100644 scripts/xdist/pytest_worker_manager.py delete mode 100755 scripts/xdist/setup_worker.sh delete mode 100644 scripts/xdist/terminate_xdist_nodes.sh diff --git a/scripts/Jenkinsfiles/python b/scripts/Jenkinsfiles/python deleted file mode 100644 index 5c0010568fac..000000000000 --- a/scripts/Jenkinsfiles/python +++ /dev/null @@ -1,272 +0,0 @@ -def runPythonTests() { - // Determine git refspec, branch, and clone type - git_branch = xdist_git_branch() - git_refspec = xdist_git_refspec() - sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) { - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]], - doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true, - noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] - sh 'bash scripts/all-tests.sh' - stash includes: 'reports/**/*coverage*,test_root/log/*.json', name: "${TEST_SUITE}-reports" - } -} - -def pythonTestCleanup() { - archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,test_root/log/*.json,**/nosetests.xml,*.log' - sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' - junit '**/nosetests.xml' - sh '''source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate - bash scripts/xdist/terminate_xdist_nodes.sh''' -} - -def createWarningsReport(fileExtension){ - println "Creating warnings report for ${fileExtension}" - warning_filename = "warning_report_${fileExtension}.html" - sh """source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate - python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/${warning_filename}""" - - publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports/pytest_warnings', reportFiles: "${warning_filename}", - reportName: "${warning_filename}", reportTitles: '']) - archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/pytest_warnings/*.html' -} - - -def xdist_git_branch() { - if (env.ghprbActualCommit) { - return "${ghprbActualCommit}" - } else { - return "${BRANCH_NAME}" - } -} - -def xdist_git_refspec() { - if (env.ghprbActualCommit) { - return "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" - } else { - return "+refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}" - } -} - -pipeline { - agent { label "jenkins-worker" } - options { - sendSplunkConsoleLog() - timestamps() - timeout(60) - } - environment { - XDIST_GIT_BRANCH = xdist_git_branch() - XDIST_GIT_REFSPEC = xdist_git_refspec() - XDIST_INSTANCE_TYPE = "c5d.large" - XDIST_WORKER_AMI = credentials('XDIST_WORKER_AMI') - XDIST_WORKER_IAM_PROFILE_ARN = credentials('XDIST_WORKER_IAM_PROFILE_ARN') - XDIST_WORKER_KEY_NAME = "jenkins-worker" - XDIST_WORKER_SUBNET = credentials('XDIST_WORKER_SUBNET') - XDIST_WORKER_SECURITY_GROUP = credentials('XDIST_WORKER_SECURITY_GROUP') - WTW_CONTEXT = "python" - } - stages { - stage('Mark build as pending on Github') { - when { - // Only run github-build-status for master builds - expression { env.ghprbActualCommit == null } - } - steps { - script { - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', - parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: 'Pending'), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: 'false'), - string(name: 'BUILD_STATUS', value: 'pending') - ], - propagate: false, wait: false - } - } - } - stage('Run Tests') { - parallel { - stage("lms-unit") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "lms-unit" - XDIST_NUM_WORKERS = 10 - XDIST_REMOTE_NUM_PROCESSES = 1 - } - steps { - script { - runPythonTests() - } - } - post { - always { - script { - pythonTestCleanup() - } - } - } - } - stage("cms-unit") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "cms-unit" - XDIST_NUM_WORKERS = 3 - XDIST_REMOTE_NUM_PROCESSES = 1 - } - steps { - script { - runPythonTests() - } - } - post { - always { - script { - pythonTestCleanup() - } - } - } - } - stage("commonlib-unit") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "commonlib-unit" - XDIST_NUM_WORKERS = 3 - XDIST_REMOTE_NUM_PROCESSES = 1 - } - steps { - script { - runPythonTests() - } - } - post { - always { - script { - pythonTestCleanup() - } - } - } - } - } - } - - - stage('Run coverage') { - environment { - CODE_COV_TOKEN = credentials('CODE_COV_TOKEN') - SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc - COLLECT_WHO_TESTS_WHAT = "${COLLECT_WHO_TESTS_WHAT}" - } - steps { - script { - if (env.ghprbActualCommit) { - git_branch = "${ghprbActualCommit}" - git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" - ci_branch = "${ghprbSourceBranch}" - target_branch = "origin/${ghprbTargetBranch}" - } else { - git_branch = "${BRANCH_NAME}" - git_refspec = "+refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}" - ci_branch = "${BRANCH_NAME}" - target_branch = "origin/${BRANCH_NAME}" - } - sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) { - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]], - doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', - honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [], - userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] - unstash 'lms-unit-reports' - unstash 'cms-unit-reports' - unstash 'commonlib-unit-reports' - sh """export CI_BRANCH=$ci_branch - export TARGET_BRANCH=$target_branch - ./scripts/jenkins-report.sh""" - createWarningsReport("all") - } - } - } - post { - always { - script { - archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/*.coverage' - sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' - publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports/cover', reportFiles: 'index.html', - reportName: 'Coverage.py Report', reportTitles: '']) - - if (env.ghprbPullId != null) { - publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports', reportFiles: 'diff_coverage_combined.html', - reportName: 'Diff Coverage Report', reportTitles: '']) - } - } - } - } - } - } - post { - always { - script { - if (env.ghprbPullId != null) { - // For PR jobs, run the edx-platform-test-notifier for PR reporting - build job: 'edx-platform-test-notifier', parameters: [string(name: 'REPO', value: "${REPO_NAME}"), string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false - } else { - // For master jobs run github-build-status and report to slack when necessary - if (currentBuild.currentResult == "SUCCESS") { - create_deployment = "true" - build_status = "success" - build_description = "Build Passed" - } - else { - create_deployment = "false" - build_status = "failure" - build_description = "Build Failed" - } - - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', - parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: build_description), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: create_deployment), - string(name: 'BUILD_STATUS', value: build_status) - ], - propagate: false, wait: false - - if (currentBuild.currentResult != "SUCCESS"){ - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - - email_body = "See: <${BUILD_URL}>\n\nChanges:\n" - change_sets = currentBuild.changeSets - for (int j = 0; j < change_sets.size(); j++) { - change_set_items = change_sets[j].items - for (int k = 0; k < change_set_items.length; k++) { - item = change_set_items[k] - email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}" - } - } - emailext body: email_body, - subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") { - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - emailext body: "See <${BUILD_URL}>", - subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } - } - } - } - } -} diff --git a/scripts/Jenkinsfiles/quality b/scripts/Jenkinsfiles/quality deleted file mode 100644 index 52324680c961..000000000000 --- a/scripts/Jenkinsfiles/quality +++ /dev/null @@ -1,228 +0,0 @@ -def runQualityTests() { - sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) { - // Determine git refspec, branch, and clone type - git_shallow_clone = true - git_extensions = [] - if (env.ghprbActualCommit) { - git_branch = "${ghprbActualCommit}" - refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" - if (SHARD == "4") { - git_shallow_clone = false - git_extensions.add([$class: 'WipeWorkspace']) - refspec = refspec + " +refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" - } - } else { - git_branch = "${BRANCH_NAME}" - refspec = "+refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}" - } - git_extensions.add([$class: 'CloneOption', honorRefspec: true, noTags: true, shallow: git_shallow_clone ]) - - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]], - doGenerateSubmoduleConfigurations: false, extensions: git_extensions, submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] - - sh "bash scripts/all-tests.sh" - stash includes: '**/reports/**/*', name: "${TEST_SUITE}-${SHARD}-reports" - } -} - -def getTargetBranch(job_name) { - if (env.ghprbTargetBranch) { - return env.ghprbTargetBranch - } else { - return "${BRANCH_NAME}" - } -} - -def qualityTestCleanup() { - archiveArtifacts allowEmptyArchive: true, artifacts: '**/reports/**/*,test_root/log/**/*.log,*.log' - sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' - junit 'reports/quality_junitxml/*.xml' -} - -pipeline { - agent { label "jenkins-worker" } - options { - sendSplunkConsoleLog() - timestamps() - timeout(120) - } - stages { - stage('Mark build as pending on Github') { - when { - // Only run github-build-status for master builds - expression { env.ghprbActualCommit == null } - } - steps { - script { - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', - parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: 'Pending'), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: 'false'), - string(name: 'BUILD_STATUS', value: 'pending') - ], - propagate: false, wait: false - } - } - } - stage('Run Tests') { - parallel { - stage("common pylint") { - // "pylint common" requires 5.5 GB of RAM, so use js-worker (8 GB) instead of jenkins-worker (4 GB) - agent { label "js-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 1 - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - stage("lms pylint") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 2 - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - stage("cms/openedx/pavelib pylint") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 3 - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - stage("Other quality checks") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 4 - TARGET_BRANCH = getTargetBranch("${JOB_NAME}") - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - } - } - } - post { - always { - script{ - try { - unstash 'quality-1-reports' - unstash 'quality-2-reports' - unstash 'quality-3-reports' - unstash 'quality-4-reports' - // Check for warnings - recordIssues enabledForFailure: true, tools: [pep8(pattern: 'reports/pep8/pep8.report'), pyLint(pattern: 'reports/**/pylint.report')] - // Publish Quality report - publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports/metrics/', - reportFiles: 'pylint/*view*/,pep8/*view*/,xsscommitlint/*view*/,xsslint/*view*/,eslint/*view*/,pii/*view*/', - reportName: 'Quality Report', reportTitles: '']) - } finally { - if (env.ghprbPullId != null) { - // For PR jobs, run the edx-platform-test-notifier for PR reporting - build job: 'edx-platform-test-notifier', parameters: [string(name: 'REPO', value: "${REPO_NAME}"), string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false - } else { - // For master jobs run github-build-status and report to slack when necessary - if (currentBuild.currentResult == "SUCCESS") { - create_deployment = "true" - build_status = "success" - build_description = "Build Passed" - } - else { - create_deployment = "false" - build_status = "failure" - build_description = "Build Failed" - } - - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: build_description), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: create_deployment), - string(name: 'BUILD_STATUS', value: build_status) - ], - propagate: false, wait: false - - if (currentBuild.currentResult != "SUCCESS"){ - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - - email_body = "See: <${BUILD_URL}>\n\nChanges:\n" - change_sets = currentBuild.changeSets - for (int j = 0; j < change_sets.size(); j++) { - change_set_items = change_sets[j].items - for (int k = 0; k < change_set_items.length; k++) { - item = change_set_items[k] - email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}" - } - } - emailext body: email_body, - subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") { - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - emailext body: "See <${BUILD_URL}>", - subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } - } - } - } - } - } -} diff --git a/scripts/jenkins-common.sh b/scripts/jenkins-common.sh deleted file mode 100644 index 5eec5b5b4922..000000000000 --- a/scripts/jenkins-common.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $HOME/jenkins_env - -# Clear the mongo database -# Note that this prevents us from running jobs in parallel on a single worker. -mongo --quiet --eval 'db.getMongo().getDBNames().forEach(function(i){db.getSiblingDB(i).dropDatabase()})' - -# Ensure we have fetched a target branch (origin/master, unless testing -# a release branc) Some of the reporting tools compare the checked out -# branch to a target branch; depending on how the GitHub plugin refspec -# is configured, this may not already be fetched. -if [ ! -z ${TARGET_BRANCH+x} ]; then - TARGET_BRANCH_WITHOUT_ORIGIN=$(echo "${TARGET_BRANCH}" | sed 's:^origin/::') - git fetch origin $TARGET_BRANCH_WITHOUT_ORIGIN:refs/remotes/origin/$TARGET_BRANCH_WITHOUT_ORIGIN -fi - -# Reset the jenkins worker's virtualenv back to the -# state it was in when the instance was spun up. -if [ -z ${PYTHON_VERSION+x} ] || [[ ${PYTHON_VERSION} == 'null' ]]; then - VENV_PATH="$HOME" - ARCHIVED_VENV="edx-venv_clean.tar.gz" -else - VENV_PATH="$HOME/edx-venv-$PYTHON_VERSION" - ARCHIVED_VENV="edx-venv_clean-$PYTHON_VERSION.tar.gz" -fi - -if [ -e $HOME/$ARCHIVED_VENV ]; then - rm -rf $VENV_PATH/edx-venv - tar -C $VENV_PATH -xf $HOME/$ARCHIVED_VENV -fi - -# Load the npm packages from the time the worker was built -# into the npm cache. This is an attempt to reduce the number -# of times that npm gets stuck during an installation, by -# reducing the number of packages that npm needs to fetch. -if [ -e $HOME/edx-npm-cache_clean.tar.gz ]; then - echo "Loading archived npm packages into the local npm cache" - rm -rf $HOME/.npm - tar -C $HOME -xf $HOME/edx-npm-cache_clean.tar.gz -fi - -# Activate the Python virtualenv -source $VENV_PATH/edx-venv/bin/activate - -# Hack to fix up egg-link files given that the virtualenv is not relocatable -sed -i "s|^/home/jenkins/shallow-clone|`pwd`|" -- \ - $VENV_PATH/edx-venv/lib/python*/site-packages/*.egg-link -make test-requirements - -# add the node packages dir to PATH -PATH=$PATH:node_modules/.bin - -echo "node version is `node --version`" -echo "npm version is `npm --version`" - -# Log any paver or ansible command timing -TIMESTAMP=$(date +%s) -SHARD_NUM=${SHARD:="all"} -export PAVER_TIMER_LOG="test_root/log/timing.paver.$TEST_SUITE.$SHARD_NUM.log" -export ANSIBLE_TIMER_LOG="test_root/log/timing.ansible.$TIMESTAMP.log" - -echo "This node is `curl http://169.254.169.254/latest/meta-data/hostname`" diff --git a/scripts/jenkins-quality-report.sh b/scripts/jenkins-quality-report.sh deleted file mode 100755 index ebfd29697560..000000000000 --- a/scripts/jenkins-quality-report.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script is used by the edx-platform-quality-check jenkins job. -source scripts/jenkins-common.sh -source scripts/thresholds.sh - -# Run quality task. Pass in the 'fail-under' percentage to diff-quality -echo "Generating quality report." -mkdir -p test_root/log/ -LOG_PREFIX=test_root/log/run_quality - -paver run_quality 2> $LOG_PREFIX.err.log > $LOG_PREFIX.out.log diff --git a/scripts/jenkins-report.sh b/scripts/jenkins-report.sh deleted file mode 100755 index b085826ee750..000000000000 --- a/scripts/jenkins-report.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -e -set -x - -# This script generates coverage and diff cover reports, and optionally -# reports this data to codecov.io. The following environment variables must be -# set in order to report to codecov: -# CODE_COV_TOKEN: CodeCov API token -# CI_BRANCH: The branch that the coverage report describes - -# This script is used by the edx-platform-unit-coverage jenkins job. - -source scripts/jenkins-common.sh - -# Get the diff coverage and html reports for unit tests -paver coverage -b $TARGET_BRANCH - -if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then - paver upload_coverage_to_s3 -fi - -# Test for the CodeCov API token -if [ -z $CODE_COV_TOKEN ]; then - echo "codecov.io API token not set." - echo "This must be set as an environment variable if order to report coverage" -else - # Send the coverage data to codecov - pip install codecov==2.0.5 - codecov --token=$CODE_COV_TOKEN --branch=$CI_BRANCH -fi - -# JUnit test reporter will fail the build -# if it thinks test results are old -touch `find . -name *.xml` || true diff --git a/scripts/xdist/get_worker_test_list.py b/scripts/xdist/get_worker_test_list.py deleted file mode 100644 index 268ab5ca8915..000000000000 --- a/scripts/xdist/get_worker_test_list.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -This script strips the console log of a pytest-xdist Jenkins run into the test -lists of each pytest worker. -Assumes the following format: -[test-suite] [worker] RESULT test -""" - - -import io -import os -import re -import shutil - -import click - - -@click.command() -@click.option( - '--log-file', - help="File name of console log .txt file from a Jenkins build " - "that ran pytest-xdist. This can be acquired by running: " - "curl -o console.txt https://build.testeng.edx.org/job/JOBNAME/BUILDNUMBER/consoleText", - required=True -) -@click.option( - '--test-suite', - help="Test suite that the pytest worker ran.", - type=click.Choice(['lms-unit', 'cms-unit']), - required=True -) -def main(log_file, test_suite): - worker_test_dict = {} - with open(log_file) as console_file: - for line in console_file: - regex_search = re.search(fr'\[gw(\d+)] (PASSED|FAILED|SKIPPED|ERROR) (\S+)', line) - if regex_search: - worker_num_string = regex_search.group(1) - if worker_num_string not in worker_test_dict: - worker_test_dict[worker_num_string] = [] - test = regex_search.group(3) - worker_test_dict[worker_num_string].append(test) - - output_folder_name = "worker_list_files" - if os.path.isdir(output_folder_name): - shutil.rmtree(output_folder_name) - os.mkdir(output_folder_name) - - for worker_num in worker_test_dict: - output_file_name = f"{output_folder_name}/{test_suite}_gw{worker_num}_test_list.txt" - with open(output_file_name, 'w') as output_file: - for line in worker_test_dict[worker_num]: - output_file.write(line + "\n") - - -if __name__ == "__main__": - main() diff --git a/scripts/xdist/prepare_xdist_nodes.sh b/scripts/xdist/prepare_xdist_nodes.sh deleted file mode 100644 index 8346f2d4619c..000000000000 --- a/scripts/xdist/prepare_xdist_nodes.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e - -echo "Spinning up xdist workers with pytest_worker_manager.py" -python scripts/xdist/pytest_worker_manager.py -a up -n ${XDIST_NUM_WORKERS} \ --ami ${XDIST_WORKER_AMI} \ --type ${XDIST_INSTANCE_TYPE} \ --s ${XDIST_WORKER_SUBNET} \ --sg ${XDIST_WORKER_SECURITY_GROUP} \ --key ${XDIST_WORKER_KEY_NAME} \ --iam ${XDIST_WORKER_IAM_PROFILE_ARN} - -ip_list=$( 0: - logger.info(f"Still waiting on {len(not_running)} workers to spin up") - time.sleep(5) - else: - logger.info("Finished spinning up workers") - all_running = True - break - - if not all_running: - raise Exception( - "Timed out waiting to spin up all workers." - ) - logger.info(f"Successfully booted up {number_of_workers} workers.") - - not_ready_ip_addresses = ip_addresses[:] - logger.info("Checking ssh connection to workers.") - pool = Pool(processes=number_of_workers) - for ssh_try in range(0, self.WORKER_SSH_ATTEMPTS): - results = pool.map(_check_worker_ready, not_ready_ip_addresses) - deleted_ips = 0 - for num in range(0, len(results)): - if results[num] == 0: - del(not_ready_ip_addresses[num - deleted_ips]) - deleted_ips += 1 - - if len(not_ready_ip_addresses) == 0: - logger.info("All workers are ready for tests.") - break - - if ssh_try == self.WORKER_SSH_ATTEMPTS - 1: - raise Exception( - "Max ssh tries to remote workers reached." - ) - - logger.info("Not all workers are ready. Sleeping for 5 seconds then retrying.") - time.sleep(5) - - # Generate .txt files containing IP addresses and instance ids - ip_list_string = ",".join(ip_addresses) - logger.info(f"Worker IP list: {ip_list_string}") - ip_list_file = open("pytest_worker_ips.txt", "w") - ip_list_file.write(ip_list_string) - ip_list_file.close() - - worker_instance_id_list_string = ",".join(worker_instance_ids) - logger.info(f"Worker Instance Id list: {worker_instance_id_list_string}") - worker_arn_file = open("pytest_worker_instance_ids.txt", "w") - worker_arn_file.write(worker_instance_id_list_string) - worker_arn_file.close() - - def terminate_workers(self, worker_instance_ids): - """ - Terminates workers based on a list of worker_instance_ids. - """ - instance_id_list = worker_instance_ids.split(',') - response = self.ec2.terminate_instances( - InstanceIds=instance_id_list - ) - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser( - description="PytestWorkerManager, manages EC2 workers in an AWS cluster." - ) - - parser.add_argument('--action', '-a', choices=['up', 'down'], default=None, - help="Action for PytestWorkerManager to perform. " - "Either up for spinning up AWS EC2 workers or down for terminating them") - - parser.add_argument('--region', '-g', default='us-east-1', - help="AWS region where EC2 infrastructure lives. Defaults to us-east-1") - - # Spinning up workers - parser.add_argument('--num-workers', '-n', type=int, default=None, - help="Number of EC2 workers to spin up") - - parser.add_argument('--ami', '-ami', default=None, - help="AMI for workers") - - parser.add_argument('--instance-type', '-type', default=None, - help="Desired EC2 instance type") - - parser.add_argument('--subnet-id', '-s', default=None, - help="Subnet for the workers to exist in") - - parser.add_argument('--security_groups', '-sg', nargs='+', default=None, - help="List of security group ids to apply to workers") - - parser.add_argument('--key-name', '-key', default=None, - help="Key pair name for sshing to worker") - - parser.add_argument('--iam-arn', '-iam', default=None, - help="Iam Instance Profile ARN for the workers") - - # Terminating workers - parser.add_argument('--instance-ids', '-ids', default=None, - help="Instance ids terminate") - - args = parser.parse_args() - workerManager = PytestWorkerManager(args.region) - - if args.action == 'up': - workerManager.spin_up_workers( - args.num_workers, - args.ami, - args.instance_type, - args.subnet_id, - args.security_groups, - args.key_name, - args.iam_arn - ) - elif args.action == 'down': - workerManager.terminate_workers( - args.instance_ids - ) - else: - logger.info("No action specified for PytestWorkerManager") diff --git a/scripts/xdist/setup_worker.sh b/scripts/xdist/setup_worker.sh deleted file mode 100755 index 7c8acb5ad03e..000000000000 --- a/scripts/xdist/setup_worker.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Set up worker node. - -while getopts 'p:' opt; do - case "$opt" in - p) PYTHON_VERSION="$OPTARG";; - [?]) - print >&2 "Usage: $0 -p python-version" - exit 1 - ;; - esac -done - -venv_parent=/home/jenkins/edx-venv-${PYTHON_VERSION} -venv=$venv_parent/edx-venv -rm -rf $venv -tar -C $venv_parent -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz -source $venv/bin/activate - -# Hack to fix up egg-link files given that the virtualenv is not relocatable -sed -i "s|\(^/home/jenkins\)/shallow-clone|\1/edx-platform|" -- \ - $venv/lib/python*/site-packages/*.egg-link - -make test-requirements - -mkdir reports diff --git a/scripts/xdist/terminate_xdist_nodes.sh b/scripts/xdist/terminate_xdist_nodes.sh deleted file mode 100644 index e2997da6f7c5..000000000000 --- a/scripts/xdist/terminate_xdist_nodes.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -if [ -f pytest_worker_instance_ids.txt ]; then - echo "Terminating xdist workers with pytest_worker_manager.py" - xdist_worker_ids=$(