From f3b658dc0b4f32ffea97a00c3a9bc703a6de2194 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 22 Feb 2023 09:14:48 -0500 Subject: [PATCH] build!: remove Jenkins CI scripts (#31812) These files were used to run tests on build.testeng.edx.org, a Jenkins instance. All tests have since been moved over to GitHub Actions. The build.testeng.edx.org instance is decomissioned. Removed files include: - scripts/jenkins-*.sh - scripts/Jenkinsfiles/* - scripts/xdist/* Note on xdist scripts: pytest-xdist in general is still useful for local test runs in order to take advantage of all available CPU cores, but the scripts here were specifically for running tests on multiple remote machines, particularly via Jenkins. --- scripts/Jenkinsfiles/python | 272 ------------------------- scripts/Jenkinsfiles/quality | 228 --------------------- scripts/jenkins-common.sh | 65 ------ scripts/jenkins-quality-report.sh | 13 -- scripts/jenkins-report.sh | 34 ---- scripts/xdist/get_worker_test_list.py | 56 ----- scripts/xdist/prepare_xdist_nodes.sh | 26 --- scripts/xdist/pytest_worker_manager.py | 219 -------------------- scripts/xdist/setup_worker.sh | 26 --- scripts/xdist/terminate_xdist_nodes.sh | 10 - 10 files changed, 949 deletions(-) delete mode 100644 scripts/Jenkinsfiles/python delete mode 100644 scripts/Jenkinsfiles/quality delete mode 100644 scripts/jenkins-common.sh delete mode 100755 scripts/jenkins-quality-report.sh delete mode 100755 scripts/jenkins-report.sh delete mode 100644 scripts/xdist/get_worker_test_list.py delete mode 100644 scripts/xdist/prepare_xdist_nodes.sh delete mode 100644 scripts/xdist/pytest_worker_manager.py delete mode 100755 scripts/xdist/setup_worker.sh delete mode 100644 scripts/xdist/terminate_xdist_nodes.sh diff --git a/scripts/Jenkinsfiles/python b/scripts/Jenkinsfiles/python deleted file mode 100644 index 5c0010568fac..000000000000 --- a/scripts/Jenkinsfiles/python +++ /dev/null @@ -1,272 +0,0 @@ -def runPythonTests() { - // Determine git refspec, branch, and clone type - git_branch = xdist_git_branch() - git_refspec = xdist_git_refspec() - sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) { - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]], - doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true, - noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] - sh 'bash scripts/all-tests.sh' - stash includes: 'reports/**/*coverage*,test_root/log/*.json', name: "${TEST_SUITE}-reports" - } -} - -def pythonTestCleanup() { - archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,test_root/log/*.json,**/nosetests.xml,*.log' - sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' - junit '**/nosetests.xml' - sh '''source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate - bash scripts/xdist/terminate_xdist_nodes.sh''' -} - -def createWarningsReport(fileExtension){ - println "Creating warnings report for ${fileExtension}" - warning_filename = "warning_report_${fileExtension}.html" - sh """source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate - python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/${warning_filename}""" - - publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports/pytest_warnings', reportFiles: "${warning_filename}", - reportName: "${warning_filename}", reportTitles: '']) - archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/pytest_warnings/*.html' -} - - -def xdist_git_branch() { - if (env.ghprbActualCommit) { - return "${ghprbActualCommit}" - } else { - return "${BRANCH_NAME}" - } -} - -def xdist_git_refspec() { - if (env.ghprbActualCommit) { - return "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" - } else { - return "+refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}" - } -} - -pipeline { - agent { label "jenkins-worker" } - options { - sendSplunkConsoleLog() - timestamps() - timeout(60) - } - environment { - XDIST_GIT_BRANCH = xdist_git_branch() - XDIST_GIT_REFSPEC = xdist_git_refspec() - XDIST_INSTANCE_TYPE = "c5d.large" - XDIST_WORKER_AMI = credentials('XDIST_WORKER_AMI') - XDIST_WORKER_IAM_PROFILE_ARN = credentials('XDIST_WORKER_IAM_PROFILE_ARN') - XDIST_WORKER_KEY_NAME = "jenkins-worker" - XDIST_WORKER_SUBNET = credentials('XDIST_WORKER_SUBNET') - XDIST_WORKER_SECURITY_GROUP = credentials('XDIST_WORKER_SECURITY_GROUP') - WTW_CONTEXT = "python" - } - stages { - stage('Mark build as pending on Github') { - when { - // Only run github-build-status for master builds - expression { env.ghprbActualCommit == null } - } - steps { - script { - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', - parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: 'Pending'), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: 'false'), - string(name: 'BUILD_STATUS', value: 'pending') - ], - propagate: false, wait: false - } - } - } - stage('Run Tests') { - parallel { - stage("lms-unit") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "lms-unit" - XDIST_NUM_WORKERS = 10 - XDIST_REMOTE_NUM_PROCESSES = 1 - } - steps { - script { - runPythonTests() - } - } - post { - always { - script { - pythonTestCleanup() - } - } - } - } - stage("cms-unit") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "cms-unit" - XDIST_NUM_WORKERS = 3 - XDIST_REMOTE_NUM_PROCESSES = 1 - } - steps { - script { - runPythonTests() - } - } - post { - always { - script { - pythonTestCleanup() - } - } - } - } - stage("commonlib-unit") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "commonlib-unit" - XDIST_NUM_WORKERS = 3 - XDIST_REMOTE_NUM_PROCESSES = 1 - } - steps { - script { - runPythonTests() - } - } - post { - always { - script { - pythonTestCleanup() - } - } - } - } - } - } - - - stage('Run coverage') { - environment { - CODE_COV_TOKEN = credentials('CODE_COV_TOKEN') - SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc - COLLECT_WHO_TESTS_WHAT = "${COLLECT_WHO_TESTS_WHAT}" - } - steps { - script { - if (env.ghprbActualCommit) { - git_branch = "${ghprbActualCommit}" - git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" - ci_branch = "${ghprbSourceBranch}" - target_branch = "origin/${ghprbTargetBranch}" - } else { - git_branch = "${BRANCH_NAME}" - git_refspec = "+refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}" - ci_branch = "${BRANCH_NAME}" - target_branch = "origin/${BRANCH_NAME}" - } - sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) { - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]], - doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', - honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [], - userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] - unstash 'lms-unit-reports' - unstash 'cms-unit-reports' - unstash 'commonlib-unit-reports' - sh """export CI_BRANCH=$ci_branch - export TARGET_BRANCH=$target_branch - ./scripts/jenkins-report.sh""" - createWarningsReport("all") - } - } - } - post { - always { - script { - archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/*.coverage' - sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' - publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports/cover', reportFiles: 'index.html', - reportName: 'Coverage.py Report', reportTitles: '']) - - if (env.ghprbPullId != null) { - publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports', reportFiles: 'diff_coverage_combined.html', - reportName: 'Diff Coverage Report', reportTitles: '']) - } - } - } - } - } - } - post { - always { - script { - if (env.ghprbPullId != null) { - // For PR jobs, run the edx-platform-test-notifier for PR reporting - build job: 'edx-platform-test-notifier', parameters: [string(name: 'REPO', value: "${REPO_NAME}"), string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false - } else { - // For master jobs run github-build-status and report to slack when necessary - if (currentBuild.currentResult == "SUCCESS") { - create_deployment = "true" - build_status = "success" - build_description = "Build Passed" - } - else { - create_deployment = "false" - build_status = "failure" - build_description = "Build Failed" - } - - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', - parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: build_description), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: create_deployment), - string(name: 'BUILD_STATUS', value: build_status) - ], - propagate: false, wait: false - - if (currentBuild.currentResult != "SUCCESS"){ - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - - email_body = "See: <${BUILD_URL}>\n\nChanges:\n" - change_sets = currentBuild.changeSets - for (int j = 0; j < change_sets.size(); j++) { - change_set_items = change_sets[j].items - for (int k = 0; k < change_set_items.length; k++) { - item = change_set_items[k] - email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}" - } - } - emailext body: email_body, - subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") { - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - emailext body: "See <${BUILD_URL}>", - subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } - } - } - } - } -} diff --git a/scripts/Jenkinsfiles/quality b/scripts/Jenkinsfiles/quality deleted file mode 100644 index 52324680c961..000000000000 --- a/scripts/Jenkinsfiles/quality +++ /dev/null @@ -1,228 +0,0 @@ -def runQualityTests() { - sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) { - // Determine git refspec, branch, and clone type - git_shallow_clone = true - git_extensions = [] - if (env.ghprbActualCommit) { - git_branch = "${ghprbActualCommit}" - refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" - if (SHARD == "4") { - git_shallow_clone = false - git_extensions.add([$class: 'WipeWorkspace']) - refspec = refspec + " +refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" - } - } else { - git_branch = "${BRANCH_NAME}" - refspec = "+refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}" - } - git_extensions.add([$class: 'CloneOption', honorRefspec: true, noTags: true, shallow: git_shallow_clone ]) - - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]], - doGenerateSubmoduleConfigurations: false, extensions: git_extensions, submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] - - sh "bash scripts/all-tests.sh" - stash includes: '**/reports/**/*', name: "${TEST_SUITE}-${SHARD}-reports" - } -} - -def getTargetBranch(job_name) { - if (env.ghprbTargetBranch) { - return env.ghprbTargetBranch - } else { - return "${BRANCH_NAME}" - } -} - -def qualityTestCleanup() { - archiveArtifacts allowEmptyArchive: true, artifacts: '**/reports/**/*,test_root/log/**/*.log,*.log' - sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' - junit 'reports/quality_junitxml/*.xml' -} - -pipeline { - agent { label "jenkins-worker" } - options { - sendSplunkConsoleLog() - timestamps() - timeout(120) - } - stages { - stage('Mark build as pending on Github') { - when { - // Only run github-build-status for master builds - expression { env.ghprbActualCommit == null } - } - steps { - script { - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', - parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: 'Pending'), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: 'false'), - string(name: 'BUILD_STATUS', value: 'pending') - ], - propagate: false, wait: false - } - } - } - stage('Run Tests') { - parallel { - stage("common pylint") { - // "pylint common" requires 5.5 GB of RAM, so use js-worker (8 GB) instead of jenkins-worker (4 GB) - agent { label "js-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 1 - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - stage("lms pylint") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 2 - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - stage("cms/openedx/pavelib pylint") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 3 - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - stage("Other quality checks") { - agent { label "jenkins-worker" } - environment { - TEST_SUITE = "quality" - SHARD = 4 - TARGET_BRANCH = getTargetBranch("${JOB_NAME}") - } - steps { - script { - runQualityTests() - } - } - post { - always { - script { - qualityTestCleanup() - } - } - } - } - } - } - } - post { - always { - script{ - try { - unstash 'quality-1-reports' - unstash 'quality-2-reports' - unstash 'quality-3-reports' - unstash 'quality-4-reports' - // Check for warnings - recordIssues enabledForFailure: true, tools: [pep8(pattern: 'reports/pep8/pep8.report'), pyLint(pattern: 'reports/**/pylint.report')] - // Publish Quality report - publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'reports/metrics/', - reportFiles: 'pylint/*view*/,pep8/*view*/,xsscommitlint/*view*/,xsslint/*view*/,eslint/*view*/,pii/*view*/', - reportName: 'Quality Report', reportTitles: '']) - } finally { - if (env.ghprbPullId != null) { - // For PR jobs, run the edx-platform-test-notifier for PR reporting - build job: 'edx-platform-test-notifier', parameters: [string(name: 'REPO', value: "${REPO_NAME}"), string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false - } else { - // For master jobs run github-build-status and report to slack when necessary - if (currentBuild.currentResult == "SUCCESS") { - create_deployment = "true" - build_status = "success" - build_description = "Build Passed" - } - else { - create_deployment = "false" - build_status = "failure" - build_description = "Build Failed" - } - - commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - build job: 'github-build-status', parameters: [ - string(name: 'GIT_SHA', value: commit_sha), - string(name: 'GITHUB_ORG', value: 'edx'), - string(name: 'GITHUB_REPO', value: "${REPO_NAME}"), - string(name: 'TARGET_URL', value: "${BUILD_URL}"), - string(name: 'DESCRIPTION', value: build_description), - string(name: 'CONTEXT', value: "${GITHUB_CONTEXT}"), - string(name: 'CREATE_DEPLOYMENT', value: create_deployment), - string(name: 'BUILD_STATUS', value: build_status) - ], - propagate: false, wait: false - - if (currentBuild.currentResult != "SUCCESS"){ - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - - email_body = "See: <${BUILD_URL}>\n\nChanges:\n" - change_sets = currentBuild.changeSets - for (int j = 0; j < change_sets.size(); j++) { - change_set_items = change_sets[j].items - for (int k = 0; k < change_set_items.length; k++) { - item = change_set_items[k] - email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}" - } - } - emailext body: email_body, - subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") { - slackSend botUser: true, - message: "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}" - emailext body: "See <${BUILD_URL}>", - subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org' - } - } - } - } - } - } -} diff --git a/scripts/jenkins-common.sh b/scripts/jenkins-common.sh deleted file mode 100644 index 5eec5b5b4922..000000000000 --- a/scripts/jenkins-common.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $HOME/jenkins_env - -# Clear the mongo database -# Note that this prevents us from running jobs in parallel on a single worker. -mongo --quiet --eval 'db.getMongo().getDBNames().forEach(function(i){db.getSiblingDB(i).dropDatabase()})' - -# Ensure we have fetched a target branch (origin/master, unless testing -# a release branc) Some of the reporting tools compare the checked out -# branch to a target branch; depending on how the GitHub plugin refspec -# is configured, this may not already be fetched. -if [ ! -z ${TARGET_BRANCH+x} ]; then - TARGET_BRANCH_WITHOUT_ORIGIN=$(echo "${TARGET_BRANCH}" | sed 's:^origin/::') - git fetch origin $TARGET_BRANCH_WITHOUT_ORIGIN:refs/remotes/origin/$TARGET_BRANCH_WITHOUT_ORIGIN -fi - -# Reset the jenkins worker's virtualenv back to the -# state it was in when the instance was spun up. -if [ -z ${PYTHON_VERSION+x} ] || [[ ${PYTHON_VERSION} == 'null' ]]; then - VENV_PATH="$HOME" - ARCHIVED_VENV="edx-venv_clean.tar.gz" -else - VENV_PATH="$HOME/edx-venv-$PYTHON_VERSION" - ARCHIVED_VENV="edx-venv_clean-$PYTHON_VERSION.tar.gz" -fi - -if [ -e $HOME/$ARCHIVED_VENV ]; then - rm -rf $VENV_PATH/edx-venv - tar -C $VENV_PATH -xf $HOME/$ARCHIVED_VENV -fi - -# Load the npm packages from the time the worker was built -# into the npm cache. This is an attempt to reduce the number -# of times that npm gets stuck during an installation, by -# reducing the number of packages that npm needs to fetch. -if [ -e $HOME/edx-npm-cache_clean.tar.gz ]; then - echo "Loading archived npm packages into the local npm cache" - rm -rf $HOME/.npm - tar -C $HOME -xf $HOME/edx-npm-cache_clean.tar.gz -fi - -# Activate the Python virtualenv -source $VENV_PATH/edx-venv/bin/activate - -# Hack to fix up egg-link files given that the virtualenv is not relocatable -sed -i "s|^/home/jenkins/shallow-clone|`pwd`|" -- \ - $VENV_PATH/edx-venv/lib/python*/site-packages/*.egg-link -make test-requirements - -# add the node packages dir to PATH -PATH=$PATH:node_modules/.bin - -echo "node version is `node --version`" -echo "npm version is `npm --version`" - -# Log any paver or ansible command timing -TIMESTAMP=$(date +%s) -SHARD_NUM=${SHARD:="all"} -export PAVER_TIMER_LOG="test_root/log/timing.paver.$TEST_SUITE.$SHARD_NUM.log" -export ANSIBLE_TIMER_LOG="test_root/log/timing.ansible.$TIMESTAMP.log" - -echo "This node is `curl http://169.254.169.254/latest/meta-data/hostname`" diff --git a/scripts/jenkins-quality-report.sh b/scripts/jenkins-quality-report.sh deleted file mode 100755 index ebfd29697560..000000000000 --- a/scripts/jenkins-quality-report.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script is used by the edx-platform-quality-check jenkins job. -source scripts/jenkins-common.sh -source scripts/thresholds.sh - -# Run quality task. Pass in the 'fail-under' percentage to diff-quality -echo "Generating quality report." -mkdir -p test_root/log/ -LOG_PREFIX=test_root/log/run_quality - -paver run_quality 2> $LOG_PREFIX.err.log > $LOG_PREFIX.out.log diff --git a/scripts/jenkins-report.sh b/scripts/jenkins-report.sh deleted file mode 100755 index b085826ee750..000000000000 --- a/scripts/jenkins-report.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -e -set -x - -# This script generates coverage and diff cover reports, and optionally -# reports this data to codecov.io. The following environment variables must be -# set in order to report to codecov: -# CODE_COV_TOKEN: CodeCov API token -# CI_BRANCH: The branch that the coverage report describes - -# This script is used by the edx-platform-unit-coverage jenkins job. - -source scripts/jenkins-common.sh - -# Get the diff coverage and html reports for unit tests -paver coverage -b $TARGET_BRANCH - -if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then - paver upload_coverage_to_s3 -fi - -# Test for the CodeCov API token -if [ -z $CODE_COV_TOKEN ]; then - echo "codecov.io API token not set." - echo "This must be set as an environment variable if order to report coverage" -else - # Send the coverage data to codecov - pip install codecov==2.0.5 - codecov --token=$CODE_COV_TOKEN --branch=$CI_BRANCH -fi - -# JUnit test reporter will fail the build -# if it thinks test results are old -touch `find . -name *.xml` || true diff --git a/scripts/xdist/get_worker_test_list.py b/scripts/xdist/get_worker_test_list.py deleted file mode 100644 index 268ab5ca8915..000000000000 --- a/scripts/xdist/get_worker_test_list.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -This script strips the console log of a pytest-xdist Jenkins run into the test -lists of each pytest worker. -Assumes the following format: -[test-suite] [worker] RESULT test -""" - - -import io -import os -import re -import shutil - -import click - - -@click.command() -@click.option( - '--log-file', - help="File name of console log .txt file from a Jenkins build " - "that ran pytest-xdist. This can be acquired by running: " - "curl -o console.txt https://build.testeng.edx.org/job/JOBNAME/BUILDNUMBER/consoleText", - required=True -) -@click.option( - '--test-suite', - help="Test suite that the pytest worker ran.", - type=click.Choice(['lms-unit', 'cms-unit']), - required=True -) -def main(log_file, test_suite): - worker_test_dict = {} - with open(log_file) as console_file: - for line in console_file: - regex_search = re.search(fr'\[gw(\d+)] (PASSED|FAILED|SKIPPED|ERROR) (\S+)', line) - if regex_search: - worker_num_string = regex_search.group(1) - if worker_num_string not in worker_test_dict: - worker_test_dict[worker_num_string] = [] - test = regex_search.group(3) - worker_test_dict[worker_num_string].append(test) - - output_folder_name = "worker_list_files" - if os.path.isdir(output_folder_name): - shutil.rmtree(output_folder_name) - os.mkdir(output_folder_name) - - for worker_num in worker_test_dict: - output_file_name = f"{output_folder_name}/{test_suite}_gw{worker_num}_test_list.txt" - with open(output_file_name, 'w') as output_file: - for line in worker_test_dict[worker_num]: - output_file.write(line + "\n") - - -if __name__ == "__main__": - main() diff --git a/scripts/xdist/prepare_xdist_nodes.sh b/scripts/xdist/prepare_xdist_nodes.sh deleted file mode 100644 index 8346f2d4619c..000000000000 --- a/scripts/xdist/prepare_xdist_nodes.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e - -echo "Spinning up xdist workers with pytest_worker_manager.py" -python scripts/xdist/pytest_worker_manager.py -a up -n ${XDIST_NUM_WORKERS} \ --ami ${XDIST_WORKER_AMI} \ --type ${XDIST_INSTANCE_TYPE} \ --s ${XDIST_WORKER_SUBNET} \ --sg ${XDIST_WORKER_SECURITY_GROUP} \ --key ${XDIST_WORKER_KEY_NAME} \ --iam ${XDIST_WORKER_IAM_PROFILE_ARN} - -ip_list=$( 0: - logger.info(f"Still waiting on {len(not_running)} workers to spin up") - time.sleep(5) - else: - logger.info("Finished spinning up workers") - all_running = True - break - - if not all_running: - raise Exception( - "Timed out waiting to spin up all workers." - ) - logger.info(f"Successfully booted up {number_of_workers} workers.") - - not_ready_ip_addresses = ip_addresses[:] - logger.info("Checking ssh connection to workers.") - pool = Pool(processes=number_of_workers) - for ssh_try in range(0, self.WORKER_SSH_ATTEMPTS): - results = pool.map(_check_worker_ready, not_ready_ip_addresses) - deleted_ips = 0 - for num in range(0, len(results)): - if results[num] == 0: - del(not_ready_ip_addresses[num - deleted_ips]) - deleted_ips += 1 - - if len(not_ready_ip_addresses) == 0: - logger.info("All workers are ready for tests.") - break - - if ssh_try == self.WORKER_SSH_ATTEMPTS - 1: - raise Exception( - "Max ssh tries to remote workers reached." - ) - - logger.info("Not all workers are ready. Sleeping for 5 seconds then retrying.") - time.sleep(5) - - # Generate .txt files containing IP addresses and instance ids - ip_list_string = ",".join(ip_addresses) - logger.info(f"Worker IP list: {ip_list_string}") - ip_list_file = open("pytest_worker_ips.txt", "w") - ip_list_file.write(ip_list_string) - ip_list_file.close() - - worker_instance_id_list_string = ",".join(worker_instance_ids) - logger.info(f"Worker Instance Id list: {worker_instance_id_list_string}") - worker_arn_file = open("pytest_worker_instance_ids.txt", "w") - worker_arn_file.write(worker_instance_id_list_string) - worker_arn_file.close() - - def terminate_workers(self, worker_instance_ids): - """ - Terminates workers based on a list of worker_instance_ids. - """ - instance_id_list = worker_instance_ids.split(',') - response = self.ec2.terminate_instances( - InstanceIds=instance_id_list - ) - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser( - description="PytestWorkerManager, manages EC2 workers in an AWS cluster." - ) - - parser.add_argument('--action', '-a', choices=['up', 'down'], default=None, - help="Action for PytestWorkerManager to perform. " - "Either up for spinning up AWS EC2 workers or down for terminating them") - - parser.add_argument('--region', '-g', default='us-east-1', - help="AWS region where EC2 infrastructure lives. Defaults to us-east-1") - - # Spinning up workers - parser.add_argument('--num-workers', '-n', type=int, default=None, - help="Number of EC2 workers to spin up") - - parser.add_argument('--ami', '-ami', default=None, - help="AMI for workers") - - parser.add_argument('--instance-type', '-type', default=None, - help="Desired EC2 instance type") - - parser.add_argument('--subnet-id', '-s', default=None, - help="Subnet for the workers to exist in") - - parser.add_argument('--security_groups', '-sg', nargs='+', default=None, - help="List of security group ids to apply to workers") - - parser.add_argument('--key-name', '-key', default=None, - help="Key pair name for sshing to worker") - - parser.add_argument('--iam-arn', '-iam', default=None, - help="Iam Instance Profile ARN for the workers") - - # Terminating workers - parser.add_argument('--instance-ids', '-ids', default=None, - help="Instance ids terminate") - - args = parser.parse_args() - workerManager = PytestWorkerManager(args.region) - - if args.action == 'up': - workerManager.spin_up_workers( - args.num_workers, - args.ami, - args.instance_type, - args.subnet_id, - args.security_groups, - args.key_name, - args.iam_arn - ) - elif args.action == 'down': - workerManager.terminate_workers( - args.instance_ids - ) - else: - logger.info("No action specified for PytestWorkerManager") diff --git a/scripts/xdist/setup_worker.sh b/scripts/xdist/setup_worker.sh deleted file mode 100755 index 7c8acb5ad03e..000000000000 --- a/scripts/xdist/setup_worker.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Set up worker node. - -while getopts 'p:' opt; do - case "$opt" in - p) PYTHON_VERSION="$OPTARG";; - [?]) - print >&2 "Usage: $0 -p python-version" - exit 1 - ;; - esac -done - -venv_parent=/home/jenkins/edx-venv-${PYTHON_VERSION} -venv=$venv_parent/edx-venv -rm -rf $venv -tar -C $venv_parent -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz -source $venv/bin/activate - -# Hack to fix up egg-link files given that the virtualenv is not relocatable -sed -i "s|\(^/home/jenkins\)/shallow-clone|\1/edx-platform|" -- \ - $venv/lib/python*/site-packages/*.egg-link - -make test-requirements - -mkdir reports diff --git a/scripts/xdist/terminate_xdist_nodes.sh b/scripts/xdist/terminate_xdist_nodes.sh deleted file mode 100644 index e2997da6f7c5..000000000000 --- a/scripts/xdist/terminate_xdist_nodes.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -if [ -f pytest_worker_instance_ids.txt ]; then - echo "Terminating xdist workers with pytest_worker_manager.py" - xdist_worker_ids=$(