From 55fd9d4f920646f091596718bc350812a817e89c Mon Sep 17 00:00:00 2001 From: Micah Snyder Date: Wed, 22 May 2024 15:42:23 -0400 Subject: [PATCH 1/3] Jenkins: separate build pipeline and package test pipeline The build pipeline used to build clamav packages and then test those packages with a rudimentary test set. This change will build the clamav packages in one pipeline - then test the packages in a new test pipeline. The new test pipeline will use the larger test suite that we use for testing from-source builds in the "regular" test pipeline. --- Jenkinsfile | 53 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 90a26e77d6..ec77d51f70 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,19 +17,25 @@ properties( description: 'test-framework branch'), string(name: 'TESTS_BRANCH', defaultValue: '1.4', - description: 'tests branch'), + description: 'tests branch for the package and regular tests'), string(name: 'TESTS_CUSTOM_BRANCH', defaultValue: '1.4', description: 'tests-custom branch'), string(name: 'TESTS_FUZZ_BRANCH', defaultValue: '1.4', description: 'tests-fuzz-regression branch'), + string(name: 'BUILD_PIPELINES_PATH', + defaultValue: 'ClamAV/build-pipelines', + description: 'build-pipelines path for clamav in Jenkins'), string(name: 'TEST_PIPELINES_PATH', defaultValue: 'ClamAV/test-pipelines', description: 'test-pipelines path for clamav in Jenkins'), string(name: 'BUILD_PIPELINE', defaultValue: 'build-1.4', description: 'test-pipelines branch for build acceptance'), + string(name: 'PACKAGE_PIPELINE', + defaultValue: 'package-1.4', + description: 'test-pipelines branch for package tests.'), string(name: 'REGULAR_PIPELINE', defaultValue: 'regular-1.4', description: 'test-pipelines branch for regular tests.'), @@ -50,7 +56,7 @@ properties( ] ) -node('ubuntu-18-x64') { +node('default') { stage('Generate Tarball') { cleanWs() @@ -82,10 +88,11 @@ node('ubuntu-18-x64') { def buildResult stage('Build') { - buildResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.BUILD_PIPELINE}", + buildResult = build(job: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}", propagate: true, wait: true, parameters: [ + [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.BUILD_PIPELINE}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], @@ -93,21 +100,45 @@ node('ubuntu-18-x64') { [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] ] ) - echo "${params.TEST_PIPELINES_PATH}/${params.BUILD_PIPELINE} #${buildResult.number} succeeded." + echo "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE} #${buildResult.number} succeeded." } stage('Test') { def tasks = [:] - tasks["regular_and_custom"] = { - def regularResult + tasks["package_regular_custom"] = { def exception = null try { - stage("Regular Pipeline") { - regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE}", + stage("Package") { + final regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE}", propagate: true, wait: true, parameters: [ + [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.PACKAGE_PIPELINE}"], + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], + [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.TEST_PIPELINES_PATH}/${params.BUILD_PIPELINE}"], + [$class: 'StringParameterValue', name: 'BUILD_JOB_NUMBER', value: "${buildResult.number}"], + [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"], + [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], + [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], + [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] + ] + ) + echo "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE} #${regularResult.number} succeeded." + } + } catch (exc) { + echo "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE} failed." + exception = exc + } + + try { + stage("Regular From-Source") { + final regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE}", + propagate: true, + wait: true, + parameters: [ + [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.REGULAR_PIPELINE}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.TEST_PIPELINES_PATH}/${params.BUILD_PIPELINE}"], @@ -124,11 +155,13 @@ node('ubuntu-18-x64') { echo "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE} failed." exception = exc } - stage("Custom Pipeline") { + + stage("Custom From-Source") { final customResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE}", propagate: true, wait: true, parameters: [ + [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.CUSTOM_PIPELINE}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_CUSTOM_BRANCH}"], @@ -140,7 +173,7 @@ node('ubuntu-18-x64') { echo "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE} #${customResult.number} succeeded." } if(exception != null) { - echo "Custom Pipeline passed, but Regular pipeline failed!" + echo "Custom Pipeline passed, but prior pipelines failed!" throw exception } } From c8e85e9d95ec615d0f2a3b836997152ccd6ec500 Mon Sep 17 00:00:00 2001 From: Micah Snyder Date: Thu, 23 May 2024 15:44:44 -0400 Subject: [PATCH 2/3] Jenkins: Specify dependencies for CPack We switched hosts for this stage to one that doesn't have system-installed libs. --- Jenkinsfile | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ec77d51f70..9486dbdc73 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,27 @@ node('default') { dir(path: 'build') { sh """# CPack - cmake .. -D VENDOR_DEPENDENCIES=ON + cmake .. -D VENDOR_DEPENDENCIES=ON \ + -D JSONC_INCLUDE_DIR="$HOME/.mussels/install/host-static/include/json-c" \ + -D JSONC_LIBRARY="$HOME/.mussels/install/host-static/lib/libjson-c.a" \ + -D ENABLE_JSON_SHARED=OFF \ + -D BZIP2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D BZIP2_LIBRARY_RELEASE="$HOME/bzip2-1.0.8-install/lib/libbz2.a" \ + -D OPENSSL_ROOT_DIR="$HOME/.mussels/install/host-static" \ + -D OPENSSL_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D OPENSSL_CRYPTO_LIBRARY="$HOME/.mussels/install/host-static/lib/libcrypto.a" \ + -D OPENSSL_SSL_LIBRARY="$HOME/.mussels/install/host-static/lib/libssl.a" \ + -D LIBXML2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include/libxml2" \ + -D LIBXML2_LIBRARY="$HOME/.mussels/install/host-static/lib/libxml2.a" \ + -D PCRE2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D PCRE2_LIBRARY="$HOME/.mussels/install/host-static/lib/libpcre2-8.a" \ + -D CURSES_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D CURSES_LIBRARY="$HOME/.mussels/install/host-static/lib/libncurses.a;$HOME/.mussels/install/host-static/lib/libtinfo.a" \ + -D ZLIB_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D ZLIB_LIBRARY="$HOME/.mussels/install/host-static/lib/libz.a" \ + -D LIBCHECK_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D LIBCHECK_LIBRARY="$HOME/.mussels/install/host-static/lib/libcheck.a" + cpack --config CPackSourceConfig.cmake """ archiveArtifacts(artifacts: "clamav-${params.VERSION}*.tar.gz", onlyIfSuccessful: true) } From d84ec62363dd1a961262ac85dd7bccb51072a443 Mon Sep 17 00:00:00 2001 From: Micah Snyder Date: Thu, 23 May 2024 21:01:53 -0400 Subject: [PATCH 3/3] Jenkins: Fixup build-pipeline path --- Jenkinsfile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9486dbdc73..d54a956b0f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,7 +112,6 @@ node('default') { propagate: true, wait: true, parameters: [ - [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.BUILD_PIPELINE}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], @@ -134,10 +133,9 @@ node('default') { propagate: true, wait: true, parameters: [ - [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.PACKAGE_PIPELINE}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], - [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.TEST_PIPELINES_PATH}/${params.BUILD_PIPELINE}"], + [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}"], [$class: 'StringParameterValue', name: 'BUILD_JOB_NUMBER', value: "${buildResult.number}"], [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"], [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], @@ -158,11 +156,8 @@ node('default') { propagate: true, wait: true, parameters: [ - [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.REGULAR_PIPELINE}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], - [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.TEST_PIPELINES_PATH}/${params.BUILD_PIPELINE}"], - [$class: 'StringParameterValue', name: 'BUILD_JOB_NUMBER', value: "${buildResult.number}"], [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"], [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], @@ -181,7 +176,6 @@ node('default') { propagate: true, wait: true, parameters: [ - [$class: 'StringParameterValue', name: 'PIPELINE_BRANCH_NAME', value: "${params.CUSTOM_PIPELINE}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_CUSTOM_BRANCH}"],