Merge pull request Cisco-Talos#1346 from micahsnyder/CLAM-2638-0.103.…

…12-and-backports

Clam 2638 0.103.12 and backports

CMakeLists.txt

@@ -15,7 +15,7 @@ string(TIMESTAMP TODAY "%Y%m%d")
 set(VERSION_SUFFIX "")
 
 project( ClamAV
-         VERSION "0.103.11"
+         VERSION "0.103.12"
          DESCRIPTION "ClamAV open source email, web, and end-point anti-virus toolkit." )
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})

Jenkins/Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get install -y \
+    gcc make automake autoconf m4 pkg-config libtool flex bison valgrind \
+    check libbz2-dev libcurl4-openssl-dev libjson-c-dev libmilter-dev \
+    libncurses5-dev libpcre2-dev libssl-dev libxml2-dev zlib1g-dev

Jenkinsfile

@@ -4,23 +4,32 @@ properties(
         parameters(
             [
                 string(name: 'VERSION',
-                       defaultValue: '0.103.11',
+                       defaultValue: '0.103.12',
                        description: 'ClamAV version string'),
                 string(name: 'FRAMEWORK_BRANCH',
                        defaultValue: '0.103',
                        description: 'test-framework branch'),
                 string(name: 'TESTS_BRANCH',
                        defaultValue: '0.103',
-                       description: 'tests branch'),
+                       description: 'tests branch for the package and regular tests'),
                 string(name: 'TESTS_CUSTOM_BRANCH',
                        defaultValue: '0.103',
                        description: 'tests-custom branch'),
                 string(name: 'TESTS_FUZZ_BRANCH',
                        defaultValue: '0.103',
                        description: 'tests-fuzz-regression branch'),
+                string(name: 'BUILD_PIPELINES_PATH',
+                       defaultValue: 'ClamAV/build-pipelines',
+                       description: 'build-pipelines path for clamav in Jenkins'),
+                string(name: 'TEST_PIPELINES_PATH',
+                       defaultValue: 'ClamAV/test-pipelines',
+                       description: 'test-pipelines path for clamav in Jenkins'),
                 string(name: 'BUILD_PIPELINE',
                        defaultValue: 'build-0.103',
                        description: 'test-pipelines branch for build acceptance'),
+                string(name: 'PACKAGE_PIPELINE',
+                       defaultValue: 'package-0.103',
+                       description: 'test-pipelines branch for package tests.'),
                 string(name: 'REGULAR_PIPELINE',
                        defaultValue: 'regular-0.103',
                        description: 'test-pipelines branch for regular tests.'),
@@ -44,7 +53,7 @@ properties(
     ]
 )
 
-node('ubuntu-18-x64') {
+node('docker') {
     stage('Generate Tarball') {
         cleanWs()
 
@@ -63,13 +72,24 @@ node('ubuntu-18-x64') {
                 '''
         }
 
-        dir(path: 'build') {
-            sh """# Make Dist
-                if [ -f '../autogen.sh' ] ; then /bin/chmod +x ../autogen.sh && ../autogen.sh ; fi
-                ../configure --enable-milter --disable-clamav --disable-silent-rules --enable-llvm --with-system-llvm=no
-                make distcheck
-                mv clamav-${params.VERSION}*.tar.gz clamav-${params.VERSION}.tar.gz || true"""
-            archiveArtifacts(artifacts: "clamav-${params.VERSION}.tar.gz", onlyIfSuccessful: true)
+        // start up docker image
+        def dockerImage = docker.build("autoconf", "./Jenkins")
+
+        try {
+            dockerImage.inside { c ->
+                dir(path: "build") {
+                    sh """# Make Dist
+                        if [ -f '../autogen.sh' ] ; then /bin/chmod +x ../autogen.sh && ../autogen.sh ; fi
+                        ../configure --enable-milter --disable-clamav --disable-silent-rules --enable-llvm --with-system-llvm=no
+                        make dist
+                        mv clamav-${params.VERSION}*.tar.gz clamav-${params.VERSION}.tar.gz || true"""
+                    archiveArtifacts(artifacts: "clamav-${params.VERSION}.tar.gz", onlyIfSuccessful: true)
+                }
+            }
+        }
+        catch(IOException err) {
+            cleanWs()
+            throw err
         }
 
         cleanWs()
@@ -78,7 +98,7 @@ node('ubuntu-18-x64') {
     def buildResult
 
     stage('Build') {
-        buildResult = build(job: "test-pipelines/${params.BUILD_PIPELINE}",
+        buildResult = build(job: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}",
             propagate: true,
             wait: true,
             parameters: [
@@ -89,39 +109,60 @@ node('ubuntu-18-x64') {
                 [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"]
             ]
         )
-        echo "test-pipelines/${params.BUILD_PIPELINE} #${buildResult.number} succeeded."
+        echo "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE} #${buildResult.number} succeeded."
     }
 
     stage('Test') {
         def tasks = [:]
 
-        tasks["regular_and_custom"] = {
-            def regularResult
+        tasks["package_regular_custom"] = {
             def exception = null
             try {
-                stage("Regular Pipeline") {
-                    regularResult = build(job: "test-pipelines/${params.REGULAR_PIPELINE}",
+                stage("Package") {
+                    final regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE}",
                         propagate: true,
                         wait: true,
                         parameters: [
                             [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"],
                             [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"],
-                            [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "test-pipelines/${params.BUILD_PIPELINE}"],
+                            [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}"],
                             [$class: 'StringParameterValue', name: 'BUILD_JOB_NUMBER', value: "${buildResult.number}"],
                             [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"],
                             [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"],
                             [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"],
                             [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"]
                         ]
                     )
-                    echo "test-pipelines/${params.REGULAR_PIPELINE} #${regularResult.number} succeeded."
+                    echo "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE} #${regularResult.number} succeeded."
+                }
+            } catch (exc) {
+                echo "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE} failed."
+                exception = exc
+            }
+
+            try {
+                stage("Regular From-Source") {
+                    final regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE}",
+                        propagate: true,
+                        wait: true,
+                        parameters: [
+                            [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"],
+                            [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"],
+                            [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"],
+                            [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"],
+                            [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"],
+                            [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"]
+                        ]
+                    )
+                    echo "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE} #${regularResult.number} succeeded."
                 }
             } catch (exc) {
-                echo "test-pipelines/${params.REGULAR_PIPELINE} failed."
+                echo "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE} failed."
                 exception = exc
             }
-            stage("Custom Pipeline") {
-                final customResult = build(job: "test-pipelines/${params.CUSTOM_PIPELINE}",
+
+            stage("Custom From-Source") {
+                final customResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE}",
                     propagate: true,
                     wait: true,
                     parameters: [
@@ -133,17 +174,17 @@ node('ubuntu-18-x64') {
                         [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"]
                     ]
                 )
-                echo "test-pipelines/${params.CUSTOM_PIPELINE} #${customResult.number} succeeded."
+                echo "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE} #${customResult.number} succeeded."
             }
             if(exception != null) {
-                echo "Custom Pipeline passed, but Regular pipeline failed!"
+                echo "Custom Pipeline passed, but prior pipelines failed!"
                 throw exception
             }
         }
 
         tasks["fuzz_regression"] = {
             stage("Fuzz Regression") {
-                final fuzzResult = build(job: "test-pipelines/${params.FUZZ_PIPELINE}",
+                final fuzzResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.FUZZ_PIPELINE}",
                     propagate: true,
                     wait: true,
                     parameters: [
@@ -154,7 +195,7 @@ node('ubuntu-18-x64') {
                         [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"]
                     ]
                 )
-                echo "test-pipelines/${params.FUZZ_PIPELINE} #${fuzzResult.number} succeeded."
+                echo "${params.TEST_PIPELINES_PATH}/${params.FUZZ_PIPELINE} #${fuzzResult.number} succeeded."
             }
         }
 

NEWS.md

@@ -3,6 +3,10 @@
 Note: This file refers to the source tarball. Things described here may differ
  slightly from the binary packages.
 
+## 0.103.12
+
+ClamAV 0.103.12 is a patch release with the following fixes:
+
 ## 0.103.11
 
 ClamAV 0.103.11 is a patch release with the following fixes:

configure.ac

@@ -22,7 +22,7 @@ AC_PREREQ([2.59])
 
 dnl For a release change [devel] to the real version [0.xy]
 dnl also change VERSION below
-AC_INIT([ClamAV], [0.103.11], [https://github.com/Cisco-Talos/clamav/issues], [clamav], [https://www.clamav.net/])
+AC_INIT([ClamAV], [0.103.12], [https://github.com/Cisco-Talos/clamav/issues], [clamav], [https://www.clamav.net/])
 
 dnl put configure auxiliary into config
 AC_CONFIG_AUX_DIR([config])

freshclam/freshclam.c

@@ -998,7 +998,7 @@ fc_error_t get_official_database_lists(
     uint32_t i;
 
     const char *hardcodedStandardDatabaseList[] = {"daily", "main", "bytecode"};
-    const char *hardcodedOptionalDatabaseList[] = {"safebrowsing", "test"};
+    const char *hardcodedOptionalDatabaseList[] = {"safebrowsing", "test", "valhalla"};
 
     if ((NULL == standardDatabases) || (NULL == nStandardDatabases) || (NULL == optionalDatabases) || (NULL == nOptionalDatabases)) {
         mprintf("!get_official_database_lists: Invalid arguments.\n");
@@ -1424,6 +1424,10 @@ fc_error_t perform_database_update(
     uint32_t nUpdated      = 0;
     uint32_t nTotalUpdated = 0;
 
+    uint32_t i;
+    char **doNotPruneDatabaseList = NULL;
+    uint32_t nDoNotPruneDatabases = 0;
+
     STATBUF statbuf;
 
     if (NULL == serverList) {
@@ -1444,7 +1448,38 @@ fc_error_t perform_database_update(
          * Prune database directory of official databases
          * that are no longer available or no longer desired.
          */
-        (void)fc_prune_database_directory(databaseList, nDatabases);
+
+        // include the URL databases in the prune process
+        doNotPruneDatabaseList = (char **)malloc(sizeof(char *) * (nDatabases + nUrlDatabases));
+        if (NULL == doNotPruneDatabaseList) {
+            logg("!perform_database_update: Can't allocate memory for doNotPruneDatabaseList\n");
+            status = FC_EMEM;
+            goto done;
+        }
+
+        for (i = 0; i < nDatabases; i++) {
+            doNotPruneDatabaseList[i] = strdup(databaseList[i]);
+            if (doNotPruneDatabaseList[i] == NULL) {
+                logg("!perform_database_update: Can't allocate memory for database name in doNotPruneDatabaseList\n");
+                status = FC_EMEM;
+                goto done;
+            }
+        }
+        nDoNotPruneDatabases = nDatabases;
+
+        for (i = 0; i < nUrlDatabases; i++) {
+            // Only append the URL databases that end with '.cvd'
+            if (strlen(urlDatabaseList[i]) > 4 && 0 == strcasecmp(urlDatabaseList[i] + strlen(urlDatabaseList[i]) - 4, ".cvd")) {
+                const char *startOfFilename = strrchr(urlDatabaseList[i], '/') + 1;
+                if (NULL != startOfFilename) {
+                    // Add the base database name to the do-not-prune list, excluding the '.cvd' extension.
+                    doNotPruneDatabaseList[nDatabases + i] = CLI_STRNDUP(startOfFilename, strlen(startOfFilename) - strlen(".cvd"));
+                    nDoNotPruneDatabases++;
+                }
+            }
+        }
+
+        (void)fc_prune_database_directory(doNotPruneDatabaseList, nDoNotPruneDatabases);
     }
 
     /*
@@ -1515,6 +1550,16 @@ fc_error_t perform_database_update(
 
 done:
 
+    // Free up the database list
+    if (NULL != doNotPruneDatabaseList) {
+        for (i = 0; i < nDoNotPruneDatabases; i++) {
+            free(doNotPruneDatabaseList[i]);
+            doNotPruneDatabaseList[i] = NULL;
+        }
+        free(doNotPruneDatabaseList);
+        doNotPruneDatabaseList = NULL;
+    }
+
     if (LSTAT(g_freshclamTempDirectory, &statbuf) != -1) {
         /* Remove temp directory */
         if (*g_freshclamTempDirectory) {

libclamav/bytecode_api.h

@@ -157,6 +157,7 @@ enum FunctionalityLevels {
     FUNC_LEVEL_0103_9    = 130, /**< LibClamAV release 0.103.9 */
     FUNC_LEVEL_0103_10   = 131, /**< LibClamAV release 0.103.10 */
     FUNC_LEVEL_0103_11   = 132, /**< LibClamAV release 0.103.11 */
+    FUNC_LEVEL_0103_12   = 133, /**< LibClamAV release 0.103.12 */
 };
 
 /**

libclamav/others.h

@@ -73,7 +73,7 @@
  * in re-enabling affected modules.
  */
 
-#define CL_FLEVEL 132
+#define CL_FLEVEL 133
 #define CL_FLEVEL_DCONF CL_FLEVEL
 #define CL_FLEVEL_SIGTOOL CL_FLEVEL
 

libclamav/special.c

@@ -48,7 +48,8 @@
 
 int cli_check_mydoom_log(cli_ctx *ctx)
 {
-    const uint32_t *record;
+    uint32_t record[16];
+    const uint32_t *ptr;
     uint32_t check, key;
     fmap_t *map         = ctx->fmap;
     unsigned int blocks = map->len / (8 * 4);
@@ -59,14 +60,24 @@ int cli_check_mydoom_log(cli_ctx *ctx)
     if (blocks > 5)
         blocks = 5;
 
-    record = fmap_need_off_once(map, 0, 8 * 4 * blocks);
-    if (!record)
+    /*
+     * The following pointer might not be properly aligned. There there is
+     * memcmp() + memcpy() workaround to avoid performing an unaligned access
+     * while reading the uint32_t.
+     */
+    ptr = fmap_need_off_once(map, 0, 8 * 4 * blocks);
+    if (!ptr)
         return CL_CLEAN;
+
     while (blocks) { /* This wasn't probably intended but that's what the current code does anyway */
-        if (record[--blocks] == 0xffffffff)
+        const uint32_t marker_ff = 0xffffffff;
+
+	if (!memcmp(&ptr[--blocks], &marker_ff, sizeof(uint32_t)))
             return CL_CLEAN;
     }
 
+    memcpy(record, ptr, sizeof(record));
+
     key   = ~be32_to_host(record[0]);
     check = (be32_to_host(record[1]) ^ key) +
             (be32_to_host(record[2]) ^ key) +

m4/reorganization/version.m4

@@ -3,7 +3,7 @@ dnl During active development, set: VERSION="<version>-devel-`date +%Y%m%d`"
 dnl For beta,                  set: VERSION="<version>-beta"
 dnl For release candidate,     set: VERSION="<version>-rc"
 dnl For release,               set: VERSION="<version>"
-VERSION="0.103.11"
+VERSION="0.103.12"
 
 major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"`
 minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"`

win32/ClamAV-Installer.iss

@@ -8,7 +8,7 @@
 
 [Setup]
 AppName=ClamAV
-AppVersion=0.103.11
+AppVersion=0.103.12
 DefaultDirName={pf}\ClamAV
 DefaultGroupName=ClamAV
 AppCopyright=2021 Cisco Systems, Inc.
@@ -20,7 +20,7 @@ UninstallDisplayName=ClamAV
 Compression=lzma2
 SolidCompression=yes
 OutputDir=.
-OutputBaseFilename=ClamAV-0.103.11
+OutputBaseFilename=ClamAV-0.103.12
 WizardImageFile=demon.bmp
 WizardSmallImageFile=talos.bmp