Clam 2498 make image fuzzy hashing optional; Clam 2532 --force-to-disk missing documentation

clamd/server-th.c

@@ -1118,6 +1118,20 @@ int recvloop(int *socketds, unsigned nsockets, struct cl_engine *engine, unsigne
         logg(LOGG_INFO, "Archive support disabled.\n");
     }
 
+    if (optget(opts, "ScanImage")->enabled) {
+        logg(LOGG_INFO, "Image (graphics) scanning support enabled.\n");
+        options.parse |= CL_SCAN_PARSE_IMAGE;
+    } else {
+        logg(LOGG_INFO, "Image (graphics) scanning support disabled.\n");
+    }
+
+    if (optget(opts, "ScanImageFuzzyHash")->enabled) {
+        logg(LOGG_INFO, "Detection using image fuzzy hash enabled.\n");
+        options.parse |= CL_SCAN_PARSE_IMAGE_FUZZY_HASH;
+    } else {
+        logg(LOGG_INFO, "Detection using image fuzzy hash disabled.\n");
+    }
+
     /* TODO: Remove deprecated option in a future feature release. */
     if (optget(opts, "ArchiveBlockEncrypted")->enabled) {
         if (options.parse & CL_SCAN_PARSE_ARCHIVE) {

clamscan/clamscan.c

@@ -250,6 +250,7 @@ void help(void)
     mprintf(LOGG_INFO, "\n");
     mprintf(LOGG_INFO, "    --tempdir=DIRECTORY                  Create temporary files in DIRECTORY\n");
     mprintf(LOGG_INFO, "    --leave-temps[=yes/no(*)]            Do not remove temporary files\n");
+    mprintf(LOGG_INFO, "    --force-to-disk[=yes/no(*)]          Create temporary files for nested file scans that would otherwise be in-memory only\n");
     mprintf(LOGG_INFO, "    --gen-json[=yes/no(*)]               Generate JSON metadata for the scanned file(s). For testing & development use ONLY.\n");
     mprintf(LOGG_INFO, "                                         JSON will be printed if --debug is enabled.\n");
     mprintf(LOGG_INFO, "                                         A JSON file will dropped to the temp directory if --leave-temps is enabled.\n");
@@ -306,6 +307,8 @@ void help(void)
     mprintf(LOGG_INFO, "    --scan-hwp3[=yes(*)/no]              Scan HWP3 files\n");
     mprintf(LOGG_INFO, "    --scan-onenote[=yes(*)/no]           Scan OneNote files\n");
     mprintf(LOGG_INFO, "    --scan-archive[=yes(*)/no]           Scan archive files (supported by libclamav)\n");
+    mprintf(LOGG_INFO, "    --scan-image[=yes(*)/no]             Scan image (graphics) files\n");
+    mprintf(LOGG_INFO, "    --scan-image-fuzzy-hash[=yes(*)/no]  Detect files by calculating image (graphics) fuzzy hashes\n");
     mprintf(LOGG_INFO, "    --alert-broken[=yes/no(*)]           Alert on broken executable files (PE & ELF)\n");
     mprintf(LOGG_INFO, "    --alert-broken-media[=yes/no(*)]     Alert on broken graphics files (JPEG, TIFF, PNG, GIF)\n");
     mprintf(LOGG_INFO, "    --alert-encrypted[=yes/no(*)]        Alert on encrypted archives and documents\n");

clamscan/manager.c

@@ -1555,6 +1555,12 @@ int scanmanager(const struct optstruct *opts)
     if (optget(opts, "scan-onenote")->enabled)
         options.parse |= CL_SCAN_PARSE_ONENOTE;
 
+    if (optget(opts, "scan-image")->enabled)
+        options.parse |= CL_SCAN_PARSE_IMAGE;
+
+    if (optget(opts, "scan-image-fuzzy-hash")->enabled)
+        options.parse |= CL_SCAN_PARSE_IMAGE_FUZZY_HASH;
+
     /* TODO: Remove deprecated option in a future feature release */
     if ((optget(opts, "algorithmic-detection")->enabled) && /* && used due to default-yes for both options */
         (optget(opts, "heuristic-alerts")->enabled)) {

common/optparser.c

@@ -438,6 +438,10 @@ const struct clam_option __clam_options[] = {
 
     {"ScanArchive", "scan-archive", 0, CLOPT_TYPE_BOOL, MATCH_BOOL, 1, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "Scan within archives and compressed files.\nIf you turn off this option, the original files will still be scanned, but\nwithout unpacking and additional processing.", "yes"},
 
+    {"ScanImage", "scan-image", 0, CLOPT_TYPE_BOOL, MATCH_BOOL, 1, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "This option enables scanning of image (graphics).\nIf you turn off this option, the original files will still be scanned, but without additional processing.", "yes"},
+
+    {"ScanImageFuzzyHash", "scan-image-fuzzy-hash", 0, CLOPT_TYPE_BOOL, MATCH_BOOL, 1, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "This option enables detection by calculating a fuzzy hash of image (graphics)\nfiles\nSignatures using image fuzzy hashes typically match files and documents by\nidentifying images embedded or attached to those files.\nIf you turn off this option, then some files may no longer be detected.", "yes"},
+
     {"ForceToDisk", "force-to-disk", 0, CLOPT_TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "This option causes memory or nested map scans to dump the content to disk.\nIf you turn on this option, more data is written to disk and is available\nwhen the leave-temps option is enabled at the cost of more disk writes.", "no"},
 
     {"MaxScanTime", "max-scantime", 0, CLOPT_TYPE_NUMBER, MATCH_NUMBER, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "This option sets the maximum amount of time a scan may take to complete.\nThe value of 0 disables the limit.\nWARNING: disabling this limit or setting it too high may result allow scanning\nof certain files to lock up the scanning process/threads resulting in a Denial of Service.\nThe value is in milliseconds.", "120000"},

common/optparser.h

@@ -33,8 +33,8 @@
 #define OPT_CLAMCONF        64
 #define OPT_CLAMDTOP        128
 #define OPT_CLAMBC          256
-#define OPT_CLAMONACC    512
-#define OPT_DEPRECATED	1024
+#define OPT_CLAMONACC       512
+#define OPT_DEPRECATED	    1024
 
 #define CLOPT_TYPE_STRING   1    /* quoted/regular string */
 #define CLOPT_TYPE_NUMBER   2    /* raw number */

docs/man/clamd.conf.5.in

@@ -499,6 +499,20 @@ If you turn off this option, the original files will still be scanned, but witho
 .br
 Default: yes
 .TP
+\fBScanImage BOOL\fR
+This option enables scanning of image (graphics).
+.br
+If you turn off this option, the original files will still be scanned, but without unpacking and additional processing.
+.br
+Default: yes
+.TP
+\fBScanImageFuzzyHash BOOL\fR
+This option enables detection by calculating a fuzzy hash of image (graphics) files. Signatures using image fuzzy hashes typically match files and documents by identifying images embedded or attached to those files.
+.br
+If you turn off this option, then some files may no longer be detected.
+.br
+Default: yes
+.TP
 \fBAlertBrokenExecutables BOOL\fR
 Alert on broken executable files (PE & ELF).
 .br

docs/man/clamscan.1.in

@@ -51,6 +51,9 @@ Create temporary files in DIRECTORY. Directory must be writable for the '@CLAMAV
 \fB\-\-leave\-temps\fR
 Do not remove temporary files.
 .TP
+\fB\-\-force\-to\-disk\fR
+This option causes memory or nested map scans to dump the content to disk. If you turn on this option, more data is written to disk and is available when the LeaveTemporaryFiles option is enabled.
+.TP
 \fB\-\-gen\-json\fR
 Generate JSON description of scanned file(s). JSON will be printed and also dropped to the temp directory if --leave-temps is enabled.
 .TP
@@ -177,6 +180,12 @@ Scan HWP3 files. If you turn off this option, the original files will still be s
 \fB\-\-scan\-archive[=yes(*)/no]\fR
 Scan archives supported by libclamav. If you turn off this option, the original files will still be scanned, but without unpacking and additional processing.
 .TP
+\fB\-\-scan\-image[=yes(*)/no]\fR
+This option enables scanning of image (graphics). If you turn off this option, the original files will still be scanned, but without additional processing.
+.TP
+\fB\-\-scan\-image\-fuzzy\-hash[=yes(*)/no]\fR
+This option enables detection by calculating a fuzzy hash of image (graphics) files. Signatures using image fuzzy hashes typically match files and documents by identifying images embedded or attached to those files. If you turn off this option, then some files may no longer be detected.
+.TP
 \fB\-\-alert\-broken[=yes/no(*)]\fR
 Alert on broken executable files (PE & ELF).
 .TP

etc/clamd.conf.sample

@@ -111,7 +111,7 @@ Example
 
 # Remove stale socket after unclean shutdown.
 # Default: yes
-#FixStaleSocket yes
+#FixStaleSocket no
 
 # TCP port address.
 # Default: no
@@ -199,7 +199,7 @@ Example
 
 # Scan files and directories on other filesystems.
 # Default: yes
-#CrossFilesystems yes
+#CrossFilesystems no
 
 # Perform a database check.
 # Default: 600 (10 min)
@@ -299,7 +299,7 @@ Example
 # may be malicious.  This option enables alerting on such heuristically
 # detected potential threats.
 # Default: yes
-#HeuristicAlerts yes
+#HeuristicAlerts no
 
 # Allow heuristic alerts to take precedence.
 # When enabled, if a heuristic scan (such as phishingScan) detects
@@ -377,7 +377,7 @@ Example
 # and Petite. If you turn off this option, the original files will still be
 # scanned, but without additional processing.
 # Default: yes
-#ScanPE yes
+#ScanPE no
 
 # Certain PE files contain an authenticode signature. By default, we check
 # the signature chain in the PE file against a database of trusted and
@@ -394,7 +394,7 @@ Example
 # If you turn off this option, the original files will still be scanned, but
 # without additional processing.
 # Default: yes
-#ScanELF yes
+#ScanELF no
 
 
 ##
@@ -406,37 +406,56 @@ Example
 # If you turn off this option, the original files will still be scanned, but
 # without additional processing.
 # Default: yes
-#ScanOLE2 yes
+#ScanOLE2 no
 
 # This option enables scanning within PDF files.
 # If you turn off this option, the original files will still be scanned, but
 # without decoding and additional processing.
 # Default: yes
-#ScanPDF yes
+#ScanPDF no
 
 # This option enables scanning within SWF files.
 # If you turn off this option, the original files will still be scanned, but
 # without decoding and additional processing.
 # Default: yes
-#ScanSWF yes
+#ScanSWF no
 
 # This option enables scanning xml-based document files supported by libclamav.
 # If you turn off this option, the original files will still be scanned, but
 # without additional processing.
 # Default: yes
-#ScanXMLDOCS yes
+#ScanXMLDOCS no
 
 # This option enables scanning of HWP3 files.
 # If you turn off this option, the original files will still be scanned, but
 # without additional processing.
 # Default: yes
-#ScanHWP3 yes
+#ScanHWP3 no
 
 # This option enables scanning of OneNote files.
 # If you turn off this option, the original files will still be scanned, but
 # without additional processing.
 # Default: yes
-#ScanOneNote yes
+#ScanOneNote no
+
+
+##
+## Other file types
+##
+
+# This option enables scanning of image (graphics).
+# If you turn off this option, the original files will still be scanned, but
+# without additional processing.
+# Default: yes
+#ScanImage no
+
+# This option enables detection by calculating a fuzzy hash of image (graphics)
+# files.
+# Signatures using image fuzzy hashes typically match files and documents by
+# identifying images embedded or attached to those files.
+# If you turn off this option, then some files may no longer be detected.
+# Default: yes
+#ScanImageFuzzyHash no
 
 
 ##
@@ -447,7 +466,7 @@ Example
 # If you turn off this option, the original files will still be scanned, but
 # without parsing individual messages/attachments.
 # Default: yes
-#ScanMail yes
+#ScanMail no
 
 # Scan RFC1341 messages split over many emails.
 # You will need to periodically clean up $TemporaryDirectory/clamav-partial
@@ -494,7 +513,7 @@ Example
 # With this option enabled the DLP module will search for valid
 # SSNs formatted as xxx-yy-zzzz
 # Default: yes
-#StructuredSSNFormatNormal yes
+#StructuredSSNFormatNormal no
 
 # With this option enabled the DLP module will search for valid
 # SSNs formatted as xxxyyzzzz
@@ -510,7 +529,7 @@ Example
 # Default: yes
 # If you turn off this option, the original files will still be scanned, but
 # without additional processing.
-#ScanHTML yes
+#ScanHTML no
 
 
 ##
@@ -521,7 +540,7 @@ Example
 # If you turn off this option, the original files will still be scanned, but
 # without unpacking and additional processing.
 # Default: yes
-#ScanArchive yes
+#ScanArchive no
 
 
 ##
@@ -793,7 +812,7 @@ Example
 # It is highly recommended you keep this option on, otherwise you'll miss
 # detections for many new viruses.
 # Default: yes
-#Bytecode yes
+#Bytecode no
 
 # Set bytecode security level.
 # Possible values:

libclamav/clamav.h

@@ -181,6 +181,8 @@ struct cl_scan_options {
 #define CL_SCAN_PARSE_HTML                          0x100
 #define CL_SCAN_PARSE_PE                            0x200
 #define CL_SCAN_PARSE_ONENOTE                       0x400
+#define CL_SCAN_PARSE_IMAGE                         0x800  /* option to enable/disable parsing images (graphics) */
+#define CL_SCAN_PARSE_IMAGE_FUZZY_HASH              0x1000 /* option to enable/disable image fuzzy hash calculation. */
 
 /* heuristic alerting options */
 #define CL_SCAN_HEURISTIC_BROKEN                    0x2    /* alert on broken PE and broken ELF files */

libclamav/dconf.c

@@ -137,6 +137,7 @@ static struct dconf_module modules[] = {
     {"OTHER", "GIF", OTHER_CONF_GIF, 1},
     {"OTHER", "PNG", OTHER_CONF_PNG, 1},
     {"OTHER", "TIFF", OTHER_CONF_TIFF, 1},
+    {"OTHER", "IMAGE FUZZY HASH", OTHER_CONF_IMAGE_FUZZY_HASH, 1},
 
     {"PHISHING", "ENGINE", PHISHING_CONF_ENGINE, 1},
     {"PHISHING", "ENTCONV", PHISHING_CONF_ENTCONV, 1},

libclamav/dconf.h

@@ -116,20 +116,21 @@ struct cli_dconf {
 #define MAIL_CONF_TNEF 0x2
 
 /* Other flags */
-#define OTHER_CONF_UUENC        0x1
-#define OTHER_CONF_SCRENC       0x2
-#define OTHER_CONF_RIFF         0x4
-#define OTHER_CONF_JPEG         0x8
-#define OTHER_CONF_CRYPTFF      0x10
-#define OTHER_CONF_DLP          0x20
-#define OTHER_CONF_MYDOOMLOG    0x40
-#define OTHER_CONF_PREFILTERING 0x80
-#define OTHER_CONF_PDFNAMEOBJ   0x100
-#define OTHER_CONF_PRTNINTXN    0x200
-#define OTHER_CONF_LZW          0x400
-#define OTHER_CONF_PNG          0x800
-#define OTHER_CONF_GIF          0x1000
-#define OTHER_CONF_TIFF         0x2000
+#define OTHER_CONF_UUENC            0x1
+#define OTHER_CONF_SCRENC           0x2
+#define OTHER_CONF_RIFF             0x4
+#define OTHER_CONF_JPEG             0x8
+#define OTHER_CONF_CRYPTFF          0x10
+#define OTHER_CONF_DLP              0x20
+#define OTHER_CONF_MYDOOMLOG        0x40
+#define OTHER_CONF_PREFILTERING     0x80
+#define OTHER_CONF_PDFNAMEOBJ       0x100
+#define OTHER_CONF_PRTNINTXN        0x200
+#define OTHER_CONF_LZW              0x400
+#define OTHER_CONF_PNG              0x800
+#define OTHER_CONF_GIF              0x1000
+#define OTHER_CONF_TIFF             0x2000
+#define OTHER_CONF_IMAGE_FUZZY_HASH 0x4000
 
 /* Phishing flags */
 #define PHISHING_CONF_ENGINE  0x1

libclamav/others.h

@@ -570,6 +570,8 @@ extern LIBCLAMAV_EXPORT int have_rar;
 #define SCAN_PARSE_HTML (ctx->options->parse & CL_SCAN_PARSE_HTML)
 #define SCAN_PARSE_PE (ctx->options->parse & CL_SCAN_PARSE_PE)
 #define SCAN_PARSE_ONENOTE (ctx->options->parse & CL_SCAN_PARSE_ONENOTE)
+#define SCAN_PARSE_IMAGE (ctx->options->parse & CL_SCAN_PARSE_IMAGE)
+#define SCAN_PARSE_IMAGE_FUZZY_HASH (ctx->options->parse & CL_SCAN_PARSE_IMAGE_FUZZY_HASH)
 
 #define SCAN_HEURISTIC_BROKEN (ctx->options->heuristic & CL_SCAN_HEURISTIC_BROKEN)
 #define SCAN_HEURISTIC_BROKEN_MEDIA (ctx->options->heuristic & CL_SCAN_HEURISTIC_BROKEN_MEDIA)
@@ -591,16 +593,16 @@ extern LIBCLAMAV_EXPORT int have_rar;
 
 /* based on macros from A. Melnikoff */
 #define cbswap16(v) (((v & 0xff) << 8) | (((v) >> 8) & 0xff))
-#define cbswap32(v) ((((v)&0x000000ff) << 24) | (((v)&0x0000ff00) << 8) | \
-                     (((v)&0x00ff0000) >> 8) | (((v)&0xff000000) >> 24))
-#define cbswap64(v) ((((v)&0x00000000000000ffULL) << 56) | \
-                     (((v)&0x000000000000ff00ULL) << 40) | \
-                     (((v)&0x0000000000ff0000ULL) << 24) | \
-                     (((v)&0x00000000ff000000ULL) << 8) |  \
-                     (((v)&0x000000ff00000000ULL) >> 8) |  \
-                     (((v)&0x0000ff0000000000ULL) >> 24) | \
-                     (((v)&0x00ff000000000000ULL) >> 40) | \
-                     (((v)&0xff00000000000000ULL) >> 56))
+#define cbswap32(v) ((((v) & 0x000000ff) << 24) | (((v) & 0x0000ff00) << 8) | \
+                     (((v) & 0x00ff0000) >> 8) | (((v) & 0xff000000) >> 24))
+#define cbswap64(v) ((((v) & 0x00000000000000ffULL) << 56) | \
+                     (((v) & 0x000000000000ff00ULL) << 40) | \
+                     (((v) & 0x0000000000ff0000ULL) << 24) | \
+                     (((v) & 0x00000000ff000000ULL) << 8) |  \
+                     (((v) & 0x000000ff00000000ULL) >> 8) |  \
+                     (((v) & 0x0000ff0000000000ULL) >> 24) | \
+                     (((v) & 0x00ff000000000000ULL) >> 40) | \
+                     (((v) & 0xff00000000000000ULL) >> 56))
 
 #ifndef HAVE_ATTRIB_PACKED
 #define __attribute__(x)
@@ -826,8 +828,8 @@ size_t cli_recursion_stack_get_size(cli_ctx *ctx, int index);
 /* used by: spin, yc (C) aCaB */
 #define __SHIFTBITS(a) (sizeof(a) << 3)
 #define __SHIFTMASK(a) (__SHIFTBITS(a) - 1)
-#define CLI_ROL(a, b) a = (a << ((b)&__SHIFTMASK(a))) | (a >> ((__SHIFTBITS(a) - (b)) & __SHIFTMASK(a)))
-#define CLI_ROR(a, b) a = (a >> ((b)&__SHIFTMASK(a))) | (a << ((__SHIFTBITS(a) - (b)) & __SHIFTMASK(a)))
+#define CLI_ROL(a, b) a = (a << ((b) & __SHIFTMASK(a))) | (a >> ((__SHIFTBITS(a) - (b)) & __SHIFTMASK(a)))
+#define CLI_ROR(a, b) a = (a >> ((b) & __SHIFTMASK(a))) | (a << ((__SHIFTBITS(a) - (b)) & __SHIFTMASK(a)))
 
 /* Implementation independent sign-extended signed right shift */
 #ifdef HAVE_SAR