Merge pull request bluesentry#96 from upsidetravel/cg_clamscan_fixes

clamav.py Download File Fixes
CitizenLabDotCo · Oct 29, 2019 · 4a34306 · 4a34306
2 parents 7aa1471 + 586e6d8
commit 4a34306
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 19 deletions.
diff --git a/README.md b/README.md
@@ -87,6 +87,7 @@ following policy document
        "Version":"2012-10-17",
        "Statement":[
           {
+             "Sid":"WriteCloudWatchLogs",
              "Effect":"Allow",
              "Action":[
                 "logs:CreateLogGroup",
@@ -96,18 +97,26 @@ following policy document
              "Resource":"*"
           },
           {
+             "Sid":"s3GetAndPutWithTagging",
              "Action":[
                 "s3:GetObject",
                 "s3:GetObjectTagging",
                 "s3:PutObject",
                 "s3:PutObjectTagging",
-                "s3:PutObjectVersionTagging",
-                "s3:ListBucket"
+                "s3:PutObjectVersionTagging"
              ],
              "Effect":"Allow",
              "Resource":[
-                "arn:aws:s3:::<bucket-name>",
-                "arn:aws:s3:::<bucket-name>/*"
+                "arn:aws:s3:::<av-definition-s3-bucket>/*"
+             ]
+          },
+          {
+             "Sid": "s3HeadObject",
+             "Effect": "Allow",
+             "Action": "s3:ListBucket",
+             "Resource": [
+                 "arn:aws:s3:::<av-definition-s3-bucket>/*",
+                 "arn:aws:s3:::<av-definition-s3-bucket>"
              ]
           }
        ]
@@ -144,6 +153,7 @@ following policy document
        "Version":"2012-10-17",
        "Statement":[
           {
+             "Sid":"WriteCloudWatchLogs",
              "Effect":"Allow",
              "Action":[
                 "logs:CreateLogGroup",
@@ -153,11 +163,13 @@ following policy document
              "Resource":"*"
           },
           {
+             "Sid":"s3AntiVirusScan",
              "Action":[
                 "s3:GetObject",
                 "s3:GetObjectTagging",
+                "s3:GetObjectVersion",
                 "s3:PutObjectTagging",
-                "s3:PutObjectVersionTagging",
+                "s3:PutObjectVersionTagging"
              ],
              "Effect":"Allow",
              "Resource": [
@@ -166,18 +178,20 @@ following policy document
              ]
           },
           {
+             "Sid":"s3AntiVirusDefinitions",
              "Action":[
                 "s3:GetObject",
-                "s3:GetObjectTagging",
+                "s3:GetObjectTagging"
              ],
              "Effect":"Allow",
              "Resource": [
                "arn:aws:s3:::<av-definition-s3-bucket>/*"
              ]
           },
           {
+             "Sid":"kmsDecrypt",
              "Action":[
-                "kms:Decrypt",
+                "kms:Decrypt"
              ],
              "Effect":"Allow",
              "Resource": [
@@ -186,14 +200,24 @@ following policy document
              ]
           },
           {
-             "Action":[
-                "sns:Publish",
+             "Sid":"snsPublish",
+             "Action": [
+                "sns:Publish"
              ],
              "Effect":"Allow",
              "Resource": [
                "arn:aws:sns:::<av-scan-start>",
                "arn:aws:sns:::<av-status>"
              ]
+          },
+          {
+             "Sid":"s3HeadObject",
+             "Effect":"Allow",
+             "Action":"s3:ListBucket",
+             "Resource":[
+                 "arn:aws:s3:::<av-definition-s3-bucket>/*",
+                 "arn:aws:s3:::<av-definition-s3-bucket>"
+             ]
           }
        ]
     }

diff --git a/clamav.py b/clamav.py
@@ -28,6 +28,8 @@
 from common import AV_DEFINITION_PATH
 from common import AV_DEFINITION_FILE_PREFIXES
 from common import AV_DEFINITION_FILE_SUFFIXES
+from common import AV_SIGNATURE_OK
+from common import AV_SIGNATURE_UNKNOWN
 from common import AV_STATUS_CLEAN
 from common import AV_STATUS_INFECTED
 from common import CLAMAVLIB_PATH
@@ -67,10 +69,6 @@ def update_defs_from_s3(s3_client, bucket, prefix):
                 print("Not downloading %s because local md5 matches s3." % filename)
                 continue
             if s3_md5:
-                print(
-                    "Downloading definition file %s from s3://%s"
-                    % (filename, os.path.join(bucket, prefix))
-                )
                 to_download[file_prefix] = {
                     "s3_path": s3_path,
                     "local_path": local_path,
@@ -196,10 +194,10 @@ def scan_file(path):
 
     # Turn the output into a data source we can read
     summary = scan_output_to_json(output)
-    signature = summary[path]
     if av_proc.returncode == 0:
-        return AV_STATUS_CLEAN, signature
+        return AV_STATUS_CLEAN, AV_SIGNATURE_OK
     elif av_proc.returncode == 1:
+        signature = summary.get(path, AV_SIGNATURE_UNKNOWN)
         return AV_STATUS_INFECTED, signature
     else:
         msg = "Unexpected exit code from clamscan: %s.\n" % av_proc.returncode

diff --git a/scan.py b/scan.py
@@ -222,9 +222,17 @@ def lambda_handler(event, context):
     file_path = get_local_path(s3_object, "/tmp")
     create_dir(os.path.dirname(file_path))
     s3_object.download_file(file_path)
-    clamav.update_defs_from_s3(
+
+    to_download = clamav.update_defs_from_s3(
         s3_client, AV_DEFINITION_S3_BUCKET, AV_DEFINITION_S3_PREFIX
     )
+
+    for download in to_download.values():
+        s3_path = download["s3_path"]
+        local_path = download["local_path"]
+        print("Downloading definition file %s from s3://%s" % (local_path, s3_path))
+        s3.Bucket(AV_DEFINITION_S3_BUCKET).download_file(s3_path, local_path)
+        print("Downloading definition file %s complete!" % (local_path))
     scan_result, scan_signature = clamav.scan_file(file_path)
     print(
         "Scan of s3://%s resulted in %s\n"

diff --git a/update.py b/update.py
@@ -35,9 +35,11 @@ def lambda_handler(event, context):
     )
 
     for download in to_download.values():
-        s3.Bucket(AV_DEFINITION_S3_BUCKET).download_file(
-            download["s3_path"], download["local_path"]
-        )
+        s3_path = download["s3_path"]
+        local_path = download["local_path"]
+        print("Downloading definition file %s from s3://%s" % (local_path, s3_path))
+        s3.Bucket(AV_DEFINITION_S3_BUCKET).download_file(s3_path, local_path)
+        print("Downloading definition file %s complete!" % (local_path))
 
     clamav.update_defs_from_freshclam(AV_DEFINITION_PATH, CLAMAVLIB_PATH)
     # If main.cvd gets updated (very rare), we will need to force freshclam