moved supporting code to libs, updated readme, added required gems

Signed-off-by: GitHub <[email protected]>

README.md

@@ -2,12 +2,25 @@
 
 A micro-baseline to check for insecure or public S3 buckets and bucket objects in your AWS Environment. This [InSpec](https://github.com/chef/inspec) compliance profile verifies that you do not have any insure or open to public S3 Bucket or Bucket Objects in your AWS Environment in an automated way.
 
+### Required Gems:
+
+This profile requires the following gems:
+
+  - `inspec`
+  - `inspec-bin`
+  - `aws-sdk-s3`
+  - `concurrent-ruby`
+
 #### Warning: Large amounts of Bucket Objects
 
 The `s3-objects-no-public-access` control iterates through and verifies every objects in each bucket in your AWS Environment, thus its runtime will depend on the number of objects in your S3 Buckets.
 
+On average the profile can process around 500 - 1000 objects/sec.
+
 If you have buckets with large numbers of objects, we suggest you script a loop and use the `single_bucket` input to parallelize the workload.
 
+To see the processing in more details use the `-l debug` flag to get verbose output.
+
 Then you can load all your HDF JSON results into [Heimdall Lite](https://heimdall-lite.mitre.org) to easily review all your scan results.
 
 ## Getting Started

controls/aws_s3_bucket.rb

@@ -1,4 +1,4 @@
-control 's3-buckets-no-public-access' do
+control 'Public_S3_Buckets' do
   impact 0.7
   title 'Ensure there are no publicly accessible S3 buckets'
   desc 'Ensure there are no publicly accessible S3 buckets'
@@ -23,15 +23,15 @@
     describe 'This control is Non Applicable since no S3 buckets were found.' do
       skip 'This control is Non Applicable since no S3 buckets were found.'
     end
-  elsif !input('single_bucket').to_s.empty?
-    describe aws_s3_bucket(input('single_bucket').to_s) do
+  elsif input('single_bucket').present?
+    describe aws_s3_bucket(input('single_bucket')) do
       it { should_not be_public }
     end
   else
     aws_s3_buckets.bucket_names.each do |bucket|
       next if exception_bucket_list.include?(bucket)
-
-      describe aws_s3_bucket(bucket) do
+      describe "#{bucket}" do
+        subject { aws_s3_bucket(bucket) }
         it { should_not be_public }
       end
     end

controls/aws_s3_bucket_objects.rb

@@ -1,38 +1,6 @@
-require 'concurrent'
-module Aws::S3
-  class Bucket
-    def objects(options = {})
-      batches = Enumerator.new do |y|
-        options = options.merge(bucket: @name)
-        resp = @client.list_objects_v2(options)
-        resp.each_page do |page|
-          batch = []
-          pool = Concurrent::FixedThreadPool.new(16)
-          mutex = Mutex.new
-          page.data.contents.each do |c|
-            #binding.pry
-            pool.post do
-              mutex.synchronize do
-                batch << ObjectSummary.new(
-                  bucket_name: @name,
-                  key: c.key,
-                  data: c,
-                  client: @client
-                )
-              end
-            end
-          end
-          pool.shutdown
-          pool.wait_for_termination
-          y.yield(batch)
-        end
-      end
-      ObjectSummary::Collection.new(batches)
-    end
-  end
-end
+require_relative '../libraries/concurent_s3.rb'
 
-control 's3-objects-no-public-access' do
+control 'Public_S3_Objects' do
   impact 0.7
   title 'Ensure there are no publicly accessible S3 objects'
   desc 'Ensure there are no publicly accessible S3 objects'
@@ -50,51 +18,28 @@ def objects(options = {})
 
   exception_bucket_list = input('exception_bucket_list')
 
-  def has_public_objects(myBucket)
-
-    myPublicKeys = []
-    s3 = Aws::S3::Resource.new()
-    pool = Concurrent::FixedThreadPool.new(56)
-    mutex = Mutex.new
-    s3.bucket(myBucket).objects.each do |object|
-      pool.post do
-        grants = object.acl.grants 
-        if grants.map { |x| x.grantee.type }.any? { |x| x =~ %r{Group} }
-          if grants.map { |x| x.grantee.uri }.any? { |x| x =~ %r{AllUsers|AuthenticatedUsers} }
-            mutex.synchronize do
-            myPublicKeys << object.key
-            end
-              end
-        end
-        end
-    end  
-    pool.shutdown
-    pool.wait_for_termination
-    myPublicKeys
-  end
-
   if aws_s3_buckets.bucket_names.empty?
     impact 0.0
     desc 'This control is Non Applicable since no S3 buckets were found.'
 
     describe 'This control is Non Applicable since no S3 buckets were found.' do
       skip 'This control is Non Applicable since no S3 buckets were found.'
     end
-  elsif !input('single_bucket').to_s.empty?
+  elsif input('single_bucket').present?
     public_objects = has_public_objects(input('single_bucket').to_s)
-    describe "This bucket #{input('single_bucket').to_s}" do
-      it 'should not have Public Objects' do
-        failure_message = "The following items are public: #{public_objects.join(', ')}"
+    describe "#{input('single_bucket').to_s}" do
+      it 'should not have any public objects' do
+        failure_message = public_objects.count > 1 ? "#{public_objects.join(', ')} are public" : "#{public_objects.join(', ')} is public"
         expect(public_objects).to be_empty, failure_message
       end
     end
   else
     aws_s3_buckets.bucket_names.each do |bucket|
       next if exception_bucket_list.include?(bucket)
       public_objects_multi = has_public_objects(bucket.to_s)
-      describe "This bucket #{bucket}" do
-        it 'should not have Public Objects' do
-          failure_message = "The following items are public: #{public_objects_multi.join(', ')}"
+      describe "#{bucket}" do
+        it 'should not have any public objects' do
+          failure_message = "#{public_objects_multi.join(', ')} is public"
           expect(public_objects_multi).to be_empty, failure_message
         end
       end

libraries/concurent_s3.rb

@@ -0,0 +1,23 @@
+def has_public_objects(myBucket)
+    Inspec::Log.debug "Processing Bucket: #{myBucket}"
+    myPublicKeys = []
+    s3 = Aws::S3::Resource.new()
+    pool = Concurrent::FixedThreadPool.new(56)
+    mutex = Mutex.new
+    s3.bucket(myBucket).objects.each do |object|
+      Inspec::Log.debug "Examining Key: #{object.key}"
+      pool.post do
+        grants = object.acl.grants 
+        if grants.map { |x| x.grantee.type }.any? { |x| x =~ %r{Group} }
+          if grants.map { |x| x.grantee.uri }.any? { |x| x =~ %r{AllUsers|AuthenticatedUsers} }
+            mutex.synchronize do
+            myPublicKeys << object.key
+            end
+              end
+        end
+        end
+    end  
+    pool.shutdown
+    pool.wait_for_termination
+    myPublicKeys
+  end