diff --git a/README.md b/README.md index 87d7216..57f47c4 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,25 @@ A micro-baseline to check for insecure or public S3 buckets and bucket objects in your AWS Environment. This [InSpec](https://github.com/chef/inspec) compliance profile verifies that you do not have any insure or open to public S3 Bucket or Bucket Objects in your AWS Environment in an automated way. +### Required Gems: + +This profile requires the following gems: + + - `inspec` + - `inspec-bin` + - `aws-sdk-s3` + - `concurrent-ruby` + #### Warning: Large amounts of Bucket Objects The `s3-objects-no-public-access` control iterates through and verifies every objects in each bucket in your AWS Environment, thus its runtime will depend on the number of objects in your S3 Buckets. +On average the profile can process around 500 - 1000 objects/sec. + If you have buckets with large numbers of objects, we suggest you script a loop and use the `single_bucket` input to parallelize the workload. +To see the processing in more details use the `-l debug` flag to get verbose output. + Then you can load all your HDF JSON results into [Heimdall Lite](https://heimdall-lite.mitre.org) to easily review all your scan results. ## Getting Started diff --git a/controls/aws_s3_bucket.rb b/controls/aws_s3_bucket.rb index e3c690f..c3bd1ec 100644 --- a/controls/aws_s3_bucket.rb +++ b/controls/aws_s3_bucket.rb @@ -1,4 +1,4 @@ -control 's3-buckets-no-public-access' do +control 'Public_S3_Buckets' do impact 0.7 title 'Ensure there are no publicly accessible S3 buckets' desc 'Ensure there are no publicly accessible S3 buckets' @@ -23,15 +23,15 @@ describe 'This control is Non Applicable since no S3 buckets were found.' do skip 'This control is Non Applicable since no S3 buckets were found.' end - elsif !input('single_bucket').to_s.empty? - describe aws_s3_bucket(input('single_bucket').to_s) do + elsif input('single_bucket').present? + describe aws_s3_bucket(input('single_bucket')) do it { should_not be_public } end else aws_s3_buckets.bucket_names.each do |bucket| next if exception_bucket_list.include?(bucket) - - describe aws_s3_bucket(bucket) do + describe "#{bucket}" do + subject { aws_s3_bucket(bucket) } it { should_not be_public } end end diff --git a/controls/aws_s3_bucket_objects.rb b/controls/aws_s3_bucket_objects.rb index 50ba8c0..5794394 100644 --- a/controls/aws_s3_bucket_objects.rb +++ b/controls/aws_s3_bucket_objects.rb @@ -1,38 +1,6 @@ -require 'concurrent' -module Aws::S3 - class Bucket - def objects(options = {}) - batches = Enumerator.new do |y| - options = options.merge(bucket: @name) - resp = @client.list_objects_v2(options) - resp.each_page do |page| - batch = [] - pool = Concurrent::FixedThreadPool.new(16) - mutex = Mutex.new - page.data.contents.each do |c| - #binding.pry - pool.post do - mutex.synchronize do - batch << ObjectSummary.new( - bucket_name: @name, - key: c.key, - data: c, - client: @client - ) - end - end - end - pool.shutdown - pool.wait_for_termination - y.yield(batch) - end - end - ObjectSummary::Collection.new(batches) - end - end -end +require_relative '../libraries/concurent_s3.rb' -control 's3-objects-no-public-access' do +control 'Public_S3_Objects' do impact 0.7 title 'Ensure there are no publicly accessible S3 objects' desc 'Ensure there are no publicly accessible S3 objects' @@ -50,29 +18,6 @@ def objects(options = {}) exception_bucket_list = input('exception_bucket_list') - def has_public_objects(myBucket) - - myPublicKeys = [] - s3 = Aws::S3::Resource.new() - pool = Concurrent::FixedThreadPool.new(56) - mutex = Mutex.new - s3.bucket(myBucket).objects.each do |object| - pool.post do - grants = object.acl.grants - if grants.map { |x| x.grantee.type }.any? { |x| x =~ %r{Group} } - if grants.map { |x| x.grantee.uri }.any? { |x| x =~ %r{AllUsers|AuthenticatedUsers} } - mutex.synchronize do - myPublicKeys << object.key - end - end - end - end - end - pool.shutdown - pool.wait_for_termination - myPublicKeys - end - if aws_s3_buckets.bucket_names.empty? impact 0.0 desc 'This control is Non Applicable since no S3 buckets were found.' @@ -80,11 +25,11 @@ def has_public_objects(myBucket) describe 'This control is Non Applicable since no S3 buckets were found.' do skip 'This control is Non Applicable since no S3 buckets were found.' end - elsif !input('single_bucket').to_s.empty? + elsif input('single_bucket').present? public_objects = has_public_objects(input('single_bucket').to_s) - describe "This bucket #{input('single_bucket').to_s}" do - it 'should not have Public Objects' do - failure_message = "The following items are public: #{public_objects.join(', ')}" + describe "#{input('single_bucket').to_s}" do + it 'should not have any public objects' do + failure_message = public_objects.count > 1 ? "#{public_objects.join(', ')} are public" : "#{public_objects.join(', ')} is public" expect(public_objects).to be_empty, failure_message end end @@ -92,9 +37,9 @@ def has_public_objects(myBucket) aws_s3_buckets.bucket_names.each do |bucket| next if exception_bucket_list.include?(bucket) public_objects_multi = has_public_objects(bucket.to_s) - describe "This bucket #{bucket}" do - it 'should not have Public Objects' do - failure_message = "The following items are public: #{public_objects_multi.join(', ')}" + describe "#{bucket}" do + it 'should not have any public objects' do + failure_message = "#{public_objects_multi.join(', ')} is public" expect(public_objects_multi).to be_empty, failure_message end end diff --git a/libraries/concurent_s3.rb b/libraries/concurent_s3.rb new file mode 100644 index 0000000..66eaa76 --- /dev/null +++ b/libraries/concurent_s3.rb @@ -0,0 +1,23 @@ +def has_public_objects(myBucket) + Inspec::Log.debug "Processing Bucket: #{myBucket}" + myPublicKeys = [] + s3 = Aws::S3::Resource.new() + pool = Concurrent::FixedThreadPool.new(56) + mutex = Mutex.new + s3.bucket(myBucket).objects.each do |object| + Inspec::Log.debug "Examining Key: #{object.key}" + pool.post do + grants = object.acl.grants + if grants.map { |x| x.grantee.type }.any? { |x| x =~ %r{Group} } + if grants.map { |x| x.grantee.uri }.any? { |x| x =~ %r{AllUsers|AuthenticatedUsers} } + mutex.synchronize do + myPublicKeys << object.key + end + end + end + end + end + pool.shutdown + pool.wait_for_termination + myPublicKeys + end \ No newline at end of file