From 4d3c968f368bb4ad1c7367ce339cde4638344318 Mon Sep 17 00:00:00 2001 From: Aaron Lippold Date: Sun, 8 Sep 2024 23:15:45 -0400 Subject: [PATCH] should be all fixed and working now Signed-off-by: Aaron Lippold --- .rubocop.yml | 60 +++++++ Gemfile | 17 +- Gemfile.lock | 8 + LICENSE.md | 18 +- NOTICE.md | 7 + README.md | 127 ++++++++------ Rakefile | 16 ++ controls/aws_s3_bucket.rb | 59 ++++--- controls/aws_s3_bucket_objects.rb | 95 +++++----- inspec.yml | 36 ++-- libraries/list_public_s3_objects.rb | 105 +++++++++++ .../concurrent_s3-cop | 0 .../concurrent_s3-orig | 3 +- old-library-examples/concurrent_s3.rb | 165 ++++++++++++++++++ old-library-examples/public-objects-2-done.rb | 92 ++++++++++ old-library-examples/public-objects.rb | 57 ++++++ spec/benchmark.rb | 83 +++++++++ 17 files changed, 795 insertions(+), 153 deletions(-) create mode 100644 .rubocop.yml create mode 100644 NOTICE.md create mode 100644 Rakefile create mode 100644 libraries/list_public_s3_objects.rb rename {libraries => old-library-examples}/concurrent_s3-cop (100%) rename libraries/concurrent_s3.rb => old-library-examples/concurrent_s3-orig (95%) create mode 100644 old-library-examples/concurrent_s3.rb create mode 100644 old-library-examples/public-objects-2-done.rb create mode 100644 old-library-examples/public-objects.rb create mode 100644 spec/benchmark.rb diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..0d026ea --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,60 @@ +AllCops: + Exclude: + - "libraries/**/*" + - "old-library-examples/**/*" + - "spec/**/*" + +Layout/LineLength: + Max: 1500 + AllowURI: true + IgnoreCopDirectives: true + +Naming/FileName: + Enabled: false + +Metrics/BlockLength: + Max: 1000 + +Lint/ConstantDefinitionInBlock: + Enabled: false + +# Required for Profiles as it can introduce profile errors +Style/NumericPredicate: + Enabled: false + +Style/WordArray: + Description: "Use %w or %W for an array of words. (https://rubystyle.guide#percent-w)" + Enabled: false + +Style/RedundantPercentQ: + Enabled: true + +Style/NestedParenthesizedCalls: + Enabled: false + +Style/TrailingCommaInHashLiteral: + Description: "https://docs.rubocop.org/rubocop/cops_style.html#styletrailingcommainhashliteral" + Enabled: true + EnforcedStyleForMultiline: no_comma + +Style/TrailingCommaInArrayLiteral: + Enabled: true + EnforcedStyleForMultiline: no_comma + +Style/BlockDelimiters: + Enabled: false + +Lint/AmbiguousBlockAssociation: + Enabled: false + +Metrics/BlockNesting: + Enabled: false + +Lint/ShadowingOuterLocalVariable: + Enabled: false + +Style/FormatStringToken: + Enabled: false + +Style/FrozenStringLiteralComment: + Enabled: false diff --git a/Gemfile b/Gemfile index 321a97b..5919328 100644 --- a/Gemfile +++ b/Gemfile @@ -1,8 +1,15 @@ # frozen_string_literal: true -source "https://rubygems.org" +source 'https://rubygems.org' -gem "inspec" -gem "inspec-bin" -gem "concurrent-ruby" -gem "pry-byebug" +gem 'concurrent-ruby' +gem 'inspec' +gem 'inspec-bin' +gem 'pry-byebug' +gem 'rubocop-rake' +gem 'rubocop-rspec' + +group :development, :test do + gem 'rake' + gem 'rspec' +end diff --git a/Gemfile.lock b/Gemfile.lock index cdbe08c..d042927 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -551,6 +551,10 @@ GEM unicode-display_width (>= 1.4.0, < 3.0) rubocop-ast (1.32.3) parser (>= 3.3.1.0) + rubocop-rake (0.6.0) + rubocop (~> 1.0) + rubocop-rspec (2.11.1) + rubocop (~> 1.19) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyntlm (0.6.5) @@ -744,6 +748,10 @@ DEPENDENCIES inspec inspec-bin pry-byebug + rake + rspec + rubocop-rake + rubocop-rspec BUNDLED WITH 2.4.22 diff --git a/LICENSE.md b/LICENSE.md index d1af25f..f2222e8 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,20 +1,10 @@ Licensed under the apache-2.0 license, except as noted below. -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright/ digital rights -legend, this list of conditions and the following Notice. +* Redistributions of source code must retain the above copyright/ digital rights legend, this list of conditions and the following Notice. -* Redistributions in binary form must reproduce the above copyright copyright/digital -rights legend, this list of conditions and the following Notice in the documentation -and/or other materials provided with the distribution. +* Redistributions in binary form must reproduce the above copyright copyright/ digital rights legend, this list of conditions and the following Notice in the documentation and/or other materials provided with the distribution. -* Neither the name of The MITRE Corporation nor the names of its contributors may be -used to endorse or promote products derived from this software without specific prior -written permission. - -MITRE’s licensed products incorporate third-party materials that are subject to open source or free software licenses (“Open Source Materials”). The Open Source Materials are as follows: - -The Open Source Materials are licensed under the terms of the applicable third-party licenses that accompany the Open Source Materials. MITRE’s license does not limit a licensee’s rights under the terms of the Open Source Materials license. MITRE’s license also does not grant licensee rights to the Open Source Materials that supersede the terms and conditions of the Open Source Materials license. +* Neither the name of The MITRE Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. \ No newline at end of file diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 0000000..080d2e8 --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,7 @@ +# NOTICE + +MITRE grants permission to reproduce, distribute, modify, and otherwise use this software to the extent permitted by the licensed terms provided in the LICENSE.md file included with this project. + +This software was produced by The MITRE Corporation for the U. S. Government under contract. As such the U.S. Government has certain use and data rights in this software. No use other than those granted to the U. S. Government, or to those acting on behalf of the U. S. Government, under these contract arrangements is authorized without the express written permission of The MITRE Corporation. + +For further information, please contact The MITRE Corporation, Contracts Management Office, 7515 Colshire Drive, McLean, VA 22102-7539, (703) 983-6000. diff --git a/README.md b/README.md index 1e42d9e..a1ca11d 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,60 @@ # aws-s3-baseline -A micro-baseline to check for insecure or public S3 buckets and bucket objects in your AWS Environment. This [InSpec](https://github.com/chef/inspec) compliance profile verifies that you do not have any insure or open to public S3 Bucket or Bucket Objects in your AWS Environment in an automated way. +A micro-baseline is provided to check for insecure or public S3 buckets and bucket objects in your AWS environment. This [InSpec](https://github.com/chef/inspec) compliance profile verifies that you do not have any insecure or publicly accessible S3 buckets or bucket objects in your AWS environment in an automated way. -### Required Gems +## Required Gems This profile requires the following gems: -- `inspec` -- `inspec-bin` -- `aws-sdk-s3` +- `inspec` (v5 or higher) +- `inspec-bin` (v5 or higher) +- `aws-sdk-s3` (v2 or higher, v3 recommended) - `concurrent-ruby` (v1.1.0 or higher) -Please **install these gems** in the ruby environment that InSpec is using prior to executing the profile. +Please **install these gems** in the Ruby environment that InSpec is using before executing the profile. ### Large Buckets and Profile Runtime -The `s3-objects-no-public-access` control iterates through every object in each bucket in your AWS environment. The runtime will depend on the number of objects in your S3 Buckets. +The `public-s3-bucket-objects` control - and its support library `list_public_s3_objects` - iterates through every object in each bucket in your AWS environment. The runtime will depend on the number of objects in your S3 buckets. -On average the profile can process around ~1000 objects/sec. +On average, the profile can process around ~1000 objects/sec. -If you have buckets with large numbers of objects, we suggest you script a loop and use the `single_bucket` input to parallelize the workload. +If you have buckets with a large number of objects, we suggest scripting a loop and using the `single_bucket` input to parallelize the workload, or you can use the `test_buckets` input to provide an array of buckets to test. -To see the processing in more details use the `-l debug` flag to get verbose output. +## Profile Inputs -Then you can load all your HDF JSON results into [Heimdall Lite](https://heimdall-lite.mitre.org) to easily review all your scan results from the multiple runs by loading them in Heimdall. +- `single_bucket`: The name of the single bucket you wish to scan. This input is useful for testing a specific bucket. +- `exempt_buckets`: A list of buckets that should be exempted from review. This input allows you to skip certain buckets from being tested. +- `test_buckets`: A list of buckets to test. This input allows you to specify multiple buckets to be tested. +- `list_public_s3_objects_params`: A hash of parameters for the `list_public_s3_objects` function. This input allows you to configure the following parameters: + - `thread_pool_size`: The size of the thread pool for concurrent processing. Default is 50. Increasing this value can improve performance for buckets with a large number of objects by allowing more concurrent processing, but it may also increase the load on your system. + - `batch_size`: The number of objects to process in each batch. Default is 200. Adjusting this value can affect the balance between memory usage and processing speed. + - `max_retries`: The maximum number of retries for S3 requests. Default is 5. This can help handle transient errors but may increase the overall runtime if set too high. + - `retry_delay`: The delay between retries in seconds. Default is 0.5. This can help handle transient errors by spacing out retry attempts. + +### Performance Considerations + +- **Threading and Concurrency**: The `thread_pool_size` parameter controls the number of concurrent threads used to process objects. Increasing this value can improve performance by allowing more objects to be processed simultaneously, but it may also increase the load on your system and potentially lead to throttling by AWS. +- **Batch Processing**: The `batch_size` parameter controls the number of objects processed in each batch. Larger batch sizes can reduce the number of API calls to AWS, but they may also increase memory usage. +- **Retries and Delays**: The `max_retries` and `retry_delay` parameters control how the function handles transient errors. Increasing the number of retries and the delay between retries can improve the robustness of the function but may also increase the overall runtime. + +To see the processing in more detail, use the `-l debug` flag to get verbose output. + +You can then load all your HDF JSON results into [Heimdall Lite](https://heimdall-lite.mitre.org) to easily review all your scan results from multiple runs by loading them in Heimdall. ## Getting Started -It is intended and recommended that InSpec and this profile be run from a **"runner"** host (such as a DevOps orchestration server, an administrative management system, or a developer's workstation/laptop) against the target remotely over **AWS CLI**. +It is intended and recommended that InSpec and this profile be run from a **"runner"** host (such as a DevOps orchestration server, an administrative management system, or a developer's workstation/laptop) against the target remotely using **AWS CLI**. -**For the best security of the runner, always install on the runner the _latest version_ of InSpec and supporting Ruby language components.** +**For the best security of the runner, always install the _latest version_ of InSpec and supporting Ruby language components on the runner.** -The latest versions and installation options are available at the [InSpec](http://inspec.io/) site. +The latest versions and installation options are available on the [InSpec](http://inspec.io/) site. -This baseline also requires the AWS Command Line Interface (CLI) which is available at the [AWS CLI](https://aws.amazon.com/cli/) site. +This baseline also requires the AWS Command Line Interface (CLI), which is available on the [AWS CLI](https://aws.amazon.com/cli/) site. -### Getting MFA Aware AWS Access, Secret and Session Tokens +### Getting MFA Aware AWS Access, Secret, and Session Tokens -You will need to ensure your AWS CLI environment has the right system environment variables set with your AWS region and credentials and session token to use the AWS CLI and InSpec resources in the AWS environment. InSpec supports the following standard AWS variables: +You need to ensure that your AWS CLI environment has the correct system environment variables set with your AWS region, credentials, and session token to use the AWS CLI and InSpec resources in the AWS environment. InSpec supports the following standard AWS variables: - `AWS_REGION` - `AWS_ACCESS_KEY_ID` @@ -46,47 +63,53 @@ You will need to ensure your AWS CLI environment has the right system environmen ### Notes on MFA -In any AWS MFA enabled environment - you need to use `derived credentials` to use the CLI. Your default `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` will not satisfy the MFA Policies in AWS environments. +In any AWS MFA-enabled environment, you need to use `derived credentials` to use the CLI. Your default `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` will not satisfy the MFA Policies in AWS environments. -- The AWS documentation is here: -- The AWS profile documentation is here: -- A useful bash script for automating this is here: +The AWS documentation is available [here](https://docs.aws.amazon.com/cli/latest/reference/sts/get-session-token.html). -To generate credentials using an AWS Profile you will need to use the following AWS CLI commands. +The AWS profile documentation is available [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-multiple-profiles.html). + +A useful bash script for automating this is available [here](https://gist.github.com/dinvlad/d1bc0a45419abc277eb86f2d1ce70625). + +To generate credentials using an AWS Profile, you will need to use the following AWS CLI commands: a. `aws sts get-session-token --serial-number arn:aws:iam::<$YOUR-MFA-SERIAL> --token-code <$YOUR-CURRENT-MFA-TOKEN> --profile=<$YOUR-AWS-PROFILE>` -b. Then export the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` that was generated by the above command. +b. Then export the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` that were generated by the above command. ## Tailoring to Your Environment -The following inputs must be configured in an inputs ".yml" file for the profile to run correctly for your specific environment. More information about InSpec inputs can be found in the [InSpec Profile Documentation](https://www.inspec.io/docs/reference/profiles/). +The following inputs must be configured in an inputs ".yml" file for the profile to run correctly in your specific environment. More information about InSpec inputs can be found in the [InSpec Profile Documentation](https://www.inspec.io/docs/reference/profiles/). ```yaml # List of buckets exempted from inspection. exception_bucket_list: - - bucket1 - - bucket2 - ... + - bucket1 + - bucket2 + ... # Test only one bucket single_bucket: 'my-bucket' -``` - -## Note -When you use the `single_bucket` input, the profile will _***ONLY scan***_ that bucket. +# Test specific buckets +test_buckets: + - bucket3 + - bucket4 + ... +``` # Usage -``` -# Set required ENV variables -$ export AWS_ACCESS_KEY_ID=key-id -$ export AWS_SECRET_ACCESS_KEY=access-key -$ export AWS_SESSION_TOKEN=session-token # if MFA is enabled +```bash +# Set required ENV variables as per your environment + +$ export AWS_REGION=us-east-1 +$ export AWS_ACCESS_KEY_ID=... +$ export AWS_SECRET_ACCESS_KEY=... +$ export AWS_SESSION_TOKEN=... # if MFA is enabled ``` -## Installing the needed Gems +## Installing the Needed Gems ### Plain Old Ruby Environment @@ -96,7 +119,7 @@ $ export AWS_SESSION_TOKEN=session-token # if MFA is enabled - `chef gem install concurrent-ruby` -## Running This Baseline Directly from Github +## Running This Baseline Directly from GitHub ### Testing all your buckets except those defined in your `excluded buckets` @@ -106,19 +129,23 @@ $ export AWS_SESSION_TOKEN=session-token # if MFA is enabled `inspec exec https://github.com/mitre/aws-s3-baseline/archive/master.tar.gz --target aws:// --input single_bucket=your_bucket --reporter=cli json:your_output_file.json` +### Testing specific buckets + +`inspec exec https://github.com/mitre/aws-s3-baseline/archive/master.tar.gz --target aws:// --input-file=your_inputs_file.yml --reporter=cli json:your_output_file.json` + ### Different Run Options [Full exec options](https://docs.chef.io/inspec/cli/#options-3) -## Running This Baseline from a local Archive copy +## Running This Baseline from a Local Archive Copy -If your runner is not always expected to have direct access to GitHub, use the following steps to create an archive bundle of this baseline and all of its dependent tests: +If your runner does not always have direct access to GitHub, use the following steps to create an archive bundle of this baseline and all its dependent tests: (Git is required to clone the InSpec profile using the instructions below. Git can be downloaded from the [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) site.) When the **"runner"** host uses this profile baseline for the first time, follow these steps: -### Create your Archieve of the Profile +### Create your Archive of the Profile ```bash mkdir profiles @@ -127,11 +154,11 @@ git clone https://github.com/mitre/aws-s3-baseline inspec archive aws-s3-baseline ``` -### Run your scan using the Archieved Copy +### Run your scan using the Archived Copy `inspec exec --target aws:// --input-file= --reporter=cli json:` -### Updating your Archieved Copy +### Updating your Archived Copy For every successive run, follow these steps to always have the latest version of this baseline: @@ -142,15 +169,15 @@ cd .. inspec archive aws-s3-baseline --overwrite ``` -### Run your updated Archieved Copy +### Run your updated Archived Copy `inspec exec --target aws:// --input-file= --reporter=cli json:` ## Using Heimdall for Viewing the JSON Results -The JSON results output file can be loaded into **[heimdall-lite](https://heimdall-lite.mitre.org/)** for a user-interactive, graphical view of the InSpec results. +The JSON results output file can be loaded into **[Heimdall Lite](https://heimdall-lite.mitre.org/)** for a user-interactive, graphical view of the InSpec results. -The JSON InSpec results file may also be loaded into a **[full heimdall server](https://github.com/mitre/heimdall)**, allowing for additional functionality such as to store and compare multiple profile runs. +The JSON InSpec results file can also be loaded into a **[full Heimdall server](https://github.com/mitre/heimdall)**, allowing for additional functionality such as storing and comparing multiple profile runs. ## Authors @@ -164,7 +191,7 @@ The JSON InSpec results file may also be loaded into a **[full heimdall server]( ### NOTICE -© 2018-2022 The MITRE Corporation. +© 2018-2024 The MITRE Corporation. Approved for Public Release; Distribution Unlimited. Case Number 18-3678. @@ -174,8 +201,8 @@ MITRE hereby grants express written permission to use, reproduce, distribute, mo ### NOTICE -This software was produced for the U. S. Government under Contract Number HHSM-500-2012-00008I, and is subject to Federal Acquisition Regulation Clause 52.227-14, Rights in Data-General. +This software was produced for the U.S. Government under Contract Number HHSM-500-2012-00008I and is subject to Federal Acquisition Regulation Clause 52.227-14, Rights in Data-General. -No other use other than that granted to the U. S. Government, or to those acting on behalf of the U. S. Government under that Clause is authorized without the express written permission of The MITRE Corporation. +No other use other than that granted to the U.S. Government, or to those acting on behalf of the U.S. Government under that Clause, is authorized without the express written permission of The MITRE Corporation. -For further information, please contact The MITRE Corporation, Contracts Management Office, 7515 Colshire Drive, McLean, VA 22102-7539, (703) 983-6000. +For further information, please contact The MITRE Corporation, Contracts Management Office, 7515 Colshire Drive, McLean, VA 22102-7539, (703) 983-6000. \ No newline at end of file diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..f0fc557 --- /dev/null +++ b/Rakefile @@ -0,0 +1,16 @@ +# Rakefile + +# Define a task to run benchmarks +namespace :benchmark do + desc 'Run the benchmark script' + task :run, %i[bucket_name strategy] do |_t, args| + args.with_defaults(bucket_name: 'saf-site', strategy: 'lite') + sh "ruby spec/benchmark.rb '#{args[:bucket_name]}' '#{args[:strategy]}'" + end +end + +desc "Run default benchmark with 'saf-site' bucket and 'lite' strategy" +task default: 'benchmark:run' + +desc 'Alias for default task' +task test: :default diff --git a/controls/aws_s3_bucket.rb b/controls/aws_s3_bucket.rb index f3dc174..6e4cd14 100644 --- a/controls/aws_s3_bucket.rb +++ b/controls/aws_s3_bucket.rb @@ -1,39 +1,46 @@ -control 'Public_S3_Buckets' do +control 'public-s3-buckets' do impact 0.7 title 'Ensure there are no publicly accessible S3 buckets' desc 'Ensure there are no publicly accessible S3 buckets' - tag "nist": ['AC-6'] - tag "severity": 'high' + tag nist: ['AC-6'] + tag severity: 'high' - tag "check": "Review your AWS console and note if any S3 buckets are set to - 'Public'. If any buckets are listed as 'Public', then this is - a finding." + desc 'check', + "Review your AWS console and note if any S3 buckets are set to 'Public'. If any buckets are listed as 'Public', then this is a finding." + desc 'fix', + 'Log into your AWS console and select the S3 buckets section. Select the buckets found in your review. Select the permissions tab for the bucket and remove the Public access permission.' - tag "fix": "Log into your AWS console and select the S3 buckets section. Select - the buckets found in your review. Select the permisssions tab for - the bucket and remove the Public access permission." + exempt_buckets = input('exempt_buckets') + test_buckets = input('test_buckets') + single_bucket = input('single_bucket') - exception_bucket_list = input('exception_bucket_list') + only_if( + 'This control is Non Applicable since no S3 buckets were found.', + impact: 0.0 + ) { !aws_s3_buckets.bucket_names.empty? } - if aws_s3_buckets.bucket_names.empty? - impact 0.0 - desc 'This control is Non Applicable since no S3 buckets were found.' + bucket_names = if single_bucket.present? + [single_bucket.to_s] + elsif test_buckets.present? + test_buckets + else + aws_s3_buckets.bucket_names + end - describe 'This control is Non Applicable since no S3 buckets were found.' do - skip 'This control is Non Applicable since no S3 buckets were found.' - end - elsif input('single_bucket').present? - describe aws_s3_bucket(input('single_bucket')) do - it { should_not be_public } - end - else - aws_s3_buckets.bucket_names.each do |bucket| - next if exception_bucket_list.include?(bucket) - - describe bucket.to_s do + bucket_names.sort.each do |bucket| + if exempt_buckets.include?(bucket) + describe "Bucket #{bucket}" do + it 'should be exempted from evaluation' do + skip "Bucket #{bucket} was not evaluated because it was exempted" + end + end + else + describe bucket do subject { aws_s3_bucket(bucket) } - it { should_not be_public } + it 'should not be publicly accessible' do + expect(subject).not_to be_public, "\tand is configured to be public." + end end end end diff --git a/controls/aws_s3_bucket_objects.rb b/controls/aws_s3_bucket_objects.rb index 5bf6e6d..88db693 100644 --- a/controls/aws_s3_bucket_objects.rb +++ b/controls/aws_s3_bucket_objects.rb @@ -1,60 +1,61 @@ -require_relative "../libraries/concurrent_s3" +require_relative '../libraries/list_public_s3_objects' -control "Public_S3_Objects" do +control 'public-s3-bucket-objects' do impact 0.7 - title "Ensure there are no publicly accessible S3 objects" - desc "Ensure there are no publicly accessible S3 objects" + title 'Ensure there are no publicly accessible S3 objects' + desc 'Ensure there are no publicly accessible S3 objects' tag nist: %w[AC-6] - tag severity: "high" + tag severity: 'high' + desc 'check', + "Review your AWS console and note if any S3 bucket objects are set to 'Public'. If any objects are listed as 'Public', then this is a finding." + desc 'fix', + 'Log into your AWS console and select the S3 buckets section. Select the buckets found in your review. For each object in the bucket select the permissions tab for the object and remove the Public Access permission.' - tag check: - "Review your AWS console and note if any S3 bucket objects are set to - 'Public'. If any objects are listed as 'Public', then this is - a finding." + exempt_buckets = input('exempt_buckets') + test_buckets = input('test_buckets') + single_bucket = input('single_bucket') + list_public_s3_objects_params = input('list_public_s3_objects_params') - tag fix: - "Log into your AWS console and select the S3 buckets section. Select - the buckets found in your review. For each object in the bucket - select the permissions tab for the object and remove - the Public Access permission." + only_if( + 'This control is Non Applicable since no S3 buckets were found.', + impact: 0.0 + ) { !aws_s3_buckets.bucket_names.empty? } - exception_bucket_list = input("exception_bucket_list") - - if aws_s3_buckets.bucket_names.empty? - impact 0.0 - desc "This control is Non Applicable since no S3 buckets were found." - - describe "This control is Non Applicable since no S3 buckets were found." do - skip "This control is Non Applicable since no S3 buckets were found." + bucket_names = + if single_bucket.present? + [single_bucket.to_s] + elsif test_buckets.present? + test_buckets + else + aws_s3_buckets.bucket_names end - elsif input("single_bucket").present? - public_objects = get_public_objects(input("single_bucket").to_s) - describe input("single_bucket").to_s do - it "should not have any public objects" do - failure_message = - ( - if public_objects.count > 1 - "#{public_objects.join(", ")} are public" - else - "#{public_objects.join(", ")} is public" - end - ) - expect(public_objects).to be_empty, failure_message + + bucket_names.sort.each do |bucket| + if exempt_buckets.include?(bucket) + describe "Bucket #{bucket}" do + it "#{bucket} was not evaluated because it was exempted" do + skip "#{bucket} was not evaluated because it was exempted" + end end - end - else - aws_s3_buckets.bucket_names.each do |bucket| - next if exception_bucket_list.include?(bucket) + else + public_objects = + list_public_s3_objects( + bucket, + thread_pool_size: list_public_s3_objects_params['thread_pool_size'], + batch_size: list_public_s3_objects_params['batch_size'], + max_retries: list_public_s3_objects_params['max_retries'], + retry_delay: list_public_s3_objects_params['retry_delay'] + ) - public_objects_multi = get_public_objects(bucket.to_s) - describe bucket.to_s do - it "should not have any public objects" do - failure_message = - "\t- #{public_objects_multi.join("\n\t- ")} \n\tis public" + describe bucket do + it 'should not have any public objects' do failure_message = - "\t- #{public_objects_multi.join("\n\t- ")} \n\tare public" if public_objects_multi.count > - 1 - expect(public_objects_multi).to be_empty, failure_message + if public_objects.count > 1 + "\t- #{public_objects.join("\n\t- ")} \n\tare public" + elsif public_objects.count == 1 + "\t- #{public_objects.join("\n\t- ")} \n\tis public" + end + expect(public_objects).to be_empty, failure_message end end end diff --git a/inspec.yml b/inspec.yml index ff2b95e..817f1ac 100644 --- a/inspec.yml +++ b/inspec.yml @@ -1,13 +1,13 @@ name: aws-s3-baseline title: aws-s3-baseline -maintainer: MITRE InSpec Team -copyright: MITRE, 2022 -copyright_email: inspec@mitre.org +maintainer: MITRE Security Automation Framework Team +copyright: The MITRE Corporation, 2024 +copyright_email: saf@groups.mitre.org license: Apache-2.0 -summary: "InSpec profile to test if you have public buckets or objects" -version: 1.5.2 +summary: "Example inspec profile to test for any public s3 buckets or buckets with public objects and use of a local support library to speed up testing using concurrent-ruby" +version: 2.0.0 -inspec_version: ">= 4.0" +inspec_version: ">= 5.0" supports: - platform: aws @@ -16,14 +16,30 @@ depends: - name: inspec-aws git: https://github.com/inspec/inspec-aws.git +gems: +- name: concurrent-ruby + inputs: - name: single_bucket description: "The name of the single bucket you wish to scan" - type: string + type: String value: "" -- name: exception_bucket_list +- name: exempt_buckets description: "List of buckets that should be exempted from review" - type: array + type: Array + value: [] + +- name: test_buckets + description: "List of buckets to test" + type: Array + value: [] + +- name: list_public_s3_objects_params + description: "Parameters for the list_public_s3_objects function" + type: Hash value: - - "" + thread_pool_size: 20 + batch_size: 100 + max_retries: 1 + retry_delay: 0.5 diff --git a/libraries/list_public_s3_objects.rb b/libraries/list_public_s3_objects.rb new file mode 100644 index 0000000..df29263 --- /dev/null +++ b/libraries/list_public_s3_objects.rb @@ -0,0 +1,105 @@ +# Conditionally require the needed libraries unless they are already loaded +require "aws-sdk-s3" unless defined?(Aws::S3::Client) +require "concurrent-ruby" unless defined?(Concurrent::FixedThreadPool) + +## +# Lists all publicly accessible objects in an S3 bucket. +# +# This method iterates through all objects in the specified S3 bucket and checks +# their access control lists (ACLs) to determine if they are publicly accessible. +# It uses a thread pool to process objects concurrently for improved performance. +# +# @param bucket_name [String] The name of the S3 bucket. +# @param thread_pool_size [Integer] The size of the thread pool for concurrent processing. Default is 50. +# @param batch_size [Integer] The number of objects to process in each batch. Default is 200. +# @param max_retries [Integer] The maximum number of retries for S3 requests. Default is 5. +# @param retry_delay [Float] The delay between retries in seconds. Default is 0.5. +# @param s3_client [Aws::S3::Client, nil] An optional S3 client. If not provided, a new client will be created. +# @return [Array] A list of keys for publicly accessible objects. +# +# @example List public objects in a bucket +# public_objects = list_public_s3_objects('my-bucket') +# puts "Public objects: #{public_objects.join(', ')}" + +def list_public_s3_objects( + bucket_name, + thread_pool_size: 20, + batch_size: 100, + max_retries: 1, + retry_delay: 0.1, + s3_client: nil +) + public_objects = [] + continuation_token = nil + + # Use the provided S3 client or create a new one + s3 = s3_client || Aws::S3::Client.new + + # Determine the bucket's region + bucket_location = + s3.get_bucket_location(bucket: bucket_name).location_constraint + bucket_region = bucket_location.empty? ? "us-east-1" : bucket_location + + # Create a new S3 client in the bucket's region if not provided + s3 = s3_client || Aws::S3::Client.new(region: bucket_region) + + # Create a thread pool for concurrent processing + thread_pool = Concurrent::FixedThreadPool.new(thread_pool_size) + + loop do + # List objects in the bucket with pagination support + response = + s3.list_objects_v2( + bucket: bucket_name, + continuation_token: continuation_token, + ) + response + .contents + .each_slice(batch_size) do |object_batch| + # Process each batch of objects concurrently + futures = + object_batch.map do |object| + Concurrent::Future.execute(executor: thread_pool) do + retries = 0 + begin + # Get the ACL for each object + acl = s3.get_object_acl(bucket: bucket_name, key: object.key) + # Check if the object is publicly accessible + if acl.grants.any? do |grant| + grant.grantee.type == "Group" && + (grant.grantee.uri =~ /AllUsers|AuthenticatedUsers/) + end + object.key + end + rescue Aws::S3::Errors::ServiceError + retries += 1 + if retries <= max_retries + sleep(retry_delay) + retry + end + end + end + end + + # Collect the results from the futures + futures.each do |future| + key = future.value + public_objects << key if key + end + end + + # Check if there are more objects to list + break unless response.is_truncated + + continuation_token = response.next_continuation_token + rescue Aws::S3::Errors::PermanentRedirect + # This block should not be reached if we correctly determine the bucket's region + break + end + + # Shutdown the thread pool and wait for termination + thread_pool.shutdown + thread_pool.wait_for_termination + + public_objects +end diff --git a/libraries/concurrent_s3-cop b/old-library-examples/concurrent_s3-cop similarity index 100% rename from libraries/concurrent_s3-cop rename to old-library-examples/concurrent_s3-cop diff --git a/libraries/concurrent_s3.rb b/old-library-examples/concurrent_s3-orig similarity index 95% rename from libraries/concurrent_s3.rb rename to old-library-examples/concurrent_s3-orig index 8a36551..9016dbe 100644 --- a/libraries/concurrent_s3.rb +++ b/old-library-examples/concurrent_s3-orig @@ -1,5 +1,4 @@ require "concurrent" -require "aws-sdk-s3" module Aws::S3 class Bucket @@ -80,9 +79,11 @@ def get_public_objects(myBucket) end rescue Aws::S3::Errors::PermanentRedirect => e Inspec::Log.warn "Permanent redirect for object #{object.key}: #{e.message}" + skip_resource "Skipping object #{object.key} due to permanent redirect: #{e.message}" rescue => e Inspec::Log.warn "Error processing object #{object.key}: #{e.message}" Inspec::Log.warn "Backtrace: #{e.backtrace.join("\n")}" + skip_resource "Skipping object #{object.key} due to error: #{e.message}" end end end diff --git a/old-library-examples/concurrent_s3.rb b/old-library-examples/concurrent_s3.rb new file mode 100644 index 0000000..810f03b --- /dev/null +++ b/old-library-examples/concurrent_s3.rb @@ -0,0 +1,165 @@ +# Conditionally require the concurrent library +require 'concurrent' + +module Aws::S3 + class Bucket + def objects(options = {}) + batches = + Enumerator.new do |y| + options = options.merge(bucket: @name) + begin + resp = @client.list_objects_v2(options) + resp.each_page do |page| + batch = [] + pool = Concurrent::FixedThreadPool.new(16) + mutex = Mutex.new + page.data.contents.each do |c| + pool.post { process_object(c, batch, mutex) } + end + pool.shutdown + pool.wait_for_termination + y.yield(batch) + end + rescue Aws::S3::Errors::PermanentRedirect => e + handle_bucket_error(e, @name) + rescue StandardError => e + handle_generic_bucket_error(e, @name) + end + end + ObjectSummary::Collection.new(batches) + end + + private + + def process_object(c, batch, mutex) + mutex.synchronize do + batch << ObjectSummary.new( + bucket_name: @name, + key: c.key, + data: c, + client: @client + ) + end + rescue Aws::S3::Errors::PermanentRedirect => e + handle_object_error(e, c.key) + rescue StandardError => e + handle_generic_object_error(e, c.key) + end + + def handle_bucket_error(e, bucket_name) + Inspec::Log.warn "Permanent redirect for bucket #{bucket_name}: #{e.message}" + end + + def handle_generic_bucket_error(e, bucket_name) + Inspec::Log.warn "Error accessing bucket #{bucket_name}: #{e.message}" + Inspec::Log.warn "Backtrace: #{e.backtrace.join("\n")}" + end + + def handle_object_error(e, key) + Inspec::Log.warn "Permanent redirect for object #{key}: #{e.message}" + skip_resource "Skipping object #{key} due to permanent redirect: #{e.message}" + end + + def handle_generic_object_error(e, key) + Inspec::Log.warn "Error processing object #{key}: #{e.message}" + Inspec::Log.warn "Backtrace: #{e.backtrace.join("\n")}" + skip_resource "Skipping object #{key} due to error: #{e.message}" + end + end +end + +def get_public_objects(myBucket) + results = { public_keys: [], redirect_buckets: [] } + s3 = Aws::S3::Resource.new + pool = Concurrent::FixedThreadPool.new(56) + mutex = Mutex.new + + begin + bucket = s3.bucket(myBucket) + object_count = bucket.objects.count + + Inspec::Log.debug "### Processing Bucket ### : #{myBucket} with #{object_count} objects" if Inspec::Log.level == :debug + + # Check if the bucket has no objects + return results if object_count.zero? + + bucket.objects.each do |object| + Inspec::Log.debug " Examining Key: #{object.key}" if Inspec::Log.level == :debug + pool.post do + grants = object.acl.grants + if grants.map { |x| x.grantee.type }.any? { |x| x =~ /Group/ } && + grants + .map { |x| x.grantee.uri } + .any? { |x| x =~ /AllUsers|AuthenticatedUsers/ } + mutex.synchronize { results[:public_keys] << object.key } + end + rescue Aws::S3::Errors::PermanentRedirect => e + Inspec::Log.warn "Permanent redirect for object #{object.key}: #{e.message}" + mutex.synchronize do + results[:redirect_buckets] << myBucket unless results[:redirect_buckets].include?(myBucket) + end + rescue StandardError => e + Inspec::Log.warn "Error processing object #{object.key}: #{e.message}" + Inspec::Log.warn "Backtrace: #{e.backtrace.join("\n")}" + end + end + + # Ensure all tasks are completed before shutting down the pool + pool.shutdown + pool.wait_for_termination + rescue Aws::S3::Errors::PermanentRedirect => e + Inspec::Log.warn "Permanent redirect for bucket #{myBucket}: #{e.message}" + results[:redirect_buckets] << myBucket unless results[:redirect_buckets].include?(myBucket) + rescue StandardError => e + Inspec::Log.warn "Error accessing bucket #{myBucket}: #{e.message}" + Inspec::Log.warn "Backtrace: #{e.backtrace.join("\n")}" + ensure + pool.shutdown if pool + end + + results +end + +def process_public_object(object, myPublicKeys, mutex) + grants = object.acl.grants + if grants.map { |x| x.grantee.type }.any? { |x| x =~ /Group/ } && + grants + .map { |x| x.grantee.uri } + .any? { |x| x =~ /AllUsers|AuthenticatedUsers/ } + mutex.synchronize { myPublicKeys << object.key } + end +rescue Aws::S3::Errors::PermanentRedirect => e + handle_object_error(e, object.key) +rescue StandardError => e + handle_generic_object_error(e, object.key) +end + +def log_bucket_processing(bucket_name, object_count) + Inspec::Log.debug "### Processing Bucket ### : #{bucket_name} with #{object_count} objects" if Inspec::Log.level == :debug +end + +def log_object_examination(key) + Inspec::Log.debug " Examining Key: #{key}" if Inspec::Log.level == :debug +end + +def handle_bucket_error(e, bucket_name) + Inspec::Log.warn "Permanent redirect for bucket #{bucket_name}: #{e.message}" + skip_resource "Skipping bucket #{bucket_name} due to permanent redirect: #{e.message}" +end + +def handle_generic_bucket_error(e, bucket_name) + Inspec::Log.warn "Error accessing bucket #{bucket_name}: #{e.message}" + Inspec::Log.warn "Backtrace: #{e.backtrace.join("\n")}" + skip_resource "Skipping bucket #{bucket_name} due to error: #{e.message}" +end + +def handle_object_error(e, key) + Inspec::Log.warn "Permanent redirect for object #{key}: #{e.message}" + skip_resource "Skipping object #{key} due to permanent redirect: #{e.message}" +end + +def handle_generic_object_error(e, key) + Inspec::Log.warn "Error processing object #{key}: #{e.message}" + Inspec::Log.warn "Backtrace: #{e.backtrace.join("\n")}" + skip_resource "Skipping object #{key} due to error: #{e.message}" +end diff --git a/old-library-examples/public-objects-2-done.rb b/old-library-examples/public-objects-2-done.rb new file mode 100644 index 0000000..32c5384 --- /dev/null +++ b/old-library-examples/public-objects-2-done.rb @@ -0,0 +1,92 @@ +require 'aws-sdk-s3' +require 'concurrent-ruby' + +def list_public_objects( + bucket_name, + thread_pool_size: 20, + batch_size: 50, + max_retries: 3, + retry_delay: 1.0 +) + public_objects = [] + continuation_token = nil + + # Determine the bucket's region + s3 = Aws::S3::Client.new + bucket_location = + s3.get_bucket_location(bucket: bucket_name).location_constraint + bucket_region = bucket_location.empty? ? 'us-east-1' : bucket_location + + # Create a new S3 client in the bucket's region + s3 = Aws::S3::Client.new(region: bucket_region) + + # Create a thread pool for concurrent processing + thread_pool = Concurrent::FixedThreadPool.new(thread_pool_size) + + loop do + response = + s3.list_objects_v2( + bucket: bucket_name, + continuation_token: continuation_token + ) + response + .contents + .each_slice(batch_size) do |object_batch| + futures = + object_batch.map do |object| + Concurrent::Future.execute(executor: thread_pool) do + retries = 0 + begin + acl = s3.get_object_acl(bucket: bucket_name, key: object.key) + if acl.grants.any? do |grant| + grant.grantee.type == 'Group' && + (grant.grantee.uri =~ /AllUsers|AuthenticatedUsers/) + end + object.key + end + rescue Aws::S3::Errors::ServiceError => e + retries += 1 + if retries <= max_retries + sleep(retry_delay) + retry + else + puts "Failed to get ACL for #{object.key}: #{e.message}" + nil + end + end + end + end + + futures.each do |future| + key = future.value + public_objects << key if key + end + end + + break unless response.is_truncated + + continuation_token = response.next_continuation_token + rescue Aws::S3::Errors::PermanentRedirect => e + # This block should not be reached if we correctly determine the bucket's region + puts "PermanentRedirect error: #{e.message}" + break + end + + thread_pool.shutdown + thread_pool.wait_for_termination + + public_objects +end + +# Example usage with parameterized values +bucket_name = 'saf-site' +public_objects = + list_public_objects( + bucket_name, + thread_pool_size: 20, + batch_size: 50, + max_retries: 3, + retry_delay: 1.0 + ) +puts "Public objects in bucket '#{bucket_name}':" +puts public_objects diff --git a/old-library-examples/public-objects.rb b/old-library-examples/public-objects.rb new file mode 100644 index 0000000..60d7688 --- /dev/null +++ b/old-library-examples/public-objects.rb @@ -0,0 +1,57 @@ +require 'aws-sdk-s3' +require 'concurrent-ruby' + +def list_public_objects(bucket_name) + public_objects = [] + continuation_token = nil + + # Determine the bucket's region + s3 = Aws::S3::Client.new + bucket_location = + s3.get_bucket_location(bucket: bucket_name).location_constraint + bucket_region = bucket_location.empty? ? 'us-east-1' : bucket_location + + # Create a new S3 client in the bucket's region + s3 = Aws::S3::Client.new(region: bucket_region) + + loop do + response = + s3.list_objects_v2( + bucket: bucket_name, + continuation_token: continuation_token + ) + futures = + response.contents.map do |object| + Concurrent::Future.execute do + acl = s3.get_object_acl(bucket: bucket_name, key: object.key) + if acl.grants.any? do |grant| + grant.grantee.type == 'Group' && + (grant.grantee.uri =~ /AllUsers|AuthenticatedUsers/) + end + object.key + end + end + end + + futures.each do |future| + key = future.value + public_objects << key if key + end + + break unless response.is_truncated + + continuation_token = response.next_continuation_token + rescue Aws::S3::Errors::PermanentRedirect => e + # This block should not be reached if we correctly determine the bucket's region + puts "PermanentRedirect error: #{e.message}" + break + end + + public_objects +end + +# Example usage +bucket_name = 'saf-site' +public_objects = list_public_objects(bucket_name) +puts "Public objects in bucket '#{bucket_name}':" +puts public_objects diff --git a/spec/benchmark.rb b/spec/benchmark.rb new file mode 100644 index 0000000..d69d109 --- /dev/null +++ b/spec/benchmark.rb @@ -0,0 +1,83 @@ +require 'aws-sdk-s3' unless defined?(Aws::S3::Client) +require 'concurrent-ruby' unless defined?(Concurrent::FixedThreadPool) +require 'benchmark' unless defined?(Benchmark) + +# Require the list_public_s3_objects.rb file +require_relative '../libraries/list_public_s3_objects' + +# List all S3 buckets +s3 = Aws::S3::Client.new +buckets = s3.list_buckets.buckets.map(&:name) + +# Get the bucket to benchmark from command line arguments or default to the first bucket +bucket_name = ARGV[0] || buckets.first + +# Get the testing strategy from command line arguments or default to "lite" +strategy = ARGV[1] || 'lite' + +# Define configurations for lite and deep strategies +configurations = { + lite: { + thread_pool_sizes: [10, 20], + batch_sizes: [50, 100], + max_retries_values: [1, 3], + retry_delays: [0.1, 0.5] + }, + deep: { + thread_pool_sizes: [10, 20, 30, 40, 50], + batch_sizes: [50, 100, 200, 300], + max_retries_values: [1, 3, 5], + retry_delays: [0.1, 0.5, 1.0] + } +} + +# Select configurations based on the strategy +selected_config = configurations[strategy.to_sym] + +results = [] + +selected_config[:thread_pool_sizes].each do |thread_pool_size| + selected_config[:batch_sizes].each do |batch_size| + selected_config[:max_retries_values].each do |max_retries| + selected_config[:retry_delays].each do |retry_delay| + time = + Benchmark.measure do + # Call the list_public_s3_objects function without printing the results + list_public_s3_objects( + bucket_name, + thread_pool_size: thread_pool_size, + batch_size: batch_size, + max_retries: max_retries, + retry_delay: retry_delay + ) + end + results << { + bucket: bucket_name, + thread_pool_size: thread_pool_size, + batch_size: batch_size, + max_retries: max_retries, + retry_delay: retry_delay, + time: time.real + } + end + end + end +end + +# Print results +results + .sort_by { |result| result[:time] } + .each do |result| + puts "Bucket: #{result[:bucket]}, Thread Pool Size: #{result[:thread_pool_size]}, Batch Size: #{result[:batch_size]}, Max Retries: #{result[:max_retries]}, Retry Delay: #{result[:retry_delay]}, Time: #{result[:time]} seconds" + end + +# Find the best configuration +best_result = results.min_by { |result| result[:time] } + +puts "\nBest Configuration:" +puts "Bucket: #{best_result[:bucket]}" +puts "Thread Pool Size: #{best_result[:thread_pool_size]}" +puts "Batch Size: #{best_result[:batch_size]}" +puts "Max Retries: #{best_result[:max_retries]}" +puts "Retry Delay: #{best_result[:retry_delay]}" +puts "Time: #{best_result[:time]} seconds"