Merge pull request #406 from owasp-noir/feature/endpoint-status-codes

Add --status-codes, --exclude-codes flag

docs/_advanced/configuration.md

@@ -51,6 +51,12 @@ exclude_techs: ""
 # The format to use for the output
 format: "plain"
 
+# Whether to display HTTP status codes in the output
+status_codes: "false"
+
+# Whether to exclude HTTP status codes from the output
+exclude_codes: ""
+
 # Whether to include the path in the output
 include_path: "false"
 

docs/_includes/usage.md

@@ -11,7 +11,15 @@ FLAGS:
                                        * curl httpie oas2 oas3
                                        * only-url only-param only-header only-cookie only-tag
     -o PATH, --output out.txt        Write result to file
-    --set-pvalue VALUE               Specifies the value of the identified parameter
+    --set-pvalue VALUE               Specifies the value of the identified parameter for all types
+    --set-pvalue-header VALUE        Specifies the value of the identified parameter for headers
+    --set-pvalue-cookie VALUE        Specifies the value of the identified parameter for cookies
+    --set-pvalue-query VALUE         Specifies the value of the identified parameter for query parameters
+    --set-pvalue-form VALUE          Specifies the value of the identified parameter for form data
+    --set-pvalue-json VALUE          Specifies the value of the identified parameter for JSON data
+    --set-pvalue-path VALUE          Specifies the value of the identified parameter for path parameters
+    --status-codes                   Display HTTP status codes for discovered endpoints
+    --exclude-codes                  Exclude specific HTTP status code
     --include-path                   Include file path in the plain result
     --no-color                       Disable color output
     --no-log                         Displaying only the results

src/completions.cr

@@ -8,6 +8,14 @@ def generate_zsh_completion_script
         '-f[Set output format]:format:(plain yaml json jsonl markdown-table curl httpie oas2 oas3 only-url only-param only-header only-cookie)' \\
         '-o[Write result to file]:path:_files' \\
         '--set-pvalue[Specifies the value of the identified parameter]:value:' \\
+        '--set-pvalue-header[Specifies the value of the identified parameter for headers]:value:' \\
+        '--set-pvalue-cookie[Specifies the value of the identified parameter for cookies]:value:' \\
+        '--set-pvalue-query[Specifies the value of the identified parameter for query parameters]:value:' \\
+        '--set-pvalue-form[Specifies the value of the identified parameter for form data]:value:' \\
+        '--set-pvalue-json[Specifies the value of the identified parameter for JSON data]:value:' \\
+        '--set-pvalue-path[Specifies the value of the identified parameter for path parameters]:value:' \\
+        '--status-codes[Display HTTP status codes for discovered endpoints]' \\
+        '--exclude-codes[Exclude specific HTTP response codes (comma-separated)]:status:' \\
         '--include-path[Include file path in the plain result]' \\
         '--no-color[Disable color output]' \\
         '--no-log[Displaying only the results]' \\
@@ -46,6 +54,14 @@ def generate_bash_completion_script
                 -f --format
                 -o --output
                 --set-pvalue
+                --set-pvalue-header
+                --set-pvalue-cookie
+                --set-pvalue-query
+                --set-pvalue-form
+                --set-pvalue-json
+                --set-pvalue-path
+                --status-codes
+                --exclude-codes
                 --include-path
                 --no-color
                 --no-log

src/config_initializer.cr

@@ -94,6 +94,8 @@ class ConfigInitializer
       "debug"             => YAML::Any.new(false),
       "exclude_techs"     => YAML::Any.new(""),
       "format"            => YAML::Any.new("plain"),
+      "status_codes"      => YAML::Any.new(false),
+      "exclude_codes"     => YAML::Any.new(""),
       "include_path"      => YAML::Any.new(false),
       "nolog"             => YAML::Any.new(false),
       "output"            => YAML::Any.new(""),

src/models/endpoint.cr

@@ -30,7 +30,8 @@ struct Endpoint
     @tags = [] of Tag
   end
 
-  def details=(@details : Details)
+  def details=(details : Details)
+    @details = details
   end
 
   def protocol=(protocol : String)
@@ -52,6 +53,7 @@ struct Endpoint
     params_hash["form"] = {} of String => String
     params_hash["header"] = {} of String => String
     params_hash["cookie"] = {} of String => String
+    params_hash["path"] = {} of String => String
 
     @params.each do |param|
       params_hash[param.param_type][param.name] = param.value
@@ -107,6 +109,7 @@ struct Details
   include JSON::Serializable
   include YAML::Serializable
   property code_paths : Array(PathInfo) = [] of PathInfo
+  property status_code : Int32 | Nil
 
   # + New details types to be added in the future..
 
@@ -121,7 +124,12 @@ struct Details
     @code_paths << code_path
   end
 
+  def status_code=(status_code : Int32)
+    @status_code = status_code
+  end
+
   def ==(other : Details) : Bool
+    return false if @status_code != other.status_code
     return false if @code_paths.size != other.code_paths.size
     return false unless @code_paths.all? { |path| other.code_paths.any? { |other_path| path == other_path } }
     true

src/models/noir.cr

@@ -85,6 +85,11 @@ class NoirRunner
     combine_url_and_endpoints
     add_path_parameters
 
+    # Set status code
+    if any_to_bool(@options["status_codes"]) == true || any_to_bool(@options["exclude_codes"]) != ""
+      update_status_codes
+    end
+
     # Run tagger
     if any_to_bool(@options["all_taggers"]) == true
       @logger.success "Running all taggers."
@@ -219,32 +224,31 @@ class NoirRunner
   end
 
   def add_path_parameters
-    @logger.info "Adding path parameters by URL"
+    @logger.info "Adding path parameters by URL."
     final = [] of Endpoint
 
     @endpoints.each do |endpoint|
       new_endpoint = endpoint
 
       scans = endpoint.url.scan(/\/\{([^}]+)\}/).flatten
       scans.each do |match|
-        param = match[1].split(":")[-1]
+        param = match[1].split(":")[0]
         new_value = apply_pvalue("path", param, "")
         if new_value != ""
-          new_endpoint.url = new_endpoint.url.gsub("{#{param}}", new_value)
+          new_endpoint.url = new_endpoint.url.gsub("{#{match[1]}}", new_value)
         end
 
         new_endpoint.params << Param.new(param, "", "path")
       end
 
       scans = endpoint.url.scan(/\/:([^\/]+)/).flatten
       scans.each do |match|
-        param = match[1].split(":")[-1]
-        new_value = apply_pvalue("path", param, "")
+        new_value = apply_pvalue("path", match[1], "")
         if new_value != ""
           new_endpoint.url = new_endpoint.url.gsub(":#{match[1]}", new_value)
         end
 
-        new_endpoint.params << Param.new(param, "", "path")
+        new_endpoint.params << Param.new(match[1], "", "path")
       end
 
       scans = endpoint.url.scan(/\/<([^>]+)>/).flatten
@@ -263,6 +267,68 @@ class NoirRunner
     @endpoints = final
   end
 
+  def update_status_codes
+    @logger.info "Updating status codes."
+    final = [] of Endpoint
+
+    exclude_codes = [] of Int32
+    if @options["exclude_codes"].to_s != ""
+      @options["exclude_codes"].to_s.split(",").each do |code|
+        exclude_codes << code.strip.to_i
+      end
+    end
+
+    @endpoints.each do |endpoint|
+      begin
+        if endpoint.params.size > 0
+          endpoint_hash = endpoint.params_to_hash
+          body = {} of String => String
+          is_json = false
+          if endpoint_hash["json"].size > 0
+            is_json = true
+            body = endpoint_hash["json"]
+          else
+            body = endpoint_hash["form"]
+          end
+
+          response = Crest::Request.execute(
+            method: get_symbol(endpoint.method),
+            url: endpoint.url,
+            tls: OpenSSL::SSL::Context::Client.insecure,
+            user_agent: "Noir/#{Noir::VERSION}",
+            params: endpoint_hash["query"],
+            form: body,
+            json: is_json,
+            handle_errors: false,
+            read_timeout: 5.second
+          )
+          endpoint.details.status_code = response.status_code
+          unless exclude_codes.includes?(response.status_code)
+            final << endpoint
+          end
+        else
+          response = Crest::Request.execute(
+            method: get_symbol(endpoint.method),
+            url: endpoint.url,
+            tls: OpenSSL::SSL::Context::Client.insecure,
+            user_agent: "Noir/#{Noir::VERSION}",
+            handle_errors: false,
+            read_timeout: 5.second
+          )
+          endpoint.details.status_code = response.status_code
+          unless exclude_codes.includes?(response.status_code)
+            final << endpoint
+          end
+        end
+      rescue e
+        @logger.error "Failed to get status code for #{endpoint.url} (#{e.message})."
+        final << endpoint
+      end
+    end
+
+    @endpoints = final
+  end
+
   def deliver
     if @send_proxy != ""
       @logger.info "Sending requests with proxy #{@send_proxy}."

src/noir.cr

@@ -23,6 +23,39 @@ if noir_options["base"] == ""
   exit(1)
 end
 
+if noir_options["url"] != "" && !noir_options["url"].to_s.includes?("://")
+  STDERR.puts "WARNING: The protocol (http or https) is missing in the URL '#{noir_options["url"]}'.".colorize(Colorize::Color256.new(208))
+  noir_options["url"] = YAML::Any.new("http://#{noir_options["url"]}")
+end
+
+# Check URL
+if noir_options["status_codes"] == true && noir_options["url"] == ""
+  STDERR.puts "ERROR: The --status-codes option requires the -u or --url flag to be specified.".colorize(:yellow)
+  STDERR.puts "Please use -u or --url to set the URL."
+  STDERR.puts "If you need help, use -h or --help."
+  exit(1)
+end
+
+# Check URL
+if noir_options["exclude_codes"] != ""
+  if noir_options["url"] == ""
+    STDERR.puts "ERROR: The --exclude-codes option requires the -u or --url flag to be specified.".colorize(:yellow)
+    STDERR.puts "Please use -u or --url to set the URL."
+    STDERR.puts "If you need help, use -h or --help."
+    exit(1)
+  end
+
+  noir_options["exclude_codes"].to_s.split(",").each do |code|
+    begin
+      code.strip.to_i
+    rescue
+      STDERR.puts "ERROR: Invalid --exclude-codes option: '#{code}'".colorize(:yellow)
+      STDERR.puts "Please use comma-separated numbers."
+      exit(1)
+    end
+  end
+end
+
 # Run Noir
 app = NoirRunner.new noir_options
 start_time = Time.monotonic

src/options.cr

@@ -54,12 +54,20 @@ def run_options_parser
       append_to_yaml_array(noir_options, set_pvalue_path, var)
     end
 
+    parser.on "--status-codes", "Display HTTP status codes for discovered endpoints" do
+      noir_options["status_codes"] = YAML::Any.new(true)
+    end
+
+    parser.on "--exclude-codes 404,500", "Exclude specific HTTP response codes (comma-separated)" { |var| noir_options["exclude_codes"] = YAML::Any.new(var) }
+
     parser.on "--include-path", "Include file path in the plain result" do
       noir_options["include_path"] = YAML::Any.new(true)
     end
+
     parser.on "--no-color", "Disable color output" do
       noir_options["color"] = YAML::Any.new(false)
     end
+
     parser.on "--no-log", "Displaying only the results" do
       noir_options["nolog"] = YAML::Any.new(true)
     end
@@ -125,7 +133,7 @@ def run_options_parser
         puts "\n"
         puts "> Instructions: Copy the content above and save it in the .bashrc file as noir.".colorize(:yellow)
       else
-        puts "ERROR: Invalid completion type."
+        puts "ERROR: Invalid completion type.".colorize(:yellow)
         puts "e.g., noir --generate-completion zsh"
         puts "e.g., noir --generate-completion bash"
       end
@@ -163,12 +171,12 @@ def run_options_parser
       exit
     end
     parser.invalid_option do |flag|
-      STDERR.puts "ERROR: #{flag} is not a valid option."
+      STDERR.puts "ERROR: #{flag} is not a valid option.".colorize(:yellow)
       STDERR.puts parser
       exit(1)
     end
     parser.missing_option do |flag|
-      STDERR.puts "ERROR: #{flag} is missing an argument."
+      STDERR.puts "ERROR: #{flag} is missing an argument.".colorize(:yellow)
       exit(1)
     end
   end

src/output_builder/common.cr

@@ -20,6 +20,25 @@ class OutputBuilderCommon < OutputBuilder
       r_ws = ""
       r_buffer = "\n#{r_method} #{r_url}"
 
+      if any_to_bool(@options["status_codes"]) == true || @options["exclude_codes"] != ""
+        status_color = :light_green
+        status_code = endpoint.details.status_code
+        if status_code
+          if status_code >= 500
+            status_color = :light_magenta
+          elsif status_code >= 400
+            status_color = :light_red
+          elsif status_code >= 300
+            status_color = :cyan
+          end
+        else
+          status_code = "error"
+          status_color = :light_red
+        end
+
+        r_buffer += " [#{status_code}]".to_s.colorize(status_color).toggle(@is_color).to_s
+      end
+
       if endpoint.protocol == "ws"
         r_ws = "[websocket]".colorize(:light_red).toggle(@is_color)
         r_buffer += " #{r_ws}"