♻️ Improved organization; more complete logic

- Single filtering of test cases for all crash scenarios
- Added significant samples of test executable output and test result files in comments
- Simplfied unity_utils removing unneeded functionality and renamed modules and methods

lib/ceedling/generator.rb

@@ -26,7 +26,7 @@ class Generator
               :loginator,
               :plugin_manager,
               :file_wrapper,
-              :unity_utils
+              :test_runner_manager
 
 
   def setup()
@@ -291,13 +291,14 @@ def generate_test_results(tool:, context:, test_name:, test_filepath:, executabl
     msg = @reportinator.generate_progress( "Running #{File.basename(arg_hash[:executable])}" )
     @loginator.log( msg )
 
-    # Unity's exit code is equivalent to the number of failed tests, so we tell @tool_executor not to fail out if there are failures
-    # so that we can run all tests and collect all results
+    # Unity's exit code is equivalent to the number of failed tests.
+    # We tell @tool_executor not to fail out if there are failures
+    # so that we can run all tests and collect all results.
     command = 
       @tool_executor.build_command_line(
         arg_hash[:tool],
         # Apply additional test case filters 
-        @unity_utils.collect_test_runner_additional_args(),
+        @test_runner_manager.collect_cmdline_args(),
         arg_hash[:executable]
       )
 
@@ -312,7 +313,10 @@ def generate_test_results(tool:, context:, test_name:, test_filepath:, executabl
       @helper.log_test_results_crash( test_name, executable, shell_result )
 
       filename = File.basename( test_filepath )
+
+      # Lookup test cases and filter based on any matchers specified for the build task
       test_cases = @test_context_extractor.lookup_test_cases( test_filepath )
+      test_cases = @generator_test_results.filter_test_cases( test_cases )
 
       case @configurator.project_config_hash[:project_use_backtrace]
       # If we have the options and tools to learn more, dig into the details

lib/ceedling/generator_test_results.rb

@@ -10,6 +10,81 @@
 require 'ceedling/constants'
 require 'ceedling/exceptions'
 
+
+##
+## Sample Unity Test Executable Output
+## ===================================
+##
+## - Output is line-oriented. Anything outside the recognized lines is assumed to be from `printf()`
+##   or equivalent calls and collected for presentation as a collection of $stdout lines.
+## - Multiline output (i.e. failure messages) can be achieved by "encoding" newlines as literal 
+##   "\n"s (slash-n). `extract_line_elements()` handles converting newline markers into real newlines.
+## - :PASS has no trailing message unless Unity's test case execution duration feature is enabled.
+##   If enabled, a numeric value with 'ms' as a units signifier trails, ":PASS 1.2 ms".
+## - :IGNORE optionally can include a trailing message.
+## - :FAIL has a trailing message that relays an assertion failure or crash condition.
+## - The statistics line always has the same format with only the count values varying.
+## - If there are no failed test cases, the final line is 'OK'. Otherwise, it is 'FAIL'.
+##
+## $stdout:
+## -----------------------------------------------------------------------------------------------------
+## TestUsartModel.c:24:testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting:PASS
+## TestUsartModel.c:34:testIgnore:IGNORE
+## TestUsartModel.c:39:testFail:FAIL: Expected 2 Was 3
+## TestUsartModel.c:49:testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately:PASS
+## TestUsartModel.c:55:testShouldReturnErrorMessageUponInvalidTemperatureValue:PASS
+## TestUsartModel.c:61:testShouldReturnWakeupMessage:PASS
+##
+## -----------------------
+## 6 Tests 1 Failures 1 Ignored
+## FAIL
+
+##
+## Sample Test Results Output File (YAML)
+## ======================================
+## The following corresponds to the test executable output above.
+##
+## TestUsartModel.fail:
+## ---
+## :source:
+##   :file: test/TestUsartModel.c
+##   :dirname: test
+##   :basename: TestUsartModel.c
+## :successes:
+## - :test: testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting
+##   :line: 24
+##   :message: ''
+##   :unity_test_time: 0
+## - :test: testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately
+##   :line: 49
+##   :message: ''
+##   :unity_test_time: 0
+## - :test: testShouldReturnErrorMessageUponInvalidTemperatureValue
+##   :line: 55
+##   :message: ''
+##   :unity_test_time: 0
+## - :test: testShouldReturnWakeupMessage
+##   :line: 61
+##   :message: ''
+##   :unity_test_time: 0
+## :failures:
+## - :test: testFail
+##   :line: 39
+##   :message: Expected 2 Was 3
+##   :unity_test_time: 0
+## :ignores:
+## - :test: testIgnore
+##   :line: 34
+##   :message: ''
+##   :unity_test_time: 0
+## :counts:
+##   :total: 6
+##   :passed: 4
+##   :failed: 1
+##   :ignored: 1
+## :stdout: []
+## :time: 0.006512000225484371
+
 class GeneratorTestResults
 
   constructor :configurator, :generator_test_results_sanity_checker, :yaml_wrapper
@@ -40,41 +115,68 @@ def process_and_write_results(executable, unity_shell_result, results_file, test
     end
 
     # Remove test statistics lines
-    output_string = unity_shell_result[:output].sub(TEST_STDOUT_STATISTICS_PATTERN, '')
+    output_string = unity_shell_result[:output].sub( TEST_STDOUT_STATISTICS_PATTERN, '' )
+
+    # Process test executable results line-by-line
     output_string.lines do |line|
-      # Process Unity output
+      # Process Unity test executable output
       case line.chomp
       when /(:IGNORE)/
         elements = extract_line_elements( executable, line, results[:source][:file] )
         results[:ignores] << elements[0] 
         results[:stdout] << elements[1] if (!elements[1].nil?)
+
       when /(:PASS$)/
         elements = extract_line_elements( executable, line, results[:source][:file] )
         results[:successes] << elements[0] 
         results[:stdout] << elements[1] if (!elements[1].nil?)
+
       when /(:PASS \(.* ms\)$)/
         elements = extract_line_elements( executable, line, results[:source][:file] )
         results[:successes] << elements[0] 
         results[:stdout] << elements[1] if (!elements[1].nil?)
+
       when /(:FAIL)/
         elements = extract_line_elements( executable, line, results[:source][:file] )
         results[:failures] << elements[0]
         results[:stdout] << elements[1] if (!elements[1].nil?)
-      else # Collect up all other output
-        results[:stdout] << line.chomp
+
+      # Collect up all other output
+      else
+        results[:stdout] << line.chomp # Ignores blank lines
       end
     end
 
     @sanity_checker.verify( results, unity_shell_result[:exit_code] )
 
-    output_file = results_file.ext(@configurator.extension_testfail) if (results[:counts][:failed] > 0)
+    output_file = results_file.ext( @configurator.extension_testfail ) if (results[:counts][:failed] > 0)
 
     @yaml_wrapper.dump(output_file, results)
 
     return { :result_file => output_file, :result => results }
   end
 
-  # TODO: Filter test cases with command line test case matchers
+  # Filter list of test cases:
+  #  --test_case
+  #  --exclude_test_case
+  #
+  # @return Array - list of the test_case hashses {:test, :line_number}
+  def filter_test_cases(test_cases)
+    _test_cases = test_cases.clone 
+
+    # Filter tests which contain test_case_name passed by `--test_case` argument
+    if !@configurator.include_test_case.empty?
+      _test_cases.delete_if { |i| !(i[:test] =~ /#{@configurator.include_test_case}/) }
+    end
+
+    # Filter tests which contain test_case_name passed by `--exclude_test_case` argument
+    if !@configurator.exclude_test_case.empty?
+      _test_cases.delete_if { |i| i[:test] =~ /#{@configurator.exclude_test_case}/ }
+    end
+
+    return _test_cases
+  end
+
   def create_crash_failure(source, shell_result, test_cases)
     count = test_cases.size()
 
@@ -96,6 +198,7 @@ def create_crash_failure(source, shell_result, test_cases)
     return shell_result
   end
 
+  # Fill out a template to mimic Unity's test executable output
   def regenerate_test_executable_stdout(total:, failed:, ignored:, output:[])
     values = {
       :total => total,

lib/ceedling/generator_test_results_backtrace.rb

@@ -20,8 +20,7 @@ def do_simple(filename, executable, shell_result, test_cases)
     # Reset time
     shell_result[:time] = 0
 
-    # Revise test case list with any matches and excludes and iterate
-    test_cases = filter_test_cases( test_cases )
+    # Iterate on test cases
     test_cases.each do |test_case|
       # Build the test fixture to run with our test case of interest
       command = @tool_executor.build_command_line(
@@ -83,9 +82,7 @@ def do_gdb(filename, executable, shell_result, test_cases)
     # Reset time
     shell_result[:time] = 0
 
-    # Revise test case list with any matches and excludes and iterate
-    test_cases = filter_test_cases( test_cases )
-
+    # Iterate on test cases
     test_cases.each do |test_case|
       # Build the test fixture to run with our test case of interest
       command = @tool_executor.build_command_line(
@@ -126,20 +123,19 @@ def do_gdb(filename, executable, shell_result, test_cases)
         # Collect file_name and line in which crash occurred
         matched = crash_result[:output].match( /#{test_case[:test]}\s*\(\)\sat.+#{filename}:(\d+)\n/ )
 
-        # If we find an error report line containing `test_case() at filename.c:###`
+        # If we found an error report line containing `test_case() at filename.c:###` in `gdb` output
         if matched
           # Line number
           line_number = matched[1]
 
-          # Filter the `gdb` $stdout report
+          # Filter the `gdb` $stdout report to find most important lines of text
           crash_report = filter_gdb_test_report( crash_result[:output], test_case[:test], filename )
 
-          # Replace:
-          # - '\n' by @new_line_tag to make gdb output flat
-          # - ':' by @colon_tag to avoid test results problems
-          # to enable parsing output for default generator_test_results regex
-          # test_output = crash_report.gsub("\n", @new_line_tag).gsub(':', @colon_tag)
-          test_output = crash_report.gsub( "\n", '\n')
+          # Unity’s test executable output is line oriented.
+          # Multi-line output is not possible (it looks like random `printf()` statements to the results parser)
+          # "Encode" actual newlines as literal "\n"s (slash-n) to be handled by the test results parser.
+          test_output = crash_report.gsub( "\n", '\n' )
+
           test_output = "#{filename}:#{line_number}:#{test_case[:test]}:FAIL: Test case crashed >> #{test_output}"
 
         # Otherwise communicate that `gdb` failed to produce a usable report
@@ -160,34 +156,13 @@ def do_gdb(filename, executable, shell_result, test_cases)
         failed:  test_case_results[:failed],
         output:  test_case_results[:output]
       )
-    puts shell_result[:output]
+
     return shell_result
   end
 
   ### Private ###
   private
 
-  # Filter list of test cases:
-  #  --test_case
-  #  --exclude_test_case
-  #
-  # @return Array - list of the test_case hashses {:test, :line_number}
-  def filter_test_cases(test_cases)
-    _test_cases = test_cases.clone 
-
-    # Filter tests which contain test_case_name passed by `--test_case` argument
-    if !@configurator.include_test_case.empty?
-      _test_cases.delete_if { |i| !(i[:test] =~ /#{@configurator.include_test_case}/) }
-    end
-
-    # Filter tests which contain test_case_name passed by `--exclude_test_case` argument
-    if !@configurator.exclude_test_case.empty?
-      _test_cases.delete_if { |i| i[:test] =~ /#{@configurator.exclude_test_case}/ }
-    end
-
-    return _test_cases
-  end
-
   def filter_gdb_test_report( report, test_case, filename )
     lines = report.split( "\n" )
 

lib/ceedling/objects.yml

@@ -40,7 +40,7 @@ file_path_collection_utils:
   compose:
     - file_wrapper
 
-unity_utils:
+test_runner_manager:
   compose:
      - configurator
 
@@ -213,7 +213,7 @@ generator:
     - loginator
     - plugin_manager
     - file_wrapper
-    - unity_utils
+    - test_runner_manager
     - generator_test_results_backtrace
 
 generator_helper:
@@ -324,7 +324,7 @@ test_invoker_helper:
     - file_path_utils
     - file_wrapper
     - generator
-    - unity_utils
+    - test_runner_manager
 
 release_invoker:
   compose:

lib/ceedling/plugin_reportinator.rb

@@ -37,6 +37,51 @@ def generate_heading(message)
     return @reportinator.generate_heading(message)
   end
 
+  ##
+  ## Sample Test Results Output File (YAML)
+  ## ======================================
+  ##
+  ## TestUsartModel.fail:
+  ## ---
+  ## :source:
+  ##   :file: test/TestUsartModel.c
+  ##   :dirname: test
+  ##   :basename: TestUsartModel.c
+  ## :successes:
+  ## - :test: testGetBaudRateRegisterSettingShouldReturnAppropriateBaudRateRegisterSetting
+  ##   :line: 24
+  ##   :message: ''
+  ##   :unity_test_time: 0
+  ## - :test: testGetFormattedTemperatureFormatsTemperatureFromCalculatorAppropriately
+  ##   :line: 49
+  ##   :message: ''
+  ##   :unity_test_time: 0
+  ## - :test: testShouldReturnErrorMessageUponInvalidTemperatureValue
+  ##   :line: 55
+  ##   :message: ''
+  ##   :unity_test_time: 0
+  ## - :test: testShouldReturnWakeupMessage
+  ##   :line: 61
+  ##   :message: ''
+  ##   :unity_test_time: 0
+  ## :failures:
+  ## - :test: testFail
+  ##   :line: 39
+  ##   :message: Expected 2 Was 3
+  ##   :unity_test_time: 0
+  ## :ignores:
+  ## - :test: testIgnore
+  ##   :line: 34
+  ##   :message: ''
+  ##   :unity_test_time: 0
+  ## :counts:
+  ##   :total: 6
+  ##   :passed: 4
+  ##   :failed: 1
+  ##   :ignored: 1
+  ## :stdout: []
+  ## :time: 0.006512000225484371
+
   def assemble_test_results(results_list, options={:boom => false})
     aggregated_results = new_results()
 
@@ -53,10 +98,11 @@ def run_test_results_report(hash, verbosity=Verbosity::NORMAL, &block)
       raise CeedlingException.new( "No test results report template has been set." )
     end
 
-    run_report( @test_results_template,
-                hash,
-                verbosity,
-                &block
+    run_report(
+      @test_results_template,
+      hash,
+      verbosity,
+      &block
     )
   end
 

lib/ceedling/plugin_reportinator_helper.rb

@@ -75,7 +75,7 @@ def process_results(aggregate, results)
   end
 
   def run_report(template, hash, verbosity)
-    output = ERB.new( template, trim_mode: "%<>" )
+    output = ERB.new( template, trim_mode: "%<>-" )
 
     # Run the report template and log result with no log level heading
     @loginator.log( output.result(binding()), verbosity, LogLabels::NONE )