From 7a84a65ad7654850fe969df8919e1c74135f75f2 Mon Sep 17 00:00:00 2001 From: Rob Nadin Date: Fri, 13 Mar 2020 11:24:36 +0000 Subject: [PATCH] Fix missing test failures * Fix xcpretty_naming for xcresult * Add test result object hierarchy * Refactored group and name parsing function --- Gemfile.lock | 2 + lib/trainer.rb | 1 + lib/trainer/test_parser.rb | 165 ++++++++++++++++++------------------- lib/trainer/test_result.rb | 139 +++++++++++++++++++++++++++++++ lib/trainer/xcresult.rb | 81 +++++++++++++----- trainer.gemspec | 1 + 6 files changed, 284 insertions(+), 105 deletions(-) create mode 100644 lib/trainer/test_result.rb diff --git a/Gemfile.lock b/Gemfile.lock index e720d70..403d2f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,6 +3,7 @@ PATH specs: trainer (0.9.1) fastlane (>= 2.25.0) + parallel (>= 1.17.0) plist (>= 3.1.0, < 4.0.0) GEM @@ -246,6 +247,7 @@ GEM PLATFORMS arm64-darwin-21 + x86_64-darwin-20 DEPENDENCIES bundler diff --git a/lib/trainer.rb b/lib/trainer.rb index 2ffc0c7..9fd1170 100644 --- a/lib/trainer.rb +++ b/lib/trainer.rb @@ -4,6 +4,7 @@ require 'trainer/options' require 'trainer/test_parser' require 'trainer/junit_generator' +require 'trainer/test_result' require 'trainer/xcresult' module Trainer diff --git a/lib/trainer/test_parser.rb b/lib/trainer/test_parser.rb index e477749..45016b2 100644 --- a/lib/trainer/test_parser.rb +++ b/lib/trainer/test_parser.rb @@ -1,3 +1,5 @@ +require 'parallel' + module Trainer class TestParser attr_accessor :data @@ -66,15 +68,9 @@ def initialize(path, config = {}) UI.user_error!("File not found at path '#{path}'") unless File.exist?(path) if File.directory?(path) && path.end_with?(".xcresult") - parse_xcresult(path) + parse_xcresult(path, config[:xcpretty_naming]) else - self.file_content = File.read(path) - self.raw_json = Plist.parse_xml(self.file_content) - - return if self.raw_json["FormatVersion"].to_s.length.zero? # maybe that's a useless plist file - - ensure_file_valid! - parse_content(config[:xcpretty_naming]) + parse_test_result(path, config[:xcpretty_naming]) end end @@ -96,46 +92,15 @@ def ensure_file_valid! UI.user_error!("Format version '#{format_version}' is not supported, must be #{supported_versions.join(', ')}") unless supported_versions.include?(format_version) end - # Converts the raw plist test structure into something that's easier to enumerate - def unfold_tests(data) - # `data` looks like this - # => [{"Subtests"=> - # [{"Subtests"=> - # [{"Subtests"=> - # [{"Duration"=>0.4, - # "TestIdentifier"=>"Unit/testExample()", - # "TestName"=>"testExample()", - # "TestObjectClass"=>"IDESchemeActionTestSummary", - # "TestStatus"=>"Success", - # "TestSummaryGUID"=>"4A24BFED-03E6-4FBE-BC5E-2D80023C06B4"}, - # {"FailureSummaries"=> - # [{"FileName"=>"/Users/krausefx/Developer/themoji/Unit/Unit.swift", - # "LineNumber"=>34, - # "Message"=>"XCTAssertTrue failed - ", - # "PerformanceFailure"=>false}], - # "TestIdentifier"=>"Unit/testExample2()", - - tests = [] - data.each do |current_hash| - if current_hash["Subtests"] - tests += unfold_tests(current_hash["Subtests"]) - end - if current_hash["TestStatus"] - tests << current_hash - end - end - return tests - end - # Returns the test group and test name from the passed summary and test # Pass xcpretty_naming = true to get the test naming aligned with xcpretty def test_group_and_name(testable_summary, test, xcpretty_naming) if xcpretty_naming - group = testable_summary["TargetName"] + "." + test["TestIdentifier"].split("/")[0..-2].join(".") - name = test["TestName"][0..-3] + group = testable_summary.target_name + "." + test.identifier.split("/")[0..-2].join(".") + name = test.name[0..-3] else - group = test["TestIdentifier"].split("/")[0..-2].join(".") - name = test["TestName"] + group = test.identifier.split("/")[0..-2].join(".") + name = test.name end return group, name end @@ -146,50 +111,82 @@ def execute_cmd(cmd) return output end - def parse_xcresult(path) + def parse_test_result(path, xcpretty_naming) + self.file_content = File.read(path) + self.raw_json = Plist.parse_xml(self.file_content) + + return if self.raw_json["FormatVersion"].to_s.length.zero? # maybe that's a useless plist file + + ensure_file_valid! + parse_content(xcpretty_naming) + end + + def xcresulttool_get_json(path, id = nil) + cmd = "xcrun xcresulttool get --format json --path #{path}" + cmd << " --id #{id}" unless id.nil? + raw = execute_cmd(cmd) + JSON.parse(raw) + end + + def parse_xcresult(path, xcpretty_naming) require 'shellwords' path = Shellwords.escape(path) # Executes xcresulttool to get JSON format of the result bundle object - result_bundle_object_raw = execute_cmd("xcrun xcresulttool get --format json --path #{path}") - result_bundle_object = JSON.parse(result_bundle_object_raw) + result_bundle_object = xcresulttool_get_json(path) # Parses JSON into ActionsInvocationRecord to find a list of all ids for ActionTestPlanRunSummaries actions_invocation_record = Trainer::XCResult::ActionsInvocationRecord.new(result_bundle_object) test_refs = actions_invocation_record.actions.map do |action| action.action_result.tests_ref end.compact - ids = test_refs.map(&:id) + test_ids = test_refs.map(&:id) # Maps ids into ActionTestPlanRunSummaries by executing xcresulttool to get JSON # containing specific information for each test summary, - summaries = ids.map do |id| - raw = execute_cmd("xcrun xcresulttool get --format json --path #{path} --id #{id}") - json = JSON.parse(raw) + summaries = Parallel.map(test_ids) do |id| + json = xcresulttool_get_json(path, id) Trainer::XCResult::ActionTestPlanRunSummaries.new(json) end - # Converts the ActionTestPlanRunSummaries to data for junit generator - failures = actions_invocation_record.issues.test_failure_summaries || [] - summaries_to_data(summaries, failures) - end - - def summaries_to_data(summaries, failures) # Gets flat list of all ActionTestableSummary all_summaries = summaries.map(&:summaries).flatten testable_summaries = all_summaries.map(&:testable_summaries).flatten + # Gets flat list of all ActionTestMetadata that failed + failed_tests = testable_summaries.map do |testable_summary| + testable_summary.all_tests.find_all { |a| a.test_status == 'Failure' } + end.flatten + + # Find a list of all ids for ActionTestSummary + summary_ids = failed_tests.map do |test| + test.summary_ref.id + end + + # Maps summary references into array of ActionTestSummary by executing xcresulttool to get JSON + # containing more information for each test failure, + failures = Parallel.map(summary_ids) do |id| + json = xcresulttool_get_json(path, id) + Trainer::XCResult::ActionTestSummary.new(json) + end + + # Converts the ActionTestPlanRunSummaries to data for junit generator + summaries_to_data(testable_summaries, failures, xcpretty_naming) + end + + def summaries_to_data(testable_summaries, failures, xcpretty_naming) # Maps ActionTestableSummary to rows for junit generator rows = testable_summaries.map do |testable_summary| all_tests = testable_summary.all_tests.flatten test_rows = all_tests.map do |test| + test_group, test_name = test_group_and_name(testable_summary, test, xcpretty_naming) test_row = { identifier: "#{test.parent.name}.#{test.name}", - name: test.name, + name: test_name, duration: test.duration, status: test.test_status, - test_group: test.parent.name, + test_group: test_group, # These don't map to anything but keeping empty strings guid: "" @@ -199,10 +196,10 @@ def summaries_to_data(summaries, failures) failure = test.find_failure(failures) if failure test_row[:failures] = [{ - file_name: "", - line_number: 0, - message: "", - performance_failure: {}, + file_name: failure.file_name, + line_number: failure.line_number, + message: failure.message, + performance_failure: failure.performance_failure, failure_message: failure.failure_message }] end @@ -229,33 +226,35 @@ def summaries_to_data(summaries, failures) # Convert the Hashes and Arrays in something more useful def parse_content(xcpretty_naming) - self.data = self.raw_json["TestableSummaries"].collect do |testable_summary| + testable_summaries = self.raw_json['TestableSummaries'].collect do |summary_data| + Trainer::TestResult::ActionTestableSummary.new(summary_data) + end + + self.data = testable_summaries.map do |testable_summary| summary_row = { - project_path: testable_summary["ProjectPath"], - target_name: testable_summary["TargetName"], - test_name: testable_summary["TestName"], - duration: testable_summary["Tests"].map { |current_test| current_test["Duration"] }.inject(:+), - tests: unfold_tests(testable_summary["Tests"]).collect do |current_test| + project_path: testable_summary.project_path, + target_name: testable_summary.target_name, + test_name: testable_summary.test_name, + duration: testable_summary.tests.map { |current_test| current_test.duration }.inject(:+), + tests: testable_summary.all_tests.map do |current_test| test_group, test_name = test_group_and_name(testable_summary, current_test, xcpretty_naming) current_row = { - identifier: current_test["TestIdentifier"], + identifier: current_test.identifier, test_group: test_group, name: test_name, - object_class: current_test["TestObjectClass"], - status: current_test["TestStatus"], - guid: current_test["TestSummaryGUID"], - duration: current_test["Duration"] + object_class: current_test.object_class, + status: current_test.status, + guid: current_test.summary_guid, + duration: current_test.duration } - if current_test["FailureSummaries"] - current_row[:failures] = current_test["FailureSummaries"].collect do |current_failure| - { - file_name: current_failure['FileName'], - line_number: current_failure['LineNumber'], - message: current_failure['Message'], - performance_failure: current_failure['PerformanceFailure'], - failure_message: "#{current_failure['Message']} (#{current_failure['FileName']}:#{current_failure['LineNumber']})" - } - end + current_row[:failures] = current_test.failure_summaries.map do |current_failure| + { + file_name: current_failure.file_name, + line_number: current_failure.line_number, + message: current_failure.message, + performance_failure: current_failure.performance_failure, + failure_message: current_failure.failure_message + } end current_row end diff --git a/lib/trainer/test_result.rb b/lib/trainer/test_result.rb new file mode 100644 index 0000000..e62b802 --- /dev/null +++ b/lib/trainer/test_result.rb @@ -0,0 +1,139 @@ +module Trainer + module TestResult + + class AbstractObject + attr_accessor :object_class + def initialize(data) + self.object_class = data['TestObjectClass'] + end + end + + # - ActionTestableSummary + # * Kind: object + # * Properties: + # + project_path: String + # + target_name: String + # + test_name: String + # + tests: [Test] + class ActionTestableSummary < AbstractObject + attr_accessor :project_path + attr_accessor :target_name + attr_accessor :test_name + attr_accessor :tests + def initialize(data) + self.project_path = data['ProjectPath'] + self.target_name = data['TargetName'] + self.test_name = data['TestName'] + self.tests = data['Tests'].collect do |test_data| + ActionTestSummaryIdentifiableObject.create(test_data) + end + super + end + + def all_tests + tests.map(&:all_subtests).flatten + end + end + + # - ActionTestSummaryIdentifiableObject + # * Kind: object + # * Properties: + # + identifier: String + # + name: String + # + duration: Double + class ActionTestSummaryIdentifiableObject < AbstractObject + attr_accessor :identifier + attr_accessor :name + attr_accessor :duration + def initialize(data) + self.identifier = data['TestIdentifier'] + self.name = data['TestName'] + self.duration = data['Duration'] + super + end + + def all_subtests + raise 'Not overridden' + end + + def self.create(data) + type = data['TestObjectClass'] + if type == 'IDESchemeActionTestSummaryGroup' + ActionTestSummaryGroup.new(data) + elsif type == 'IDESchemeActionTestSummary' + ActionTestSummary.new(data) + else + raise "Unsupported type: #{type}" + end + end + end + + # - ActionTestSummaryGroup + # * Kind: object + # * Properties: + # + subtests: [Test] + class ActionTestSummaryGroup < ActionTestSummaryIdentifiableObject + attr_accessor :subtests + def initialize(data) + self.subtests = (data['Subtests'] || []).collect do |subtests_data| + ActionTestSummaryIdentifiableObject.create(subtests_data) + end + super + end + + def all_subtests + subtests.map(&:all_subtests).flatten + end + end + + # - ActionTestSummary + # * Kind: object + # * Properties: + # + status: String + # + summary_guid: String + # + activity_summaries: [ActivitySummaries]? + # + failure_summaries: [FailureSummary]? + class ActionTestSummary < ActionTestSummaryIdentifiableObject + attr_accessor :status + attr_accessor :summary_guid + attr_accessor :failure_summaries + def initialize(data) + self.status = data['TestStatus'] + self.summary_guid = data['TestSummaryGUID'] + self.failure_summaries = (data['FailureSummaries'] || []).collect do |summary_data| + ActionTestFailureSummary.new(summary_data) + end + super + end + + def all_subtests + [self] + end + end + + # - ActionTestFailureSummary + # * Kind: object + # * Properties: + # + file_name: String + # + line_number: Int + # + message: String + # + performance_failure: Bool + class ActionTestFailureSummary < AbstractObject + attr_accessor :file_name + attr_accessor :line_number + attr_accessor :message + attr_accessor :performance_failure + def initialize(data) + self.file_name = data['FileName'] + self.line_number = data['LineNumber'] + self.message = data['Message'] + self.performance_failure = data['PerformanceFailure'] + super + end + + def failure_message + "#{message} (#{file_name}:#{line_number})" + end + end + end +end diff --git a/lib/trainer/xcresult.rb b/lib/trainer/xcresult.rb index 8c33d4b..18d84d0 100644 --- a/lib/trainer/xcresult.rb +++ b/lib/trainer/xcresult.rb @@ -76,6 +76,7 @@ class ActionTestableSummary < ActionAbstractTestSummary attr_accessor :target_name attr_accessor :test_kind attr_accessor :tests + attr_accessor :failure_summaries def initialize(data) self.project_relative_path = fetch_value(data, "projectRelativePath") self.target_name = fetch_value(data, "targetName") @@ -83,6 +84,7 @@ def initialize(data) self.tests = fetch_values(data, "tests").map do |tests_data| ActionTestSummaryIdentifiableObject.create(tests_data, self) end + self.failure_summaries = fetch_values(data, 'failureSummaries') super end @@ -153,18 +155,21 @@ def all_subtests # + performanceMetricsCount: Int # + failureSummariesCount: Int # + activitySummariesCount: Int + # + summaryRef: Reference class ActionTestMetadata < ActionTestSummaryIdentifiableObject attr_accessor :test_status attr_accessor :duration attr_accessor :performance_metrics_count attr_accessor :failure_summaries_count attr_accessor :activity_summaries_count + attr_accessor :summary_ref def initialize(data, parent) self.test_status = fetch_value(data, "testStatus") self.duration = fetch_value(data, "duration").to_f self.performance_metrics_count = fetch_value(data, "performanceMetricsCount") self.failure_summaries_count = fetch_value(data, "failureSummariesCount") self.activity_summaries_count = fetch_value(data, "activitySummariesCount") + self.summary_ref = Reference.new(data['summaryRef']) if data['summaryRef'] super(data, parent) end @@ -172,36 +177,68 @@ def all_subtests return [self] end - def find_failure(failures) + def find_failure(summaries) if self.test_status == "Failure" - # Tries to match failure on test case name - # Example TestFailureIssueSummary: - # producingTarget: "TestThisDude" - # test_case_name: "TestThisDude.testFailureJosh2()" (when Swift) - # or "-[TestThisDudeTests testFailureJosh2]" (when Objective-C) - # Example ActionTestMetadata - # identifier: "TestThisDude/testFailureJosh2()" (when Swift) - # or identifier: "TestThisDude/testFailureJosh2" (when Objective-C) - - found_failure = failures.find do |failure| - # Clean test_case_name to match identifier format - # Sanitize for Swift by replacing "." for "/" - # Sanitize for Objective-C by removing "-", "[", "]", and replacing " " for ?/ - sanitized_test_case_name = failure.test_case_name - .tr(".", "/") - .tr("-", "") - .tr("[", "") - .tr("]", "") - .tr(" ", "/") - self.identifier == sanitized_test_case_name + found_summary = summaries.find do |summary| + self.identifier == summary.identifier + end + if found_summary + return found_summary.failure_summaries.first + else + return nil end - return found_failure else return nil end end end + # - ActionTestSummary + # * Kind: object + # * Properties: + # + testStatus: String + # + duration: Double + # + failureSummaries: [ActionTestFailureSummary] + # + activitySummaries: [ActionTestActivitySummary] + class ActionTestSummary < ActionTestSummaryIdentifiableObject + attr_accessor :test_status + attr_accessor :duration + attr_accessor :failure_summaries + def initialize(data) + self.test_status = fetch_value(data, 'testStatus') + self.duration = fetch_value(data, 'duration').to_f + self.failure_summaries = fetch_values(data, 'failureSummaries').map do |summary_data| + ActionTestFailureSummary.new(summary_data) + end + super(data, nil) + end + end + + # - ActionTestFailureSummary + # * Kind: object + # * Properties: + # + message: String? + # + fileName: String + # + lineNumber: Int + # + isPerformanceFailure: Boolean? + class ActionTestFailureSummary < AbstractObject + attr_accessor :message + attr_accessor :file_name + attr_accessor :line_number + attr_accessor :performance_failure + def initialize(data) + self.message = fetch_value(data, 'message') + self.file_name = fetch_value(data, 'fileName') + self.line_number = fetch_value(data, 'lineNumber').to_i + self.performance_failure = fetch_value(data, 'isPerformanceFailure').to_boolean if data['isPerformanceFailure'] + super + end + + def failure_message + "#{message} (#{file_name}:#{line_number})" + end + end + # - ActionsInvocationRecord # * Kind: object # * Properties: diff --git a/trainer.gemspec b/trainer.gemspec index 7cc7e6a..b36e46f 100644 --- a/trainer.gemspec +++ b/trainer.gemspec @@ -24,6 +24,7 @@ Gem::Specification.new do |spec| spec.add_dependency 'plist', ">= 3.1.0", "< 4.0.0" spec.add_dependency 'fastlane', '>= 2.25.0' + spec.add_dependency 'parallel', '>= 1.17.0' # Development only spec.add_development_dependency 'bundler'