From c79dc0d2a55cc096c52d78e01505449b0bf1a431 Mon Sep 17 00:00:00 2001 From: Charles Oliveira Date: Wed, 24 Apr 2024 11:15:16 -0300 Subject: [PATCH] tradefed: support fetching tradefed file from tuxsuite Signed-off-by: Charles Oliveira --- test/test_tradefed.py | 128 +++++++++++++++++++++++++++++++++++++++++- tradefed/__init__.py | 96 ++++++++++++++++++++++++------- 2 files changed, 204 insertions(+), 20 deletions(-) diff --git a/test/test_tradefed.py b/test/test_tradefed.py index 0854bd0..c8ec203 100644 --- a/test/test_tradefed.py +++ b/test/test_tradefed.py @@ -493,7 +493,7 @@ def tearDown(self): @patch("tradefed.Tradefed._get_from_artifactorial") @patch("tradefed.update_testjob_status.delay") @patch("tradefed.Tradefed.tradefed_results_url", new_callable=PropertyMock) - def test_postprocess_testjob( + def test_postprocess_testjob_lava( self, results_url_mock, update_testjob_status_mock, @@ -528,6 +528,48 @@ def test_postprocess_testjob( create_testrun_attachment_mock.assert_not_called() update_testjob_status_mock.assert_called() + @patch("tradefed.Tradefed._get_tradefed_url_from_tuxsuite") + @patch("tradefed.Tradefed._download_results") + @patch("tradefed.Tradefed._extract_cts_results") + @patch("tradefed.Tradefed._create_testrun_attachment") + @patch("tradefed.Tradefed._assign_test_log") + @patch("tradefed.update_testjob_status.delay") + @patch("tradefed.Tradefed.tradefed_results_url", new_callable=PropertyMock) + def test_postprocess_testjob_tuxsuite( + self, + results_url_mock, + update_testjob_status_mock, + assign_test_log_mock, + create_testrun_attachment_mock, + extract_cts_results_mock, + download_results_mock, + get_tradefed_url_from_tuxsuite_mock, + ): + results_url_mock.return_value = "http://foo.com" + download_results_mock.return_value = ResultFiles() + testjob_mock = MagicMock() + id_mock = PropertyMock(return_value="999111") + type(testjob_mock).pk = id_mock + job_id_mock = PropertyMock(return_value="1234") + type(testjob_mock).job_id = job_id_mock + testjob_mock.backend = MagicMock() + implementation_type_mock = PropertyMock(return_value="tuxsuite") + type(testjob_mock.backend).implementation_type = implementation_type_mock + testjob_target = MagicMock() + project_settings_mock = PropertyMock(return_value='{}') + type(testjob_target).project_settings = project_settings_mock + type(testjob_mock).target = testjob_target + self.plugin.postprocess_testjob(testjob_mock) + implementation_type_mock.assert_called_once_with() + results_url_mock.assert_called_with() + testjob_mock.testrun.metadata.__setitem__.assert_called_with('tradefed_results_url_1234', 'http://foo.com') + testjob_mock.testrun.save.assert_called_with() + assign_test_log_mock.assert_not_called() + create_testrun_attachment_mock.assert_not_called() + update_testjob_status_mock.assert_called() + get_tradefed_url_from_tuxsuite_mock.assert_called() + download_results_mock.assert_called() + @patch("tradefed.Tradefed._create_testrun_attachment") @patch("tradefed.Tradefed._assign_test_log") @patch("tradefed.Tradefed._get_from_artifactorial") @@ -1169,3 +1211,87 @@ def test_extract_tarball_filename_from_url(self): filename = self.plugin._extract_tarball_filename_from_url("http://some.url/?param1=val1&filename=tradefed.tar.xz") self.assertEqual("tradefed.tar.xz", filename) + + def test_get_tradefed_url_from_tuxsuite_bad_request(self): + job_url = "http://tuxsuite.com/job#123" + + testjob = MagicMock() + testjob.url = job_url + + with requests_mock.Mocker() as fake_request: + fake_request.get( + f"{job_url}/results", + status_code=404, + ) + + url = self.plugin._get_tradefed_url_from_tuxsuite(testjob) + self.assertIsNone(url) + + def test_get_tradefed_url_from_tuxsuite_errored_json(self): + job_url = "http://tuxsuite.com/job#123" + + testjob = MagicMock() + testjob.url = job_url + + with requests_mock.Mocker() as fake_request: + fake_request.get( + f"{job_url}/results", + status_code=200, + content=b'{"error": "some error"}', + headers={"Content-Type": "application/json"}, + ) + + url = self.plugin._get_tradefed_url_from_tuxsuite(testjob) + self.assertIsNone(url) + + def test_get_tradefed_url_from_tuxsuite_no_test_attachment(self): + job_url = "http://tuxsuite.com/job#123" + + testjob = MagicMock() + testjob.url = job_url + + with requests_mock.Mocker() as fake_request: + fake_request.get( + f"{job_url}/results", + status_code=200, + content=b'{"suite": {"test1": {"result": "pass"}}}', + headers={"Content-Type": "application/json"}, + ) + + url = self.plugin._get_tradefed_url_from_tuxsuite(testjob) + self.assertIsNone(url) + + def test_get_tradefed_url_from_tuxsuite_no_reference(self): + job_url = "http://tuxsuite.com/job#123" + + testjob = MagicMock() + testjob.url = job_url + + with requests_mock.Mocker() as fake_request: + fake_request.get( + f"{job_url}/results", + status_code=200, + content=b'{"suite": {"test-attachment": {"result": "pass"}}}', + headers={"Content-Type": "application/json"}, + ) + + url = self.plugin._get_tradefed_url_from_tuxsuite(testjob) + self.assertIsNone(url) + + def test_get_tradefed_url_from_tuxsuite(self): + job_url = "http://tuxsuite.com/job#123" + tradefed_url = "http://tradefed.url" + + testjob = MagicMock() + testjob.url = job_url + + with requests_mock.Mocker() as fake_request: + fake_request.get( + f"{job_url}/results", + status_code=200, + content=b'{"suite": {"test-attachment": {"result": "pass", "reference": "http://tradefed.url"}}}', + headers={"Content-Type": "application/json"}, + ) + + url = self.plugin._get_tradefed_url_from_tuxsuite(testjob) + self.assertEqual(tradefed_url, url) diff --git a/tradefed/__init__.py b/tradefed/__init__.py index e2ef22e..d9a8d6a 100644 --- a/tradefed/__init__.py +++ b/tradefed/__init__.py @@ -550,34 +550,91 @@ def _extract_tradefed_from_job_definition(self, testjob): return tradefed_files + def _get_tradefed_url_from_tuxsuite(self, testjob): + """ + Android jobs coming from Tuxsuite should have a test names "test-attachment" + and it should have a reference attached to it, same as regular LAVA. + + The test looks like this: + + { + 'test-attachment': { + 'reference': 'https://qa-reports.linaro.org/api/testruns/23505462/attachments/?filename=tradefed-output-20240417085054.tar.xz', + 'result': 'pass' + } + } + + The tricky part is that SQUAD only reads "result" and disregard "reference". + So Tradefed plugin needs to request results once more from Tuxsuite just to + get the "reference" value. + """ + + session = get_session() + results_url = f"{testjob.url}/results" + response = session.get(results_url) + if response.status_code != 200: + logger.error(f"Failed to retrieve results from Tuxsuite: {response.content}") + return None + + results = response.json() + error = results.get("error", None) + if error is not None: + logger.error(f"Failed to retrieve results from Tuxsuite: {error}") + return None + + for suite, suite_tests in results.items(): + test = suite_tests.get("test-attachment") + if test: + + reference = test.get("reference") + if reference is None: + logger.error("Failed to retrieve results from Tuxsuite: there is no 'reference' in 'test-attachment'") + return None + + return reference + + logger.info("No 'test-attachment' found for this Tuxsuite job") + return None + def postprocess_testjob(self, testjob): self.extra_args["job_id"] = testjob.id logger.info("Starting CTS/VTS plugin for test job: %s" % testjob) - if not testjob.backend.implementation_type == 'lava': - logger.error(f"Test job {testjob.id} doesn't come from LAVA") + backend_type = testjob.backend.implementation_type + if backend_type not in ['lava', 'tuxsuite']: + logger.error(f"Test job {testjob.id} does not come from LAVA nor Tuxsuite") update_testjob_status.delay(testjob.id, self.extra_args.get("job_status")) return - tradefed_files = self._extract_tradefed_from_job_definition(testjob) - if len(tradefed_files) != 1: - logger.info(f"Job {testjob.id} has {len(tradefed_files)} tradefed files in the definition, it should have 1, aborting") - update_testjob_status.delay(testjob.id, self.extra_args.get("job_status")) - logger.info("Finishing CTS/VTS plugin for test run: %s" % testjob) - return - - tradefed_name, results_format = tradefed_files[0] - - results_extracted = False results = None - try: - results = self._get_from_artifactorial(testjob, tradefed_name) - except xmlrpc.client.ProtocolError as err: - error_cleaned = 'Failed to process CTS/VTS tests: %s - %s' % (err.errcode, testjob.backend.get_implementation().url_remove_token(str(err.errmsg))) - logger.error(error_cleaned) + if backend_type == 'lava': + tradefed_files = self._extract_tradefed_from_job_definition(testjob) + if len(tradefed_files) != 1: + logger.error(f"Job {testjob.id} has {len(tradefed_files)} tradefed files in the definition, it should have 1") + update_testjob_status.delay(testjob.id, self.extra_args.get("job_status")) + logger.info("Finishing CTS/VTS plugin for test run: %s" % testjob) + return + + try: + tradefed_name, results_format = tradefed_files[0] + results = self._get_from_artifactorial(testjob, tradefed_name) + except xmlrpc.client.ProtocolError as err: + error_cleaned = 'Failed to process CTS/VTS tests: %s - %s' % (err.errcode, testjob.backend.get_implementation().url_remove_token(str(err.errmsg))) + logger.error(error_cleaned) + + testjob.failure += error_cleaned + testjob.save() + else: + # Get it from Tuxsuite + url = self._get_tradefed_url_from_tuxsuite(testjob) + if url is None: + logger.info("Aborting CTS/VTS, no tradefed URL found within Tuxsuite results") + update_testjob_status.delay(testjob.id, self.extra_args.get("job_status")) + return - testjob.failure += error_cleaned - testjob.save() + results = self._download_results(url) + tradefed_name = "vts-lkft" + results_format = "aggregated" if results is None: logger.info("Aborting CTS/VTS, no tradefed file found") @@ -588,6 +645,7 @@ def postprocess_testjob(self, testjob): testjob.testrun.metadata["tradefed_results_url_%s" % testjob.job_id] = self.tradefed_results_url testjob.testrun.save() + results_extracted = False if results.test_results is not None: if testjob.target.get_setting("PLUGINS_TRADEFED_EXTRACT_AGGREGATED", False) and results_format == "aggregated": self._extract_cts_results(results.test_results.contents, testjob.testrun, tradefed_name)