From 21ae8b2431e2f9bd610069127b08bd1f33138982 Mon Sep 17 00:00:00 2001 From: Dani Alcala <112832187+clavedeluna@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:25:34 -0300 Subject: [PATCH] Report failed parsing (#75) * report even if no results * pygoat findings only if changset * update unit test --- .github/workflows/codemod_pygoat.yml | 2 +- Makefile | 4 ++-- ...at_findings.py => test_pygoat_findings.py} | 12 +++++------ integration_tests/base_test.py | 2 +- src/codemodder/context.py | 9 ++++----- tests/{test_main.py => test_codemodder.py} | 20 ++++++++++++++----- 6 files changed, 29 insertions(+), 20 deletions(-) rename ci_tests/{test_webgoat_findings.py => test_pygoat_findings.py} (65%) rename tests/{test_main.py => test_codemodder.py} (88%) diff --git a/.github/workflows/codemod_pygoat.yml b/.github/workflows/codemod_pygoat.yml index e4eada36..aabe7bae 100644 --- a/.github/workflows/codemod_pygoat.yml +++ b/.github/workflows/codemod_pygoat.yml @@ -38,4 +38,4 @@ jobs: - name: Run Codemodder run: codemodder --output output.codetf pygoat - name: Check PyGoat Findings - run: make webgoat-test + run: make pygoat-test diff --git a/Makefile b/Makefile index 94947e94..c25e5c53 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ test: integration-test: ${PYTEST} integration_tests -webgoat-test: - ${PYTEST} -v ci_tests/test_webgoat_findings.py +pygoat-test: + ${PYTEST} -v ci_tests/test_pygoat_findings.py lint: pylint -v codemodder core_codemods tests integration_tests diff --git a/ci_tests/test_webgoat_findings.py b/ci_tests/test_pygoat_findings.py similarity index 65% rename from ci_tests/test_webgoat_findings.py rename to ci_tests/test_pygoat_findings.py index 8ce1d1ad..f0701483 100644 --- a/ci_tests/test_webgoat_findings.py +++ b/ci_tests/test_pygoat_findings.py @@ -17,17 +17,17 @@ @pytest.fixture(scope="session") -def webgoat_findings(): +def pygoat_findings(): with open("output.codetf") as ff: results = json.load(ff) - yield set([x["codemod"] for x in results["results"]]) + yield set([x["codemod"] for x in results["results"] if x["changeset"]]) -def test_num_webgoat_findings(webgoat_findings): - assert len(webgoat_findings) == len(EXPECTED_FINDINGS) +def test_num_pygoat_findings(pygoat_findings): + assert len(pygoat_findings) == len(EXPECTED_FINDINGS) @pytest.mark.parametrize("finding", EXPECTED_FINDINGS) -def test_webgoat_findings(webgoat_findings, finding): - assert finding in webgoat_findings +def test_pygoat_findings(pygoat_findings, finding): + assert finding in pygoat_findings diff --git a/integration_tests/base_test.py b/integration_tests/base_test.py index e3886e72..4a2ddd53 100644 --- a/integration_tests/base_test.py +++ b/integration_tests/base_test.py @@ -175,7 +175,7 @@ def _run_idempotency_chec(self, command): if pathlib.Path(self.output_path).exists(): with open(self.output_path, "r", encoding="utf-8") as f: codetf = json.load(f) - assert codetf["results"] == [] + assert codetf["results"][0]["changeset"] == [] with open(self.code_path, "r", encoding="utf-8") as f: still_new_code = f.read() diff --git a/src/codemodder/context.py b/src/codemodder/context.py index 70ac466f..bd8d982b 100644 --- a/src/codemodder/context.py +++ b/src/codemodder/context.py @@ -75,17 +75,16 @@ def add_dependency(self, dependency: str): def compile_results(self, codemods: list[CodemodExecutorWrapper]): results = [] for codemod in codemods: - if not (changeset := self._results_by_codemod.get(codemod.id)): - continue - data = { "codemod": codemod.id, "summary": codemod.summary, "description": codemod.description, "references": codemod.references, "properties": {}, - "failedFiles": [], - "changeset": [change.to_json() for change in changeset], + "failedFiles": [str(file) for file in self.get_failures(codemod.id)], + "changeset": [ + change.to_json() for change in self.get_results(codemod.id) + ], } results.append(data) diff --git a/tests/test_main.py b/tests/test_codemodder.py similarity index 88% rename from tests/test_main.py rename to tests/test_codemodder.py index fcb7eca7..78cc1813 100644 --- a/tests/test_main.py +++ b/tests/test_codemodder.py @@ -32,6 +32,10 @@ def test_cst_parsing_fails(self, mock_reporting, mock_parse): "tests/samples/", "--output", "here.txt", + "--codemod-include", + "requests-verify", + "--path-include", + "*request.py", ] res = run(args) @@ -39,12 +43,19 @@ def test_cst_parsing_fails(self, mock_reporting, mock_parse): assert res == 0 mock_parse.assert_called() - # We still report for now even if parsing failed mock_reporting.assert_called_once() args_to_reporting = mock_reporting.call_args_list[0][0] assert len(args_to_reporting) == 4 _, _, _, results_by_codemod = args_to_reporting - assert results_by_codemod == [] + assert results_by_codemod != [] + + requests_report = results_by_codemod[0] + assert requests_report["changeset"] == [] + assert len(requests_report["failedFiles"]) == 2 + assert sorted(requests_report["failedFiles"]) == [ + "tests/samples/make_request.py", + "tests/samples/unverified_request.py", + ] @mock.patch("codemodder.codemodder.update_code") @mock.patch("codemodder.codemods.base_codemod.semgrep_run", side_effect=semgrep_run) @@ -79,10 +90,9 @@ def test_reporting(self, mock_reporting, dry_run): args_to_reporting = mock_reporting.call_args_list[0][0] assert len(args_to_reporting) == 4 _, _, _, results_by_codemod = args_to_reporting - # assert len(results_by_codemod) == 2 - for codemod_results in results_by_codemod: - assert len(codemod_results["changeset"]) > 0 + registry = load_registered_codemods() + assert len(results_by_codemod) == len(registry.codemods) @mock.patch("codemodder.codemods.base_codemod.semgrep_run") def test_no_codemods_to_run(self, mock_semgrep_run):