From ffcabf1e0bed6045518afbab0fc1fd803b0b960f Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Fri, 23 Feb 2024 15:49:28 +0100 Subject: [PATCH 1/4] [CI] Update Ana06/automatic-pull-request-review The old version was using a deprecated version of Node. --- .github/workflows/changelog.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 853a5cc4be..d8d6ad3c80 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -29,14 +29,14 @@ jobs: echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG" - name: Reject pull request if no CHANGELOG update if: ${{ always() && steps.changelog_updated.outcome == 'failure' }} - uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0 + uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} event: REQUEST_CHANGES body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`" allow_duplicate: false - name: Dismiss previous review if CHANGELOG update - uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0 + uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} event: DISMISS From 8857511e553439869a2a17783779609ca497db91 Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Fri, 23 Feb 2024 16:02:20 +0100 Subject: [PATCH 2/4] [CI] Fix CHANGELOG PR review Sending a PR review with a message about the CHANGELOG needing to be updated has been broken since July, where the permissions were changed. --- .github/workflows/changelog.yml | 3 ++- CHANGELOG.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index d8d6ad3c80..d915e67d43 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -7,7 +7,8 @@ on: pull_request_target: types: [opened, edited, synchronize] -permissions: read-all +permissions: + pull-requests: write jobs: check_changelog: diff --git a/CHANGELOG.md b/CHANGELOG.md index 514fd4d789..e6fa77d05c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ ### Development +- ci: Fix PR review in the changelog check GH action #2004 @Ana06 - ci: update github workflows to use latest version for depricated actions (checkout, setup-python, upload-artifact, download-artifact) #1967 @sjha2048 ### Raw diffs From 9a449b6bd923da7577d880cc1c13f65fe1baf84d Mon Sep 17 00:00:00 2001 From: Rohit Konakalla <92310728+Rohit1123@users.noreply.github.com> Date: Sun, 25 Feb 2024 20:31:36 +0530 Subject: [PATCH 3/4] Load .json.gz files directly (#1990) * Load .json.gz files directly * Add helper function to load .json and replace json.load references * add test and update change log * add .json.gz in EXTENSIONS_DYNAMIC Co-authored-by: Moritz --------- Co-authored-by: Moritz --- CHANGELOG.md | 1 + capa/helpers.py | 17 ++++++++++++++--- capa/loader.py | 5 ++--- tests/fixtures.py | 8 ++------ tests/test_main.py | 6 ++++++ 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e469a676b6..5cd526f5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### New Features +- add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123 ### Breaking Changes diff --git a/capa/helpers.py b/capa/helpers.py index ad27f3903e..ecf1b32005 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -6,6 +6,7 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import sys +import gzip import json import inspect import logging @@ -30,7 +31,7 @@ EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64") -EXTENSIONS_DYNAMIC = ("json", "json_") +EXTENSIONS_DYNAMIC = ("json", "json_", "json.gz") EXTENSIONS_ELF = "elf_" EXTENSIONS_FREEZE = "frz" @@ -70,9 +71,19 @@ def assert_never(value) -> NoReturn: assert False, f"Unhandled value: {value} ({type(value).__name__})" # noqa: B011 -def get_format_from_report(sample: Path) -> str: - report = json.load(sample.open(encoding="utf-8")) +def load_json_from_path(json_path: Path): + with gzip.open(json_path, "r") as compressed_report: + try: + report_json = compressed_report.read() + except gzip.BadGzipFile: + report = json.load(json_path.open(encoding="utf-8")) + else: + report = json.loads(report_json) + return report + +def get_format_from_report(sample: Path) -> str: + report = load_json_from_path(sample) if "CAPE" in report: return FORMAT_CAPE diff --git a/capa/loader.py b/capa/loader.py index e4f0a5c92b..024091e01c 100644 --- a/capa/loader.py +++ b/capa/loader.py @@ -6,7 +6,6 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import sys -import json import logging import datetime from typing import Set, Dict, List, Optional @@ -180,7 +179,7 @@ def get_extractor( if backend == BACKEND_CAPE: import capa.features.extractors.cape.extractor - report = json.loads(input_path.read_text(encoding="utf-8")) + report = capa.helpers.load_json_from_path(input_path) return capa.features.extractors.cape.extractor.CapeExtractor.from_report(report) elif backend == BACKEND_DOTNET: @@ -297,7 +296,7 @@ def get_file_extractors(input_file: Path, input_format: str) -> List[FeatureExtr elif input_format == FORMAT_CAPE: import capa.features.extractors.cape.extractor - report = json.loads(input_file.read_text(encoding="utf-8")) + report = capa.helpers.load_json_from_path(input_file) file_extractors.append(capa.features.extractors.cape.extractor.CapeExtractor.from_report(report)) return file_extractors diff --git a/tests/fixtures.py b/tests/fixtures.py index ebfe557a50..ce21d7db1e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -191,14 +191,10 @@ def get_binja_extractor(path: Path): @lru_cache(maxsize=1) def get_cape_extractor(path): - import gzip - import json - + from capa.helpers import load_json_from_path from capa.features.extractors.cape.extractor import CapeExtractor - with gzip.open(path, "r") as compressed_report: - report_json = compressed_report.read() - report = json.loads(report_json) + report = load_json_from_path(path) return CapeExtractor.from_report(report) diff --git a/tests/test_main.py b/tests/test_main.py index 6d588dda1c..2ee7e7da29 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -356,3 +356,9 @@ def test_main_cape1(tmp_path): assert capa.main.main([str(path), "-j", "-r", str(rules)]) == 0 assert capa.main.main([str(path), "-v", "-r", str(rules)]) == 0 assert capa.main.main([str(path), "-vv", "-r", str(rules)]) == 0 + + +def test_main_cape_gzip(): + # tests successful execution of .json.gz + path = str(fixtures.get_data_path_by_name("0000a657")) + assert capa.main.main([path]) == 0 From 08b3ae60d76f6f7a40ad2578d3a54135da41f638 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 27 Feb 2024 11:56:47 +0000 Subject: [PATCH 4/4] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 34e3755624..ce3e6d74b1 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 34e3755624530a6ed0da9942ad3c68ea8afa89d3 +Subproject commit ce3e6d74b1526bacd370d1c4001ff844876e3edc