Merge branch 'master' into stack-trace

.github/workflows/build.yml

@@ -11,34 +11,41 @@ permissions:
 
 jobs:
   build:
-    name: PyInstaller for ${{ matrix.os }}
+    name: PyInstaller for ${{ matrix.os }} / Py ${{ matrix.python_version }}
     runs-on: ${{ matrix.os }}
     strategy:
       # set to false for debugging
       fail-fast: true
       matrix:
+        # using Python 3.8 to support running across multiple operating systems including Windows 7
         include:
           - os: ubuntu-20.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
+            python_version: 3.8
+          - os: ubuntu-20.04
+            artifact_name: capa
+            asset_name: linux-py311
+            python_version: 3.11
           - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
+            python_version: 3.8
           - os: macos-11
             # use older macOS for assumed better portability
             artifact_name: capa
             asset_name: macos
+            python_version: 3.8
     steps:
       - name: Checkout capa
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: true
-      # using Python 3.8 to support running across multiple operating systems including Windows 7
-      - name: Set up Python 3.8
+      - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.python_version }}
       - if: matrix.os == 'ubuntu-20.04'
         run: sudo apt-get install -y libyaml-dev
       - name: Upgrade pip, setuptools
@@ -55,13 +62,17 @@ jobs:
         run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
       - name: Does it run (ELF)?
         run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
+      - name: Does it run (CAPE)?
+        run: |
+          7z e "tests/data/dynamic/cape/v2.2/d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json.gz"
+          dist/capa "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json"
       - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: ${{ matrix.asset_name }}
           path: dist/${{ matrix.artifact_name }}
 
   test_run:
-    name: Test run on ${{ matrix.os }}
+    name: Test run on ${{ matrix.os }} / ${{ matrix.asset_name }}
     runs-on: ${{ matrix.os }}
     needs: [build]
     strategy:
@@ -71,6 +82,9 @@ jobs:
           - os: ubuntu-22.04
             artifact_name: capa
             asset_name: linux
+          - os: ubuntu-22.04
+            artifact_name: capa
+            asset_name: linux-py311
           - os: windows-2022
             artifact_name: capa.exe
             asset_name: windows
@@ -96,6 +110,8 @@ jobs:
         include:
           - asset_name: linux
             artifact_name: capa
+          - asset_name: linux-py311
+            artifact_name: capa
           - asset_name: windows
             artifact_name: capa.exe
           - asset_name: macos

.github/workflows/tests.yml

@@ -39,13 +39,13 @@ jobs:
     - name: Lint with ruff
       run: pre-commit run ruff
     - name: Lint with isort
-      run: pre-commit run isort
+      run: pre-commit run isort --show-diff-on-failure
     - name: Lint with black
-      run: pre-commit run black
+      run: pre-commit run black --show-diff-on-failure
     - name: Lint with flake8
-      run: pre-commit run flake8
+      run: pre-commit run flake8 --hook-stage manual
     - name: Check types with mypy
-      run:  pre-commit run mypy
+      run:  pre-commit run mypy --hook-stage manual
 
   rule_linter:
     runs-on: ubuntu-20.04
@@ -95,6 +95,10 @@ jobs:
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
       run: pip install -e .[dev]
+    - name: Run tests (fast)
+      # this set of tests runs about 80% of the cases in 20% of the time,
+      # and should catch most errors quickly.
+      run:  pre-commit run pytest-fast --all-files --hook-stage manual
     - name: Run tests
       run: pytest -v tests/
 
@@ -103,7 +107,7 @@ jobs:
     env:
       BN_SERIAL: ${{ secrets.BN_SERIAL }}
     runs-on: ubuntu-20.04
-    needs: [code_style, rule_linter]
+    needs: [tests]
     strategy:
       fail-fast: false
       matrix:
@@ -143,7 +147,7 @@ jobs:
   ghidra-tests:
     name: Ghidra tests for ${{ matrix.python-version }}
     runs-on: ubuntu-20.04
-    needs: [code_style, rule_linter]
+    needs: [tests]
     strategy:
       fail-fast: false
       matrix:
@@ -197,4 +201,4 @@ jobs:
         cat ../output.log
         exit_code=$(cat ../output.log | grep exit | awk '{print $NF}')
         exit $exit_code
- 
+ 

.gitmodules

@@ -1,6 +1,8 @@
 [submodule "rules"]
 	path = rules
 	url = ../capa-rules.git
+	branch = dynamic-syntax
 [submodule "tests/data"]
 	path = tests/data
 	url = ../capa-testfiles.git
+	branch = dynamic-feature-extractor

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     hooks:
     -   id: isort
         name: isort
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: isort
         args: 
@@ -45,7 +45,7 @@ repos:
     hooks:
     -   id: black
         name: black
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: black
         args: 
@@ -62,7 +62,7 @@ repos:
     hooks:
     -   id: ruff
         name: ruff
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: ruff
         args: 
@@ -79,7 +79,7 @@ repos:
     hooks:
     -   id: flake8
         name: flake8
-        stages: [commit, push]
+        stages: [push, manual]
         language: system
         entry: flake8
         args: 
@@ -97,7 +97,7 @@ repos:
     hooks:
     -   id: mypy
         name: mypy
-        stages: [commit, push]
+        stages: [push, manual]
         language: system
         entry: mypy
         args: 
@@ -109,3 +109,21 @@ repos:
         -   "tests/"
         always_run: true
         pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: pytest-fast
+        name: pytest (fast)
+        stages: [manual]
+        language: system
+        entry: pytest
+        args: 
+        -   "tests/"
+        -   "--ignore=tests/test_binja_features.py"
+        -   "--ignore=tests/test_ghidra_features.py"
+        -   "--ignore=tests/test_ida_features.py"
+        -   "--ignore=tests/test_viv_features.py"
+        -   "--ignore=tests/test_main.py"
+        -   "--ignore=tests/test_scripts.py"
+        always_run: true
+        pass_filenames: false

CHANGELOG.md

@@ -3,15 +3,26 @@
 ## master (unreleased)
 
 ### New Features
-- ghidra: add Ghidra feature extractor and supporting code #1770 @colton-gabertan
-- ghidra: add entry script helping users run capa against a loaded Ghidra database #1767 @mike-hunhoff
+- add Ghidra backend #1770 #1767 @colton-gabertan @mike-hunhoff
+- add dynamic analysis via CAPE sandbox reports #48 #1535 @yelhamer
+  - add call scope #771 @yelhamer
+  - add thread scope #1517 @yelhamer
+  - add process scope #1517 @yelhamer
+  - rules: change `meta.scope` to `meta.scopes` @yelhamer
+  - protobuf: add `Metadata.flavor` @williballenthin
 - binja: add support for forwarded exports #1646 @xusheng6
 - binja: add support for symtab names #1504 @xusheng6
 - add com class/interface features #322 @Aayush-goel-04
 
 ### Breaking Changes
 
-### New Rules (19)
+- remove the `SCOPE_*` constants in favor of the `Scope` enum #1764 @williballenthin
+- protobuf: deprecate `RuleMetadata.scope` in favor of `RuleMetadata.scopes` @williballenthin
+- protobuf: deprecate `Metadata.analysis` in favor of `Metadata.analysis2` that is dynamic analysis aware @williballenthin
+- update freeze format to v3, adding support for dynamic analysis @williballenthin
+- extractor: ignore DLL name for api features #1815 @mr-tz
+
+### New Rules (34)
 
 - nursery/get-ntoskrnl-base-address @mr-tz
 - host-interaction/network/connectivity/set-tcp-connection-state @johnk3r
@@ -32,12 +43,26 @@
 - host-interaction/process/inject/allocate-or-change-rwx-memory @mr-tz
 - lib/allocate-or-change-rw-memory [email protected] @mr-tz
 - lib/change-memory-protection @mr-tz
+- anti-analysis/anti-av/patch-antimalware-scan-interface-function [email protected]
+- executable/dotnet-singlefile/bundled-with-dotnet-single-file-deployment [email protected]
+- internal/limitation/file/internal-dotnet-single-file-deployment-limitation [email protected]
+- data-manipulation/encoding/encode-data-using-add-xor-sub-operations [email protected]
+- nursery/access-camera-in-dotnet-on-android [email protected]
+- nursery/capture-microphone-audio-in-dotnet-on-android [email protected]
+- nursery/capture-screenshot-in-dotnet-on-android [email protected]
+- nursery/check-for-incoming-call-in-dotnet-on-android [email protected]
+- nursery/check-for-outgoing-call-in-dotnet-on-android [email protected]
+- nursery/compiled-with-xamarin [email protected]
+- nursery/get-os-version-in-dotnet-on-android [email protected]
+- data-manipulation/compression/create-cabinet-on-windows [email protected] [email protected]
+- data-manipulation/compression/extract-cabinet-on-windows [email protected]
+- lib/create-file-decompression-interface-context-on-windows [email protected]
 -
 
 ### Bug Fixes
-- ghidra: fix ints_to_bytes performance #1761 @mike-hunhoff
+- ghidra: fix `ints_to_bytes` performance #1761 @mike-hunhoff
 - binja: improve function call site detection @xusheng6
-- binja: use binaryninja.load to open files @xusheng6
+- binja: use `binaryninja.load` to open files @xusheng6
 - binja: bump binja version to 3.5 #1789 @xusheng6
 
 ### capa explorer IDA Pro plugin