From 01b35e75826dab91230c17a500387dc812fe354e Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 23 Sep 2024 13:14:03 +0200 Subject: [PATCH 01/24] pyproject.toml: bump min python version to 3.8.1 fixed #2387 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c95ad6522..2ceeed3f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ authors = [ description = "The FLARE team's open-source tool to identify capabilities in executable files." readme = {file = "README.md", content-type = "text/markdown"} license = {file = "LICENSE.txt"} -requires-python = ">=3.8" +requires-python = ">=3.8.1" keywords = ["malware analysis", "reverse engineering", "capability detection", "software behaviors", "capa", "FLARE"] classifiers = [ "Development Status :: 5 - Production/Stable", From 74777ad23e87ff913e2ebb59c82672efb06bf9f6 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 23 Sep 2024 13:15:02 +0200 Subject: [PATCH 02/24] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4bf9a8f1..176d97266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ ### Bug Fixes +- bump minimum Python version to 3.8.1 to satisfy uv #2387 @williballenthin + ### capa explorer IDA Pro plugin ### Development From 783e14b949ade861624974482a9041cec8863fcb Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 23 Sep 2024 20:33:23 +0000 Subject: [PATCH 03/24] pyinstaller: use Python 3.12 for standalone build (#2385) * pyinstaller: use Python 3.12 for standalone build closes #2383 * changelog * ci: build: fix test filename --- .github/workflows/build.yml | 6 +++--- CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f595f9ac1..e81749a57 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,8 +30,8 @@ jobs: python_version: 3.8 - os: ubuntu-20.04 artifact_name: capa - asset_name: linux-py311 - python_version: 3.11 + asset_name: linux-py312 + python_version: 3.12 - os: windows-2019 artifact_name: capa.exe asset_name: windows @@ -88,7 +88,7 @@ jobs: asset_name: linux - os: ubuntu-22.04 artifact_name: capa - asset_name: linux-py311 + asset_name: linux-py312 - os: windows-2022 artifact_name: capa.exe asset_name: windows diff --git a/CHANGELOG.md b/CHANGELOG.md index 176d97266..b7825e0bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ### Bug Fixes +- use Python 3.12 to build extra standalone build on Linux #2383 @williballenthin - bump minimum Python version to 3.8.1 to satisfy uv #2387 @williballenthin ### capa explorer IDA Pro plugin From 4891fd750fef20c876238e9455d81a67cdb1bbf7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 04:55:36 +0000 Subject: [PATCH 04/24] build(deps): bump rollup from 4.21.3 to 4.22.4 in /web/explorer Bumps [rollup](https://github.com/rollup/rollup) from 4.21.3 to 4.22.4. - [Release notes](https://github.com/rollup/rollup/releases) - [Changelog](https://github.com/rollup/rollup/blob/master/CHANGELOG.md) - [Commits](https://github.com/rollup/rollup/compare/v4.21.3...v4.22.4) --- updated-dependencies: - dependency-name: rollup dependency-type: indirect ... Signed-off-by: dependabot[bot] --- web/explorer/package-lock.json | 134 ++++++++++++++++----------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/web/explorer/package-lock.json b/web/explorer/package-lock.json index 1eb756fa2..2129353d7 100644 --- a/web/explorer/package-lock.json +++ b/web/explorer/package-lock.json @@ -680,9 +680,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.21.3.tgz", - "integrity": "sha512-MmKSfaB9GX+zXl6E8z4koOr/xU63AMVleLEa64v7R0QF/ZloMs5vcD1sHgM64GXXS1csaJutG+ddtzcueI/BLg==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.22.4.tgz", + "integrity": "sha512-Fxamp4aEZnfPOcGA8KSNEohV8hX7zVHOemC8jVBoBUHu5zpJK/Eu3uJwt6BMgy9fkvzxDaurgj96F/NiLukF2w==", "cpu": [ "arm" ], @@ -693,9 +693,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.21.3.tgz", - "integrity": "sha512-zrt8ecH07PE3sB4jPOggweBjJMzI1JG5xI2DIsUbkA+7K+Gkjys6eV7i9pOenNSDJH3eOr/jLb/PzqtmdwDq5g==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.22.4.tgz", + "integrity": "sha512-VXoK5UMrgECLYaMuGuVTOx5kcuap1Jm8g/M83RnCHBKOqvPPmROFJGQaZhGccnsFtfXQ3XYa4/jMCJvZnbJBdA==", "cpu": [ "arm64" ], @@ -706,9 +706,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.21.3.tgz", - "integrity": "sha512-P0UxIOrKNBFTQaXTxOH4RxuEBVCgEA5UTNV6Yz7z9QHnUJ7eLX9reOd/NYMO3+XZO2cco19mXTxDMXxit4R/eQ==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.22.4.tgz", + "integrity": "sha512-xMM9ORBqu81jyMKCDP+SZDhnX2QEVQzTcC6G18KlTQEzWK8r/oNZtKuZaCcHhnsa6fEeOBionoyl5JsAbE/36Q==", "cpu": [ "arm64" ], @@ -719,9 +719,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.21.3.tgz", - "integrity": "sha512-L1M0vKGO5ASKntqtsFEjTq/fD91vAqnzeaF6sfNAy55aD+Hi2pBI5DKwCO+UNDQHWsDViJLqshxOahXyLSh3EA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.22.4.tgz", + "integrity": "sha512-aJJyYKQwbHuhTUrjWjxEvGnNNBCnmpHDvrb8JFDbeSH3m2XdHcxDd3jthAzvmoI8w/kSjd2y0udT+4okADsZIw==", "cpu": [ "x64" ], @@ -732,9 +732,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.21.3.tgz", - "integrity": "sha512-btVgIsCjuYFKUjopPoWiDqmoUXQDiW2A4C3Mtmp5vACm7/GnyuprqIDPNczeyR5W8rTXEbkmrJux7cJmD99D2g==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.22.4.tgz", + "integrity": "sha512-j63YtCIRAzbO+gC2L9dWXRh5BFetsv0j0va0Wi9epXDgU/XUi5dJKo4USTttVyK7fGw2nPWK0PbAvyliz50SCQ==", "cpu": [ "arm" ], @@ -745,9 +745,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.21.3.tgz", - "integrity": "sha512-zmjbSphplZlau6ZTkxd3+NMtE4UKVy7U4aVFMmHcgO5CUbw17ZP6QCgyxhzGaU/wFFdTfiojjbLG3/0p9HhAqA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.22.4.tgz", + "integrity": "sha512-dJnWUgwWBX1YBRsuKKMOlXCzh2Wu1mlHzv20TpqEsfdZLb3WoJW2kIEsGwLkroYf24IrPAvOT/ZQ2OYMV6vlrg==", "cpu": [ "arm" ], @@ -758,9 +758,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.21.3.tgz", - "integrity": "sha512-nSZfcZtAnQPRZmUkUQwZq2OjQciR6tEoJaZVFvLHsj0MF6QhNMg0fQ6mUOsiCUpTqxTx0/O6gX0V/nYc7LrgPw==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.22.4.tgz", + "integrity": "sha512-AdPRoNi3NKVLolCN/Sp4F4N1d98c4SBnHMKoLuiG6RXgoZ4sllseuGioszumnPGmPM2O7qaAX/IJdeDU8f26Aw==", "cpu": [ "arm64" ], @@ -771,9 +771,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.21.3.tgz", - "integrity": "sha512-MnvSPGO8KJXIMGlQDYfvYS3IosFN2rKsvxRpPO2l2cum+Z3exiExLwVU+GExL96pn8IP+GdH8Tz70EpBhO0sIQ==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.22.4.tgz", + "integrity": "sha512-Gl0AxBtDg8uoAn5CCqQDMqAx22Wx22pjDOjBdmG0VIWX3qUBHzYmOKh8KXHL4UpogfJ14G4wk16EQogF+v8hmA==", "cpu": [ "arm64" ], @@ -784,9 +784,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.21.3.tgz", - "integrity": "sha512-+W+p/9QNDr2vE2AXU0qIy0qQE75E8RTwTwgqS2G5CRQ11vzq0tbnfBd6brWhS9bCRjAjepJe2fvvkvS3dno+iw==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.22.4.tgz", + "integrity": "sha512-3aVCK9xfWW1oGQpTsYJJPF6bfpWfhbRnhdlyhak2ZiyFLDaayz0EP5j9V1RVLAAxlmWKTDfS9wyRyY3hvhPoOg==", "cpu": [ "ppc64" ], @@ -797,9 +797,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.21.3.tgz", - "integrity": "sha512-yXH6K6KfqGXaxHrtr+Uoy+JpNlUlI46BKVyonGiaD74ravdnF9BUNC+vV+SIuB96hUMGShhKV693rF9QDfO6nQ==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.22.4.tgz", + "integrity": "sha512-ePYIir6VYnhgv2C5Xe9u+ico4t8sZWXschR6fMgoPUK31yQu7hTEJb7bCqivHECwIClJfKgE7zYsh1qTP3WHUA==", "cpu": [ "riscv64" ], @@ -810,9 +810,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.21.3.tgz", - "integrity": "sha512-R8cwY9wcnApN/KDYWTH4gV/ypvy9yZUHlbJvfaiXSB48JO3KpwSpjOGqO4jnGkLDSk1hgjYkTbTt6Q7uvPf8eg==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.22.4.tgz", + "integrity": "sha512-GqFJ9wLlbB9daxhVlrTe61vJtEY99/xB3C8e4ULVsVfflcpmR6c8UZXjtkMA6FhNONhj2eA5Tk9uAVw5orEs4Q==", "cpu": [ "s390x" ], @@ -823,9 +823,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.21.3.tgz", - "integrity": "sha512-kZPbX/NOPh0vhS5sI+dR8L1bU2cSO9FgxwM8r7wHzGydzfSjLRCFAT87GR5U9scj2rhzN3JPYVC7NoBbl4FZ0g==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.22.4.tgz", + "integrity": "sha512-87v0ol2sH9GE3cLQLNEy0K/R0pz1nvg76o8M5nhMR0+Q+BBGLnb35P0fVz4CQxHYXaAOhE8HhlkaZfsdUOlHwg==", "cpu": [ "x64" ], @@ -836,9 +836,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.21.3.tgz", - "integrity": "sha512-S0Yq+xA1VEH66uiMNhijsWAafffydd2X5b77eLHfRmfLsRSpbiAWiRHV6DEpz6aOToPsgid7TI9rGd6zB1rhbg==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.22.4.tgz", + "integrity": "sha512-UV6FZMUgePDZrFjrNGIWzDo/vABebuXBhJEqrHxrGiU6HikPy0Z3LfdtciIttEUQfuDdCn8fqh7wiFJjCNwO+g==", "cpu": [ "x64" ], @@ -849,9 +849,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.21.3.tgz", - "integrity": "sha512-9isNzeL34yquCPyerog+IMCNxKR8XYmGd0tHSV+OVx0TmE0aJOo9uw4fZfUuk2qxobP5sug6vNdZR6u7Mw7Q+Q==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.22.4.tgz", + "integrity": "sha512-BjI+NVVEGAXjGWYHz/vv0pBqfGoUH0IGZ0cICTn7kB9PyjrATSkX+8WkguNjWoj2qSr1im/+tTGRaY+4/PdcQw==", "cpu": [ "arm64" ], @@ -862,9 +862,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.21.3.tgz", - "integrity": "sha512-nMIdKnfZfzn1Vsk+RuOvl43ONTZXoAPUUxgcU0tXooqg4YrAqzfKzVenqqk2g5efWh46/D28cKFrOzDSW28gTA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.22.4.tgz", + "integrity": "sha512-SiWG/1TuUdPvYmzmYnmd3IEifzR61Tragkbx9D3+R8mzQqDBz8v+BvZNDlkiTtI9T15KYZhP0ehn3Dld4n9J5g==", "cpu": [ "ia32" ], @@ -875,9 +875,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.21.3.tgz", - "integrity": "sha512-fOvu7PCQjAj4eWDEuD8Xz5gpzFqXzGlxHZozHP4b9Jxv9APtdxL6STqztDzMLuRXEc4UpXGGhx029Xgm91QBeA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.22.4.tgz", + "integrity": "sha512-j8pPKp53/lq9lMXN57S8cFz0MynJk8OWNuUnXct/9KCpKU7DgU3bYMJhwWmcqC0UU29p8Lr0/7KEVcaM6bf47Q==", "cpu": [ "x64" ], @@ -3195,9 +3195,9 @@ } }, "node_modules/rollup": { - "version": "4.21.3", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.21.3.tgz", - "integrity": "sha512-7sqRtBNnEbcBtMeRVc6VRsJMmpI+JU1z9VTvW8D4gXIYQFz0aLcsE6rRkyghZkLfEgUZgVvOG7A5CVz/VW5GIA==", + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.22.4.tgz", + "integrity": "sha512-vD8HJ5raRcWOyymsR6Z3o6+RzfEPCnVLMFJ6vRslO1jt4LO6dUo5Qnpg7y4RkZFM2DMe3WUirkI5c16onjrc6A==", "dev": true, "dependencies": { "@types/estree": "1.0.5" @@ -3210,22 +3210,22 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.21.3", - "@rollup/rollup-android-arm64": "4.21.3", - "@rollup/rollup-darwin-arm64": "4.21.3", - "@rollup/rollup-darwin-x64": "4.21.3", - "@rollup/rollup-linux-arm-gnueabihf": "4.21.3", - "@rollup/rollup-linux-arm-musleabihf": "4.21.3", - "@rollup/rollup-linux-arm64-gnu": "4.21.3", - "@rollup/rollup-linux-arm64-musl": "4.21.3", - "@rollup/rollup-linux-powerpc64le-gnu": "4.21.3", - "@rollup/rollup-linux-riscv64-gnu": "4.21.3", - "@rollup/rollup-linux-s390x-gnu": "4.21.3", - "@rollup/rollup-linux-x64-gnu": "4.21.3", - "@rollup/rollup-linux-x64-musl": "4.21.3", - "@rollup/rollup-win32-arm64-msvc": "4.21.3", - "@rollup/rollup-win32-ia32-msvc": "4.21.3", - "@rollup/rollup-win32-x64-msvc": "4.21.3", + "@rollup/rollup-android-arm-eabi": "4.22.4", + "@rollup/rollup-android-arm64": "4.22.4", + "@rollup/rollup-darwin-arm64": "4.22.4", + "@rollup/rollup-darwin-x64": "4.22.4", + "@rollup/rollup-linux-arm-gnueabihf": "4.22.4", + "@rollup/rollup-linux-arm-musleabihf": "4.22.4", + "@rollup/rollup-linux-arm64-gnu": "4.22.4", + "@rollup/rollup-linux-arm64-musl": "4.22.4", + "@rollup/rollup-linux-powerpc64le-gnu": "4.22.4", + "@rollup/rollup-linux-riscv64-gnu": "4.22.4", + "@rollup/rollup-linux-s390x-gnu": "4.22.4", + "@rollup/rollup-linux-x64-gnu": "4.22.4", + "@rollup/rollup-linux-x64-musl": "4.22.4", + "@rollup/rollup-win32-arm64-msvc": "4.22.4", + "@rollup/rollup-win32-ia32-msvc": "4.22.4", + "@rollup/rollup-win32-x64-msvc": "4.22.4", "fsevents": "~2.3.2" } }, From 7884248022a4c9c51583c3395a2418be1d1d28f3 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 24 Sep 2024 11:25:18 +0000 Subject: [PATCH 05/24] Sync capa rules submodule --- CHANGELOG.md | 3 ++- rules | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7825e0bc..33ebf93b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Breaking Changes -### New Rules (6) +### New Rules (7) - nursery/access-unmanaged-com-objects-in-dotnet mehunhoff@google.com - nursery/implement-ui-automation-client-in-dotnet mehunhoff@google.com @@ -14,6 +14,7 @@ - nursery/interact-with-windows-scripting-host-in-dotnet mehunhoff@google.com - nursery/use-dotnet-library-simplejson mehunhoff@google.com - nursery/use-dotnet-library-websocket-sharp mehunhoff@google.com +- linking/runtime-linking/populate-syswhispers2-syscall-list still@teamt5.org - ### Bug Fixes diff --git a/rules b/rules index 1bf58dc0e..1fd0d8e14 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 1bf58dc0eea3d778b8209bce48254a82cb8417ac +Subproject commit 1fd0d8e1410d074e74ccf73e9cfee6b208f75ab1 From 22e88c860f13ae2ed1964295f231adf328ea661d Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 24 Sep 2024 11:25:28 +0000 Subject: [PATCH 06/24] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 93dd0f904..0298f0198 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 93dd0f904a2c0c56f0cbe3f3ac7153044333f644 +Subproject commit 0298f01989ffad824061a5f3f11dc6e12ebc1b63 From 6eda8c9713afd3e518ebb6c22f12f02aca5a5439 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 24 Sep 2024 11:29:53 +0000 Subject: [PATCH 07/24] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 0298f0198..8ea9ff9fd 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 0298f01989ffad824061a5f3f11dc6e12ebc1b63 +Subproject commit 8ea9ff9fda6ec5382f50d042ff4a98e64ecbeb4c From f024e1d54cf0533aad0b802331487cd778adbaa9 Mon Sep 17 00:00:00 2001 From: mr-tz Date: Tue, 24 Sep 2024 12:09:38 +0000 Subject: [PATCH 08/24] bump upload size limit to 100MB from 10MB --- web/explorer/src/components/UploadOptions.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/explorer/src/components/UploadOptions.vue b/web/explorer/src/components/UploadOptions.vue index f9ad6bf42..4cf3be209 100644 --- a/web/explorer/src/components/UploadOptions.vue +++ b/web/explorer/src/components/UploadOptions.vue @@ -5,7 +5,7 @@ mode="basic" name="model[]" accept=".json,.gz" - :max-file-size="10000000" + :max-file-size="100000000" :auto="true" :custom-upload="true" choose-label="Upload from local" From 12337be2b7ee764180f095658a79004b91f74292 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Wed, 25 Sep 2024 09:17:50 +0000 Subject: [PATCH 09/24] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 8ea9ff9fd..04b763b47 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 8ea9ff9fda6ec5382f50d042ff4a98e64ecbeb4c +Subproject commit 04b763b4784d520cc8c8d8d91068c38b1c4710f9 From bcd57a9af16a1007d5389147eb4ce6f291ab2465 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Thu, 26 Sep 2024 11:21:55 +0000 Subject: [PATCH 10/24] detect and use third-party analysis backends when possible (#2380) * introduce script to detect 3P backends ref #2376 * add idalib backend * binary ninja: search for API using XDG desktop entry ref #2376 * binja: search more XDG locations for desktop entry * binary ninja: optimize embedded PE scanning closes #2397 * add script for comparing the performance of analysis backends --- .github/pyinstaller/pyinstaller.spec | 3 + .gitignore | 1 + CHANGELOG.md | 3 + capa/features/extractors/binja/file.py | 54 +-- .../extractors/binja/find_binja_api.py | 160 ++++++++- capa/features/extractors/ida/extractor.py | 5 +- capa/features/extractors/ida/idalib.py | 113 +++++++ capa/helpers.py | 56 ++++ capa/ida/helpers.py | 13 + capa/loader.py | 60 ++-- capa/main.py | 2 + pyproject.toml | 2 + scripts/compare-backends.py | 316 ++++++++++++++++++ scripts/detect-backends.py | 106 ++++++ 14 files changed, 820 insertions(+), 74 deletions(-) create mode 100644 capa/features/extractors/ida/idalib.py create mode 100644 scripts/compare-backends.py create mode 100644 scripts/detect-backends.py diff --git a/.github/pyinstaller/pyinstaller.spec b/.github/pyinstaller/pyinstaller.spec index 021a2b294..e392eb5ae 100644 --- a/.github/pyinstaller/pyinstaller.spec +++ b/.github/pyinstaller/pyinstaller.spec @@ -70,7 +70,10 @@ a = Analysis( "qt5", "pyqtwebengine", "pyasn1", + # don't pull in Binary Ninja/IDA bindings that should + # only be installed locally. "binaryninja", + "ida", ], ) diff --git a/.gitignore b/.gitignore index ce07daf4a..997cef4cc 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,4 @@ Pipfile.lock .github/binja/download_headless.py .github/binja/BinaryNinja-headless.zip justfile +data/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 33ebf93b7..cc93d2d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### New Features +- add IDA v9.0 backend via idalib #2376 @williballenthin +- locate Binary Ninja API using XDG Desktop Entries #2376 @williballenthin + ### Breaking Changes ### New Rules (7) diff --git a/capa/features/extractors/binja/file.py b/capa/features/extractors/binja/file.py index cd340e77d..d5bb5a7c5 100644 --- a/capa/features/extractors/binja/file.py +++ b/capa/features/extractors/binja/file.py @@ -5,8 +5,6 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. - -import struct from typing import Tuple, Iterator from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding @@ -20,56 +18,24 @@ from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name -def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]: - """check segment for embedded PE - - adapted for binja from: - https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19 - """ - mz_xor = [ - ( - capa.features.extractors.helpers.xor_static(b"MZ", i), - capa.features.extractors.helpers.xor_static(b"PE", i), - i, - ) - for i in range(256) - ] - - todo = [] - # If this is the first segment of the binary, skip the first bytes. Otherwise, there will always be a matched - # PE at the start of the binaryview. - start = seg.start - if bv.view_type == "PE" and start == bv.start: +def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[Feature, Address]]: + """check segment for embedded PE""" + start = 0 + if bv.view_type == "PE" and seg.start == bv.start: + # If this is the first segment of the binary, skip the first bytes. + # Otherwise, there will always be a matched PE at the start of the binaryview. start += 1 - for mzx, pex, i in mz_xor: - for off, _ in bv.find_all_data(start, seg.end, mzx): - todo.append((off, mzx, pex, i)) - - while len(todo): - off, mzx, pex, i = todo.pop() - - # The MZ header has one field we will check e_lfanew is at 0x3c - e_lfanew = off + 0x3C - - if seg.end < (e_lfanew + 4): - continue - - newoff = struct.unpack(" Iterator[Tuple[Feature, Address]]: """extract embedded PE features""" for seg in bv.segments: - for ea, _ in check_segment_for_pe(bv, seg): - yield Characteristic("embedded pe"), FileOffsetAddress(ea) + yield from check_segment_for_pe(bv, seg) def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]: diff --git a/capa/features/extractors/binja/find_binja_api.py b/capa/features/extractors/binja/find_binja_api.py index 7412259f2..2a5dc6a93 100644 --- a/capa/features/extractors/binja/find_binja_api.py +++ b/capa/features/extractors/binja/find_binja_api.py @@ -5,31 +5,175 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import os +import sys +import logging import subprocess +import importlib.util +from typing import Optional from pathlib import Path +logger = logging.getLogger(__name__) + + # When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because # we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try # to find out the path of the binaryninja module that has been installed. # Note, including the binaryninja module in the `pyinstaller.spec` would not work, since the binaryninja module tries to # find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the # binaryninja module is extracted by the PyInstaller. -code = r""" +CODE = r""" from pathlib import Path from importlib import util spec = util.find_spec('binaryninja') if spec is not None: if len(spec.submodule_search_locations) > 0: - path = Path(spec.submodule_search_locations[0]) - # encode the path with utf8 then convert to hex, make sure it can be read and restored properly - print(str(path.parent).encode('utf8').hex()) + path = Path(spec.submodule_search_locations[0]) + # encode the path with utf8 then convert to hex, make sure it can be read and restored properly + print(str(path.parent).encode('utf8').hex()) """ -def find_binja_path() -> Path: - raw_output = subprocess.check_output(["python", "-c", code]).decode("ascii").strip() - return Path(bytes.fromhex(raw_output).decode("utf8")) +def find_binaryninja_path_via_subprocess() -> Optional[Path]: + raw_output = subprocess.check_output(["python", "-c", CODE]).decode("ascii").strip() + output = bytes.fromhex(raw_output).decode("utf8") + if not output.strip(): + return None + return Path(output) + + +def get_desktop_entry(name: str) -> Optional[Path]: + """ + Find the path for the given XDG Desktop Entry name. + + Like: + + >> get_desktop_entry("com.vector35.binaryninja.desktop") + Path("~/.local/share/applications/com.vector35.binaryninja.desktop") + """ + assert sys.platform in ("linux", "linux2") + assert name.endswith(".desktop") + + data_dirs = os.environ.get("XDG_DATA_DIRS", "/usr/share") + f":{Path.home()}/.local/share" + for data_dir in data_dirs.split(":"): + applications = Path(data_dir) / "applications" + for application in applications.glob("*.desktop"): + if application.name == name: + return application + + return None + + +def get_binaryninja_path(desktop_entry: Path) -> Optional[Path]: + # from: Exec=/home/wballenthin/software/binaryninja/binaryninja %u + # to: /home/wballenthin/software/binaryninja/ + for line in desktop_entry.read_text(encoding="utf-8").splitlines(): + if not line.startswith("Exec="): + continue + + if not line.endswith("binaryninja %u"): + continue + + binaryninja_path = Path(line[len("Exec=") : -len("binaryninja %u")]) + if not binaryninja_path.exists(): + return None + + return binaryninja_path + + return None + + +def validate_binaryninja_path(binaryninja_path: Path) -> bool: + if not binaryninja_path: + return False + + module_path = binaryninja_path / "python" + if not module_path.is_dir(): + return False + + if not (module_path / "binaryninja" / "__init__.py").is_file(): + return False + + return True + + +def find_binaryninja() -> Optional[Path]: + binaryninja_path = find_binaryninja_path_via_subprocess() + if not binaryninja_path or not validate_binaryninja_path(binaryninja_path): + if sys.platform == "linux" or sys.platform == "linux2": + # ok + logger.debug("detected OS: linux") + elif sys.platform == "darwin": + logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform) + return False + elif sys.platform == "win32": + logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform) + return False + else: + logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform) + return False + + desktop_entry = get_desktop_entry("com.vector35.binaryninja.desktop") + if not desktop_entry: + logger.debug("failed to find Binary Ninja application") + return None + logger.debug("found Binary Ninja application: %s", desktop_entry) + + binaryninja_path = get_binaryninja_path(desktop_entry) + if not binaryninja_path: + logger.debug("failed to determine Binary Ninja installation path") + return None + + if not validate_binaryninja_path(binaryninja_path): + logger.debug("failed to validate Binary Ninja installation") + return None + + logger.debug("found Binary Ninja installation: %s", binaryninja_path) + + return binaryninja_path / "python" + + +def is_binaryninja_installed() -> bool: + """Is the binaryninja module ready to import?""" + try: + return importlib.util.find_spec("binaryninja") is not None + except ModuleNotFoundError: + return False + + +def has_binaryninja() -> bool: + if is_binaryninja_installed(): + logger.debug("found installed Binary Ninja API") + return True + + logger.debug("Binary Ninja API not installed, searching...") + + binaryninja_path = find_binaryninja() + if not binaryninja_path: + logger.debug("failed to find Binary Ninja installation") + + logger.debug("found Binary Ninja API: %s", binaryninja_path) + return binaryninja_path is not None + + +def load_binaryninja() -> bool: + try: + import binaryninja + + return True + except ImportError: + binaryninja_path = find_binaryninja() + if not binaryninja_path: + return False + + sys.path.append(binaryninja_path.absolute().as_posix()) + try: + import binaryninja # noqa: F401 unused import + + return True + except ImportError: + return False if __name__ == "__main__": - print(find_binja_path()) + print(find_binaryninja_path_via_subprocess()) diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index 806ef8e78..a2b4f7913 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -8,7 +8,6 @@ from typing import List, Tuple, Iterator import idaapi -import ida_nalt import capa.ida.helpers import capa.features.extractors.elf @@ -32,7 +31,9 @@ class IdaFeatureExtractor(StaticFeatureExtractor): def __init__(self): super().__init__( hashes=SampleHashes( - md5=ida_nalt.retrieve_input_file_md5(), sha1="(unknown)", sha256=ida_nalt.retrieve_input_file_sha256() + md5=capa.ida.helpers.retrieve_input_file_md5(), + sha1="(unknown)", + sha256=capa.ida.helpers.retrieve_input_file_sha256(), ) ) self.global_features: List[Tuple[Feature, Address]] = [] diff --git a/capa/features/extractors/ida/idalib.py b/capa/features/extractors/ida/idalib.py new file mode 100644 index 000000000..df1e3172e --- /dev/null +++ b/capa/features/extractors/ida/idalib.py @@ -0,0 +1,113 @@ +# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: [package root]/LICENSE.txt +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under the License. +import os +import sys +import json +import logging +import importlib.util +from typing import Optional +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def is_idalib_installed() -> bool: + try: + return importlib.util.find_spec("ida") is not None + except ModuleNotFoundError: + return False + + +def get_idalib_user_config_path() -> Optional[Path]: + """Get the path to the user's config file based on platform following IDA's user directories.""" + # derived from `py-activate-idalib.py` from IDA v9.0 Beta 4 + + if sys.platform == "win32": + # On Windows, use the %APPDATA%\Hex-Rays\IDA Pro directory + config_dir = Path(os.getenv("APPDATA")) / "Hex-Rays" / "IDA Pro" + else: + # On macOS and Linux, use ~/.idapro + config_dir = Path.home() / ".idapro" + + # Return the full path to the config file (now in JSON format) + user_config_path = config_dir / "ida-config.json" + if not user_config_path.exists(): + return None + return user_config_path + + +def find_idalib() -> Optional[Path]: + config_path = get_idalib_user_config_path() + if not config_path: + return None + + config = json.loads(config_path.read_text(encoding="utf-8")) + + try: + ida_install_dir = Path(config["Paths"]["ida-install-dir"]) + except KeyError: + return None + + if not ida_install_dir.exists(): + return None + + libname = { + "win32": "idalib.dll", + "linux": "libidalib.so", + "linux2": "libidalib.so", + "darwin": "libidalib.dylib", + }[sys.platform] + + if not (ida_install_dir / "ida.hlp").is_file(): + return None + + if not (ida_install_dir / libname).is_file(): + return None + + idalib_path = ida_install_dir / "idalib" / "python" + if not idalib_path.exists(): + return None + + if not (idalib_path / "ida" / "__init__.py").is_file(): + return None + + return idalib_path + + +def has_idalib() -> bool: + if is_idalib_installed(): + logger.debug("found installed IDA idalib API") + return True + + logger.debug("IDA idalib API not installed, searching...") + + idalib_path = find_idalib() + if not idalib_path: + logger.debug("failed to find IDA idalib installation") + + logger.debug("found IDA idalib API: %s", idalib_path) + return idalib_path is not None + + +def load_idalib() -> bool: + try: + import ida + + return True + except ImportError: + idalib_path = find_idalib() + if not idalib_path: + return False + + sys.path.append(idalib_path.absolute().as_posix()) + try: + import ida # noqa: F401 unused import + + return True + except ImportError: + return False diff --git a/capa/helpers.py b/capa/helpers.py index 237a67f62..f185db9e6 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -5,11 +5,14 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import io import os import sys import gzip +import ctypes import inspect import logging +import tempfile import contextlib import importlib.util from typing import Dict, List, Union, BinaryIO, Iterator, NoReturn @@ -81,6 +84,59 @@ def assert_never(value) -> NoReturn: assert False, f"Unhandled value: {value} ({type(value).__name__})" # noqa: B011 +@contextlib.contextmanager +def stdout_redirector(stream): + """ + Redirect stdout at the C runtime level, + which lets us handle native libraries that spam stdout. + + *But*, this only works on Linux! Otherwise will silently still write to stdout. + So, try to upstream the fix when possible. + + Via: https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ + """ + if sys.platform not in ("linux", "linux2"): + logger.warning("Unable to capture STDOUT on non-Linux (begin)") + yield + logger.warning("Unable to capture STDOUT on non-Linux (end)") + return + + # libc is only on Linux + LIBC = ctypes.CDLL(None) + C_STDOUT = ctypes.c_void_p.in_dll(LIBC, "stdout") + + # The original fd stdout points to. Usually 1 on POSIX systems. + original_stdout_fd = sys.stdout.fileno() + + def _redirect_stdout(to_fd): + """Redirect stdout to the given file descriptor.""" + # Flush the C-level buffer stdout + LIBC.fflush(C_STDOUT) + # Flush and close sys.stdout - also closes the file descriptor (fd) + sys.stdout.close() + # Make original_stdout_fd point to the same file as to_fd + os.dup2(to_fd, original_stdout_fd) + # Create a new sys.stdout that points to the redirected fd + sys.stdout = io.TextIOWrapper(os.fdopen(original_stdout_fd, "wb")) + + # Save a copy of the original stdout fd in saved_stdout_fd + saved_stdout_fd = os.dup(original_stdout_fd) + try: + # Create a temporary file and redirect stdout to it + tfile = tempfile.TemporaryFile(mode="w+b") + _redirect_stdout(tfile.fileno()) + # Yield to caller, then redirect stdout back to the saved fd + yield + _redirect_stdout(saved_stdout_fd) + # Copy contents of temporary file to the given stream + tfile.flush() + tfile.seek(0, io.SEEK_SET) + stream.write(tfile.read()) + finally: + tfile.close() + os.close(saved_stdout_fd) + + def load_json_from_path(json_path: Path): with gzip.open(json_path, "r") as compressed_report: try: diff --git a/capa/ida/helpers.py b/capa/ida/helpers.py index 547099f47..91f29f05e 100644 --- a/capa/ida/helpers.py +++ b/capa/ida/helpers.py @@ -14,6 +14,7 @@ import idc import idaapi import ida_ida +import ida_nalt import idautils import ida_bytes import ida_loader @@ -64,6 +65,12 @@ def is_64bit() -> bool: info: idaapi.idainfo = idaapi.get_inf_structure() return info.is_64bit() + def retrieve_input_file_md5() -> str: + return ida_nalt.retrieve_input_file_md5() + + def retrieve_input_file_sha256() -> str: + return ida_nalt.retrieve_input_file_sha256() + else: def get_filetype() -> "ida_ida.filetype_t": @@ -78,6 +85,12 @@ def is_32bit() -> bool: def is_64bit() -> bool: return idaapi.inf_is_64bit() + def retrieve_input_file_md5() -> str: + return ida_nalt.retrieve_input_file_md5().hex() + + def retrieve_input_file_sha256() -> str: + return ida_nalt.retrieve_input_file_sha256().hex() + def inform_user_ida_ui(message): # this isn't a logger, this is IDA's logging facility diff --git a/capa/loader.py b/capa/loader.py index 818198710..6dfc6be42 100644 --- a/capa/loader.py +++ b/capa/loader.py @@ -5,8 +5,8 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import io import os -import sys import logging import datetime import contextlib @@ -69,6 +69,7 @@ BACKEND_VMRAY = "vmray" BACKEND_FREEZE = "freeze" BACKEND_BINEXPORT2 = "binexport2" +BACKEND_IDA = "ida" class CorruptFile(ValueError): @@ -237,24 +238,15 @@ def get_extractor( return capa.features.extractors.dnfile.extractor.DnfileFeatureExtractor(input_path) elif backend == BACKEND_BINJA: - import capa.helpers - from capa.features.extractors.binja.find_binja_api import find_binja_path - - # When we are running as a standalone executable, we cannot directly import binaryninja - # We need to fist find the binja API installation path and add it into sys.path - if capa.helpers.is_running_standalone(): - bn_api = find_binja_path() - if bn_api.exists(): - sys.path.append(str(bn_api)) - - try: - import binaryninja - from binaryninja import BinaryView - except ImportError: - raise RuntimeError( - "Cannot import binaryninja module. Please install the Binary Ninja Python API first: " - + "https://docs.binary.ninja/dev/batch.html#install-the-api)." - ) + import capa.features.extractors.binja.find_binja_api as finder + + if not finder.has_binaryninja(): + raise RuntimeError("cannot find Binary Ninja API module.") + + if not finder.load_binaryninja(): + raise RuntimeError("failed to load Binary Ninja API module.") + + import binaryninja import capa.features.extractors.binja.extractor @@ -269,7 +261,7 @@ def get_extractor( raise UnsupportedOSError() with console.status("analyzing program...", spinner="dots"): - bv: BinaryView = binaryninja.load(str(input_path)) + bv: binaryninja.BinaryView = binaryninja.load(str(input_path)) if bv is None: raise RuntimeError(f"Binary Ninja cannot open file {input_path}") @@ -321,6 +313,34 @@ def get_extractor( return capa.features.extractors.binexport2.extractor.BinExport2FeatureExtractor(be2, buf) + elif backend == BACKEND_IDA: + import capa.features.extractors.ida.idalib as idalib + + if not idalib.has_idalib(): + raise RuntimeError("cannot find IDA idalib module.") + + if not idalib.load_idalib(): + raise RuntimeError("failed to load IDA idalib module.") + + import ida + import ida_auto + + import capa.features.extractors.ida.extractor + + logger.debug("idalib: opening database...") + # idalib writes to stdout (ugh), so we have to capture that + # so as not to screw up structured output. + with capa.helpers.stdout_redirector(io.BytesIO()): + with console.status("analyzing program...", spinner="dots"): + if ida.open_database(str(input_path), run_auto_analysis=True): + raise RuntimeError("failed to analyze input file") + + logger.debug("idalib: waiting for analysis...") + ida_auto.auto_wait() + logger.debug("idalib: opened database.") + + return capa.features.extractors.ida.extractor.IdaFeatureExtractor() + else: raise ValueError("unexpected backend: " + backend) diff --git a/capa/main.py b/capa/main.py index 8035eafa2..d7b45e03a 100644 --- a/capa/main.py +++ b/capa/main.py @@ -43,6 +43,7 @@ from capa.rules import RuleSet from capa.engine import MatchResults from capa.loader import ( + BACKEND_IDA, BACKEND_VIV, BACKEND_CAPE, BACKEND_BINJA, @@ -283,6 +284,7 @@ def install_common_args(parser, wanted=None): backends = [ (BACKEND_AUTO, "(default) detect appropriate backend automatically"), (BACKEND_VIV, "vivisect"), + (BACKEND_IDA, "IDA via idalib"), (BACKEND_PEFILE, "pefile (file features only)"), (BACKEND_BINJA, "Binary Ninja"), (BACKEND_DOTNET, ".NET"), diff --git a/pyproject.toml b/pyproject.toml index 2ceeed3f8..c60893c49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,7 +183,9 @@ known_first_party = [ "binaryninja", "flirt", "ghidra", + "ida", "ida_ida", + "ida_auto", "ida_bytes", "ida_entry", "ida_funcs", diff --git a/scripts/compare-backends.py b/scripts/compare-backends.py new file mode 100644 index 000000000..1c000bade --- /dev/null +++ b/scripts/compare-backends.py @@ -0,0 +1,316 @@ +# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: [package root]/LICENSE.txt +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +import sys +import json +import time +import logging +import argparse +import contextlib +import statistics +import subprocess +import multiprocessing +from typing import Set, Dict, List, Optional +from pathlib import Path +from collections import Counter +from dataclasses import dataclass +from multiprocessing import Pool + +import rich +import rich.box +import rich.table + +import capa.main + +logger = logging.getLogger("capa.compare-backends") + +BACKENDS = ("vivisect", "ida", "binja") + + +@dataclass +class CapaInvocation: + path: Path + backend: str + duration: float + returncode: int + stdout: Optional[str] + stderr: Optional[str] + err: Optional[str] + + +def invoke_capa(file: Path, backend: str) -> CapaInvocation: + stdout = None + stderr = None + err = None + returncode: int + try: + logger.debug("run capa: %s: %s", backend, file.name) + t1 = time.time() + child = subprocess.run( + ["python", "-m", "capa.main", "--json", "--backend=" + backend, str(file)], + capture_output=True, + check=True, + text=True, + encoding="utf-8", + ) + returncode = child.returncode + stdout = child.stdout + stderr = child.stderr + except subprocess.CalledProcessError as e: + returncode = e.returncode + stdout = e.stdout + stderr = e.stderr + + logger.debug("%s:%s: error", backend, file.name) + err = str(e) + else: + pass + finally: + t2 = time.time() + + return CapaInvocation( + path=file, + backend=backend, + duration=t2 - t1, + returncode=returncode, + stdout=stdout, + stderr=stderr, + err=err, + ) + + +def wrapper_invoke_capa(args): + file, backend = args + return invoke_capa(file, backend) + + +def collect(args): + results_path = args.results_path + if not results_path.is_file(): + default_doc = {backend: {} for backend in BACKENDS} # type: ignore + results_path.write_text(json.dumps(default_doc), encoding="utf-8") + + testfiles = Path(__file__).parent.parent / "tests" / "data" + + for file in sorted(p for p in testfiles.glob("*")): + # remove leftover analysis files + # because IDA doesn't cleanup after itself, currently. + if file.suffix in (".til", ".id0", ".id1", ".id2", ".nam", ".viv"): + logger.debug("removing: %s", file) + with contextlib.suppress(IOError): + file.unlink() + + doc = json.loads(results_path.read_text(encoding="utf-8")) + + plan = [] + for file in sorted(p for p in testfiles.glob("*")): + if not file.is_file(): + continue + + if file.is_dir(): + continue + + if file.name.startswith("."): + continue + + if file.suffix not in (".exe_", ".dll_", ".elf_", ""): + continue + + logger.debug("%s", file.name) + key = str(file) + + for backend in BACKENDS: + + if (backend, file.name) in { + ("binja", "0953cc3b77ed2974b09e3a00708f88de931d681e2d0cb64afbaf714610beabe6.exe_") + }: + # this file takes 38GB+ and 20hrs+ + # https://github.com/Vector35/binaryninja-api/issues/5951 + continue + + if key in doc[backend]: + if not args.retry_failures: + continue + + if not doc[backend][key]["err"]: + # didn't previously fail, don't repeat work + continue + + else: + # want to retry this previous failure + pass + + plan.append((file, backend)) + + pool_size = multiprocessing.cpu_count() // 2 + logger.info("work pool size: %d", pool_size) + with Pool(processes=pool_size) as pool: + for i, result in enumerate(pool.imap_unordered(wrapper_invoke_capa, plan)): + doc[result.backend][str(result.path)] = { + "path": str(result.path), + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "err": result.err, + "duration": result.duration, + } + + if i % 8 == 0: + logger.info("syncing output database") + results_path.write_text(json.dumps(doc)) + + logger.info( + "%.1f\t%s %s %s", + result.duration, + "(err)" if result.err else " ", + result.backend.ljust(8), + result.path.name, + ) + + results_path.write_text(json.dumps(doc)) + return + + +def report(args): + doc = json.loads(args.results_path.read_text(encoding="utf-8")) + + samples = set() + for backend in BACKENDS: + samples.update(doc[backend].keys()) + + failures_by_backend: Dict[str, Set[str]] = {backend: set() for backend in BACKENDS} + durations_by_backend: Dict[str, List[float]] = {backend: [] for backend in BACKENDS} + + console = rich.get_console() + for key in sorted(samples): + sample = Path(key).name + console.print(sample, style="bold") + + seen_rules: Counter[str] = Counter() + + rules_by_backend: Dict[str, Set[str]] = {backend: set() for backend in BACKENDS} + + for backend in BACKENDS: + if key not in doc[backend]: + continue + + entry = doc[backend][key] + duration = entry["duration"] + + if not entry["err"]: + matches = json.loads(entry["stdout"])["rules"].keys() + seen_rules.update(matches) + rules_by_backend[backend].update(matches) + durations_by_backend[backend].append(duration) + + console.print(f" {backend: >8}: {duration: >6.1f}s {len(matches): >3d} matches") + + else: + failures_by_backend[backend].add(sample) + console.print(f" {backend: >8}: {duration: >6.1f}s (error)") + + if not seen_rules: + console.print() + continue + + t = rich.table.Table(box=rich.box.SIMPLE, header_style="default") + t.add_column("viv") + t.add_column("ida") + t.add_column("bn") + t.add_column("rule") + + for rule, _ in seen_rules.most_common(): + t.add_row( + "x" if rule in rules_by_backend["vivisect"] else " ", + "x" if rule in rules_by_backend["ida"] else " ", + "x" if rule in rules_by_backend["binja"] else " ", + rule, + ) + + console.print(t) + + for backend in BACKENDS: + console.print(f"failures for {backend}:", style="bold") + for failure in sorted(failures_by_backend[backend]): + console.print(f" - {failure}") + + if not failures_by_backend[backend]: + console.print(" (none)", style="green") + console.print() + + console.print("durations:", style="bold") + console.print(" (10-quantiles, in seconds)", style="grey37") + for backend in BACKENDS: + q = statistics.quantiles(durations_by_backend[backend], n=10) + console.print(f" {backend: <8}: ", end="") + for i in range(9): + if i in (4, 8): + style = "bold" + else: + style = "default" + console.print(f"{q[i]: >6.1f}", style=style, end=" ") + console.print() + console.print(" ^-- 10% of samples took less than this ^", style="grey37") + console.print(" 10% of samples took more than this -----------------+", style="grey37") + + console.print() + for backend in BACKENDS: + total = sum(durations_by_backend[backend]) + successes = len(durations_by_backend[backend]) + avg = statistics.mean(durations_by_backend[backend]) + console.print( + f" {backend: <8}: {total: >7.0f} seconds across {successes: >4d} successful runs, {avg: >4.1f} average" + ) + console.print() + + console.print("slowest samples:", style="bold") + for backend in BACKENDS: + console.print(backend) + for duration, path in sorted( + ((d["duration"], Path(d["path"]).name) for d in doc[backend].values()), reverse=True + )[:5]: + console.print(f" - {duration: >6.1f} {path}") + + return + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + default_samples_path = Path(__file__).resolve().parent.parent / "tests" / "data" + + parser = argparse.ArgumentParser(description="Compare analysis backends.") + capa.main.install_common_args( + parser, + wanted=set(), + ) + + subparsers = parser.add_subparsers() + collect_parser = subparsers.add_parser("collect") + collect_parser.add_argument("results_path", type=Path, help="Path to output JSON file") + collect_parser.add_argument("--samples", type=Path, default=default_samples_path, help="Path to samples") + collect_parser.add_argument("--retry-failures", action="store_true", help="Retry previous failures") + collect_parser.set_defaults(func=collect) + + report_parser = subparsers.add_parser("report") + report_parser.add_argument("results_path", type=Path, help="Path to JSON file") + report_parser.set_defaults(func=report) + + args = parser.parse_args(args=argv) + + try: + capa.main.handle_common_args(args) + except capa.main.ShouldExitError as e: + return e.status_code + + args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/detect-backends.py b/scripts/detect-backends.py new file mode 100644 index 000000000..2840058f5 --- /dev/null +++ b/scripts/detect-backends.py @@ -0,0 +1,106 @@ +# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: [package root]/LICENSE.txt +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +import sys +import logging +import argparse +import importlib.util + +import rich +import rich.table + +import capa.main +from capa.features.extractors.ida.idalib import find_idalib, load_idalib, is_idalib_installed +from capa.features.extractors.binja.find_binja_api import find_binaryninja, load_binaryninja, is_binaryninja_installed + +logger = logging.getLogger(__name__) + + +def is_vivisect_installed() -> bool: + try: + return importlib.util.find_spec("vivisect") is not None + except ModuleNotFoundError: + return False + + +def load_vivisect() -> bool: + try: + import vivisect # noqa: F401 unused import + + return True + except ImportError: + return False + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + parser = argparse.ArgumentParser(description="Detect analysis backends.") + capa.main.install_common_args(parser, wanted=set()) + args = parser.parse_args(args=argv) + + try: + capa.main.handle_common_args(args) + except capa.main.ShouldExitError as e: + return e.status_code + + if args.debug: + logging.getLogger("capa").setLevel(logging.DEBUG) + logging.getLogger("viv_utils").setLevel(logging.DEBUG) + else: + logging.getLogger("capa").setLevel(logging.ERROR) + logging.getLogger("viv_utils").setLevel(logging.ERROR) + + table = rich.table.Table() + table.add_column("backend") + table.add_column("already installed?") + table.add_column("found?") + table.add_column("loads?") + + if True: + row = ["vivisect"] + if is_vivisect_installed(): + row.append("True") + row.append("-") + else: + row.append("False") + row.append("False") + + row.append(str(load_vivisect())) + table.add_row(*row) + + if True: + row = ["Binary Ninja"] + if is_binaryninja_installed(): + row.append("True") + row.append("-") + else: + row.append("False") + row.append(str(find_binaryninja() is not None)) + + row.append(str(load_binaryninja())) + table.add_row(*row) + + if True: + row = ["IDA idalib"] + if is_idalib_installed(): + row.append("True") + row.append("-") + else: + row.append("False") + row.append(str(find_idalib() is not None)) + + row.append(str(load_idalib())) + table.add_row(*row) + + rich.print(table) + + +if __name__ == "__main__": + sys.exit(main()) From c5d8f99d6f046fd880ab0af997f259b6c2bb0a05 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 26 Sep 2024 12:25:36 +0000 Subject: [PATCH 11/24] Sync capa rules submodule --- CHANGELOG.md | 3 ++- rules | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc93d2d76..d549cfac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ### Breaking Changes -### New Rules (7) +### New Rules (8) - nursery/access-unmanaged-com-objects-in-dotnet mehunhoff@google.com - nursery/implement-ui-automation-client-in-dotnet mehunhoff@google.com @@ -18,6 +18,7 @@ - nursery/use-dotnet-library-simplejson mehunhoff@google.com - nursery/use-dotnet-library-websocket-sharp mehunhoff@google.com - linking/runtime-linking/populate-syswhispers2-syscall-list still@teamt5.org +- host-interaction/os/hide-shutdown-actions-via-policy still@teamt5.org - ### Bug Fixes diff --git a/rules b/rules index 1fd0d8e14..9da73becd 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 1fd0d8e1410d074e74ccf73e9cfee6b208f75ab1 +Subproject commit 9da73becdba96e0b4378b674db4cd06cf2c649fd From 9975f769f975bd4f45acad6d0e0bd0a0e9a1ffc9 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 26 Sep 2024 17:34:51 +0000 Subject: [PATCH 12/24] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 04b763b47..c2c53165d 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 04b763b4784d520cc8c8d8d91068c38b1c4710f9 +Subproject commit c2c53165dda44e875f88b859ba9b19a71fcb4abc From c48bccf623b05a8d33fe34b66a8b127eac5a1357 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 26 Sep 2024 17:38:34 +0000 Subject: [PATCH 13/24] Sync capa rules submodule --- CHANGELOG.md | 3 ++- rules | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d549cfac0..6cd8b487c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ### Breaking Changes -### New Rules (8) +### New Rules (9) - nursery/access-unmanaged-com-objects-in-dotnet mehunhoff@google.com - nursery/implement-ui-automation-client-in-dotnet mehunhoff@google.com @@ -19,6 +19,7 @@ - nursery/use-dotnet-library-websocket-sharp mehunhoff@google.com - linking/runtime-linking/populate-syswhispers2-syscall-list still@teamt5.org - host-interaction/os/hide-shutdown-actions-via-policy still@teamt5.org +- host-interaction/process/get-process-filename matthew.williams@mandiant.com - ### Bug Fixes diff --git a/rules b/rules index 9da73becd..627395d6f 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 9da73becdba96e0b4378b674db4cd06cf2c649fd +Subproject commit 627395d6f19dc76894bbf7d365d28d083fab67cf From 06271a88d4ad17c6502e76fd168e316442b29634 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 26 Sep 2024 21:57:30 +0200 Subject: [PATCH 14/24] Fix VMRay missing process data (#2396) * get all processes, see #2394 * add tests for process recording * rename symbols for clarification * handle single and list entries * update changelog * dynamic: vmray: use monitor IDs to track processes and threads * dynamic: vmray: code refactor * dynamic: vmray: add sanity checks when processing monitor processes * dynamic: vmray: remove unnecessary keys() access * dynamic: vmray: clarify comments * Update CHANGELOG.md Co-authored-by: Willi Ballenthin * dynamic: vmray: update CHANGELOG --------- Co-authored-by: Mike Hunhoff Co-authored-by: Willi Ballenthin --- CHANGELOG.md | 1 + capa/features/extractors/vmray/__init__.py | 95 ++++++++++++++------- capa/features/extractors/vmray/extractor.py | 31 ++++--- capa/features/extractors/vmray/file.py | 23 +---- capa/features/extractors/vmray/models.py | 40 ++++++++- tests/fixtures.py | 8 ++ tests/test_vmray_features.py | 39 +++++---- 7 files changed, 158 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cd8b487c..2edd08c5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - use Python 3.12 to build extra standalone build on Linux #2383 @williballenthin - bump minimum Python version to 3.8.1 to satisfy uv #2387 @williballenthin +- vmray: collect more process information from flog.xml #2394 @mr-tz @mike-hunhoff ### capa explorer IDA Pro plugin diff --git a/capa/features/extractors/vmray/__init__.py b/capa/features/extractors/vmray/__init__.py index 06d581cc9..4a004af61 100644 --- a/capa/features/extractors/vmray/__init__.py +++ b/capa/features/extractors/vmray/__init__.py @@ -10,6 +10,7 @@ from pathlib import Path from zipfile import ZipFile from collections import defaultdict +from dataclasses import dataclass from capa.exceptions import UnsupportedFormatError from capa.features.extractors.vmray.models import File, Flog, SummaryV2, StaticData, FunctionCall, xml_to_dict @@ -21,6 +22,21 @@ SUPPORTED_FLOG_VERSIONS = ("2",) +@dataclass +class VMRayMonitorThread: + tid: int # thread ID assigned by OS + monitor_id: int # unique ID assigned to thread by VMRay + process_monitor_id: int # unqiue ID assigned to containing process by VMRay + + +@dataclass +class VMRayMonitorProcess: + pid: int # process ID assigned by OS + ppid: int # parent process ID assigned by OS + monitor_id: int # unique ID assigned to process by VMRay + image_name: str + + class VMRayAnalysis: def __init__(self, zipfile_path: Path): self.zipfile = ZipFile(zipfile_path, "r") @@ -45,9 +61,15 @@ def __init__(self, zipfile_path: Path): self.exports: Dict[int, str] = {} self.imports: Dict[int, Tuple[str, str]] = {} self.sections: Dict[int, str] = {} - self.process_ids: Dict[int, int] = {} - self.process_threads: Dict[int, List[int]] = defaultdict(list) - self.process_calls: Dict[int, Dict[int, List[FunctionCall]]] = defaultdict(lambda: defaultdict(list)) + self.monitor_processes: Dict[int, VMRayMonitorProcess] = {} + self.monitor_threads: Dict[int, VMRayMonitorThread] = {} + + # map monitor thread IDs to their associated monitor process ID + self.monitor_threads_by_monitor_process: Dict[int, List[int]] = defaultdict(list) + + # map function calls to their associated monitor thread ID mapped to its associated monitor process ID + self.monitor_process_calls: Dict[int, Dict[int, List[FunctionCall]]] = defaultdict(lambda: defaultdict(list)) + self.base_address: int self.sample_file_name: Optional[str] = None @@ -79,13 +101,14 @@ def __init__(self, zipfile_path: Path): self.sample_file_buf: bytes = self.zipfile.read(sample_file_path, pwd=DEFAULT_ARCHIVE_PASSWORD) + # do not change order, it matters self._compute_base_address() self._compute_imports() self._compute_exports() self._compute_sections() - self._compute_process_ids() - self._compute_process_threads() - self._compute_process_calls() + self._compute_monitor_processes() + self._compute_monitor_threads() + self._compute_monitor_process_calls() def _find_sample_file(self): for file_name, file_analysis in self.sv2.files.items(): @@ -128,34 +151,48 @@ def _compute_sections(self): for elffile_section in self.sample_file_static_data.elf.sections: self.sections[elffile_section.header.sh_addr] = elffile_section.header.sh_name - def _compute_process_ids(self): + def _compute_monitor_processes(self): for process in self.sv2.processes.values(): - # we expect VMRay's monitor IDs to be unique, but OS PIDs may be reused - assert process.monitor_id not in self.process_ids.keys() - self.process_ids[process.monitor_id] = process.os_pid + # we expect monitor IDs to be unique + assert process.monitor_id not in self.monitor_processes - def _compute_process_threads(self): - # logs/flog.xml appears to be the only file that contains thread-related data - # so we use it here to map processes to threads - for function_call in self.flog.analysis.function_calls: - pid: int = self.get_process_os_pid(function_call.process_id) # flog.xml uses process monitor ID, not OS PID - tid: int = function_call.thread_id + ppid: int = ( + self.sv2.processes[process.ref_parent_process.path[1]].os_pid if process.ref_parent_process else 0 + ) + self.monitor_processes[process.monitor_id] = VMRayMonitorProcess( + process.os_pid, ppid, process.monitor_id, process.image_name + ) - assert isinstance(pid, int) - assert isinstance(tid, int) + # not all processes are recorded in SummaryV2.json, get missing data from flog.xml, see #2394 + for monitor_process in self.flog.analysis.monitor_processes: + vmray_monitor_process: VMRayMonitorProcess = VMRayMonitorProcess( + monitor_process.os_pid, + monitor_process.os_parent_pid, + monitor_process.process_id, + monitor_process.image_name, + ) - if tid not in self.process_threads[pid]: - self.process_threads[pid].append(tid) + if monitor_process.process_id not in self.monitor_processes: + self.monitor_processes[monitor_process.process_id] = vmray_monitor_process + else: + # we expect monitor processes recorded in both SummaryV2.json and flog.xml to equal + assert self.monitor_processes[monitor_process.process_id] == vmray_monitor_process - def _compute_process_calls(self): - for function_call in self.flog.analysis.function_calls: - pid: int = self.get_process_os_pid(function_call.process_id) # flog.xml uses process monitor ID, not OS PID - tid: int = function_call.thread_id + def _compute_monitor_threads(self): + for monitor_thread in self.flog.analysis.monitor_threads: + # we expect monitor IDs to be unique + assert monitor_thread.thread_id not in self.monitor_threads - assert isinstance(pid, int) - assert isinstance(tid, int) + self.monitor_threads[monitor_thread.thread_id] = VMRayMonitorThread( + monitor_thread.os_tid, monitor_thread.thread_id, monitor_thread.process_id + ) + + # we expect each monitor thread ID to be unique for its associated monitor process ID e.g. monitor + # thread ID 10 should not be captured twice for monitor process ID 1 + assert monitor_thread.thread_id not in self.monitor_threads_by_monitor_process[monitor_thread.thread_id] - self.process_calls[pid][tid].append(function_call) + self.monitor_threads_by_monitor_process[monitor_thread.process_id].append(monitor_thread.thread_id) - def get_process_os_pid(self, monitor_id: int) -> int: - return self.process_ids[monitor_id] + def _compute_monitor_process_calls(self): + for function_call in self.flog.analysis.function_calls: + self.monitor_process_calls[function_call.process_id][function_call.thread_id].append(function_call) diff --git a/capa/features/extractors/vmray/extractor.py b/capa/features/extractors/vmray/extractor.py index 735c646b9..36a0b430f 100644 --- a/capa/features/extractors/vmray/extractor.py +++ b/capa/features/extractors/vmray/extractor.py @@ -15,9 +15,16 @@ import capa.features.extractors.vmray.file import capa.features.extractors.vmray.global_ from capa.features.common import Feature, Characteristic -from capa.features.address import NO_ADDRESS, Address, ThreadAddress, DynamicCallAddress, AbsoluteVirtualAddress -from capa.features.extractors.vmray import VMRayAnalysis -from capa.features.extractors.vmray.models import PARAM_TYPE_STR, Process, ParamList, FunctionCall +from capa.features.address import ( + NO_ADDRESS, + Address, + ThreadAddress, + ProcessAddress, + DynamicCallAddress, + AbsoluteVirtualAddress, +) +from capa.features.extractors.vmray import VMRayAnalysis, VMRayMonitorThread, VMRayMonitorProcess +from capa.features.extractors.vmray.models import PARAM_TYPE_STR, ParamList, FunctionCall from capa.features.extractors.base_extractor import ( CallHandle, SampleHashes, @@ -69,20 +76,24 @@ def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]: yield from self.global_features def get_processes(self) -> Iterator[ProcessHandle]: - yield from capa.features.extractors.vmray.file.get_processes(self.analysis) + for monitor_process in self.analysis.monitor_processes.values(): + address: ProcessAddress = ProcessAddress(pid=monitor_process.pid, ppid=monitor_process.ppid) + yield ProcessHandle(address, inner=monitor_process) def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]: # we have not identified process-specific features for VMRay yet yield from [] def get_process_name(self, ph) -> str: - process: Process = ph.inner - return process.image_name + monitor_process: VMRayMonitorProcess = ph.inner + return monitor_process.image_name def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]: - for thread in self.analysis.process_threads[ph.address.pid]: - address: ThreadAddress = ThreadAddress(process=ph.address, tid=thread) - yield ThreadHandle(address=address, inner={}) + for monitor_thread_id in self.analysis.monitor_threads_by_monitor_process[ph.inner.monitor_id]: + monitor_thread: VMRayMonitorThread = self.analysis.monitor_threads[monitor_thread_id] + + address: ThreadAddress = ThreadAddress(process=ph.address, tid=monitor_thread.tid) + yield ThreadHandle(address=address, inner=monitor_thread) def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]: if False: @@ -92,7 +103,7 @@ def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterat return def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]: - for function_call in self.analysis.process_calls[ph.address.pid][th.address.tid]: + for function_call in self.analysis.monitor_process_calls[ph.inner.monitor_id][th.inner.monitor_id]: addr = DynamicCallAddress(thread=th.address, id=function_call.fncall_id) yield CallHandle(address=addr, inner=function_call) diff --git a/capa/features/extractors/vmray/file.py b/capa/features/extractors/vmray/file.py index 38ac9db01..7f4ba0395 100644 --- a/capa/features/extractors/vmray/file.py +++ b/capa/features/extractors/vmray/file.py @@ -6,37 +6,18 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import logging -from typing import Dict, Tuple, Iterator +from typing import Tuple, Iterator import capa.features.extractors.common from capa.features.file import Export, Import, Section from capa.features.common import String, Feature -from capa.features.address import NO_ADDRESS, Address, ProcessAddress, AbsoluteVirtualAddress +from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress from capa.features.extractors.vmray import VMRayAnalysis from capa.features.extractors.helpers import generate_symbols -from capa.features.extractors.vmray.models import Process -from capa.features.extractors.base_extractor import ProcessHandle logger = logging.getLogger(__name__) -def get_processes(analysis: VMRayAnalysis) -> Iterator[ProcessHandle]: - processes: Dict[str, Process] = analysis.sv2.processes - - for process in processes.values(): - # we map VMRay's monitor ID to the OS PID to make it easier for users - # to follow the processes in capa's output - pid: int = analysis.get_process_os_pid(process.monitor_id) - ppid: int = ( - analysis.get_process_os_pid(processes[process.ref_parent_process.path[1]].monitor_id) - if process.ref_parent_process - else 0 - ) - - addr: ProcessAddress = ProcessAddress(pid=pid, ppid=ppid) - yield ProcessHandle(address=addr, inner=process) - - def extract_export_names(analysis: VMRayAnalysis) -> Iterator[Tuple[Feature, Address]]: for addr, name in analysis.exports.items(): yield Export(name), AbsoluteVirtualAddress(addr) diff --git a/capa/features/extractors/vmray/models.py b/capa/features/extractors/vmray/models.py index a599dc420..f5371bec1 100644 --- a/capa/features/extractors/vmray/models.py +++ b/capa/features/extractors/vmray/models.py @@ -87,7 +87,7 @@ class Param(BaseModel): deref: Optional[ParamDeref] = None -def validate_param_list(value: Union[List[Param], Param]) -> List[Param]: +def validate_ensure_is_list(value: Union[List[Param], Param]) -> List[Param]: if isinstance(value, list): return value else: @@ -97,7 +97,7 @@ def validate_param_list(value: Union[List[Param], Param]) -> List[Param]: # params may be stored as a list of Param or a single Param so we convert # the input value to Python list type before the inner validation (List[Param]) # is called -ParamList = Annotated[List[Param], BeforeValidator(validate_param_list)] +ParamList = Annotated[List[Param], BeforeValidator(validate_ensure_is_list)] class Params(BaseModel): @@ -137,12 +137,46 @@ class FunctionReturn(BaseModel): from_addr: HexInt = Field(alias="from") +class MonitorProcess(BaseModel): + ts: HexInt + process_id: int + image_name: str + filename: str + # page_root: HexInt + os_pid: HexInt + # os_integrity_level: HexInt + # os_privileges: HexInt + monitor_reason: str + parent_id: int + os_parent_pid: HexInt + # cmd_line: str + # cur_dir: str + # os_username: str + # bitness: int + # os_groups: str + + +class MonitorThread(BaseModel): + ts: HexInt + thread_id: int + process_id: int + os_tid: HexInt + + +# handle if there's only single entries, but the model expects a list +MonitorProcessList = Annotated[List[MonitorProcess], BeforeValidator(validate_ensure_is_list)] +MonitorThreadList = Annotated[List[MonitorThread], BeforeValidator(validate_ensure_is_list)] +FunctionCallList = Annotated[List[FunctionCall], BeforeValidator(validate_ensure_is_list)] + + class Analysis(BaseModel): log_version: str # tested 2 analyzer_version: str # tested 2024.2.1 # analysis_date: str - function_calls: List[FunctionCall] = Field(alias="fncall", default=[]) + monitor_processes: MonitorProcessList = Field(alias="monitor_process", default=[]) + monitor_threads: MonitorThreadList = Field(alias="monitor_thread", default=[]) + function_calls: FunctionCallList = Field(alias="fncall", default=[]) # function_returns: List[FunctionReturn] = Field(alias="fnret", default=[]) diff --git a/tests/fixtures.py b/tests/fixtures.py index e4d0a6fa0..1912a456a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -431,6 +431,14 @@ def get_data_path_by_name(name) -> Path: / "vmray" / "93b2d1840566f45fab674ebc79a9d19c88993bcb645e0357f3cb584d16e7c795_min_archive.zip" ) + elif name.startswith("2f8a79-vmray"): + return ( + CD + / "data" + / "dynamic" + / "vmray" + / "2f8a79b12a7a989ac7e5f6ec65050036588a92e65aeb6841e08dc228ff0e21b4_min_archive.zip" + ) elif name.startswith("ea2876"): return CD / "data" / "ea2876e9175410b6f6719f80ee44b9553960758c7d0f7bed73c0fe9a78d8e669.dll_" elif name.startswith("1038a2"): diff --git a/tests/test_vmray_features.py b/tests/test_vmray_features.py index d92a75e49..02eb683ec 100644 --- a/tests/test_vmray_features.py +++ b/tests/test_vmray_features.py @@ -20,21 +20,21 @@ # file/imports ("93b2d1-vmray", "file", capa.features.file.Import("GetAddrInfoW"), True), # thread/api calls - ("93b2d1-vmray", "process=(2176:0),thread=7", capa.features.insn.API("GetAddrInfoW"), True), - ("93b2d1-vmray", "process=(2176:0),thread=7", capa.features.insn.API("DoesNotExist"), False), + ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfoW"), True), + ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("DoesNotExist"), False), # call/api - ("93b2d1-vmray", "process=(2176:0),thread=7,call=2361", capa.features.insn.API("GetAddrInfoW"), True), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2361", capa.features.insn.API("GetAddrInfoW"), True), # call/string argument ( "93b2d1-vmray", - "process=(2176:0),thread=7,call=10323", + "process=(2176:0),thread=2420,call=10323", capa.features.common.String("raw.githubusercontent.com"), True, ), # call/number argument # VirtualAlloc(4096, 4) - ("93b2d1-vmray", "process=(2176:0),thread=7,call=2358", capa.features.insn.Number(4096), True), - ("93b2d1-vmray", "process=(2176:0),thread=7,call=2358", capa.features.insn.Number(4), True), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2358", capa.features.insn.Number(4096), True), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2358", capa.features.insn.Number(4), True), ], # order tests by (file, item) # so that our LRU cache is most effective. @@ -46,24 +46,24 @@ # file/imports ("93b2d1-vmray", "file", capa.features.file.Import("GetAddrInfoW"), 1), # thread/api calls - ("93b2d1-vmray", "process=(2176:0),thread=7", capa.features.insn.API("free"), 1), - ("93b2d1-vmray", "process=(2176:0),thread=7", capa.features.insn.API("GetAddrInfoW"), 5), + ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("free"), 1), + ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfoW"), 5), # call/api - ("93b2d1-vmray", "process=(2176:0),thread=7,call=2345", capa.features.insn.API("free"), 1), - ("93b2d1-vmray", "process=(2176:0),thread=7,call=2345", capa.features.insn.API("GetAddrInfoW"), 0), - ("93b2d1-vmray", "process=(2176:0),thread=7,call=2361", capa.features.insn.API("GetAddrInfoW"), 1), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2345", capa.features.insn.API("free"), 1), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2345", capa.features.insn.API("GetAddrInfoW"), 0), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2361", capa.features.insn.API("GetAddrInfoW"), 1), # call/string argument ( "93b2d1-vmray", - "process=(2176:0),thread=7,call=10323", + "process=(2176:0),thread=2420,call=10323", capa.features.common.String("raw.githubusercontent.com"), 1, ), - ("93b2d1-vmray", "process=(2176:0),thread=7,call=10323", capa.features.common.String("non_existant"), 0), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=10323", capa.features.common.String("non_existant"), 0), # call/number argument - ("93b2d1-vmray", "process=(2176:0),thread=7,call=10315", capa.features.insn.Number(4096), 1), - ("93b2d1-vmray", "process=(2176:0),thread=7,call=10315", capa.features.insn.Number(4), 1), - ("93b2d1-vmray", "process=(2176:0),thread=7,call=10315", capa.features.insn.Number(404), 0), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=10315", capa.features.insn.Number(4096), 1), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=10315", capa.features.insn.Number(4), 1), + ("93b2d1-vmray", "process=(2176:0),thread=2420,call=10315", capa.features.insn.Number(404), 0), ], # order tests by (file, item) # so that our LRU cache is most effective. @@ -87,3 +87,10 @@ def test_vmray_features(sample, scope, feature, expected): ) def test_vmray_feature_counts(sample, scope, feature, expected): fixtures.do_test_feature_count(fixtures.get_vmray_extractor, sample, scope, feature, expected) + + +def test_vmray_processes(): + # see #2394 + path = fixtures.get_data_path_by_name("2f8a79-vmray") + vmre = fixtures.get_vmray_extractor(path) + assert len(vmre.analysis.monitor_processes) == 9 From a5d9459c427a15a3790f61453afdb1c3c67cf35a Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Thu, 26 Sep 2024 14:15:21 -0600 Subject: [PATCH 15/24] dynamic: vmray: fix A/W API detection --- capa/features/extractors/vmray/call.py | 4 +++- tests/test_vmray_features.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/capa/features/extractors/vmray/call.py b/capa/features/extractors/vmray/call.py index 436b4bebb..6b87d7d89 100644 --- a/capa/features/extractors/vmray/call.py +++ b/capa/features/extractors/vmray/call.py @@ -8,6 +8,7 @@ import logging from typing import Tuple, Iterator +import capa.features.extractors.helpers from capa.features.insn import API, Number from capa.features.common import String, Feature from capa.features.address import Address @@ -41,7 +42,8 @@ def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) - for param in call.params_in.params: yield from get_call_param_features(param, ch) - yield API(call.name), ch.address + for name in capa.features.extractors.helpers.generate_symbols("", call.name): + yield API(name), ch.address def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]: diff --git a/tests/test_vmray_features.py b/tests/test_vmray_features.py index 02eb683ec..02dabafd9 100644 --- a/tests/test_vmray_features.py +++ b/tests/test_vmray_features.py @@ -19,8 +19,13 @@ ("93b2d1-vmray", "file", capa.features.common.String("\\Program Files\\WindowsApps\\does_not_exist"), False), # file/imports ("93b2d1-vmray", "file", capa.features.file.Import("GetAddrInfoW"), True), + ("93b2d1-vmray", "file", capa.features.file.Import("GetAddrInfo"), True), # thread/api calls + ("93b2d1-vmray", "process=(2176:0),thread=2180", capa.features.insn.API("LoadLibraryExA"), True), + ("93b2d1-vmray", "process=(2176:0),thread=2180", capa.features.insn.API("LoadLibraryEx"), True), ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfoW"), True), + ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfo"), True), + ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfo"), True), ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("DoesNotExist"), False), # call/api ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2361", capa.features.insn.API("GetAddrInfoW"), True), From 31ec208a9bfdab4cb1b482a49f92ca4cfd3c1a54 Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Thu, 26 Sep 2024 14:27:45 -0600 Subject: [PATCH 16/24] dynamic: cape: fix A/W API detection --- capa/features/extractors/cape/call.py | 4 +++- tests/test_cape_features.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/capa/features/extractors/cape/call.py b/capa/features/extractors/cape/call.py index 88680b3fa..0bee22fcc 100644 --- a/capa/features/extractors/cape/call.py +++ b/capa/features/extractors/cape/call.py @@ -9,6 +9,7 @@ import logging from typing import Tuple, Iterator +import capa.features.extractors.helpers from capa.helpers import assert_never from capa.features.insn import API, Number from capa.features.common import String, Feature @@ -50,7 +51,8 @@ def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) - else: assert_never(value) - yield API(call.api), ch.address + for name in capa.features.extractors.helpers.generate_symbols("", call.api): + yield API(name), ch.address def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]: diff --git a/tests/test_cape_features.py b/tests/test_cape_features.py index d72caa9ab..ade933e73 100644 --- a/tests/test_cape_features.py +++ b/tests/test_cape_features.py @@ -37,6 +37,8 @@ ), ("0000a657", "process=(1180:3052)", capa.features.common.String("nope"), False), # thread/api calls + ("0000a657", "process=(2900:2852),thread=2904", capa.features.insn.API("RegQueryValueExA"), True), + ("0000a657", "process=(2900:2852),thread=2904", capa.features.insn.API("RegQueryValueEx"), True), ("0000a657", "process=(2852:3052),thread=2804", capa.features.insn.API("NtQueryValueKey"), True), ("0000a657", "process=(2852:3052),thread=2804", capa.features.insn.API("GetActiveWindow"), False), # thread/number call argument From 834150ad1d9ea4f86459acd3314ee34422511942 Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Thu, 26 Sep 2024 14:36:16 -0600 Subject: [PATCH 17/24] dynamic: drakvuf: fix A/W API detection --- capa/features/extractors/drakvuf/call.py | 4 +++- tests/test_drakvuf_features.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/capa/features/extractors/drakvuf/call.py b/capa/features/extractors/drakvuf/call.py index 34e877acc..7d0e2a5ee 100644 --- a/capa/features/extractors/drakvuf/call.py +++ b/capa/features/extractors/drakvuf/call.py @@ -9,6 +9,7 @@ import logging from typing import Tuple, Iterator +import capa.features.extractors.helpers from capa.features.insn import API, Number from capa.features.common import String, Feature from capa.features.address import Address @@ -44,7 +45,8 @@ def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) - # but yielding the entire string would be helpful for an analyst looking at the verbose output yield String(arg_value), ch.address - yield API(call.name), ch.address + for name in capa.features.extractors.helpers.generate_symbols("", call.name): + yield API(name), ch.address def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]: diff --git a/tests/test_drakvuf_features.py b/tests/test_drakvuf_features.py index 61fe69442..132b02ddc 100644 --- a/tests/test_drakvuf_features.py +++ b/tests/test_drakvuf_features.py @@ -22,6 +22,8 @@ ("93b2d1-drakvuf", "process=(3564:4852),thread=6592", capa.features.insn.API("LdrLoadDll"), True), ("93b2d1-drakvuf", "process=(3564:4852),thread=6592", capa.features.insn.API("DoesNotExist"), False), # call/api + ("93b2d1-drakvuf", "process=(3564:4852),thread=4716,call=17", capa.features.insn.API("CreateWindowExW"), True), + ("93b2d1-drakvuf", "process=(3564:4852),thread=4716,call=17", capa.features.insn.API("CreateWindowEx"), True), ("93b2d1-drakvuf", "process=(3564:4852),thread=6592,call=1", capa.features.insn.API("LdrLoadDll"), True), ("93b2d1-drakvuf", "process=(3564:4852),thread=6592,call=1", capa.features.insn.API("DoesNotExist"), False), # call/string argument From bfcc7051173f9eac441a86fff17b0c1db0d2cc27 Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Thu, 26 Sep 2024 14:42:08 -0600 Subject: [PATCH 18/24] dynamic: vmray: remove redundant test --- tests/test_vmray_features.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_vmray_features.py b/tests/test_vmray_features.py index 02dabafd9..e0803236d 100644 --- a/tests/test_vmray_features.py +++ b/tests/test_vmray_features.py @@ -25,7 +25,6 @@ ("93b2d1-vmray", "process=(2176:0),thread=2180", capa.features.insn.API("LoadLibraryEx"), True), ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfoW"), True), ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfo"), True), - ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("GetAddrInfo"), True), ("93b2d1-vmray", "process=(2176:0),thread=2420", capa.features.insn.API("DoesNotExist"), False), # call/api ("93b2d1-vmray", "process=(2176:0),thread=2420,call=2361", capa.features.insn.API("GetAddrInfoW"), True), From 80e007787c58313d8d4171947aa67063eb03433b Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Thu, 26 Sep 2024 14:43:20 -0600 Subject: [PATCH 19/24] dynamic: update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2edd08c5c..bdd3e6248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - use Python 3.12 to build extra standalone build on Linux #2383 @williballenthin - bump minimum Python version to 3.8.1 to satisfy uv #2387 @williballenthin - vmray: collect more process information from flog.xml #2394 @mr-tz @mike-hunhoff +- dynamic: emit complete features for A/W APIs #2409 @mike-hunhoff ### capa explorer IDA Pro plugin From e9e8fe42ed161e9e06f27361d021ce0d2de6b473 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Fri, 27 Sep 2024 07:31:51 +0000 Subject: [PATCH 20/24] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 627395d6f..bd3f81242 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 627395d6f19dc76894bbf7d365d28d083fab67cf +Subproject commit bd3f8124253b618f000216145f8eddd793e8f591 From f82fc1902c0d0913b7a6d7af0d414318bd0429b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:32:08 +0200 Subject: [PATCH 21/24] build(deps): bump types-protobuf from 5.27.0.20240907 to 5.27.0.20240920 (#2393) Bumps [types-protobuf](https://github.com/python/typeshed) from 5.27.0.20240907 to 5.27.0.20240920. - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-protobuf dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c60893c49..2ab9fe016 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,7 +150,7 @@ dev = [ "types-termcolor==1.1.4", "types-psutil==6.0.0.20240901", "types_requests==2.32.0.20240712", - "types-protobuf==5.27.0.20240907", + "types-protobuf==5.27.0.20240920", "deptry==0.20.0" ] build = [ From 76aff57467a287d00e64b4c549f62db519c1ef07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:32:18 +0200 Subject: [PATCH 22/24] build(deps): bump setuptools from 70.0.0 to 75.1.0 (#2392) Bumps [setuptools](https://github.com/pypa/setuptools) from 70.0.0 to 75.1.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v70.0.0...v75.1.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2ab9fe016..0305c0467 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,7 +159,7 @@ build = [ # These dependencies are not used in production environments # and should not conflict with other libraries/tooling. "pyinstaller==6.10.0", - "setuptools==70.0.0", + "setuptools==75.1.0", "build==1.2.2" ] scripts = [ diff --git a/requirements.txt b/requirements.txt index 79f33815a..a512d7c34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ pyyaml==6.0.2 rich==13.8.0 ruamel-yaml==0.18.6 ruamel-yaml-clib==0.2.8 -setuptools==70.0.0 +setuptools==75.1.0 six==1.16.0 sortedcontainers==2.4.0 tabulate==0.9.0 From 558bf0fbf23994851224028dfcb83d564d0025ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:32:58 +0200 Subject: [PATCH 23/24] build(deps): bump protobuf from 5.27.3 to 5.28.2 (#2390) Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 5.27.3 to 5.28.2. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/protobuf_release.bzl) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v5.27.3...v5.28.2) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a512d7c34..37142bc7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ msgpack==1.0.8 networkx==3.1 pefile==2023.2.7 pip==24.2 -protobuf==5.27.3 +protobuf==5.28.2 pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.22 From 51a4eb46b8c258daac29e4716d5b146bd3c3055c Mon Sep 17 00:00:00 2001 From: Fariss Date: Fri, 27 Sep 2024 09:34:21 +0200 Subject: [PATCH 24/24] replace tqdm, termcolor, tabulate with rich (#2374) * logging: use rich handler for logging * tqdm: remove unneeded redirecting_print_to_tqdm function * tqdm: introduce `CapaProgressBar` rich `Progress` bar * tqdm: replace tqdm with rich Progress bar * tqdm: remove tqdm dependency * termcolor: replace termcolor and update `scripts/` * tests: update `test_render.py` to use rich.console.Console * termcolor: remove termcolor dependency * capa.render.utils: add `write` & `writeln` methods to subclass `Console` * update markup util functions to use fmt strings * tests: update `test_render.py` to use `capa.render.utils.Console` * replace kwarg `end=""` with `write` and `writeln` methods * tabulate: replace tabulate with `rich.table` * tabulate: remove `tabulate` and its dependency `wcwidth` * logging: handle logging in `capa.main` * logging: set up logging in `capa.main` this commit sets up logging in `capa.main` and uses a shared `log_console` in `capa.helpers` for logging purposes * changelog: replace packages with rich * remove entry from pyinstaller and unneeded progress.update call * update requirements.txt * scripts: use `capa.helpers.log_console` in `CapaProgressBar` * logging: configure root logger to use `RichHandler` * remove unused import `inspect` --- .github/mypy/mypy.ini | 3 - .github/pyinstaller/pyinstaller.spec | 13 - CHANGELOG.md | 946 ++++++++++++++------------- capa/capabilities/dynamic.py | 62 +- capa/capabilities/static.py | 126 ++-- capa/helpers.py | 97 ++- capa/main.py | 15 +- capa/render/utils.py | 33 +- capa/render/verbose.py | 79 ++- capa/render/vverbose.py | 169 ++--- pyproject.toml | 9 - requirements.txt | 4 - scripts/lint.py | 43 +- scripts/profile-time.py | 75 ++- scripts/show-unused-features.py | 26 +- tests/test_render.py | 12 +- 16 files changed, 898 insertions(+), 814 deletions(-) diff --git a/.github/mypy/mypy.ini b/.github/mypy/mypy.ini index 3bd4b2e23..163731eb6 100644 --- a/.github/mypy/mypy.ini +++ b/.github/mypy/mypy.ini @@ -1,8 +1,5 @@ [mypy] -[mypy-tqdm.*] -ignore_missing_imports = True - [mypy-ruamel.*] ignore_missing_imports = True diff --git a/.github/pyinstaller/pyinstaller.spec b/.github/pyinstaller/pyinstaller.spec index e392eb5ae..5cd514358 100644 --- a/.github/pyinstaller/pyinstaller.spec +++ b/.github/pyinstaller/pyinstaller.spec @@ -2,7 +2,6 @@ # Copyright (C) 2020 Mandiant, Inc. All Rights Reserved. import sys -import wcwidth import capa.rules.cache from pathlib import Path @@ -29,13 +28,6 @@ a = Analysis( ("../../rules", "rules"), ("../../sigs", "sigs"), ("../../cache", "cache"), - # capa.render.default uses tabulate that depends on wcwidth. - # it seems wcwidth uses a json file `version.json` - # and this doesn't get picked up by pyinstaller automatically. - # so we manually embed the wcwidth resources here. - # - # ref: https://stackoverflow.com/a/62278462/87207 - (Path(wcwidth.__file__).parent, "wcwidth"), ], # when invoking pyinstaller from the project root, # this gets run from the project root. @@ -48,11 +40,6 @@ a = Analysis( "tkinter", "_tkinter", "Tkinter", - # tqdm provides renderers for ipython, - # however, this drags in a lot of dependencies. - # since we don't spawn a notebook, we can safely remove these. - "IPython", - "ipywidgets", # these are pulled in by networkx # but we don't need to compute the strongly connected components. "numpy", diff --git a/CHANGELOG.md b/CHANGELOG.md index 2edd08c5c..77bfaa3b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,29 +27,36 @@ - use Python 3.12 to build extra standalone build on Linux #2383 @williballenthin - bump minimum Python version to 3.8.1 to satisfy uv #2387 @williballenthin - vmray: collect more process information from flog.xml #2394 @mr-tz @mike-hunhoff +- replace tabulate, tqdm, and termcolor with rich #2374 @s-ff ### capa explorer IDA Pro plugin ### Development ### Raw diffs + - [capa v7.3.0...master](https://github.com/mandiant/capa/compare/v7.3.0...master) - [capa-rules v7.3.0...master](https://github.com/mandiant/capa-rules/compare/v7.3.0...master) ## v7.3.0 + This release comes with the following three major enhancements: ### 1. Support for VMRay sandbox analysis archives + Unlock powerful malware analysis with capa's new [VMRay sandbox](https://www.vmray.com/) integration! Simply provide a VMRay analysis archive, and capa will automatically extract and match capabilities to streamline your workflow. This is the second support for the analysis of dynamic analysis results after [CAPE](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox). ### 2. Support for BinExport files generated by Ghidra + [BinExport](https://github.com/google/binexport) files store disassembled data into a Protocol Buffer format. capa now supports the analysis of BinExport files generated by Ghidra. Using Ghidra and the BinExport file format users can now analyze ARM (AARCH64) ELF files targeting Android. ### 3. Introducing the capa rules website + You can now browse capa's default rule set at https://mandiant.github.io/capa/rules. In modern terminals the CLI capa tool hyperlinks to resources on the web, including entries on the capa rules website. Furthermore, https://mandiant.github.io/capa provides a landing page for the capa tool project. ### Additional updates + - [capa Explorer Web](https://mandiant.github.io/capa/explorer/) received several enhancements and bug fixes. - Support for the IDA Pro 9.0 IDAPython API while keeping compatibility to older IDA Pro versions - Six rules have been added and two rules have been updated @@ -57,6 +64,7 @@ Furthermore, https://mandiant.github.io/capa provides a landing page for the cap Thanks to @r-sm2024 for their contribution in https://github.com/mandiant/capa/pull/2155 and their further work. And of course a big thanks to the community for reporting issues, participating in discussions, and supporting the capa tool and capa rules. ### New Features + - regenerate ruleset cache automatically on source change (only in dev mode) #2133 @s-ff - add landing page https://mandiant.github.io/capa/ @williballenthin #2310 - add rules website https://mandiant.github.io/capa/rules @DeeyaSingh #2310 @@ -82,14 +90,17 @@ Thanks to @r-sm2024 for their contribution in https://github.com/mandiant/capa/p - fix code path reference in linter @williballenthin #2350 ### capa explorer IDA Pro plugin + - update IDAPython to IDA Pro 9.0 @mr-tz - fix byte search IDA Pro 7.5 compatibility @mr-tz #2371 ### Raw diffs + - [capa v7.2.0...v7.3.0](https://github.com/mandiant/capa/compare/v7.2.0...v7.3.0) - [capa-rules v7.2.0...v7.3.0](https://github.com/mandiant/capa-rules/compare/v7.2.0...v7.3.0) ## v7.2.0 + capa v7.2.0 introduces a first version of capa explorer web: a web-based user interface to inspect capa results using your browser. Users can inspect capa result JSON documents in an online web instance or a standalone HTML page for offline usage. capa explorer supports interactive exploring of capa results to make it easier to understand them. Users can filter, sort, and see the details of all identified capabilities. capa explorer web was worked on by @s-ff as part of a [GSoC project](https://summerofcode.withgoogle.com/programs/2024/projects/cR3hjbsq), and it is available at https://mandiant.github.io/capa/explorer/#/. This release also adds a feature extractor for output from the DRAKVUF sandbox. Now, analysts can pass the resulting `drakmon.log` file to capa and extract capabilities from the artifacts captured by the sandbox. This feature extractor will also be added to the DRAKVUF sandbox as a post-processing script, and it was worked on by @yelhamer as part of a [GSoC project](https://summerofcode.withgoogle.com/programs/2024/projects/fCnBGuEC). @@ -97,9 +108,10 @@ This release also adds a feature extractor for output from the DRAKVUF sandbox. Additionally, we fixed several bugs handling ELF files, and added the ability to filter capa analysis by functions or processes. We also added support to the IDA Pro extractor to leverage analyst recovered API names. Special thanks to our repeat and new contributors: -* @lakshayletsgo for their first contribution in https://github.com/mandiant/capa/pull/2248 -* @msm-cert for their first contribution in https://github.com/mandiant/capa/pull/2143 -* @VascoSch92 for their first contribution in https://github.com/mandiant/capa/pull/2143 + +- @lakshayletsgo for their first contribution in https://github.com/mandiant/capa/pull/2248 +- @msm-cert for their first contribution in https://github.com/mandiant/capa/pull/2143 +- @VascoSch92 for their first contribution in https://github.com/mandiant/capa/pull/2143 ### New Features @@ -126,38 +138,43 @@ Special thanks to our repeat and new contributors: ### capa explorer IDA Pro plugin ### Development + - CI: use macos-12 since macos-11 is deprecated and will be removed on June 28th, 2024 #2173 @mr-tz - CI: update Binary Ninja version to 4.1 and use Python 3.9 to test it #2211 @xusheng6 - CI: update tests.yml workflow to exclude web and documentation files #2263 @s-ff - CI: update build.yml workflow to exclude web and documentation files #2270 @s-ff ### Raw diffs + - [capa v7.1.0...7.2.0](https://github.com/mandiant/capa/compare/v7.1.0...7.2.0) - [capa-rules v7.1.0...7.2.0](https://github.com/mandiant/capa-rules/compare/v7.1.0...7.2.0) ## v7.1.0 + The v7.1.0 release brings large performance improvements to capa's rule matching engine. Additionally, we've fixed various bugs and added new features for people using and developing capa. Special thanks to our repeat and new contributors: -* @sjha2048 made their first contribution in https://github.com/mandiant/capa/pull/2000 -* @Rohit1123 made their first contribution in https://github.com/mandiant/capa/pull/1990 -* @psahithireddy made their first contribution in https://github.com/mandiant/capa/pull/2020 -* @Atlas-64 made their first contribution in https://github.com/mandiant/capa/pull/2018 -* @s-ff made their first contribution in https://github.com/mandiant/capa/pull/2011 -* @samadpls made their first contribution in https://github.com/mandiant/capa/pull/2024 -* @acelynnzhang made their first contribution in https://github.com/mandiant/capa/pull/2044 -* @RainRat made their first contribution in https://github.com/mandiant/capa/pull/2058 -* @ReversingWithMe made their first contribution in https://github.com/mandiant/capa/pull/2093 -* @malwarefrank made their first contribution in https://github.com/mandiant/capa/pull/2037 + +- @sjha2048 made their first contribution in https://github.com/mandiant/capa/pull/2000 +- @Rohit1123 made their first contribution in https://github.com/mandiant/capa/pull/1990 +- @psahithireddy made their first contribution in https://github.com/mandiant/capa/pull/2020 +- @Atlas-64 made their first contribution in https://github.com/mandiant/capa/pull/2018 +- @s-ff made their first contribution in https://github.com/mandiant/capa/pull/2011 +- @samadpls made their first contribution in https://github.com/mandiant/capa/pull/2024 +- @acelynnzhang made their first contribution in https://github.com/mandiant/capa/pull/2044 +- @RainRat made their first contribution in https://github.com/mandiant/capa/pull/2058 +- @ReversingWithMe made their first contribution in https://github.com/mandiant/capa/pull/2093 +- @malwarefrank made their first contribution in https://github.com/mandiant/capa/pull/2037 ### New Features + - Emit "dotnet" as format to ResultDocument when processing .NET files #2024 @samadpls - ELF: detect OS from statically-linked Go binaries #1978 @williballenthin - add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123 - document Antivirus warnings and VirusTotal false positive detections #2028 @RionEV @mr-tz - Add json to sarif conversion script @reversingwithme -- render maec/* fields #843 @s-ff +- render maec/\* fields #843 @s-ff - replace Halo spinner with Rich #2086 @s-ff - optimize rule matching #2080 @williballenthin - add aarch64 as a valid architecture #2144 mehunhoff@google.com @williballenthin @@ -200,6 +217,7 @@ Special thanks to our repeat and new contributors: - cape: support more report formats #2035 @mr-tz ### capa explorer IDA Pro plugin + - replace deprecated IDA API find_binary with bin_search #1606 @s-ff ### Development @@ -208,11 +226,12 @@ Special thanks to our repeat and new contributors: - ci: use rules number badge stored in our bot gist and generated using `schneegans/dynamic-badges-action` #2001 capa-rules#882 @Ana06 - ci: update github workflows to use latest version of actions that were using a deprecated version of node #1967 #2003 capa-rules#883 @sjha2048 @Ana06 - ci: update binja version to stable 4.0 #2016 @xusheng6 -- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions #2020 @psahithireddy +- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions #2020 @psahithireddy - ci: include rule caching in PyInstaller build process #2097 @s-ff - add deptry support #1497 @s-ff ### Raw diffs + - [capa v7.0.1...v7.1.0](https://github.com/mandiant/capa/compare/v7.0.1...v7.1.0) - [capa-rules v7.0.1...v7.1.0](https://github.com/mandiant/capa-rules/compare/v7.0.1...v7.1.0) @@ -225,10 +244,12 @@ This release fixes a circular import error when using capa as a library. - fix potentially circular import errors #1969 @williballenthin ### Raw diffs + - [capa v7.0.0...v7.0.1](https://github.com/mandiant/capa/compare/v7.0.0...v7.0.1) - [capa-rules v7.0.0...v7.0.1](https://github.com/mandiant/capa-rules/compare/v7.0.0...v7.0.1) ## v7.0.0 + This is the v7.0.0 release of capa which was mainly worked on during the Google Summer of Code (GSoC) 2023. A huge shoutout to our GSoC contributors @colton-gabertan and @yelhamer for their amazing work. @@ -304,6 +325,7 @@ Also, a big thanks to the other contributors: @aaronatp, @Aayush-Goel-04, @bkoju - host-interaction/process/inject/process-ghostly-hollowing sara.rincon@mandiant.com ### Bug Fixes + - ghidra: fix `ints_to_bytes` performance #1761 @mike-hunhoff - binja: improve function call site detection @xusheng6 - binja: use `binaryninja.load` to open files @xusheng6 @@ -314,12 +336,15 @@ Also, a big thanks to the other contributors: @aaronatp, @Aayush-Goel-04, @bkoju - remove unnecessary scripts/vivisect-py2-vs-py3.sh file #1949 @JCoonradt ### capa explorer IDA Pro plugin + - various integration updates and minor bug fixes ### Development + - update ATT&CK/MBC data for linting #1932 @mr-tz #### Developer Notes + With this new release, many classes and concepts have been split up into static (mostly identical to the prior implementations) and dynamic ones. For example, the legacy FeatureExtractor class has been renamed to StaticFeatureExtractor and the DynamicFeatureExtractor has been added. @@ -340,6 +365,7 @@ format and backend is more consistent. We've documented that the input file is n (cape/freeze/etc.) inputs are not actually the sample. ### Raw diffs + - [capa v6.1.0...v7.0.0](https://github.com/mandiant/capa/compare/v6.1.0...v7.0.0) - [capa-rules v6.1.0...v7.0.0](https://github.com/mandiant/capa-rules/compare/v6.1.0...v7.0.0) @@ -353,6 +379,7 @@ You could use this script to find opportunities for new rules. Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Moritz, Ervin, and still@teamt5.org! ### New Features + - ELF: implement import and export name extractor #1607 #1608 @Aayush-Goel-04 - bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04 - develop script to highlight features not used during matching #331 @Aayush-Goel-04 @@ -371,16 +398,18 @@ Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Morit - rules: fix forwarded export characteristic #1656 @RonnieSalomonsen - Binary Ninja: Fix stack string detection #1473 @xusheng6 -- linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin +- linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin - OS: detect Android ELF files #1705 @williballenthin - ELF: fix parsing of symtab #1704 @williballenthin - result document: don't use deprecated pydantic functions #1718 @williballenthin - pytest: don't mark IDA tests as pytest tests #1719 @williballenthin ### capa explorer IDA Pro plugin + - fix unhandled exception when resolving rule path #1693 @mike-hunhoff ### Raw diffs + - [capa v6.0.0...v6.1.0](https://github.com/mandiant/capa/compare/v6.0.0...v6.1.0) - [capa-rules v6.0.0...v6.1.0](https://github.com/mandiant/capa-rules/compare/v6.0.0...v6.1.0) @@ -391,12 +420,14 @@ capa v6.0 brings many bug fixes and quality improvements, including 64 rule upda For those that use capa as a library, we've introduced some limited breaking changes that better represent data types (versus less-structured data like dictionaries and strings). With the recent deprecation, we've also dropped support for Python 3.7. ### New Features + - add script to detect feature overlap between new and existing capa rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04) - extract forwarded exports from PE files #1624 @williballenthin - extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736 - use fancy box drawing characters for default output #1586 @williballenthin ### Breaking Changes + - use a class to represent Metadata (not dict) #1411 @Aayush-Goel-04 @manasghandat - use pathlib.Path to represent file paths #1534 @Aayush-Goel-04 - Python 3.8 is now the minimum supported Python version #1578 @williballenthin @@ -431,10 +462,11 @@ For those that use capa as a library, we've introduced some limited breaking cha - linking/runtime-linking/resolve-function-by-brute-ratel-badger-hash jakub.jozwiak@mandiant.com ### Bug Fixes + - extractor: add a Binary Ninja test that asserts its version #1487 @xusheng6 - extractor: update Binary Ninja stack string detection after the new constant outlining feature #1473 @xusheng6 - extractor: update vivisect Arch extraction #1334 @mr-tz -- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 +- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 - symtab: fix struct.unpack() format for 64-bit ELF files @yelhamer - symtab: safeguard against ZeroDivisionError for files containing a symtab with a null entry size @yelhamer - improve ELF strtab and needed parsing @mr-tz @@ -451,6 +483,7 @@ For those that use capa as a library, we've introduced some limited breaking cha ### capa explorer IDA Pro plugin ### Development + - update ATT&CK/MBC data for linting #1568 @mr-tz - log time taken to analyze each function #1290 @williballenthin - tests: make fixture available via conftest.py #1592 @williballenthin @@ -458,12 +491,13 @@ For those that use capa as a library, we've introduced some limited breaking cha - migrate to pyproject.toml #1301 @williballenthin - use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin - ### Raw diffs + - [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0) - [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0) ## v5.1.0 + capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and an output color format that's easier to read. Over 25 capa rules have been added and improved. @@ -471,7 +505,8 @@ Over 25 capa rules have been added and improved. Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046, @manasghandat, @ooprathamm, @linpeiyu164, @yelhamer, @HongThatCong, @naikordian, @stevemk14ebr, @emtuls, @raymondlleong, @bkojusner, @joren485, and everyone else who submitted bugs and provided feedback! ### New Features -- add protobuf format for result documents #1219 @williballenthin @mr-tz + +- add protobuf format for result documents #1219 @williballenthin @mr-tz - extractor: add Binary Ninja feature extractor @xusheng6 - new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42 - change colour/highlight to "cyan" instead of "blue" for better readability #1384 @ggold7046 @@ -508,24 +543,27 @@ Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046 - nursery/contain-a-thread-local-storage-tls-section-in-dotnet michael.hunhoff@mandiant.com ### Bug Fixes + - extractor: interface of cache modified to prevent extracting file and global features multiple times @stevemk14ebr -- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr +- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr - extractor: fix vivisect loop detection corner case #1310 @mr-tz - match: extend OS characteristic to match OS_ANY to all supported OSes #1324 @mike-hunhoff - extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6 ### capa explorer IDA Pro plugin + - rule generator plugin now loads faster when jumping between functions @stevemk14ebr - fix exception when plugin loaded in IDA hosted under idat #1341 @mike-hunhoff - improve embedded PE detection performance and reduce FP potential #1344 @mike-hunhoff ### Raw diffs + - [capa v5.0.0...v5.1.0](https://github.com/mandiant/capa/compare/v5.0.0...v5.1.0) - [capa-rules v5.0.0...v5.1.0](https://github.com/mandiant/capa-rules/compare/v5.0.0...v5.1.0) - ## v5.0.0 (2023-02-08) -This capa version comes with major improvements and additions to better handle .NET binaries. To showcase this we've updated and added over 30 .NET rules. + +This capa version comes with major improvements and additions to better handle .NET binaries. To showcase this we've updated and added over 30 .NET rules. Additionally, capa now caches its rule set for better performance. The capa explorer also caches its analysis results, so that multiple IDA Pro or plugin invocations don't need to repeat the same analysis. @@ -536,6 +574,7 @@ Other improvements to highlight include better ELF OS detection, various renderi Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardweiss80, @joren485, @ryantxu1, @mwilliams31, @anushkavirgaonkar, @MalwareMechanic, @Still34, @dzbeck, @johnk3r, and everyone else who submitted bugs and provided feedback! ### New Features + - verify rule metadata format on load #1160 @mr-tz - dotnet: emit property features #1168 @anushkavirgaonkar - dotnet: emit API features for objects created via the newobj instruction #1186 @mike-hunhoff @@ -550,6 +589,7 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - update ATT&CK/MBC data for linting #1297 @mr-tz ### Breaking Changes + - remove SMDA backend #1062 @williballenthin - error return codes are now positive numbers #1269 @mr-tz @@ -629,6 +669,7 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - nursery/unmanaged-call-via-dynamic-pinvoke-in-dotnet michael.hunhoff@mandiant.com ### Bug Fixes + - render: convert feature attributes to aliased dictionary for vverbose #1152 @mike-hunhoff - decouple Token dependency / extractor and features #1139 @mr-tz - update pydantic model to guarantee type coercion #1176 @mike-hunhoff @@ -647,6 +688,7 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - extractor: don't extract byte features for strings #1293 @mr-tz ### capa explorer IDA Pro plugin + - fix: display instruction items #1154 @mr-tz - fix: accept only plaintext pasted content #1194 @williballenthin - fix: UnboundLocalError #1217 @williballenthin @@ -661,27 +703,30 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - cache capa results across IDA sessions #1279 @mr-tz ### Raw diffs + - [capa v4.0.1...v5.0.0](https://github.com/mandiant/capa/compare/v4.0.1...v5.0.0) - [capa-rules v4.0.1...v5.0.0](https://github.com/mandiant/capa-rules/compare/v4.0.1...v5.0.0) - ## v4.0.1 (2022-08-15) + Some rules contained invalid metadata fields that caused an error when rendering rule hits. We've updated all rules and enhanced the rule linter to catch such issues. ### New Rules (1) - anti-analysis/obfuscation/obfuscated-with-vs-obfuscation jakub.jozwiak@mandiant.com - ### Bug Fixes + - linter: use pydantic to validate rule metadata #1141 @mike-hunhoff - build binaries using PyInstaller no longer overwrites functions in version.py #1136 @mr-tz ### Raw diffs + - [capa v4.0.0...v4.0.1](https://github.com/mandiant/capa/compare/v4.0.0...v4.0.1) - [capa-rules v4.0.0...v4.0.1](https://github.com/mandiant/capa-rules/compare/v4.0.0...v4.0.1) ## v4.0.0 (2022-08-10) + Version 4 adds support for analyzing .NET executables. capa will autodetect .NET modules, or you can explicitly invoke the new feature extractor via `--format dotnet`. We've also extended the rule syntax for .NET features including `namespace` and `class`. Additionally, new `instruction` scope and `operand` features enable users to create more explicit rules. These features are not backwards compatible. We removed the previously used `/x32` and `/x64` flavors of number and operand features. @@ -690,33 +735,33 @@ We updated 49 existing rules and added 22 new rules leveraging these new feature More breaking changes include updates to the JSON results document, freeze file format schema (now format version v2), and the internal handling of addresses. -Thanks for all the support, especially to @htnhan, @jtothej, @sara-rn, @anushkavirgaonkar, and @_re_fox! +Thanks for all the support, especially to @htnhan, @jtothej, @sara-rn, @anushkavirgaonkar, and @\_re_fox! -*Deprecation warning: v4.0 will be the last capa version to support the SMDA backend.* +_Deprecation warning: v4.0 will be the last capa version to support the SMDA backend._ ### New Features - - add new scope "instruction" for matching mnemonics and operands #767 @williballenthin - - add new feature "operand[{0, 1, 2}].number" for matching instruction operand immediate values #767 @williballenthin - - add new feature "operand[{0, 1, 2}].offset" for matching instruction operand offsets #767 @williballenthin - - extract additional offset/number features in certain circumstances #320 @williballenthin - - add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin - - add file string extraction for dotnet files #1012 @mike-hunhoff - - add file function-name extraction for dotnet files #1015 @mike-hunhoff - - add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff - - add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff - - emit class and namespace features for dotnet files #1030 @mike-hunhoff - - render: support Addresses that aren't simple integers, like .NET token+offset #981 @williballenthin - - document rule tags and branches #1006 @williballenthin, @mr-tz +- add new scope "instruction" for matching mnemonics and operands #767 @williballenthin +- add new feature "operand[{0, 1, 2}].number" for matching instruction operand immediate values #767 @williballenthin +- add new feature "operand[{0, 1, 2}].offset" for matching instruction operand offsets #767 @williballenthin +- extract additional offset/number features in certain circumstances #320 @williballenthin +- add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin +- add file string extraction for dotnet files #1012 @mike-hunhoff +- add file function-name extraction for dotnet files #1015 @mike-hunhoff +- add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff +- add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff +- emit class and namespace features for dotnet files #1030 @mike-hunhoff +- render: support Addresses that aren't simple integers, like .NET token+offset #981 @williballenthin +- document rule tags and branches #1006 @williballenthin, @mr-tz ### Breaking Changes - - instruction scope and operand feature are new and are not backwards compatible with older versions of capa - - Python 3.7 is now the minimum supported Python version #866 @williballenthin - - remove /x32 and /x64 flavors of number and operand features #932 @williballenthin - - the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin - - extractors must use handles to identify functions/basic blocks/instructions #981 @williballenthin - - the freeze file format schema was updated, including format version bump to v2 #986 @williballenthin +- instruction scope and operand feature are new and are not backwards compatible with older versions of capa +- Python 3.7 is now the minimum supported Python version #866 @williballenthin +- remove /x32 and /x64 flavors of number and operand features #932 @williballenthin +- the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin +- extractors must use handles to identify functions/basic blocks/instructions #981 @williballenthin +- the freeze file format schema was updated, including format version bump to v2 #986 @williballenthin Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issues/937), we plan to remove the SMDA backend for v5. If you rely on this backend, please reach out so we can discuss extending the support for SMDA or transitioning your workflow to use vivisect. @@ -733,11 +778,11 @@ Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issu - anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com - nursery/execute-dotnet-assembly anushka.virgaonkar@mandiant.com - nursery/invoke-dotnet-assembly-method anushka.virgaonkar@mandiant.com -- collection/screenshot/capture-screenshot-via-keybd-event @_re_fox -- collection/browser/gather-chrome-based-browser-login-information @_re_fox +- collection/screenshot/capture-screenshot-via-keybd-event @\_re_fox +- collection/browser/gather-chrome-based-browser-login-information @\_re_fox - nursery/power-down-monitor michael.hunhoff@mandiant.com -- nursery/hash-data-using-aphash @_re_fox -- nursery/hash-data-using-jshash @_re_fox +- nursery/hash-data-using-aphash @\_re_fox +- nursery/hash-data-using-jshash @\_re_fox - host-interaction/file-system/files/list/enumerate-files-on-windows moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com - nursery/check-clipboard-data anushka.virgaonkar@mandiant.com - nursery/clear-clipboard-data anushka.virgaonkar@mandiant.com @@ -749,16 +794,18 @@ Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issu - nursery/send-keystrokes anushka.virgaonkar@mandiant.com - nursery/send-request-in-dotnet anushka.virgaonakr@mandiant.com - nursery/terminate-process-by-name-in-dotnet anushka.virgaonkar@mandiant.com -- nursery/hash-data-using-rshash @_re_fox +- nursery/hash-data-using-rshash @\_re_fox - persistence/authentication-process/act-as-credential-manager-dll jakub.jozwiak@mandiant.com - persistence/authentication-process/act-as-password-filter-dll jakub.jozwiak@mandiant.com ### Bug Fixes -- improve handling _ prefix compile/link artifact #924 @mike-hunhoff + +- improve handling \_ prefix compile/link artifact #924 @mike-hunhoff - better detect OS in ELF samples #988 @williballenthin - display number feature zero in vverbose #1097 @mike-hunhoff ### capa explorer IDA Pro plugin + - improve file format extraction #918 @mike-hunhoff - remove decorators added by IDA to ELF imports #919 @mike-hunhoff - bug fixes for Address abstraction #1091 @mike-hunhoff @@ -766,10 +813,12 @@ Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issu ### Development ### Raw diffs + - [capa v3.2.0...v4.0.0](https://github.com/mandiant/capa/compare/v3.2.0...master) - [capa-rules v3.2.0...v4.0.0](https://github.com/mandiant/capa-rules/compare/v3.2.0...master) ## v3.2.1 (2022-06-06) + This out-of-band release bumps the SMDA dependency version to enable installation on Python 3.10. ### Bug Fixes @@ -777,10 +826,12 @@ This out-of-band release bumps the SMDA dependency version to enable installatio - update SMDA dependency @mike-hunhoff #922 ### Raw diffs + - [capa v3.2.0...v3.2.1](https://github.com/mandiant/capa/compare/v3.2.0...v3.2.1) - [capa-rules v3.2.0...v3.2.1](https://github.com/mandiant/capa-rules/compare/v3.2.0...v3.2.1) ## v3.2.0 (2022-03-03) + This release adds a new characteristic `characteristic: call $+5` enabling users to create more explicit rules. The linter now also validates ATT&CK and MBC categories. Additionally, many dependencies, including the vivisect backend, have been updated. One rule has been added and many more have been improved. @@ -802,20 +853,21 @@ Thanks for all the support, especially to @kn0wl3dge and first time contributor - elf: fix OS detection for Linux kernel modules #867 @williballenthin ### Raw diffs + - [capa v3.1.0...v3.2.0](https://github.com/mandiant/capa/compare/v3.1.0...v3.2.0) - [capa-rules v3.1.0...v3.2.0](https://github.com/mandiant/capa-rules/compare/v3.1.0...v3.2.0) ## v3.1.0 (2022-01-10) + This release improves the performance of capa while also adding 23 new rules and many code quality enhancements. We profiled capa's CPU usage and optimized the way that it matches rules, such as by short circuiting when appropriate. According to our testing, the matching phase is approximately 66% faster than v3.0.3! We also added support for Python 3.10, aarch64 builds, and additional MAEC metadata in the rule headers. - + This release adds 23 new rules, including nine by Jakub Jozwiak of Mandiant. @ryantxu1 and @dzbeck updated the ATT&CK and MBC mappings for many rules. Thank you! - + And as always, welcome first time contributors! - - @kn0wl3dge - - @jtothej - - @cl30 - +- @kn0wl3dge +- @jtothej +- @cl30 ### New Features @@ -854,10 +906,10 @@ And as always, welcome first time contributors! ### Rule Changes - - update ATT&CK mappings by @ryantxu1 - - update ATT&CK and MBC mappings by @dzbeck - - aplib detection by @cdong1012 - - golang runtime detection by @stevemk14eber +- update ATT&CK mappings by @ryantxu1 +- update ATT&CK and MBC mappings by @dzbeck +- aplib detection by @cdong1012 +- golang runtime detection by @stevemk14eber ### Bug Fixes @@ -875,22 +927,23 @@ And as always, welcome first time contributors! - show features script: add backend flag #430 @kn0wl3dge ### Raw diffs + - [capa v3.0.3...v3.1.0](https://github.com/mandiant/capa/compare/v3.0.3...v3.1.0) - [capa-rules v3.0.3...v3.1.0](https://github.com/mandiant/capa-rules/compare/v3.0.3...v3.1.0) - ## v3.0.3 (2021-10-27) This is primarily a rule maintenance release: - - eight new rules, including all relevant techniques from [ATT&CK v10](https://medium.com/mitre-attack/introducing-attack-v10-7743870b37e3), and - - two rules removed, due to the prevalence of false positives + +- eight new rules, including all relevant techniques from [ATT&CK v10](https://medium.com/mitre-attack/introducing-attack-v10-7743870b37e3), and +- two rules removed, due to the prevalence of false positives We've also tweaked the status codes returned by capa.exe to be more specific and added a bit more metadata to the JSON output format. - + As always, welcome first time contributors! - - still@teamt5.org - - zander.work@mandiant.com - + +- still@teamt5.org +- zander.work@mandiant.com ### New Features @@ -909,6 +962,7 @@ As always, welcome first time contributors! - targeting/language/identify-system-language-via-api william.ballenthin@mandiant.com ## Removed rules (2) + - load-code/pe/parse-pe-exports: too many false positives in unrelated structure accesses - anti-analysis/anti-vm/vm-detection/execute-anti-vm-instructions: too many false positives in junk code @@ -917,11 +971,12 @@ As always, welcome first time contributors! - update references from FireEye to Mandiant ### Raw diffs + - [capa v3.0.2...v3.0.3](https://github.com/fireeye/capa/compare/v3.0.2...v3.0.3) - [capa-rules v3.0.2...v3.0.3](https://github.com/fireeye/capa-rules/compare/v3.0.2...v3.0.3) - + ## v3.0.2 (2021-09-28) - + This release fixes an issue with the standalone executables built with PyInstaller when running capa against ELF files. ### Bug Fixes @@ -929,6 +984,7 @@ This release fixes an issue with the standalone executables built with PyInstall - fix bug in PyInstaller config preventing ELF analysis #795 @mr-tz ### Raw diffs + - [capa v3.0.1...v3.0.2](https://github.com/fireeye/capa/compare/v3.0.1...v3.0.2) - [capa-rules v3.0.1...v3.0.2](https://github.com/fireeye/capa-rules/compare/v3.0.1...v3.0.2) @@ -943,6 +999,7 @@ Thanks to the community for highlighting issues and analysis misses. Your feedba - fix many underlying bugs in vivisect analysis and update to version v1.0.5 #786 @williballenthin ### Raw diffs + - [capa v3.0.0...v3.0.1](https://github.com/fireeye/capa/compare/v3.0.0...v3.0.1) - [capa-rules v3.0.0...v3.0.1](https://github.com/fireeye/capa-rules/compare/v3.0.0...v3.0.1) @@ -951,6 +1008,7 @@ Thanks to the community for highlighting issues and analysis misses. Your feedba We are excited to announce version 3.0! :tada: capa 3.0: + - adds support for ELF files targeting Linux thanks to [Intezer](https://www.intezer.com/) - adds new features to specify OS, CPU architecture, and file format - fixes a few bugs that may have led to false negatives (missed capabilities) in older versions @@ -959,9 +1017,10 @@ capa 3.0: A huge thanks to everyone who submitted issues, provided feedback, and contributed code and rules. Special acknowledgement to @Adir-Shemesh and @TcM1911 of [Intezer](https://www.intezer.com/) for contributing the code to enable ELF support. Also, welcome first time contributors: - - @jaredscottwilson - - @cdong1012 - - @jlepore-fe + +- @jaredscottwilson +- @cdong1012 +- @jlepore-fe ### New Features @@ -1066,14 +1125,15 @@ Also, welcome first time contributors: ### Development ### Raw diffs + - [capa v2.0.0...v3.0.0](https://github.com/mandiant/capa/compare/v2.0.0...v3.0.0) - [capa-rules v2.0.0...v3.0.0](https://github.com/mandiant/capa-rules/compare/v2.0.0...v3.0.0) - ## v2.0.0 (2021-07-19) We are excited to announce version 2.0! :tada: capa 2.0: + - enables anyone to contribute rules more easily - is the first Python 3 ONLY version - provides more concise and relevant result via identification of library functions using FLIRT @@ -1083,7 +1143,6 @@ capa 2.0: A huge thanks to everyone who submitted issues, provided feedback, and contributed code and rules. Many colleagues across dozens of organizations have volunteered their experience to improve this tool! :heart: - ### New Features - rules: update ATT&CK and MBC mappings https://github.com/mandiant/capa-rules/pull/317 @williballenthin @@ -1210,6 +1269,7 @@ A huge thanks to everyone who submitted issues, provided feedback, and contribut - main: do not process non-PE files even when --format explicitly provided #664 @mr-tz ### capa explorer IDA Pro plugin + - explorer: IDA 7.6 support #497 @williballenthin - explorer: explain how to install IDA 7.6 patch to enable the plugin #528 @williballenthin - explorer: document IDA 7.6sp1 as alternative to the patch #536 @Ana06 @@ -1230,10 +1290,10 @@ A huge thanks to everyone who submitted issues, provided feedback, and contribut ### Raw diffs + - [capa v1.6.1...v2.0.0](https://github.com/mandiant/capa/compare/v1.6.1...v2.0.0) - [capa-rules v1.6.1...v2.0.0](https://github.com/mandiant/capa-rules/compare/v1.6.1...v2.0.0) - ## v1.6.3 (2021-04-29) This release adds IDA 7.6 support to capa. @@ -1244,8 +1304,7 @@ This release adds IDA 7.6 support to capa. ### Raw diffs - - [capa v1.6.2...v1.6.3](https://github.com/mandiant/capa/compare/v1.6.2...v1.6.3) - +- [capa v1.6.2...v1.6.3](https://github.com/mandiant/capa/compare/v1.6.2...v1.6.3) ## v1.6.2 (2021-04-13) @@ -1257,8 +1316,7 @@ This release backports a fix to capa 1.6: The Windows binary was built with Pyth ### Raw diffs - - [capa v1.6.1...v1.6.2](https://github.com/mandiant/capa/compare/v1.6.1...v1.6.2) - +- [capa v1.6.1...v1.6.2](https://github.com/mandiant/capa/compare/v1.6.1...v1.6.2) ## v1.6.1 (2021-04-07) @@ -1325,9 +1383,8 @@ This release includes several bug fixes, such as a vivisect issue that prevented ### Raw diffs - - [capa v1.6.0...v1.6.1](https://github.com/mandiant/capa/compare/v1.6.0...v1.6.1) - - [capa-rules v1.6.0...v1.6.1](https://github.com/mandiant/capa-rules/compare/v1.6.0...v1.6.1) - +- [capa v1.6.0...v1.6.1](https://github.com/mandiant/capa/compare/v1.6.0...v1.6.1) +- [capa-rules v1.6.0...v1.6.1](https://github.com/mandiant/capa-rules/compare/v1.6.0...v1.6.1) ## v1.6.0 (2021-03-09) @@ -1344,6 +1401,7 @@ The capa explorer IDA plugin now helps you quickly build new capa rules using fe This version of capa adds Python 3 support in vivisect. Note that `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. When updating to Python 3 you need to delete all the `.viv` files for capa to work. If you get the following error (or a similar one), you most likely need to delete `.viv` files: + ``` UnicodeDecodeError: 'ascii' codec can't decode byte 0x90 in position 2: ordinal not in range(128) ``` @@ -1356,13 +1414,13 @@ If you have workflows that rely on the Python 2 version and need future maintena ### New features -- explorer: Add capa explorer rule generator plugin for IDA Pro. Now capa explorer helps you build new capa rules! #426, #438, #439 @mike-hunhoff +- explorer: Add capa explorer rule generator plugin for IDA Pro. Now capa explorer helps you build new capa rules! #426, #438, #439 @mike-hunhoff - python: Python 3 support in vivisect #421 @Ana06 - main: Add backend option in Python 3 to select the backend to be used (either SMDA or vivisect) #421 @Ana06 - python: Python 3 support in IDA #429, #437 @mike-hunhoff - ci: test pyinstaller CI #452 @williballenthin - scripts: enable multiple backends in `show-features.py` #429 @mike-hunhoff -- scripts: add `scripts/vivisect-py2-vs-py3.sh` to compare vivisect Python 2 vs 3 (can easily be modified to test run times and compare different versions) #421 @Ana06 +- scripts: add `scripts/vivisect-py2-vs-py3.sh` to compare vivisect Python 2 vs 3 (can easily be modified to test run times and compare different versions) #421 @Ana06 ### New Rules (12) @@ -1397,9 +1455,8 @@ If you have workflows that rely on the Python 2 version and need future maintena ### Raw diffs - - [capa v1.5.1...v1.6.0](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) - - [capa-rules v1.5.1...v1.6.0](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) - +- [capa v1.5.1...v1.6.0](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) +- [capa-rules v1.5.1...v1.6.0](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) ## v1.5.1 (2021-02-09) @@ -1411,107 +1468,105 @@ This release fixes the version number that we forgot to update for v1.5.0 (there ### Raw diffs - - [capa v1.5.0...v1.5.1](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) - - [capa-rules v1.5.0...v1.5.1](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) - +- [capa v1.5.0...v1.5.1](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) +- [capa-rules v1.5.0...v1.5.1](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) ## v1.5.0 (2021-02-05) This release brings support for running capa under Python 3 via [SMDA](https://github.com/danielplohmann/smda), more thorough CI testing and linting, better extraction of strings and byte features, and 50 (!) new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - @johnk3r - - @doomedraven - - @stvemillertime - - @itreallynick - - @0x534a - +- @johnk3r +- @doomedraven +- @stvemillertime +- @itreallynick +- @0x534a + @dzbeck also added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for many rules. Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). - ### New Features - - py3 support via SMDA #355 @danielplohmann @jcrussell - - scripts: example of using capa as a library #372, #380 @doomedraven - - ci: enable dependabot #373 @mr-tz - - ci: lint rules @mr-tz - - ci: lint rule format #401 @mr-tz - - freeze: add base address #391 @mr-tz - - json: meta: add base address #412 @mr-tz +- py3 support via SMDA #355 @danielplohmann @jcrussell +- scripts: example of using capa as a library #372, #380 @doomedraven +- ci: enable dependabot #373 @mr-tz +- ci: lint rules @mr-tz +- ci: lint rule format #401 @mr-tz +- freeze: add base address #391 @mr-tz +- json: meta: add base address #412 @mr-tz ### New Rules (50) - - 64-bit execution via heavens gate @recvfrom - - contain anti-disasm techniques @mr-tz - - check for microsoft office emulation @re-fox - - check for windows sandbox via device @re-fox - - check for windows sandbox via dns suffix @re-fox - - check for windows sandbox via genuine state @re-fox - - check for windows sandbox via process name @re-fox - - check for windows sandbox via registry @re-fox - - capture microphone audio @re-fox - - capture public ip @re-fox - - get domain trust relationships @johnk3r - - check HTTP status code @mr-tz - - compiled with perl2exe @re-fox - - compiled with ps2exe @re-fox - - compiled with pyarmor @stvemillertime, @itreallynick - - validate payment card number using luhn algorithm @re-fox - - hash data using fnv @re-fox @mr-tz - - generate random numbers via WinAPI @mike-hunhoff @johnk3r - - enumerate files recursively @re-fox - - get file system object information @mike-hunhoff - - read virtual disk @re-fox - - register minifilter driver @mike-hunhoff - - start minifilter driver @mike-hunhoff - - enumerate gui resources @johnk3r - - simulate CTRL ALT DEL @mike-hunhoff - - hijack thread execution @0x534a - - inject dll @0x534a - - inject pe @0x534a - - create or open registry key @mike-hunhoff - - delete registry value @mike-hunhoff - - query or enumerate registry key @mike-hunhoff - - query or enumerate registry value @mike-hunhoff - - resume thread @0x534a - - suspend thread @0x534a - - allocate memory @0x534a - - allocate RW memory @0x534a - - contain pusha popa sequence @mr-tz - - create or open file @mike-hunhoff - - open process @0x534a - - open thread @0x534a - - get kernel32 base address @mr-tz - - get ntdll base address @mr-tz - - encrypt or decrypt data via BCrypt @mike-hunhoff - - generate random numbers using the Delphi LCG @williballenthin - - hash data via BCrypt @mike-hunhoff - - migrate process to active window station @williballenthin - - patch process command line @williballenthin - - resolve function by hash @williballenthin - - persist via Winlogon Helper DLL registry key @0x534a - - schedule task via command line @0x534a +- 64-bit execution via heavens gate @recvfrom +- contain anti-disasm techniques @mr-tz +- check for microsoft office emulation @re-fox +- check for windows sandbox via device @re-fox +- check for windows sandbox via dns suffix @re-fox +- check for windows sandbox via genuine state @re-fox +- check for windows sandbox via process name @re-fox +- check for windows sandbox via registry @re-fox +- capture microphone audio @re-fox +- capture public ip @re-fox +- get domain trust relationships @johnk3r +- check HTTP status code @mr-tz +- compiled with perl2exe @re-fox +- compiled with ps2exe @re-fox +- compiled with pyarmor @stvemillertime, @itreallynick +- validate payment card number using luhn algorithm @re-fox +- hash data using fnv @re-fox @mr-tz +- generate random numbers via WinAPI @mike-hunhoff @johnk3r +- enumerate files recursively @re-fox +- get file system object information @mike-hunhoff +- read virtual disk @re-fox +- register minifilter driver @mike-hunhoff +- start minifilter driver @mike-hunhoff +- enumerate gui resources @johnk3r +- simulate CTRL ALT DEL @mike-hunhoff +- hijack thread execution @0x534a +- inject dll @0x534a +- inject pe @0x534a +- create or open registry key @mike-hunhoff +- delete registry value @mike-hunhoff +- query or enumerate registry key @mike-hunhoff +- query or enumerate registry value @mike-hunhoff +- resume thread @0x534a +- suspend thread @0x534a +- allocate memory @0x534a +- allocate RW memory @0x534a +- contain pusha popa sequence @mr-tz +- create or open file @mike-hunhoff +- open process @0x534a +- open thread @0x534a +- get kernel32 base address @mr-tz +- get ntdll base address @mr-tz +- encrypt or decrypt data via BCrypt @mike-hunhoff +- generate random numbers using the Delphi LCG @williballenthin +- hash data via BCrypt @mike-hunhoff +- migrate process to active window station @williballenthin +- patch process command line @williballenthin +- resolve function by hash @williballenthin +- persist via Winlogon Helper DLL registry key @0x534a +- schedule task via command line @0x534a ### Bug Fixes - - doc: pyinstaller build process @mr-tz - - ida: better bytes extraction #409 @mike-hunhoff - - viv: better unicode string extraction #364 @mike-hunhoff - - viv: better unicode string extraction #378 @mr-tz - - viv: more xor instructions #379 @mr-tz - - viv: decrease logging verbosity #381 @mr-tz - - rules: fix api description syntax #403 @mike-hunhoff - - main: disable progress background thread #410 @mike-hunhoff - +- doc: pyinstaller build process @mr-tz +- ida: better bytes extraction #409 @mike-hunhoff +- viv: better unicode string extraction #364 @mike-hunhoff +- viv: better unicode string extraction #378 @mr-tz +- viv: more xor instructions #379 @mr-tz +- viv: decrease logging verbosity #381 @mr-tz +- rules: fix api description syntax #403 @mike-hunhoff +- main: disable progress background thread #410 @mike-hunhoff + ### Changes - - rules: return lib rules for scopes #398 @mr-tz - +- rules: return lib rules for scopes #398 @mr-tz + ### Raw diffs - - [capa v1.4.1...v1.5.0](https://github.com/mandiant/capa/compare/v1.4.1...v1.5.0) - - [capa-rules v1.4.0...v1.5.0](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.5.0) +- [capa v1.4.1...v1.5.0](https://github.com/mandiant/capa/compare/v1.4.1...v1.5.0) +- [capa-rules v1.4.0...v1.5.0](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.5.0) ## v1.4.1 (2020-10-23) @@ -1519,19 +1574,19 @@ This release fixes an issue building capa on our CI server, which prevented us f ### Bug Fixes - - install VC dependencies for Python 2.7 during Windows build - +- install VC dependencies for Python 2.7 during Windows build + ### Raw diffs - - [capa v1.4.0...v1.4.1](https://github.com/mandiant/capa/compare/v1.4.0...v1.4.1) - - [capa-rules v1.4.0...v1.4.1](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.4.1) +- [capa v1.4.0...v1.4.1](https://github.com/mandiant/capa/compare/v1.4.0...v1.4.1) +- [capa-rules v1.4.0...v1.4.1](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.4.1) ## v1.4.0 (2020-10-23) This capa release includes changes to the rule parsing, enhanced feature extraction, various bug fixes, and improved capa scripts. Everyone should benefit from the improved functionality and performance. The community helped to add 69 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - @mwilliams31 - - @yt0ng +- @mwilliams31 +- @yt0ng @dzbeck added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for 86 rules. @@ -1539,277 +1594,278 @@ Download a standalone binary below and checkout the readme [here on GitHub](http ### New features - - script that demonstrates bulk processing @williballenthin #307 - - main: render MBC table @mr-tz #332 - - ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 - - viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 - - features: extract APIs called via jmp instruction @mr-tz #337 +- script that demonstrates bulk processing @williballenthin #307 +- main: render MBC table @mr-tz #332 +- ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 +- viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 +- features: extract APIs called via jmp instruction @mr-tz #337 ### New rules - - clear the Windows event log @mike-hunhoff - - crash the Windows event logging service @mike-hunhoff - - packed with kkrunchy @re-fox - - packed with nspack @re-fox - - packed with pebundle @re-fox - - packed with pelocknt @re-fox - - packed with peshield @re-fox - - packed with petite @re-fox - - packed with rlpack @re-fox - - packed with upack @re-fox - - packed with y0da crypter @re-fox - - compiled with rust @re-fox - - compute adler32 checksum @mwilliams31 - - encrypt-data-using-hc-128 @recvfrom - - manipulate console @williballenthin - - references logon banner @re-fox - - terminate process via fastfail @re-fox - - delete volume shadow copies @mr-tz - - authenticate HMAC @mr-tz - - compiled from EPL @williballenthin - - compiled with Go @williballenthin - - create Restart Manager session @mike-hunhoff - - decode data using Base64 via WinAPI @mike-hunhoff - - empty recycle bin quietly @mwilliams31 - - enumerate network shares @mike-hunhoff - - hook routines via microsoft detours @williballenthin - - hooked by API Override @williballenthin - - impersonate user @mike-hunhoff - - the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ - - packed with CCG - - packed with Crunch - - packed with Dragon Armor - - packed with enigma - - packed with Epack - - packed with MaskPE - - packed with MEW - - packed with Mpress - - packed with Neolite - - packed with PECompact - - packed with Pepack - - packed with Perplex - - packed with ProCrypt - - packed with RPCrypt - - packed with SeauSFX - - packed with Shrinker - - packed with Simple Pack - - packed with StarForce - - packed with SVKP - - packed with Themida - - packed with TSULoader - - packed with VProtect - - packed with WWPACK - - rebuilt by ImpRec - - packaged as a Pintool - - packaged as a CreateInstall installer - - packaged as a WinZip self-extracting archive - - reference 114DNS DNS server @williballenthin - - reference AliDNS DNS server @williballenthin - - reference Cloudflare DNS server @williballenthin - - reference Comodo Secure DNS server @williballenthin - - reference Google Public DNS server @williballenthin - - reference Hurricane Electric DNS server @williballenthin - - reference kornet DNS server @williballenthin - - reference L3 DNS server @williballenthin - - reference OpenDNS DNS server @williballenthin - - reference Quad9 DNS server @williballenthin - - reference Verisign DNS server @williballenthin - - run as service @mike-hunhoff - - schedule task via ITaskService @mike-hunhoff - - references DNS over HTTPS endpoints @yt0ng +- clear the Windows event log @mike-hunhoff +- crash the Windows event logging service @mike-hunhoff +- packed with kkrunchy @re-fox +- packed with nspack @re-fox +- packed with pebundle @re-fox +- packed with pelocknt @re-fox +- packed with peshield @re-fox +- packed with petite @re-fox +- packed with rlpack @re-fox +- packed with upack @re-fox +- packed with y0da crypter @re-fox +- compiled with rust @re-fox +- compute adler32 checksum @mwilliams31 +- encrypt-data-using-hc-128 @recvfrom +- manipulate console @williballenthin +- references logon banner @re-fox +- terminate process via fastfail @re-fox +- delete volume shadow copies @mr-tz +- authenticate HMAC @mr-tz +- compiled from EPL @williballenthin +- compiled with Go @williballenthin +- create Restart Manager session @mike-hunhoff +- decode data using Base64 via WinAPI @mike-hunhoff +- empty recycle bin quietly @mwilliams31 +- enumerate network shares @mike-hunhoff +- hook routines via microsoft detours @williballenthin +- hooked by API Override @williballenthin +- impersonate user @mike-hunhoff +- the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ + - packed with CCG + - packed with Crunch + - packed with Dragon Armor + - packed with enigma + - packed with Epack + - packed with MaskPE + - packed with MEW + - packed with Mpress + - packed with Neolite + - packed with PECompact + - packed with Pepack + - packed with Perplex + - packed with ProCrypt + - packed with RPCrypt + - packed with SeauSFX + - packed with Shrinker + - packed with Simple Pack + - packed with StarForce + - packed with SVKP + - packed with Themida + - packed with TSULoader + - packed with VProtect + - packed with WWPACK + - rebuilt by ImpRec + - packaged as a Pintool + - packaged as a CreateInstall installer + - packaged as a WinZip self-extracting archive +- reference 114DNS DNS server @williballenthin +- reference AliDNS DNS server @williballenthin +- reference Cloudflare DNS server @williballenthin +- reference Comodo Secure DNS server @williballenthin +- reference Google Public DNS server @williballenthin +- reference Hurricane Electric DNS server @williballenthin +- reference kornet DNS server @williballenthin +- reference L3 DNS server @williballenthin +- reference OpenDNS DNS server @williballenthin +- reference Quad9 DNS server @williballenthin +- reference Verisign DNS server @williballenthin +- run as service @mike-hunhoff +- schedule task via ITaskService @mike-hunhoff +- references DNS over HTTPS endpoints @yt0ng ### Bug fixes - - ida plugin: fix tree-view exception @mike-hunhoff #315 - - ida plugin: fix feature count @mike-hunhoff - - main: fix reported total rule count @williballenthin #325 - - features: fix handling of API names with multiple periods @mike-hunhoff #329 - - ida backend: find all byte sequences instead of only first @mike-hunhoff #335 - - features: display 0 value @mr-tz #338 - - ida backend: extract ordinal and name imports @mr-tz #343 - - show-features: improvements and support within IDA @mr-tz #342 - - main: sanity check MBC rendering @williballenthin - - main: handle sample path that contains non-ASCII characters @mr-tz #328 +- ida plugin: fix tree-view exception @mike-hunhoff #315 +- ida plugin: fix feature count @mike-hunhoff +- main: fix reported total rule count @williballenthin #325 +- features: fix handling of API names with multiple periods @mike-hunhoff #329 +- ida backend: find all byte sequences instead of only first @mike-hunhoff #335 +- features: display 0 value @mr-tz #338 +- ida backend: extract ordinal and name imports @mr-tz #343 +- show-features: improvements and support within IDA @mr-tz #342 +- main: sanity check MBC rendering @williballenthin +- main: handle sample path that contains non-ASCII characters @mr-tz #328 ### Changes - - rules: use yaml.CLoader for better performance @williballenthin #306 - - rules: parse descriptions for statements @mr-tz #312 +- rules: use yaml.CLoader for better performance @williballenthin #306 +- rules: parse descriptions for statements @mr-tz #312 ### Raw diffs - - [capa v1.3.0...v1.4.0](https://github.com/mandiant/capa/compare/v1.3.0...v1.4.0) - - [capa-rules v1.3.0...v1.4.0](https://github.com/mandiant/capa-rules/compare/v1.3.0...v1.4.0) +- [capa v1.3.0...v1.4.0](https://github.com/mandiant/capa/compare/v1.3.0...v1.4.0) +- [capa-rules v1.3.0...v1.4.0](https://github.com/mandiant/capa-rules/compare/v1.3.0...v1.4.0) ## v1.3.0 (2020-09-14) This release brings newly updated mappings to the [Malware Behavior Catalog version 2.0](https://github.com/MBCProject/mbc-markdown), many enhancements to the IDA Pro plugin, [flare-capa on PyPI](https://pypi.org/project/flare-capa/), a bunch of bug fixes to improve feature extraction, and four new rules. We received contributions from ten reverse engineers, including seven new ones: - - @dzbeck - - @recvfrom - - @toomanybananas - - @cclauss - - @adamprescott91 - - @weslambert - - @stevemk14ebr - +- @dzbeck +- @recvfrom +- @toomanybananas +- @cclauss +- @adamprescott91 +- @weslambert +- @stevemk14ebr + Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). ### Key changes to IDA Plugin The IDA Pro integration is now distributed as a real plugin, instead of a script. This enables a few things: - - keyboard shortcuts and file menu integration - - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` - - generally doing thing the "right way" +- keyboard shortcuts and file menu integration +- updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` +- generally doing thing the "right way" -How to get this new version? It's easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. +How to get this new version? It's easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. Please refer to the plugin [readme](https://github.com/mandiant/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin. Please open an issue in this repository if you notice anything weird. - + ### New features - - ida plugin: now a real plugin, not a script @mike-hunhoff - - core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin - - features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 - - ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 - - ida plugin: add search bar to results view @williballenthin #285 - - ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 - - ida plugin: add status label to tree view @mike-hunhoff - - ida plugin: add progress indicator @mike-hunhoff, @mr-tz +- ida plugin: now a real plugin, not a script @mike-hunhoff +- core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin +- features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 +- ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 +- ida plugin: add search bar to results view @williballenthin #285 +- ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 +- ida plugin: add status label to tree view @mike-hunhoff +- ida plugin: add progress indicator @mike-hunhoff, @mr-tz ### New rules - - compiled with py2exe @re-fox - - resolve path using msvcrt @re-fox - - decompress data using QuickLZ @edeca - - encrypt data using sosemanuk @recvfrom +- compiled with py2exe @re-fox +- resolve path using msvcrt @re-fox +- decompress data using QuickLZ @edeca +- encrypt data using sosemanuk @recvfrom ### Bug fixes - - rule: reduce FP in DNS resolution @toomanybananas - - engine: report correct strings matched via regex @williballenthin #262 - - formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 - - viv: better extract offsets from SibOper operands @williballenthin @edeca #276 - - import-to-ida: fix import error @cclauss - - viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 - - ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 - - ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 - - ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 +- rule: reduce FP in DNS resolution @toomanybananas +- engine: report correct strings matched via regex @williballenthin #262 +- formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 +- viv: better extract offsets from SibOper operands @williballenthin @edeca #276 +- import-to-ida: fix import error @cclauss +- viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 +- ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 +- ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 +- ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 ### Changes - - rules: update meta mapping to MBC 2.0! @dzbeck - - render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 - - ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 - - ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 - - ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff - - ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 +- rules: update meta mapping to MBC 2.0! @dzbeck +- render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 +- ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 +- ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 +- ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff +- ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 ### Raw diffs - - [capa v1.2.0...v1.3.0](https://github.com/mandiant/capa/compare/v1.2.0...v1.3.0) - - [capa-rules v1.2.0...v1.3.0](https://github.com/mandiant/capa-rules/compare/v1.2.0...v1.3.0) +- [capa v1.2.0...v1.3.0](https://github.com/mandiant/capa/compare/v1.2.0...v1.3.0) +- [capa-rules v1.2.0...v1.3.0](https://github.com/mandiant/capa-rules/compare/v1.2.0...v1.3.0) ## v1.2.0 (2020-08-31) -This release brings UI enhancements, especially for the IDA Pro plugin, +This release brings UI enhancements, especially for the IDA Pro plugin, investment towards py3 support, -fixes some bugs identified by the community, +fixes some bugs identified by the community, and 46 (!) new rules. We received contributions from ten reverse engineers, including five new ones: - - @agithubuserlol - - @recvfrom - - @D4nch3n - - @edeca - - @winniepe - +- @agithubuserlol +- @recvfrom +- @D4nch3n +- @edeca +- @winniepe + Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). - + ### New features - - ida plugin: display arch flavors @mike-hunhoff - - ida plugin: display block descriptions @mike-hunhoff - - ida backend: extract features from nested pointers @mike-hunhoff - - main: show more progress output @williballenthin - - core: pin dependency versions #258 @recvfrom +- ida plugin: display arch flavors @mike-hunhoff +- ida plugin: display block descriptions @mike-hunhoff +- ida backend: extract features from nested pointers @mike-hunhoff +- main: show more progress output @williballenthin +- core: pin dependency versions #258 @recvfrom ### New rules - - bypass UAC via AppInfo ALPC @agithubuserlol - - bypass UAC via token manipulation @agithubuserlol - - check for sandbox and av modules @re-fox - - check for sandbox username @re-fox - - check if process is running under wine @re-fox - - validate credit card number using luhn algorithm @re-fox - - validate credit card number using luhn algorithm with no lookup table @re-fox - - hash data using FNV @edeca @mr-tz - - link many functions at runtime @mr-tz - - reference public RSA key @mr-tz - - packed with ASPack @williballenthin - - delete internet cache @mike-hunhoff - - enumerate internet cache @mike-hunhoff - - send ICMP echo request @mike-hunhoff - - check for debugger via API @mike-hunhoff - - check for hardware breakpoints @mike-hunhoff - - check for kernel debugger via shared user data structure @mike-hunhoff - - check for protected handle exception @mike-hunhoff - - check for software breakpoints @mike-hunhoff - - check for trap flag exception @mike-hunhoff - - check for unexpected memory writes @mike-hunhoff - - check process job object @mike-hunhoff - - reference anti-VM strings targeting Parallels @mike-hunhoff - - reference anti-VM strings targeting Qemu @mike-hunhoff - - reference anti-VM strings targeting VirtualBox @mike-hunhoff - - reference anti-VM strings targeting VirtualPC @mike-hunhoff - - reference anti-VM strings targeting VMWare @mike-hunhoff - - reference anti-VM strings targeting Xen @mike-hunhoff - - reference analysis tools strings @mike-hunhoff - - reference WMI statements @mike-hunhoff - - get number of processor cores @mike-hunhoff - - get number of processors @mike-hunhoff - - enumerate disk properties @mike-hunhoff - - get disk size @mike-hunhoff - - get process heap flags @mike-hunhoff - - get process heap force flags @mike-hunhoff - - get Explorer PID @mike-hunhoff - - delay execution @mike-hunhoff - - check for process debug object @mike-hunhoff - - check license value @mike-hunhoff - - check ProcessDebugFlags @mike-hunhoff - - check ProcessDebugPort @mike-hunhoff - - check SystemKernelDebuggerInformation @mike-hunhoff - - check thread yield allowed @mike-hunhoff - - enumerate system firmware tables @mike-hunhoff - - get system firmware table @mike-hunhoff - - hide thread from debugger @mike-hunhoff + +- bypass UAC via AppInfo ALPC @agithubuserlol +- bypass UAC via token manipulation @agithubuserlol +- check for sandbox and av modules @re-fox +- check for sandbox username @re-fox +- check if process is running under wine @re-fox +- validate credit card number using luhn algorithm @re-fox +- validate credit card number using luhn algorithm with no lookup table @re-fox +- hash data using FNV @edeca @mr-tz +- link many functions at runtime @mr-tz +- reference public RSA key @mr-tz +- packed with ASPack @williballenthin +- delete internet cache @mike-hunhoff +- enumerate internet cache @mike-hunhoff +- send ICMP echo request @mike-hunhoff +- check for debugger via API @mike-hunhoff +- check for hardware breakpoints @mike-hunhoff +- check for kernel debugger via shared user data structure @mike-hunhoff +- check for protected handle exception @mike-hunhoff +- check for software breakpoints @mike-hunhoff +- check for trap flag exception @mike-hunhoff +- check for unexpected memory writes @mike-hunhoff +- check process job object @mike-hunhoff +- reference anti-VM strings targeting Parallels @mike-hunhoff +- reference anti-VM strings targeting Qemu @mike-hunhoff +- reference anti-VM strings targeting VirtualBox @mike-hunhoff +- reference anti-VM strings targeting VirtualPC @mike-hunhoff +- reference anti-VM strings targeting VMWare @mike-hunhoff +- reference anti-VM strings targeting Xen @mike-hunhoff +- reference analysis tools strings @mike-hunhoff +- reference WMI statements @mike-hunhoff +- get number of processor cores @mike-hunhoff +- get number of processors @mike-hunhoff +- enumerate disk properties @mike-hunhoff +- get disk size @mike-hunhoff +- get process heap flags @mike-hunhoff +- get process heap force flags @mike-hunhoff +- get Explorer PID @mike-hunhoff +- delay execution @mike-hunhoff +- check for process debug object @mike-hunhoff +- check license value @mike-hunhoff +- check ProcessDebugFlags @mike-hunhoff +- check ProcessDebugPort @mike-hunhoff +- check SystemKernelDebuggerInformation @mike-hunhoff +- check thread yield allowed @mike-hunhoff +- enumerate system firmware tables @mike-hunhoff +- get system firmware table @mike-hunhoff +- hide thread from debugger @mike-hunhoff ### Bug fixes - - ida backend: extract unmapped immediate number features @mike-hunhoff - - ida backend: fix stack cookie check #257 @mike-hunhoff - - viv backend: better extract gs segment access @williballenthin - - core: enable counting of string features #241 @D4nch3n @williballenthin - - core: enable descriptions on feature with arch flavors @mike-hunhoff - - core: update git links for non-SSH access #259 @recvfrom +- ida backend: extract unmapped immediate number features @mike-hunhoff +- ida backend: fix stack cookie check #257 @mike-hunhoff +- viv backend: better extract gs segment access @williballenthin +- core: enable counting of string features #241 @D4nch3n @williballenthin +- core: enable descriptions on feature with arch flavors @mike-hunhoff +- core: update git links for non-SSH access #259 @recvfrom ### Changes - - ida plugin: better default display showing first level nesting @winniepe - - remove unused `characteristic(switch)` feature @ana06 - - prepare testing infrastructure for multiple backends/py3 @williballenthin - - ci: zip build artifacts @ana06 - - ci: build all supported python versions @ana06 - - code style and formatting @mr-tz +- ida plugin: better default display showing first level nesting @winniepe +- remove unused `characteristic(switch)` feature @ana06 +- prepare testing infrastructure for multiple backends/py3 @williballenthin +- ci: zip build artifacts @ana06 +- ci: build all supported python versions @ana06 +- code style and formatting @mr-tz ### Raw diffs - - [capa v1.1.0...v1.2.0](https://github.com/mandiant/capa/compare/v1.1.0...v1.2.0) - - [capa-rules v1.1.0...v1.2.0](https://github.com/mandiant/capa-rules/compare/v1.1.0...v1.2.0) +- [capa v1.1.0...v1.2.0](https://github.com/mandiant/capa/compare/v1.1.0...v1.2.0) +- [capa-rules v1.1.0...v1.2.0](https://github.com/mandiant/capa-rules/compare/v1.1.0...v1.2.0) ## v1.1.0 (2020-08-05) @@ -1817,83 +1873,83 @@ This release brings new rule format updates, such as adding `offset/x32` and neg fixes some bugs identified by the community, and 28 (!) new rules. We received contributions from eight reverse engineers, including four new ones: - - @re-fox - - @psifertex - - @bitsofbinary - - @threathive - +- @re-fox +- @psifertex +- @bitsofbinary +- @threathive + Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). - + ### New features - - import: add Binary Ninja import script #205 #207 @psifertex - - rules: offsets can be negative #197 #208 @williballenthin - - rules: enable descriptions for statement nodes #194 #209 @Ana06 - - rules: add arch flavors to number and offset features #210 #216 @williballenthin - - render: show SHA1/SHA256 in default report #164 @threathive - - tests: add tests for IDA Pro backend #202 @williballenthin - +- import: add Binary Ninja import script #205 #207 @psifertex +- rules: offsets can be negative #197 #208 @williballenthin +- rules: enable descriptions for statement nodes #194 #209 @Ana06 +- rules: add arch flavors to number and offset features #210 #216 @williballenthin +- render: show SHA1/SHA256 in default report #164 @threathive +- tests: add tests for IDA Pro backend #202 @williballenthin + ### New rules - - check for unmoving mouse cursor @BitsOfBinary - - check mutex and exit @re-fox - - parse credit card information @re-fox - - read ini file @re-fox - - validate credit card number with luhn algorithm @re-fox - - change the wallpaper @re-fox - - acquire debug privileges @williballenthin - - import public key @williballenthin - - terminate process by name @williballenthin - - encrypt data using DES @re-fox - - encrypt data using DES via WinAPI @re-fox - - hash data using sha1 via x86 extensions @re-fox - - hash data using sha256 via x86 extensions @re-fox - - capture network configuration via ipconfig @re-fox - - hash data via WinCrypt @mike-hunhoff - - get file attributes @mike-hunhoff - - allocate thread local storage @mike-hunhoff - - get thread local storage value @mike-hunhoff - - set thread local storage @mike-hunhoff - - get session integrity level @mike-hunhoff - - add file to cabinet file @mike-hunhoff - - flush cabinet file @mike-hunhoff - - open cabinet file @mike-hunhoff - - gather firefox profile information @re-fox - - encrypt data using skipjack @re-fox - - encrypt data using camellia @re-fox - - hash data using tiger @re-fox - - encrypt data using blowfish @re-fox - - encrypt data using twofish @re-fox +- check for unmoving mouse cursor @BitsOfBinary +- check mutex and exit @re-fox +- parse credit card information @re-fox +- read ini file @re-fox +- validate credit card number with luhn algorithm @re-fox +- change the wallpaper @re-fox +- acquire debug privileges @williballenthin +- import public key @williballenthin +- terminate process by name @williballenthin +- encrypt data using DES @re-fox +- encrypt data using DES via WinAPI @re-fox +- hash data using sha1 via x86 extensions @re-fox +- hash data using sha256 via x86 extensions @re-fox +- capture network configuration via ipconfig @re-fox +- hash data via WinCrypt @mike-hunhoff +- get file attributes @mike-hunhoff +- allocate thread local storage @mike-hunhoff +- get thread local storage value @mike-hunhoff +- set thread local storage @mike-hunhoff +- get session integrity level @mike-hunhoff +- add file to cabinet file @mike-hunhoff +- flush cabinet file @mike-hunhoff +- open cabinet file @mike-hunhoff +- gather firefox profile information @re-fox +- encrypt data using skipjack @re-fox +- encrypt data using camellia @re-fox +- hash data using tiger @re-fox +- encrypt data using blowfish @re-fox +- encrypt data using twofish @re-fox ### Bug fixes - - linter: fix exception when examples is `None` @Ana06 - - linter: fix suggested recommendations via templating @williballenthin - - render: fix exception when rendering counts @williballenthin - - render: fix render of negative offsets @williballenthin - - extractor: fix segmentation violation from vivisect @williballenthin - - main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin - - main: fix shellcode .viv save path @williballenthin +- linter: fix exception when examples is `None` @Ana06 +- linter: fix suggested recommendations via templating @williballenthin +- render: fix exception when rendering counts @williballenthin +- render: fix render of negative offsets @williballenthin +- extractor: fix segmentation violation from vivisect @williballenthin +- main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin +- main: fix shellcode .viv save path @williballenthin ### Changes - - doc: explain how to bypass gatekeeper on macOS @psifertex - - doc: explain supported linux distributions @Ana06 - - doc: explain submodule update with --init @psifertex - - main: improve program help output @mr-tz - - main: disable progress when run in quiet mode @mr-tz - - main: assert supported IDA versions @mr-tz - - extractor: better identify nested pointers to strings @williballenthin - - setup: specify vivisect download url @Ana06 - - setup: pin vivisect version @williballenthin - - setup: bump vivisect dependency version @williballenthin - - setup: set Python project name to `flare-capa` @williballenthin - - ci: run tests and linter via GitHub Actions @Ana06 - - hooks: run style checkers and hide stashed output @Ana06 - - linter: ignore period in rule filename @williballenthin - - linter: warn on nursery rule with no changes needed @williballenthin +- doc: explain how to bypass gatekeeper on macOS @psifertex +- doc: explain supported linux distributions @Ana06 +- doc: explain submodule update with --init @psifertex +- main: improve program help output @mr-tz +- main: disable progress when run in quiet mode @mr-tz +- main: assert supported IDA versions @mr-tz +- extractor: better identify nested pointers to strings @williballenthin +- setup: specify vivisect download url @Ana06 +- setup: pin vivisect version @williballenthin +- setup: bump vivisect dependency version @williballenthin +- setup: set Python project name to `flare-capa` @williballenthin +- ci: run tests and linter via GitHub Actions @Ana06 +- hooks: run style checkers and hide stashed output @Ana06 +- linter: ignore period in rule filename @williballenthin +- linter: warn on nursery rule with no changes needed @williballenthin ### Raw diffs - - [capa v1.0.0...v1.1.0](https://github.com/mandiant/capa/compare/v1.0.0...v1.1.0) - - [capa-rules v1.0.0...v1.1.0](https://github.com/mandiant/capa-rules/compare/v1.0.0...v1.1.0) +- [capa v1.0.0...v1.1.0](https://github.com/mandiant/capa/compare/v1.0.0...v1.1.0) +- [capa-rules v1.0.0...v1.1.0](https://github.com/mandiant/capa-rules/compare/v1.0.0...v1.1.0) diff --git a/capa/capabilities/dynamic.py b/capa/capabilities/dynamic.py index 98858c581..2a433be4e 100644 --- a/capa/capabilities/dynamic.py +++ b/capa/capabilities/dynamic.py @@ -6,20 +6,16 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import sys import logging import itertools import collections -from typing import Any, Tuple - -import tqdm +from typing import Any, List, Tuple import capa.perf import capa.features.freeze as frz import capa.render.result_document as rdoc from capa.rules import Scope, RuleSet from capa.engine import FeatureSet, MatchResults -from capa.helpers import redirecting_print_to_tqdm from capa.capabilities.common import find_file_capabilities from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor @@ -139,38 +135,30 @@ def find_dynamic_capabilities( feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=()) assert isinstance(extractor, DynamicFeatureExtractor) - with redirecting_print_to_tqdm(disable_progress): - with tqdm.contrib.logging.logging_redirect_tqdm(): - pbar = tqdm.tqdm - if disable_progress: - # do not use tqdm to avoid unnecessary side effects when caller intends - # to disable progress completely - def pbar(s, *args, **kwargs): - return s - - elif not sys.stderr.isatty(): - # don't display progress bar when stderr is redirected to a file - def pbar(s, *args, **kwargs): - return s - - processes = list(extractor.get_processes()) - - pb = pbar(processes, desc="matching", unit=" processes", leave=False) - for p in pb: - process_matches, thread_matches, call_matches, feature_count = find_process_capabilities( - ruleset, extractor, p - ) - feature_counts.processes += ( - rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count), - ) - logger.debug("analyzed %s and extracted %d features", p.address, feature_count) - - for rule_name, res in process_matches.items(): - all_process_matches[rule_name].extend(res) - for rule_name, res in thread_matches.items(): - all_thread_matches[rule_name].extend(res) - for rule_name, res in call_matches.items(): - all_call_matches[rule_name].extend(res) + processes: List[ProcessHandle] = list(extractor.get_processes()) + n_processes: int = len(processes) + + with capa.helpers.CapaProgressBar( + console=capa.helpers.log_console, transient=True, disable=disable_progress + ) as pbar: + task = pbar.add_task("matching", total=n_processes, unit="processes") + for p in processes: + process_matches, thread_matches, call_matches, feature_count = find_process_capabilities( + ruleset, extractor, p + ) + feature_counts.processes += ( + rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count), + ) + logger.debug("analyzed %s and extracted %d features", p.address, feature_count) + + for rule_name, res in process_matches.items(): + all_process_matches[rule_name].extend(res) + for rule_name, res in thread_matches.items(): + all_thread_matches[rule_name].extend(res) + for rule_name, res in call_matches.items(): + all_call_matches[rule_name].extend(res) + + pbar.advance(task) # collection of features that captures the rule matches within process and thread scopes. # mapping from feature (matched rule) to set of addresses at which it matched. diff --git a/capa/capabilities/static.py b/capa/capabilities/static.py index 4f3b3b6a1..aeb710ae3 100644 --- a/capa/capabilities/static.py +++ b/capa/capabilities/static.py @@ -6,21 +6,18 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import sys import time import logging import itertools import collections -from typing import Any, Tuple - -import tqdm.contrib.logging +from typing import Any, List, Tuple import capa.perf +import capa.helpers import capa.features.freeze as frz import capa.render.result_document as rdoc from capa.rules import Scope, RuleSet from capa.engine import FeatureSet, MatchResults -from capa.helpers import redirecting_print_to_tqdm from capa.capabilities.common import find_file_capabilities from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor @@ -143,75 +140,58 @@ def find_static_capabilities( library_functions: Tuple[rdoc.LibraryFunction, ...] = () assert isinstance(extractor, StaticFeatureExtractor) - with redirecting_print_to_tqdm(disable_progress): - with tqdm.contrib.logging.logging_redirect_tqdm(): - pbar = tqdm.tqdm - if capa.helpers.is_runtime_ghidra(): - # Ghidrathon interpreter cannot properly handle - # the TMonitor thread that is created via a monitor_interval - # > 0 - pbar.monitor_interval = 0 - if disable_progress: - # do not use tqdm to avoid unnecessary side effects when caller intends - # to disable progress completely - def pbar(s, *args, **kwargs): - return s - - elif not sys.stderr.isatty(): - # don't display progress bar when stderr is redirected to a file - def pbar(s, *args, **kwargs): - return s - - functions = list(extractor.get_functions()) - n_funcs = len(functions) - - pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False) - for f in pb: - t0 = time.time() - if extractor.is_library_function(f.address): - function_name = extractor.get_function_name(f.address) - logger.debug("skipping library function 0x%x (%s)", f.address, function_name) - library_functions += ( - rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name), - ) - n_libs = len(library_functions) - percentage = round(100 * (n_libs / n_funcs)) - if isinstance(pb, tqdm.tqdm): - pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)") - continue - - function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities( - ruleset, extractor, f - ) - feature_counts.functions += ( - rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count), - ) - t1 = time.time() - - match_count = 0 - for name, matches_ in itertools.chain( - function_matches.items(), bb_matches.items(), insn_matches.items() - ): - # in practice, most matches are derived rules, - # like "check OS version/5bf4c7f39fd4492cbed0f6dc7d596d49" - # but when we log to the human, they really care about "real" rules. - if not ruleset.rules[name].is_subscope_rule(): - match_count += len(matches_) - - logger.debug( - "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs", - f.address, - feature_count, - match_count, - t1 - t0, + functions: List[FunctionHandle] = list(extractor.get_functions()) + n_funcs: int = len(functions) + n_libs: int = 0 + percentage: float = 0 + + with capa.helpers.CapaProgressBar( + console=capa.helpers.log_console, transient=True, disable=disable_progress + ) as pbar: + task = pbar.add_task( + "matching", total=n_funcs, unit="functions", postfix=f"skipped {n_libs} library functions, {percentage}%" + ) + for f in functions: + t0 = time.time() + if extractor.is_library_function(f.address): + function_name = extractor.get_function_name(f.address) + logger.debug("skipping library function 0x%x (%s)", f.address, function_name) + library_functions += ( + rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name), ) - - for rule_name, res in function_matches.items(): - all_function_matches[rule_name].extend(res) - for rule_name, res in bb_matches.items(): - all_bb_matches[rule_name].extend(res) - for rule_name, res in insn_matches.items(): - all_insn_matches[rule_name].extend(res) + n_libs = len(library_functions) + percentage = round(100 * (n_libs / n_funcs)) + pbar.update(task, postfix=f"skipped {n_libs} library functions, {percentage}%") + pbar.advance(task) + continue + + function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f) + feature_counts.functions += ( + rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count), + ) + t1 = time.time() + + match_count = 0 + for name, matches_ in itertools.chain(function_matches.items(), bb_matches.items(), insn_matches.items()): + if not ruleset.rules[name].is_subscope_rule(): + match_count += len(matches_) + + logger.debug( + "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs", + f.address, + feature_count, + match_count, + t1 - t0, + ) + + for rule_name, res in function_matches.items(): + all_function_matches[rule_name].extend(res) + for rule_name, res in bb_matches.items(): + all_bb_matches[rule_name].extend(res) + for rule_name, res in insn_matches.items(): + all_insn_matches[rule_name].extend(res) + + pbar.advance(task) # collection of features that captures the rule matches within function, BB, and instruction scopes. # mapping from feature (matched rule) to set of addresses at which it matched. diff --git a/capa/helpers.py b/capa/helpers.py index f185db9e6..4505647c4 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -10,7 +10,6 @@ import sys import gzip import ctypes -import inspect import logging import tempfile import contextlib @@ -20,8 +19,21 @@ from zipfile import ZipFile from datetime import datetime -import tqdm import msgspec.json +from rich.console import Console +from rich.progress import ( + Task, + Text, + Progress, + BarColumn, + TextColumn, + SpinnerColumn, + ProgressColumn, + TimeElapsedColumn, + MofNCompleteColumn, + TaskProgressColumn, + TimeRemainingColumn, +) from capa.exceptions import UnsupportedFormatError from capa.features.common import ( @@ -51,6 +63,10 @@ logger = logging.getLogger("capa") +# shared console used to redirect logging to stderr +log_console: Console = Console(stderr=True) + + def hex(n: int) -> str: """render the given number using upper case hex, like: 0x123ABC""" if n < 0: @@ -247,39 +263,6 @@ def get_format(sample: Path) -> str: return FORMAT_UNKNOWN -@contextlib.contextmanager -def redirecting_print_to_tqdm(disable_progress): - """ - tqdm (progress bar) expects to have fairly tight control over console output. - so calls to `print()` will break the progress bar and make things look bad. - so, this context manager temporarily replaces the `print` implementation - with one that is compatible with tqdm. - via: https://stackoverflow.com/a/42424890/87207 - """ - old_print = print # noqa: T202 [reserved word print used] - - def new_print(*args, **kwargs): - # If tqdm.tqdm.write raises error, use builtin print - if disable_progress: - old_print(*args, **kwargs) - else: - try: - tqdm.tqdm.write(*args, **kwargs) - except Exception: - old_print(*args, **kwargs) - - try: - # Globally replace print with new_print. - # Verified this works manually on Python 3.11: - # >>> import inspect - # >>> inspect.builtins - # - inspect.builtins.print = new_print # type: ignore - yield - finally: - inspect.builtins.print = old_print # type: ignore - - def log_unsupported_format_error(): logger.error("-" * 80) logger.error(" Input file does not appear to be a supported file.") @@ -433,3 +416,47 @@ def ts_to_str(ts): return False return True + + +class RateColumn(ProgressColumn): + """Renders speed column in progress bar.""" + + def render(self, task: "Task") -> Text: + speed = f"{task.speed:>.1f}" if task.speed else "00.0" + unit = task.fields.get("unit", "it") + return Text.from_markup(f"[progress.data.speed]{speed} {unit}/s") + + +class PostfixColumn(ProgressColumn): + """Renders a postfix column in progress bar.""" + + def render(self, task: "Task") -> Text: + return Text(task.fields.get("postfix", "")) + + +class MofNCompleteColumnWithUnit(MofNCompleteColumn): + """Renders completed/total count column with a unit.""" + + def render(self, task: "Task") -> Text: + ret = super().render(task) + unit = task.fields.get("unit") + return ret.append(f" {unit}") if unit else ret + + +class CapaProgressBar(Progress): + @classmethod + def get_default_columns(cls): + return ( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + TaskProgressColumn(), + BarColumn(), + MofNCompleteColumnWithUnit(), + "•", + TimeElapsedColumn(), + "<", + TimeRemainingColumn(), + "•", + RateColumn(), + PostfixColumn(), + ) diff --git a/capa/main.py b/capa/main.py index d7b45e03a..518c9ce26 100644 --- a/capa/main.py +++ b/capa/main.py @@ -22,6 +22,7 @@ import colorama from pefile import PEFormatError +from rich.logging import RichHandler from elftools.common.exceptions import ELFError import capa.perf @@ -405,15 +406,23 @@ def handle_common_args(args): ShouldExitError: if the program is invoked incorrectly and should exit. """ if args.quiet: - logging.basicConfig(level=logging.WARNING) logging.getLogger().setLevel(logging.WARNING) elif args.debug: - logging.basicConfig(level=logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) else: - logging.basicConfig(level=logging.INFO) logging.getLogger().setLevel(logging.INFO) + # use [/] after the logger name to reset any styling, + # and prevent the color from carrying over to the message + logformat = "[dim]%(name)s[/]: %(message)s" + + # set markup=True to allow the use of Rich's markup syntax in log messages + rich_handler = RichHandler(markup=True, show_time=False, show_path=True, console=capa.helpers.log_console) + rich_handler.setFormatter(logging.Formatter(logformat)) + + # use RichHandler for root logger + logging.getLogger().addHandler(rich_handler) + # disable vivisect-related logging, it's verbose and not relevant for capa users set_vivisect_log_level(logging.CRITICAL) diff --git a/capa/render/utils.py b/capa/render/utils.py index 2846e05fd..73ed1d296 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -9,28 +9,29 @@ import io from typing import Dict, List, Tuple, Union, Iterator, Optional -import termcolor +import rich.console +from rich.progress import Text import capa.render.result_document as rd -def bold(s: str) -> str: +def bold(s: str) -> Text: """draw attention to the given string""" - return termcolor.colored(s, "cyan") + return Text.from_markup(f"[cyan]{s}") -def bold2(s: str) -> str: +def bold2(s: str) -> Text: """draw attention to the given string, within a `bold` section""" - return termcolor.colored(s, "green") + return Text.from_markup(f"[green]{s}") -def mute(s: str) -> str: +def mute(s: str) -> Text: """draw attention away from the given string""" - return termcolor.colored(s, "dark_grey") + return Text.from_markup(f"[dim]{s}") -def warn(s: str) -> str: - return termcolor.colored(s, "yellow") +def warn(s: str) -> Text: + return Text.from_markup(f"[yellow]{s}") def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]): @@ -85,3 +86,17 @@ class StringIO(io.StringIO): def writeln(self, s): self.write(s) self.write("\n") + + +class Console(rich.console.Console): + def writeln(self, *args, **kwargs) -> None: + """ + prints the text with a new line at the end. + """ + return self.print(*args, **kwargs) + + def write(self, *args, **kwargs) -> None: + """ + prints the text without a new line at the end. + """ + return self.print(*args, **kwargs, end="") diff --git a/capa/render/verbose.py b/capa/render/verbose.py index 076ad2b13..54ac53e95 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -25,7 +25,8 @@ from typing import cast -import tabulate +from rich.text import Text +from rich.table import Table import capa.rules import capa.helpers @@ -34,6 +35,7 @@ import capa.render.result_document as rd from capa.rules import RuleSet from capa.engine import MatchResults +from capa.render.utils import Console def format_address(address: frz.Address) -> str: @@ -140,7 +142,7 @@ def render_call(layout: rd.DynamicLayout, addr: frz.Address) -> str: ) -def render_static_meta(ostream, meta: rd.StaticMetadata): +def render_static_meta(console: Console, meta: rd.StaticMetadata): """ like: @@ -161,12 +163,16 @@ def render_static_meta(ostream, meta: rd.StaticMetadata): total feature count 1918 """ + grid = Table.grid(padding=(0, 2)) + grid.add_column(style="dim") + grid.add_column() + rows = [ ("md5", meta.sample.md5), ("sha1", meta.sample.sha1), ("sha256", meta.sample.sha256), ("path", meta.sample.path), - ("timestamp", meta.timestamp), + ("timestamp", str(meta.timestamp)), ("capa version", meta.version), ("os", meta.analysis.os), ("format", meta.analysis.format), @@ -175,18 +181,21 @@ def render_static_meta(ostream, meta: rd.StaticMetadata): ("extractor", meta.analysis.extractor), ("base address", format_address(meta.analysis.base_address)), ("rules", "\n".join(meta.analysis.rules)), - ("function count", len(meta.analysis.feature_counts.functions)), - ("library function count", len(meta.analysis.library_functions)), + ("function count", str(len(meta.analysis.feature_counts.functions))), + ("library function count", str(len(meta.analysis.library_functions))), ( "total feature count", - meta.analysis.feature_counts.file + sum(f.count for f in meta.analysis.feature_counts.functions), + str(meta.analysis.feature_counts.file + sum(f.count for f in meta.analysis.feature_counts.functions)), ), ] - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) + for row in rows: + grid.add_row(*row) + + console.print(grid) -def render_dynamic_meta(ostream, meta: rd.DynamicMetadata): +def render_dynamic_meta(console: Console, meta: rd.DynamicMetadata): """ like: @@ -205,12 +214,16 @@ def render_dynamic_meta(ostream, meta: rd.DynamicMetadata): total feature count 1918 """ + table = Table.grid(padding=(0, 2)) + table.add_column(style="dim") + table.add_column() + rows = [ ("md5", meta.sample.md5), ("sha1", meta.sample.sha1), ("sha256", meta.sample.sha256), ("path", meta.sample.path), - ("timestamp", meta.timestamp), + ("timestamp", str(meta.timestamp)), ("capa version", meta.version), ("os", meta.analysis.os), ("format", meta.analysis.format), @@ -218,26 +231,29 @@ def render_dynamic_meta(ostream, meta: rd.DynamicMetadata): ("analysis", meta.flavor.value), ("extractor", meta.analysis.extractor), ("rules", "\n".join(meta.analysis.rules)), - ("process count", len(meta.analysis.feature_counts.processes)), + ("process count", str(len(meta.analysis.feature_counts.processes))), ( "total feature count", - meta.analysis.feature_counts.file + sum(p.count for p in meta.analysis.feature_counts.processes), + str(meta.analysis.feature_counts.file + sum(p.count for p in meta.analysis.feature_counts.processes)), ), ] - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) + for row in rows: + table.add_row(*row) + + console.print(table) -def render_meta(osstream, doc: rd.ResultDocument): +def render_meta(console: Console, doc: rd.ResultDocument): if doc.meta.flavor == rd.Flavor.STATIC: - render_static_meta(osstream, cast(rd.StaticMetadata, doc.meta)) + render_static_meta(console, cast(rd.StaticMetadata, doc.meta)) elif doc.meta.flavor == rd.Flavor.DYNAMIC: - render_dynamic_meta(osstream, cast(rd.DynamicMetadata, doc.meta)) + render_dynamic_meta(console, cast(rd.DynamicMetadata, doc.meta)) else: raise ValueError("invalid meta analysis") -def render_rules(ostream, doc: rd.ResultDocument): +def render_rules(console: Console, doc: rd.ResultDocument): """ like: @@ -254,11 +270,15 @@ def render_rules(ostream, doc: rd.ResultDocument): if count == 1: capability = rutils.bold(rule.meta.name) else: - capability = f"{rutils.bold(rule.meta.name)} ({count} matches)" + capability = Text.assemble(rutils.bold(rule.meta.name), f" ({count} matches)") - ostream.writeln(capability) + console.print(capability) had_match = True + table = Table.grid(padding=(0, 2)) + table.add_column(style="dim") + table.add_column() + rows = [] ns = rule.meta.namespace @@ -310,23 +330,26 @@ def render_rules(ostream, doc: rd.ResultDocument): rows.append(("matches", "\n".join(lines))) - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) - ostream.write("\n") + for row in rows: + table.add_row(*row) + + console.print(table) + console.print() if not had_match: - ostream.writeln(rutils.bold("no capabilities found")) + console.print(rutils.bold("no capabilities found")) def render_verbose(doc: rd.ResultDocument): - ostream = rutils.StringIO() - - render_meta(ostream, doc) - ostream.write("\n") + console = Console(highlight=False) - render_rules(ostream, doc) - ostream.write("\n") + with console.capture() as capture: + render_meta(console, doc) + console.print() + render_rules(console, doc) + console.print() - return ostream.getvalue() + return capture.get() def render(meta, rules: RuleSet, capabilities: MatchResults) -> str: diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 2ee2af499..9c45119e5 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -9,7 +9,8 @@ import textwrap from typing import Dict, Iterable, Optional -import tabulate +from rich.text import Text +from rich.table import Table import capa.rules import capa.helpers @@ -22,6 +23,7 @@ import capa.features.freeze.features as frzf from capa.rules import RuleSet from capa.engine import MatchResults +from capa.render.utils import Console logger = logging.getLogger(__name__) @@ -45,7 +47,7 @@ def hanging_indent(s: str, indent: int) -> str: return textwrap.indent(s, prefix=prefix)[len(prefix) :] -def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address], indent: int): +def render_locations(console: Console, layout: rd.Layout, locations: Iterable[frz.Address], indent: int): import capa.render.verbose as v # it's possible to have an empty locations array here, @@ -56,7 +58,7 @@ def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address if len(locations) == 0: return - ostream.write(" @ ") + console.write(" @ ") location0 = locations[0] if len(locations) == 1: @@ -64,58 +66,58 @@ def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address if location.type == frz.AddressType.CALL: assert isinstance(layout, rd.DynamicLayout) - ostream.write(hanging_indent(v.render_call(layout, location), indent + 1)) + console.write(hanging_indent(v.render_call(layout, location), indent + 1)) else: - ostream.write(v.format_address(locations[0])) + console.write(v.format_address(locations[0])) elif location0.type == frz.AddressType.CALL and len(locations) > 1: location = locations[0] assert isinstance(layout, rd.DynamicLayout) s = f"{v.render_call(layout, location)}\nand {(len(locations) - 1)} more..." - ostream.write(hanging_indent(s, indent + 1)) + console.write(hanging_indent(s, indent + 1)) elif len(locations) > 4: # don't display too many locations, because it becomes very noisy. # probably only the first handful of locations will be useful for inspection. - ostream.write(", ".join(map(v.format_address, locations[0:4]))) - ostream.write(f", and {(len(locations) - 4)} more...") + console.write(", ".join(map(v.format_address, locations[0:4]))) + console.write(f", and {(len(locations) - 4)} more...") elif len(locations) > 1: - ostream.write(", ".join(map(v.format_address, locations))) + console.write(", ".join(map(v.format_address, locations))) else: raise RuntimeError("unreachable") -def render_statement(ostream, layout: rd.Layout, match: rd.Match, statement: rd.Statement, indent: int): - ostream.write(" " * indent) +def render_statement(console: Console, layout: rd.Layout, match: rd.Match, statement: rd.Statement, indent: int): + console.write(" " * indent) if isinstance(statement, rd.SubscopeStatement): # emit `basic block:` # rather than `subscope:` - ostream.write(statement.scope) + console.write(statement.scope) - ostream.write(":") + console.write(":") if statement.description: - ostream.write(f" = {statement.description}") - ostream.writeln("") + console.write(f" = {statement.description}") + console.writeln() elif isinstance(statement, (rd.CompoundStatement)): # emit `and:` `or:` `optional:` `not:` - ostream.write(statement.type) + console.write(statement.type) - ostream.write(":") + console.write(":") if statement.description: - ostream.write(f" = {statement.description}") - ostream.writeln("") + console.write(f" = {statement.description}") + console.writeln() elif isinstance(statement, rd.SomeStatement): - ostream.write(f"{statement.count} or more:") + console.write(f"{statement.count} or more:") if statement.description: - ostream.write(f" = {statement.description}") - ostream.writeln("") + console.write(f" = {statement.description}") + console.writeln() elif isinstance(statement, rd.RangeStatement): # `range` is a weird node, its almost a hybrid of statement+feature. @@ -133,25 +135,25 @@ def render_statement(ostream, layout: rd.Layout, match: rd.Match, statement: rd. value = rutils.bold2(value) if child.description: - ostream.write(f"count({child.type}({value} = {child.description})): ") + console.write(f"count({child.type}({value} = {child.description})): ") else: - ostream.write(f"count({child.type}({value})): ") + console.write(f"count({child.type}({value})): ") else: - ostream.write(f"count({child.type}): ") + console.write(f"count({child.type}): ") if statement.max == statement.min: - ostream.write(f"{statement.min}") + console.write(f"{statement.min}") elif statement.min == 0: - ostream.write(f"{statement.max} or fewer") + console.write(f"{statement.max} or fewer") elif statement.max == (1 << 64 - 1): - ostream.write(f"{statement.min} or more") + console.write(f"{statement.min} or more") else: - ostream.write(f"between {statement.min} and {statement.max}") + console.write(f"between {statement.min} and {statement.max}") if statement.description: - ostream.write(f" = {statement.description}") - render_locations(ostream, layout, match.locations, indent) - ostream.writeln("") + console.write(f" = {statement.description}") + render_locations(console, layout, match.locations, indent) + console.writeln() else: raise RuntimeError("unexpected match statement type: " + str(statement)) @@ -162,9 +164,9 @@ def render_string_value(s: str) -> str: def render_feature( - ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, feature: frzf.Feature, indent: int + console: Console, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, feature: frzf.Feature, indent: int ): - ostream.write(" " * indent) + console.write(" " * indent) key = feature.type value: Optional[str] @@ -205,14 +207,14 @@ def render_feature( elif isinstance(feature, frzf.OperandOffsetFeature): key = f"operand[{feature.index}].offset" - ostream.write(f"{key}: ") + console.write(f"{key}: ") if value: - ostream.write(rutils.bold2(value)) + console.write(rutils.bold2(value)) if feature.description: - ostream.write(capa.rules.DESCRIPTION_SEPARATOR) - ostream.write(feature.description) + console.write(capa.rules.DESCRIPTION_SEPARATOR) + console.write(feature.description) if isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)): # don't show the location of these global features @@ -224,35 +226,32 @@ def render_feature( elif isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)): pass else: - render_locations(ostream, layout, match.locations, indent) - ostream.write("\n") + render_locations(console, layout, match.locations, indent) + console.writeln() else: # like: # regex: /blah/ = SOME_CONSTANT # - "foo blah baz" @ 0x401000 # - "aaa blah bbb" @ 0x402000, 0x403400 - ostream.write(key) - ostream.write(": ") - ostream.write(value) - ostream.write("\n") + console.writeln(f"{key}: {value}") for capture, locations in sorted(match.captures.items()): - ostream.write(" " * (indent + 1)) - ostream.write("- ") - ostream.write(rutils.bold2(render_string_value(capture))) + console.write(" " * (indent + 1)) + console.write("- ") + console.write(rutils.bold2(render_string_value(capture))) if isinstance(layout, rd.DynamicLayout) and rule.meta.scopes.dynamic == capa.rules.Scope.CALL: # like above, don't re-render calls when in call scope. pass else: - render_locations(ostream, layout, locations, indent=indent) - ostream.write("\n") + render_locations(console, layout, locations, indent=indent) + console.writeln() -def render_node(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, node: rd.Node, indent: int): +def render_node(console: Console, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, node: rd.Node, indent: int): if isinstance(node, rd.StatementNode): - render_statement(ostream, layout, match, node.statement, indent=indent) + render_statement(console, layout, match, node.statement, indent=indent) elif isinstance(node, rd.FeatureNode): - render_feature(ostream, layout, rule, match, node.feature, indent=indent) + render_feature(console, layout, rule, match, node.feature, indent=indent) else: raise RuntimeError("unexpected node type: " + str(node)) @@ -265,7 +264,9 @@ def render_node(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Matc MODE_FAILURE = "failure" -def render_match(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, indent=0, mode=MODE_SUCCESS): +def render_match( + console: Console, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, indent=0, mode=MODE_SUCCESS +): child_mode = mode if mode == MODE_SUCCESS: # display only nodes that evaluated successfully. @@ -297,13 +298,13 @@ def render_match(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Mat else: raise RuntimeError("unexpected mode: " + mode) - render_node(ostream, layout, rule, match, match.node, indent=indent) + render_node(console, layout, rule, match, match.node, indent=indent) for child in match.children: - render_match(ostream, layout, rule, child, indent=indent + 1, mode=child_mode) + render_match(console, layout, rule, child, indent=indent + 1, mode=child_mode) -def render_rules(ostream, doc: rd.ResultDocument): +def render_rules(console: Console, doc: rd.ResultDocument): """ like: @@ -350,13 +351,13 @@ def render_rules(ostream, doc: rd.ResultDocument): if count == 1: if rule.meta.lib: lib_info = " (library rule)" - capability = f"{rutils.bold(rule.meta.name)}{lib_info}" + capability = Text.assemble(rutils.bold(rule.meta.name), f"{lib_info}") else: if rule.meta.lib: lib_info = ", only showing first match of library rule" - capability = f"{rutils.bold(rule.meta.name)} ({count} matches{lib_info})" + capability = Text.assemble(rutils.bold(rule.meta.name), f" ({count} matches{lib_info})") - ostream.writeln(capability) + console.writeln(capability) had_match = True rows = [] @@ -402,7 +403,14 @@ def render_rules(ostream, doc: rd.ResultDocument): if rule.meta.description: rows.append(("description", rule.meta.description)) - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) + grid = Table.grid(padding=(0, 2)) + grid.add_column(style="dim") + grid.add_column() + + for row in rows: + grid.add_row(*row) + + console.writeln(grid) if capa.rules.Scope.FILE in rule.meta.scopes: matches = doc.rules[rule.meta.name].matches @@ -413,61 +421,58 @@ def render_rules(ostream, doc: rd.ResultDocument): # so, lets be explicit about our assumptions and raise an exception if they fail. raise RuntimeError(f"unexpected file scope match count: {len(matches)}") _, first_match = matches[0] - render_match(ostream, doc.meta.analysis.layout, rule, first_match, indent=0) + render_match(console, doc.meta.analysis.layout, rule, first_match, indent=0) else: for location, match in sorted(doc.rules[rule.meta.name].matches): if doc.meta.flavor == rd.Flavor.STATIC: assert rule.meta.scopes.static is not None - ostream.write(rule.meta.scopes.static.value) - ostream.write(" @ ") - ostream.write(capa.render.verbose.format_address(location)) + console.write(rule.meta.scopes.static.value + " @ ") + console.write(capa.render.verbose.format_address(location)) if rule.meta.scopes.static == capa.rules.Scope.BASIC_BLOCK: func = frz.Address.from_capa(functions_by_bb[location.to_capa()]) - ostream.write(f" in function {capa.render.verbose.format_address(func)}") + console.write(f" in function {capa.render.verbose.format_address(func)}") elif doc.meta.flavor == rd.Flavor.DYNAMIC: assert rule.meta.scopes.dynamic is not None assert isinstance(doc.meta.analysis.layout, rd.DynamicLayout) - ostream.write(rule.meta.scopes.dynamic.value) - - ostream.write(" @ ") + console.write(rule.meta.scopes.dynamic.value + " @ ") if rule.meta.scopes.dynamic == capa.rules.Scope.PROCESS: - ostream.write(v.render_process(doc.meta.analysis.layout, location)) + console.write(v.render_process(doc.meta.analysis.layout, location)) elif rule.meta.scopes.dynamic == capa.rules.Scope.THREAD: - ostream.write(v.render_thread(doc.meta.analysis.layout, location)) + console.write(v.render_thread(doc.meta.analysis.layout, location)) elif rule.meta.scopes.dynamic == capa.rules.Scope.CALL: - ostream.write(hanging_indent(v.render_call(doc.meta.analysis.layout, location), indent=1)) + console.write(hanging_indent(v.render_call(doc.meta.analysis.layout, location), indent=1)) else: capa.helpers.assert_never(rule.meta.scopes.dynamic) else: capa.helpers.assert_never(doc.meta.flavor) - ostream.write("\n") - render_match(ostream, doc.meta.analysis.layout, rule, match, indent=1) + console.writeln() + render_match(console, doc.meta.analysis.layout, rule, match, indent=1) if rule.meta.lib: # only show first match break - ostream.write("\n") + console.writeln() if not had_match: - ostream.writeln(rutils.bold("no capabilities found")) + console.writeln(rutils.bold("no capabilities found")) def render_vverbose(doc: rd.ResultDocument): - ostream = rutils.StringIO() - - capa.render.verbose.render_meta(ostream, doc) - ostream.write("\n") + console = Console(highlight=False) - render_rules(ostream, doc) - ostream.write("\n") + with console.capture() as capture: + capa.render.verbose.render_meta(console, doc) + console.writeln() + render_rules(console, doc) + console.writeln() - return ostream.getvalue() + return capture.get() def render(meta, rules: RuleSet, capabilities: MatchResults) -> str: diff --git a/pyproject.toml b/pyproject.toml index 0305c0467..dc658420d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,12 +65,8 @@ dependencies = [ # or minor otherwise). # As specific constraints are identified, please provide # comments and context. - "tqdm>=4", "pyyaml>=6", - "tabulate>=0.9", "colorama>=0.4", - "termcolor>=2", - "wcwidth>=0.2", "ida-settings>=2", "ruamel.yaml>=0.18", "pefile>=2023.2.7", @@ -146,8 +142,6 @@ dev = [ "types-backports==0.1.3", "types-colorama==0.4.15.11", "types-PyYAML==6.0.8", - "types-tabulate==0.9.0.20240106", - "types-termcolor==1.1.4", "types-psutil==6.0.0.20240901", "types_requests==2.32.0.20240712", "types-protobuf==5.27.0.20240920", @@ -236,10 +230,7 @@ DEP002 = [ "types-protobuf", "types-psutil", "types-PyYAML", - "types-tabulate", - "types-termcolor", "types_requests", - "wcwidth" ] # dependencies imported but missing from definitions diff --git a/requirements.txt b/requirements.txt index 37142bc7f..b63558ff1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,10 +39,6 @@ ruamel-yaml-clib==0.2.8 setuptools==75.1.0 six==1.16.0 sortedcontainers==2.4.0 -tabulate==0.9.0 -termcolor==2.4.0 -tqdm==4.66.5 viv-utils==0.7.11 vivisect==1.2.1 -wcwidth==0.2.13 msgspec==0.18.6 diff --git a/scripts/lint.py b/scripts/lint.py index ad9b79906..e96604e64 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -31,11 +31,9 @@ from pathlib import Path from dataclasses import field, dataclass -import tqdm import pydantic -import termcolor import ruamel.yaml -import tqdm.contrib.logging +from rich import print import capa.main import capa.rules @@ -51,18 +49,6 @@ logger = logging.getLogger("lint") -def red(s): - return termcolor.colored(s, "red") - - -def orange(s): - return termcolor.colored(s, "yellow") - - -def green(s): - return termcolor.colored(s, "green") - - @dataclass class Context: """ @@ -80,8 +66,8 @@ class Context: class Lint: - WARN = orange("WARN") - FAIL = red("FAIL") + WARN = "[yellow]WARN[/yellow]" + FAIL = "[red]FAIL[/red]" name = "lint" level = FAIL @@ -896,7 +882,7 @@ def lint_rule(ctx: Context, rule: Rule): if (not lints_failed) and (not lints_warned) and has_examples: print("") print(f'{" (nursery) " if is_nursery_rule(rule) else ""} {rule.name}') - print(f" {Lint.WARN}: {green('no lint failures')}: Graduate the rule") + print(f" {Lint.WARN}: '[green]no lint failures[/green]': Graduate the rule") print("") else: lints_failed = len(tuple(filter(lambda v: v.level == Lint.FAIL, violations))) @@ -921,12 +907,15 @@ def lint(ctx: Context): ret = {} source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()] - with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule", leave=False) as pbar: - with capa.helpers.redirecting_print_to_tqdm(False): - for rule in pbar: - name = rule.name - pbar.set_description(width(f"linting rule: {name}", 48)) - ret[name] = lint_rule(ctx, rule) + n_rules: int = len(source_rules) + + with capa.helpers.CapaProgressBar(transient=True, console=capa.helpers.log_console) as pbar: + task = pbar.add_task(description="linting", total=n_rules, unit="rule") + for rule in source_rules: + name = rule.name + pbar.update(task, description=width(f"linting rule: {name}", 48)) + ret[name] = lint_rule(ctx, rule) + pbar.advance(task) return ret @@ -1020,18 +1009,18 @@ def main(argv=None): logger.debug("lints ran for ~ %02d:%02dm", min, sec) if warned_rules: - print(orange("rules with WARN:")) + print("[yellow]rules with WARN:[/yellow]") for warned_rule in sorted(warned_rules): print(" - " + warned_rule) print() if failed_rules: - print(red("rules with FAIL:")) + print("[red]rules with FAIL:[/red]") for failed_rule in sorted(failed_rules): print(" - " + failed_rule) return 1 else: - logger.info(green("no lints failed, nice!")) + logger.info("[green]no lints failed, nice![/green]") return 0 diff --git a/scripts/profile-time.py b/scripts/profile-time.py index 34123a738..4ba66cc19 100644 --- a/scripts/profile-time.py +++ b/scripts/profile-time.py @@ -42,9 +42,10 @@ import argparse import subprocess -import tqdm import humanize -import tabulate +from rich import box +from rich.table import Table +from rich.console import Console import capa.main import capa.perf @@ -92,51 +93,61 @@ def main(argv=None): except capa.main.ShouldExitError as e: return e.status_code - with tqdm.tqdm(total=args.number * args.repeat, leave=False) as pbar: + with capa.helpers.CapaProgressBar(console=capa.helpers.log_console) as progress: + total_iterations = args.number * args.repeat + task = progress.add_task("profiling", total=total_iterations) def do_iteration(): capa.perf.reset() capa.capabilities.common.find_capabilities(rules, extractor, disable_progress=True) - pbar.update(1) + + progress.advance(task) samples = timeit.repeat(do_iteration, number=args.number, repeat=args.repeat) logger.debug("perf: find capabilities: min: %0.2fs", (min(samples) / float(args.number))) - logger.debug("perf: find capabilities: avg: %0.2fs", (sum(samples) / float(args.repeat) / float(args.number))) + logger.debug( + "perf: find capabilities: avg: %0.2fs", + (sum(samples) / float(args.repeat) / float(args.number)), + ) logger.debug("perf: find capabilities: max: %0.2fs", (max(samples) / float(args.number))) for counter, count in capa.perf.counters.most_common(): logger.debug("perf: counter: %s: %s", counter, count) - print( - tabulate.tabulate( - [(counter, humanize.intcomma(count)) for counter, count in capa.perf.counters.most_common()], - headers=["feature class", "evaluation count"], - tablefmt="github", - ) - ) - print() - - print( - tabulate.tabulate( - [ - ( - args.label, - "{:,}".format(capa.perf.counters["evaluate.feature"]), - # python documentation indicates that min(samples) should be preferred, - # so lets put that first. - # - # https://docs.python.org/3/library/timeit.html#timeit.Timer.repeat - f"{(min(samples) / float(args.number)):.2f}s", - f"{(sum(samples) / float(args.repeat) / float(args.number)):.2f}s", - f"{(max(samples) / float(args.number)):.2f}s", - ) - ], - headers=["label", "count(evaluations)", "min(time)", "avg(time)", "max(time)"], - tablefmt="github", - ) + console = Console() + + table1 = Table(box=box.MARKDOWN) + table1.add_column("feature class") + table1.add_column("evaluation count") + + for counter, count in capa.perf.counters.most_common(): + table1.add_row(counter, humanize.intcomma(count)) + + console.print(table1) + console.print() + + table2 = Table(box=box.MARKDOWN) + table2.add_column("label") + table2.add_column("count(evaluations)", style="magenta") + table2.add_column("min(time)", style="green") + table2.add_column("avg(time)", style="yellow") + table2.add_column("max(time)", style="red") + + table2.add_row( + args.label, + # python documentation indicates that min(samples) should be preferred, + # so lets put that first. + # + # https://docs.python.org/3/library/timeit.html#timeit.Timer.repeat + "{:,}".format(capa.perf.counters["evaluate.feature"]), + f"{(min(samples) / float(args.number)):.2f}s", + f"{(sum(samples) / float(args.repeat) / float(args.number)):.2f}s", + f"{(max(samples) / float(args.number)):.2f}s", ) + console.print(table2) + return 0 diff --git a/scripts/show-unused-features.py b/scripts/show-unused-features.py index 0390cd640..be850e927 100644 --- a/scripts/show-unused-features.py +++ b/scripts/show-unused-features.py @@ -12,11 +12,12 @@ import typing import logging import argparse -from typing import Set, Tuple +from typing import Set, List, Tuple from collections import Counter -import tabulate -from termcolor import colored +from rich import print +from rich.text import Text +from rich.table import Table import capa.main import capa.rules @@ -77,23 +78,30 @@ def get_file_features( return feature_map -def get_colored(s: str): +def get_colored(s: str) -> Text: if "(" in s and ")" in s: s_split = s.split("(", 1) - s_color = colored(s_split[1][:-1], "cyan") - return f"{s_split[0]}({s_color})" + return Text.assemble(s_split[0], "(", (s_split[1][:-1], "cyan"), ")") else: - return colored(s, "cyan") + return Text(s, style="cyan") def print_unused_features(feature_map: typing.Counter[Feature], rules_feature_set: Set[Feature]): - unused_features = [] + unused_features: List[Tuple[str, Text]] = [] for feature, count in reversed(feature_map.most_common()): if feature in rules_feature_set: continue unused_features.append((str(count), get_colored(str(feature)))) + + table = Table(title="Unused Features", box=None) + table.add_column("Count", style="dim") + table.add_column("Feature") + + for count_str, feature_text in unused_features: + table.add_row(count_str, feature_text) + print("\n") - print(tabulate.tabulate(unused_features, headers=["Count", "Feature"], tablefmt="plain")) + print(table) print("\n") diff --git a/tests/test_render.py b/tests/test_render.py index 7f896949c..16b0e6439 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -10,7 +10,6 @@ from unittest.mock import Mock import fixtures -import rich.console import capa.rules import capa.render.utils @@ -24,6 +23,7 @@ import capa.render.result_document import capa.render.result_document as rd import capa.features.freeze.features +from capa.render.utils import Console def test_render_number(): @@ -154,7 +154,7 @@ def test_render_meta_maec(): # capture the output of render_maec f = io.StringIO() - console = rich.console.Console(file=f) + console = Console(file=f) capa.render.default.render_maec(mock_rd, console) output = f.getvalue() @@ -198,7 +198,7 @@ def test_render_meta_maec(): ], ) def test_render_vverbose_feature(feature, expected): - ostream = capa.render.utils.StringIO() + console = Console(highlight=False) addr = capa.features.freeze.Address.from_capa(capa.features.address.AbsoluteVirtualAddress(0x401000)) feature = capa.features.freeze.features.feature_from_capa(feature) @@ -240,6 +240,8 @@ def test_render_vverbose_feature(feature, expected): matches=(), ) - capa.render.vverbose.render_feature(ostream, layout, rm, matches, feature, indent=0) + with console.capture() as capture: + capa.render.vverbose.render_feature(console, layout, rm, matches, feature, indent=0) - assert ostream.getvalue().strip() == expected + output = capture.get().strip() + assert output == expected