diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..60dd8416 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +test-data +research + +venv + +.idea +.pytest_cache +apkid.egg-info diff --git a/.gitignore b/.gitignore index a3d89c77..d6ae0e48 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ # precompiled rules included in dist but not source apkid/rules/rules.yarc -# Apks and dex files for testing +# APKs and DEX files for local testing test-data research + ### VirtualEnv template # Virtualenv # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ @@ -318,6 +319,7 @@ nosetests.xml coverage.xml *,cover .hypothesis/ +.pytest_cache/ # Translations *.mo diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..41fd4550 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python +dist: trusty +cache: + pip: true +# directories: +# - $HOME/.cache/samples +python: + - "2.7" + - "3.6" +install: + - git clone --recursive https://github.com/rednaga/yara-python-1 yara-python + - cd yara-python + - CFLAGS="-std=gnu99" python setup.py build --enable-dex install + - cd ../ + - pip install -e . +script: + - ./prep-release.py | true + - pytest diff --git a/docker/Dockerfile b/Dockerfile similarity index 71% rename from docker/Dockerfile rename to Dockerfile index c2551306..502db42b 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -1,11 +1,12 @@ FROM python:2.7-slim RUN apt-get update -qq && apt-get install -y git build-essential gcc pandoc - RUN pip install --upgrade pip -RUN git clone https://github.com/rednaga/yara-python.git + +RUN git clone --recursive https://github.com/rednaga/yara-python-1.git yara-python WORKDIR yara-python -RUN python setup.py install +RUN CFLAGS="-std=gnu99" python setup.py build --enable-dex install + RUN mkdir /apkid COPY ./ /apkid/ WORKDIR /apkid diff --git a/README.md b/README.md index 8e65315d..dcf7e433 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,45 @@ # APKiD +[![Build Status](https://travis-ci.org/rednaga/APKiD.svg?branch=master)](https://travis-ci.org/rednaga/APKiD) + APKiD gives you information about how an APK was made. It identifies many compilers, packers, obfuscators, and other weird stuff. It's _PEiD_ for Android. -For more information on what this tool can be used for, check out: +For more information on what this tool can be used for check out: * [Android Compiler Fingerprinting](http://hitcon.org/2016/CMT/slide/day1-r0-e-1.pdf) * [Detecting Pirated and Malicious Android Apps with APKiD](http://rednaga.io/2016/07/31/detecting_pirated_and_malicious_android_apps_with_apkid/) # Installing -The _yara-python_ clone and compile steps here are temporarily necessary because we must point directly to our modified version of a _Yara_ branch which includes our DEX Yara module. This step is nessecary until (if?) the original maintainers of _Yara_ merge our module into the master branch. When this happens, we will undate the instructions here. After the _yara-python_ fork is compiled, you can use `pip` to the most currently published `APKiD` package. +Unfortunately, you can't just `pip install` APKiD since it depends on RedNaga's custom fork of [yara-python](https://github.com/rednaga/yara-python-1). + +First, install our yara-python fork: ```bash -git clone --recursive https://github.com/rednaga/yara-python +git clone --recursive https://github.com/rednaga/yara-python-1 yara-python cd yara-python -python setup.py install +python setup.py build --enable-dex install +``` + +Then, you can install apkid normally: +```bash pip install apkid ``` -## Docker install +This extra step is necessary until yara-python is updated with a version of Yara which includes the new, experimental DEX module. + +## Docker -In an attempt to reduce the support ticket we receive from the above instructions being hard to follow, there is -a docker file and script which can be used for processing files quickly. This also serves as a proof that the above -instructions _do_ work! This usage, of course, requires that you have docker correctly installed on your machine. However the following instructions should "just work" if you have docker and git install on a machine: +If installing is too complicated, you can just use [Docker](https://www.docker.com/community-edition)! Of course, this usage requires that you have git and docker installed on your machine. + +Here's how to use Docker: ```bash git clone https://github.com/rednaga/APKiD cd APKiD/ -docker-compose build -cd docker/ -./apkid.sh ~/reverse/targets/android/example/example.apk -[+] APKiD 1.0.0 :: from RedNaga :: rednaga.io +docker build . -t rednaga:apkid +docker/apkid.sh ~/reverse/targets/android/example/example.apk +[+] APKiD 1.2.1 :: from RedNaga :: rednaga.io [*] example.apk!classes.dex |-> compiler : dx ``` @@ -38,31 +47,31 @@ cd docker/ # Usage ``` -usage: apkid [-h] [-j] [-t TIMEOUT] [-o DIR] [FILE [FILE ...]] +usage: apkid [-h] [-j] [-t TIMEOUT] [-o DIR] [-q] [FILE [FILE ...]] -APKiD - Android Application Identifier v1.0.0 +APKiD - Android Application Identifier v1.2.1 positional arguments: - FILE apk, dex, or directory + FILE apk, dex, or directory optional arguments: - -h, --help show this help message and exit - -j, --json output results in JSON format - -t TIMEOUT, --timeout TIMEOUT - Yara scan timeout (in seconds) - -o DIR, --output-dir DIR - write individual JSON results to this directory + -h, --help show this help message and exit + -j, --json output scan results in JSON format + -t TIMEOUT, --timeout TIMEOUT Yara scan timeout (in seconds) + -o DIR, --output-dir DIR write individual results to this directory (implies --json) + -q, --quiet suppress extraneous output ``` # Submitting New Packers / Compilers / Obfuscators If you come across an APK or DEX which APKiD does not recognize, please open a GitHub issue and tell us: + * what you think it is * the file hash (either MD5, SHA1, SHA256) -We are open to any type of concept you might have for "something interesting" to detect, so do not limit yourself solely to packers, compilers or obfuscators. If there is an interesting anti disassembler, anti vm, anti* trick, please make an issue. +We are open to any type of concept you might have for "something interesting" to detect, so do not limit yourself solely to packers, compilers or obfuscators. If there is an interesting anti-disassembler, anti-vm, anti-* trick, please make an issue. -You're also welcome to submit pull requests. Just be sure to include a file hash so we can check the rule. +Pull requests are welcome. If you're submitting a new rule, be sure to include a file hash of the APK / DEX so we can check the rule. # License @@ -72,15 +81,15 @@ Depending on your needs, you must choose one of them and follow its policies. A # Hacking -First you will need to install the specific version of _yara-python_ the project depends on (more information about this in the _Installing_ section): +First, you'll need to install our fork of _yara-python_: ```bash -git clone --recursive https://github.com/rednaga/yara-python +git clone --recursive https://github.com/rednaga/yara-python-1 yara-python cd yara-python -python setup.py install +python setup.py build --enable-dex install ``` -Then, clone this repo, compile the rules, and install the package in editable mode: +Then, clone this repository, compile the rules, and install the package in editable mode: ```bash git clone https://github.com/rednaga/APKiD @@ -94,3 +103,20 @@ If the above doesn't work, due to permission errors dependent on your local mach ```bash pip install -e .[dev] --user ``` + +If you update any of the rules, be sure to run `prep-release.py` to recompile them. + +# For Maintainers + +This section is for package maintainers. + +To update the PyPI package: + +```bash +./prep-release.py readme +rm dist/* +python setup.py sdist bdist_wheel +twine upload --repository-url https://upload.pypi.org/legacy/ dist/* +``` + +For more information see [Packaging Projects](https://packaging.python.org/tutorials/packaging-projects/). diff --git a/README.rst b/README.rst index 26a3fcd9..17837eed 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,13 @@ APKiD ===== +`Build Status `__ + APKiD gives you information about how an APK was made. It identifies many compilers, packers, obfuscators, and other weird stuff. It’s *PEiD* for Android. -For more information on what this tool can be used for, check out: +For more information on what this tool can be used for check out: - `Android Compiler Fingerprinting `__ @@ -15,76 +17,81 @@ For more information on what this tool can be used for, check out: Installing ========== -The *yara-python* clone and compile steps here are temporarily necessary -because we must point directly to our modified version of a *Yara* -branch which includes our DEX Yara module. This step is nessecary until -(if?) the original maintainers of *Yara* merge our module into the -master branch. When this happens, we will undate the instructions here. -After the *yara-python* fork is compiled, you can use ``pip`` to the -most currently published ``APKiD`` package. +Unfortunately, you can’t just ``pip install`` APKiD since it depends on +RedNaga’s custom fork of +`yara-python `__. + +First, install our yara-python fork: .. code:: bash - git clone --recursive https://github.com/rednaga/yara-python - cd yara-python - python setup.py install - pip install apkid + git clone --recursive https://github.com/rednaga/yara-python-1 yara-python + cd yara-python + python setup.py build --enable-dex install + +Then, you can install apkid normally: + +.. code:: bash -Docker install --------------- + pip install apkid -In an attempt to reduce the support ticket we receive from the above -instructions being hard to follow, there is a docker file and script -which can be used for processing files quickly. This also serves as a -proof that the above instructions *do* work! This usage, of course, -requires that you have docker correctly installed on your machine. -However the following instructions should “just work” if you have docker -and git install on a machine: +This extra step is necessary until yara-python is updated with a version +of Yara which includes the new, experimental DEX module. + +Docker +------ + +If installing is too complicated, you can just use +`Docker `__! Of course, this +usage requires that you have git and docker installed on your machine. + +Here’s how to use Docker: .. code:: bash - git clone https://github.com/rednaga/APKiD - cd APKiD/docker - docker-compose build - ./apkid.sh ~/reverse/targets/android/example/example.apk - [+] APKiD 1.0.0 :: from RedNaga :: rednaga.io - [*] example.apk!classes.dex - |-> compiler : dx + git clone https://github.com/rednaga/APKiD + cd APKiD/ + docker build . -t rednaga:apkid + docker/apkid.sh ~/reverse/targets/android/example/example.apk + [+] APKiD 1.2.1 :: from RedNaga :: rednaga.io + [*] example.apk!classes.dex + |-> compiler : dx Usage ===== :: - usage: apkid [-h] [-j] [-t TIMEOUT] [-o DIR] [FILE [FILE ...]] + usage: apkid [-h] [-j] [-t TIMEOUT] [-o DIR] [-q] [FILE [FILE ...]] - APKiD - Android Application Identifier v1.0.0 + APKiD - Android Application Identifier v1.2.1 - positional arguments: - FILE apk, dex, or directory + positional arguments: + FILE apk, dex, or directory - optional arguments: - -h, --help show this help message and exit - -j, --json output results in JSON format - -t TIMEOUT, --timeout TIMEOUT - Yara scan timeout (in seconds) - -o DIR, --output-dir DIR - write individual JSON results to this directory + optional arguments: + -h, --help show this help message and exit + -j, --json output scan results in JSON format + -t TIMEOUT, --timeout TIMEOUT Yara scan timeout (in seconds) + -o DIR, --output-dir DIR write individual results to this directory (implies --json) + -q, --quiet suppress extraneous output Submitting New Packers / Compilers / Obfuscators ================================================ If you come across an APK or DEX which APKiD does not recognize, please -open a GitHub issue and tell us: \* what you think it is \* the file -hash (either MD5, SHA1, SHA256) +open a GitHub issue and tell us: + +- what you think it is +- the file hash (either MD5, SHA1, SHA256) We are open to any type of concept you might have for “something interesting” to detect, so do not limit yourself solely to packers, -compilers or obfuscators. If there is an interesting anti disassembler, -anti vm, anti\* trick, please make an issue. +compilers or obfuscators. If there is an interesting anti-disassembler, +anti-vm, anti-\* trick, please make an issue. -You’re also welcome to submit pull requests. Just be sure to include a -file hash so we can check the rule. +Pull requests are welcome. If you’re submitting a new rule, be sure to +include a file hash of the APK / DEX so we can check the rule. License ======= @@ -100,25 +107,23 @@ are available in the LICENSE.COMMERCIAL and LICENSE.GPL files. Hacking ======= -First you will need to install the specific version of *yara-python* the -project depends on (more information about this in the *Installing* -section): +First, you’ll need to install our fork of *yara-python*: .. code:: bash - git clone --recursive https://github.com/rednaga/yara-python - cd yara-python - python setup.py install + git clone --recursive https://github.com/rednaga/yara-python-1 yara-python + cd yara-python + python setup.py build --enable-dex install -Then, clone this repo, compile the rules, and install the package in -editable mode: +Then, clone this repository, compile the rules, and install the package +in editable mode: .. code:: bash - git clone https://github.com/rednaga/APKiD - cd APKiD - ./prep-release.py - pip install -e .[dev] + git clone https://github.com/rednaga/APKiD + cd APKiD + ./prep-release.py + pip install -e .[dev] If the above doesn’t work, due to permission errors dependent on your local machine and where Python has been installed, try specifying the @@ -126,4 +131,24 @@ local machine and where Python has been installed, try specifying the .. code:: bash - pip install -e .[dev] --user + pip install -e .[dev] --user + +If you update any of the rules, be sure to run ``prep-release.py`` to +recompile them. + +For Maintainers +=============== + +This section is for package maintainers. + +To update the PyPI package: + +.. code:: bash + + ./prep-release.py readme + rm dist/* + python setup.py sdist bdist_wheel + twine upload --repository-url https://upload.pypi.org/legacy/ dist/* + +For more information see `Packaging +Projects `__. diff --git a/apkid/__init__.py b/apkid/__init__.py index 7e63b866..9b195bb4 100644 --- a/apkid/__init__.py +++ b/apkid/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ - Copyright (C) 2017 RedNaga. http://rednaga.io + Copyright (C) 2018 RedNaga. https://rednaga.io All rights reserved. Contact: rednaga@protonmail.com @@ -27,34 +27,48 @@ """ __title__ = 'apkid' -__version__ = '1.0.0' +__version__ = '1.2.1' __author__ = 'Caleb Fenton & Tim Strazzere' __license__ = 'GPL & Commercial' -__copyright__ = 'Copyright (C) 2017 RedNaga' +__copyright__ = 'Copyright (C) 2018 RedNaga' import argparse +import os from . import apkid -def main(): +def get_parser(): + formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=50, width=100) + parser = argparse.ArgumentParser( - description="APKiD - Android Application Identifier v{}".format(__version__)) + description="APKiD - Android Application Identifier v{}".format(__version__), + formatter_class=formatter + ) parser.add_argument('input', metavar='FILE', type=str, nargs='*', help="apk, dex, or directory") parser.add_argument('-j', '--json', action='store_true', - help="output results in JSON format", ) + help="output scan results in JSON format", ) parser.add_argument('-t', '--timeout', type=int, default=30, help="Yara scan timeout (in seconds)") parser.add_argument('-o', '--output-dir', metavar='DIR', type=str, - help="write individual JSON results to this directory") + help="write individual results to this directory (implies --json)") + parser.add_argument('-q', '--quiet', action='store_true', + help="suppress extraneous output") + return parser + + +def main(): + parser = get_parser() args = parser.parse_args() if not args.json: - print("[+] APKiD %s :: from RedNaga :: rednaga.io" % __version__) + print("[+] APKiD {} :: from RedNaga :: rednaga.io".format(__version__)) for input in args.input: if args.output_dir: - apkid.scan_singly(input, args.timeout, args.output_dir) - else: - apkid.scan(input, args.timeout, args.json) + args.json = True + if not os.path.exists(args.output_dir): + os.makedirs(args.output_dir) + + apkid.scan(input, args.timeout, args.json, args.output_dir, args.quiet) diff --git a/apkid/apkid.py b/apkid/apkid.py index 133b5c9d..4f451931 100644 --- a/apkid/apkid.py +++ b/apkid/apkid.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2017 RedNaga. http://rednaga.io + Copyright (C) 2018 RedNaga. https://rednaga.io All rights reserved. Contact: rednaga@protonmail.com @@ -31,36 +31,85 @@ import sys import tempfile import traceback -import yara import zipfile +import yara + +from . import rules, output + LOGGING_LEVEL = logging.INFO logging.basicConfig(level=LOGGING_LEVEL, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', stream=sys.stdout) -# Magic doesn't need to be perfect. Just used to filter likely scannable files. -ZIP_MAGIC = [b'PK\x03\x04', b'PK\x05\x06', b'PK\x07\x08'] -DEX_MAGIC = [b'dex\n', b'dey\n'] -ELF_MAGIC = [b'\x7fELF'] -AXML_MAGIC = [] # TODO +FILE_MAGICS = { + 'zip': [b'PK\x03\x04', b'PK\x05\x06', b'PK\x07\x08'], + 'dex': [b'dex\n', b'dey\n'], + 'elf': [b'\x7fELF'], + 'axml': [], +} + + +def scan(input, timeout, use_json, output_dir, quiet): + all_results = {} + out_file = None + for file_type, file_path in collect_files(input): + file_results = {} + + if output_dir: + filename = os.path.basename(file_path) + out_file = os.path.join(output_dir, '{}_apkid.json'.format(filename)) + if os.path.exists(out_file): + if not quiet: + print("Match result {} already exists; skipping {}".format(out_file, file_path)) + continue + + if output_dir and not quiet: + print("Processing: {}".format(file_path)) + + try: + matches = rules.match(file_path, timeout) + + if len(matches) > 0: + file_results[file_path] = matches + + if 'zip' == file_type: + apk_matches = scan_apk(file_path, timeout, use_json) + file_results.update(apk_matches) + + if output_dir: + with open(out_file, 'w') as f: + f.write(json.dumps(file_results)) + else: + all_results.update(file_results) + + if not use_json: + output.print_matches(file_path, matches) + + if output_dir and not quiet: + print("Finished: {}".format(file_path)) + except yara.Error as e: + tb = traceback.format_exc() + logging.error("error scanning {}: {}\n{}".format(file_path, e, tb)) + if output_dir and not os.path.exists(out_file): + with open(out_file, 'w') as f: + f.write(json.dumps({'error': e, 'trace': tb})) + + if not output_dir and use_json: + print(json.dumps(all_results)) def get_file_type(file_path): - # Don't scan links if not os.path.isfile(file_path): return 'invalid' + with open(file_path, 'rb') as f: magic = f.read(4) - if magic in ZIP_MAGIC: - return 'zip' - elif magic in DEX_MAGIC: - return 'dex' - elif magic in ELF_MAGIC: - return 'elf' - elif magic in AXML_MAGIC: - return 'axml' + + for file_type, magics in FILE_MAGICS.items(): + if magic in magics: + return file_type return 'invalid' @@ -78,138 +127,37 @@ def collect_files(input): yield (file_type, filepath) -def get_rules(): - rules_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'rules/rules.yarc') - return yara.load(rules_path) - - -def build_match_dict(matches): - results = {} - for match in matches: - tags = ', '.join(sorted(match.tags)) - value = match.meta.get('description', match) - if tags in results: - if value not in results[tags]: - results[tags].append(value) - else: - results[tags] = [value] - return results - - -def print_matches(key_path, match_dict): - ''' example matches dict - [{ - 'tags': ['foo', 'bar'], - 'matches': True, - 'namespace': 'default', - 'rule': 'my_rule', - 'meta': {}, - 'strings': [(81L, '$a', 'abc'), (141L, '$b', 'def')] - }] - ''' - print("[*] {}".format(key_path)) - for tags in sorted(match_dict): - values = ', '.join(sorted(match_dict[tags])) - print(" |-> {} : {}".format(tags, values)) - - -def is_target_member(name): - if name.startswith('classes') or name.startswith('AndroidManifest.xml') or name.startswith( - 'lib/'): +def is_likely_supported_file(name): + if name.startswith('classes') \ + or name.startswith('AndroidManifest.xml') \ + or name.startswith('lib/') \ + or name.endswith('.so') \ + or name.endswith('.dex') \ + or name.endswith('.apk'): return True return False -def do_yara(file_path, rules, timeout): - matches = rules.match(file_path, timeout=timeout) - return build_match_dict(matches) - - -def scan_apk(apk_path, rules, timeout, output_json): +def scan_apk(apk_path, timeout, output_json): td = None results = {} try: zf = zipfile.ZipFile(apk_path, 'r') - target_members = filter(lambda n: is_target_member(n), zf.namelist()) + target_members = filter(lambda n: is_likely_supported_file(n), zf.namelist()) td = tempfile.mkdtemp() zf.extractall(td, members=target_members) zf.close() for file_type, file_path in collect_files(td): entry_name = file_path.replace('{}/'.format(td), '') key_path = '{}!{}'.format(apk_path, entry_name) - match_dict = do_yara(file_path, rules, timeout) - if len(match_dict) > 0: - results[key_path] = match_dict + matches = rules.match(file_path, timeout) + if len(matches) > 0: + results[key_path] = matches if not output_json: - print_matches(key_path, match_dict) + output.print_matches(key_path, matches) except Exception as e: tb = traceback.format_exc() logging.error("error extracting {}: {}\n{}".format(apk_path, e, tb)) if td: shutil.rmtree(td) return results - - -def get_json_output(results): - import pkg_resources - output = { - 'apkid_version': pkg_resources.get_distribution('apkid').version, - 'files': [], - } - for filename in results: - result = { - 'filename': filename, - 'results': results[filename], - } - output['files'].append(result) - return output - - -def print_json_results(results): - output = get_json_output(results) - print(json.dumps(output)) - - -def scan(input, timeout, output_json): - rules = get_rules() - results = {} - for file_type, file_path in collect_files(input): - try: - match_dict = do_yara(file_path, rules, timeout) - if len(match_dict) > 0: - results[file_path] = match_dict - if not output_json: - print_matches(file_path, match_dict) - if 'zip' == file_type: - apk_matches = scan_apk(file_path, rules, timeout, output_json) - results.update(apk_matches) - except yara.Error as e: - logging.error("error scanning: {}".format(e)) - if output_json: - print_json_results(results) - - -def scan_singly(input, timeout, output_dir): - rules = get_rules() - for file_type, file_path in collect_files(input): - results = {} - filename = os.path.basename(file_path) - out_file = os.path.join(output_dir, filename) - if os.path.exists(out_file): - continue - print("Processing: {}".format(file_path)) - try: - match_dict = do_yara(file_path, rules, timeout) - if len(match_dict) > 0: - results[file_path] = match_dict - if 'zip' == file_type: - apk_matches = scan_apk(file_path, rules, timeout, True) - results.update(apk_matches) - if len(results) > 0: - if not os.path.exists(output_dir): - os.makedirs(output_dir) - with open(out_file, 'w') as f: - f.write(json.dumps(results)) - print("Finished: {}".format(file_path)) - except yara.Error as e: - logging.error("error scanning: {}".format(e)) diff --git a/apkid/output.py b/apkid/output.py new file mode 100644 index 00000000..81f409e5 --- /dev/null +++ b/apkid/output.py @@ -0,0 +1,105 @@ +""" + Copyright (C) 2018 RedNaga. https://rednaga.io + All rights reserved. Contact: rednaga@protonmail.com + + + This file is part of APKiD + + + Commercial License Usage + ------------------------ + Licensees holding valid commercial APKiD licenses may use this file + in accordance with the commercial license agreement provided with the + Software or, alternatively, in accordance with the terms contained in + a written agreement between you and RedNaga. + + + GNU General Public License Usage + -------------------------------- + Alternatively, this file may be used under the terms of the GNU General + Public License version 3.0 as published by the Free Software Foundation + and appearing in the file LICENSE.GPL included in the packaging of this + file. Please visit http://www.gnu.org/copyleft/gpl.html and review the + information to ensure the GNU General Public License version 3.0 + requirements will be met. +""" + +import json +import sys + + +def print_matches(key_path, matches): + """ + example matches dict + [{ + 'tags': ['foo', 'bar'], + 'matches': True, + 'namespace': 'default', + 'rule': 'my_rule', + 'meta': {}, + 'strings': [(81L, '$a', 'abc'), (141L, '$b', 'def')] + }] + """ + print("[*] {}".format(key_path)) + for tags in sorted(matches): + descriptions = ', '.join(sorted(matches[tags])) + if sys.stdout.isatty(): + tags_str = colorize_tags(tags) + else: + tags_str = tags + print(" |-> {} : {}".format(tags_str, descriptions)) + + +def colorize_tags(tags): + colored_tags = [] + for tag in tags.split(', '): + if tag == 'compiler': + colored_tag = prt_cyan(tag) + elif tag == 'manipulator': + colored_tag = prt_lightCyan(tag) + elif tag == 'abnormal': + colored_tag = prt_lightGray(tag) + elif tag in ['anti_vm', 'anti_disassembly', 'anti_debug']: + colored_tag = prt_purple(tag) + elif tag in ['packer', 'protector']: + colored_tag = prt_red(tag) + elif tag == 'obfuscator': + colored_tag = prt_yellow(tag) + else: + colored_tag = tag + colored_tags.append(colored_tag) + colored_tags = ', '.join(colored_tags) + return colored_tags + + +def get_json_output(results): + from . import __version__, rules + output = { + 'apkid_version': __version__, + 'rules_sha256': rules.sha256(), + 'files': [], + } + for filename in results: + result = { + 'filename': filename, + 'results': results[filename], + } + output['files'].append(result) + return output + + +def print_json_results(results): + output = get_json_output(results) + print(json.dumps(output)) + + +prt_red = lambda s: "\033[91m{}\033[00m".format(s) +prt_green = lambda s: "\033[92m{}\033[00m".format(s) +prt_yellow = lambda s: "\033[93m{}\033[00m".format(s) +prt_lightPurple = lambda s: "\033[94m{}\033[00m".format(s) +prt_purple = lambda s: "\033[95m{}\033[00m".format(s) +prt_cyan = lambda s: "\033[36m{}\033[00m".format(s) +prt_lightCyan = lambda s: "\033[96m{}\033[00m".format(s) +prt_lightGray = lambda s: "\033[97m{}\033[00m".format(s) +prt_orange = lambda s: "\033[33m{}\033[00m".format(s) +prt_pink = lambda s: "'\033[95m'{}\033[00m".format(s) diff --git a/apkid/rules.py b/apkid/rules.py new file mode 100644 index 00000000..b96749ff --- /dev/null +++ b/apkid/rules.py @@ -0,0 +1,87 @@ +""" + Copyright (C) 2018 RedNaga. https://rednaga.io + All rights reserved. Contact: rednaga@protonmail.com + + + This file is part of APKiD + + + Commercial License Usage + ------------------------ + Licensees holding valid commercial APKiD licenses may use this file + in accordance with the commercial license agreement provided with the + Software or, alternatively, in accordance with the terms contained in + a written agreement between you and RedNaga. + + + GNU General Public License Usage + -------------------------------- + Alternatively, this file may be used under the terms of the GNU General + Public License version 3.0 as published by the Free Software Foundation + and appearing in the file LICENSE.GPL included in the packaging of this + file. Please visit http://www.gnu.org/copyleft/gpl.html and review the + information to ensure the GNU General Public License version 3.0 + requirements will be met. +""" + +import hashlib +import os + +import yara + +RULES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'rules') +RULES_PATH = os.path.join(RULES_DIR, 'rules.yarc') +RULES_EXT = '.yara' +RULES = None + + +def load(): + global RULES + if not RULES: + RULES = yara.load(RULES_PATH) + return RULES + + +def collect_yara_files(): + files = {} + for root, dirnames, filenames in os.walk(RULES_DIR): + for filename in filenames: + if not filename.lower().endswith(RULES_EXT): + continue + path = os.path.join(root, filename) + files[path] = path + return files + + +def compile(): + yara_files = collect_yara_files() + return yara.compile(filepaths=yara_files) + + +def save(rules): + rules.save(RULES_PATH) + rules_count = sum(1 for _ in rules) + return rules_count, RULES_PATH + + +def sha256(): + hashlib.sha256(open(RULES_PATH, 'rb').read()).hexdigest() + + +def match(file_path, timeout): + load() + matches = RULES.match(file_path, timeout=timeout) + return build_match_dict(matches) + + +def build_match_dict(matches): + results = {} + for m in matches: + tags = ', '.join(sorted(m.tags)) + description = m.meta.get('description', m) + if tags in results: + if description not in results[tags]: + results[tags].append(description) + else: + results[tags] = [description] + return results diff --git a/apkid/rules/apk/common.yara b/apkid/rules/apk/common.yara index 1c51ee14..701793fb 100644 --- a/apkid/rules/apk/common.yara +++ b/apkid/rules/apk/common.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -25,7 +25,7 @@ * **/ -private rule is_apk +private rule is_apk : internal { meta: description = "Resembles an APK that is likely not corrupt" @@ -38,21 +38,23 @@ private rule is_apk $zip_head at 0 and $manifest and #manifest >= 2 } -private rule is_signed_apk +private rule is_signed_apk : internal { meta: description = "Resembles a signed APK that is likely not corrupt" strings: $meta_inf = "META-INF/" - $rsa = ".RSA" - $dsa = ".DSA" + $ext_rsa = ".RSA" + $ext_dsa = ".DSA" + $ext_ec = ".EC" condition: - is_apk and for all of ($meta_inf*) : ($rsa or $dsa in (@ + 9..@ + 9 + 100)) + is_apk and + for all of ($meta_inf*) : ($ext_rsa or $ext_dsa or $ext_ec in (@ + 9..@ + 9 + 100)) } -private rule is_unsigned_apk +private rule is_unsigned_apk : internal { meta: description = "Resembles an unsigned APK that is likely not corrupt" diff --git a/apkid/rules/apk/packers.yara b/apkid/rules/apk/packers.yara index add28274..29377f5e 100644 --- a/apkid/rules/apk/packers.yara +++ b/apkid/rules/apk/packers.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -44,16 +44,16 @@ rule appguard : packer rule appguard_new : packer { meta: - description = "AppGuard" - sample = "c5195daa5d17ba6e1755f8cb7270ae3a971eb688ee7d650d10c284d7c93b777d" - url = "http://appguard.nprotect.com/en/index.html" - author = "Eduardo Novella" + description = "AppGuard" + sample = "c5195daa5d17ba6e1755f8cb7270ae3a971eb688ee7d650d10c284d7c93b777d" + url = "http://appguard.nprotect.com/en/index.html" + author = "Eduardo Novella" strings: - $a = "assets/AppGuard0.jar" - $b = "assets/AppGuard.dgc" - $c = "libAppGuard.so" - $d = "libAppGuard-x86.so" + $a = "assets/AppGuard0.jar" + $b = "assets/AppGuard.dgc" + $c = "libAppGuard.so" + $d = "libAppGuard-x86.so" condition: is_apk and 3 of them @@ -92,8 +92,8 @@ rule dexprotector : packer { /** - * DexProtector v6.x.x :- Demo,Standard,Business Edition (https://dexprotector.com) - **/ + * DexProtector v6.x.x :- Demo, Standard, Business Edition + **/ meta: author = "Jasi2169 and Eduardo Novella" @@ -204,20 +204,19 @@ rule bangcle : packer rule bangcle_secshell : packer { - meta: - description = "Bangcle (SecShell)" - sample = "d710a24971a0cd56c5cbe62b4b926e0122704fba52821e9c888e651a2d26a05c" - url = "https://blog.fortinet.com/2017/01/26/deep-analysis-of-android-rootnik-malware-using-advanced-anti-debug-and-anti-hook-part-i-debugging-in-the-scope-of-native-layer" - author = "Eduardo Novella" - + meta: + description = "Bangcle (SecShell)" + sample = "d710a24971a0cd56c5cbe62b4b926e0122704fba52821e9c888e651a2d26a05c" + url = "https://blog.fortinet.com/2017/01/26/deep-analysis-of-android-rootnik-malware-using-advanced-anti-debug-and-anti-hook-part-i-debugging-in-the-scope-of-native-layer" + author = "Eduardo Novella" - strings: - $a = "assets/secData0.jar" - $b = "libSecShell.so" - $c = "libSecShell-x86.so" + strings: + $a = "assets/secData0.jar" + $b = "libSecShell.so" + $c = "libSecShell-x86.so" - condition: - is_apk and 2 of them + condition: + is_apk and 2 of them } rule kiro : packer @@ -283,8 +282,8 @@ rule jiagu_a : packer rule qdbh_packer : packer { meta: - description = "Unknown. Asset 'qdbh'" - example = "faf1e85f878ea52a3b3fbb67126132b527f509586706f242f39b8c1fdb4a2065" + description = "qdbh packer" + sample = "faf1e85f878ea52a3b3fbb67126132b527f509586706f242f39b8c1fdb4a2065" strings: $qdbh = "assets/qdbh" @@ -293,22 +292,6 @@ rule qdbh_packer : packer is_apk and $qdbh } -rule unknown_packer_lib : packer -{ - meta: - description = "Unknown. Random library name." - example = "faf1e85f878ea52a3b3fbb67126132b527f509586706f242f39b8c1fdb4a2065" - - strings: - $pre_jar = { 00 6F 6E 43 72 65 61 74 65 00 28 29 56 00 63 6F 6D 2F 76 } // .onCreate.()V.com/v - $jar_data = { 2E 6A 61 72 00 2F 64 61 74 61 2F 64 61 74 61 2F 00 2F } // .jar./data/data - $post_jar = { 2E 6A 61 72 00 77 00 6A 61 76 61 2F 75 74 69 6C 2F 4D 61 70 00 67 65 74 49 6E 74 00 } // .jar.w.java/util/Map.getInt. - - condition: - //is_apk and - ($pre_jar and $jar_data and $post_jar) -} - rule unicom_loader : packer { meta: @@ -459,7 +442,7 @@ rule pangxie : packer { meta: description = "PangXie" - example = "ea70a5b3f7996e9bfea2d5d99693195fdb9ce86385b7116fd08be84032d43d2c" + sample = "ea70a5b3f7996e9bfea2d5d99693195fdb9ce86385b7116fd08be84032d43d2c" strings: $lib = "libnsecure.so" @@ -472,7 +455,7 @@ rule kony : packer { meta: description = "Kony" - url = "http://www.kony.com/" + url = "http://www.kony.com/" strings: $lib = "libkonyjsvm.so" @@ -487,7 +470,7 @@ rule approov : packer { meta: description = "Aproov" - url = "https://www.approov.io/" + url = "https://www.approov.io/" strings: $lib = "libapproov.so" @@ -501,7 +484,7 @@ rule yidun : packer { meta: description = "yidun" - url = "https://dun.163.com/product/app-protect" + url = "https://dun.163.com/product/app-protect" strings: $anti_trick = "Lcom/_" // Class path of anti-trick diff --git a/apkid/rules/dex/abnormal.yara b/apkid/rules/dex/abnormal.yara index f213afd2..f73f6923 100644 --- a/apkid/rules/dex/abnormal.yara +++ b/apkid/rules/dex/abnormal.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -97,12 +97,3 @@ rule illegal_class_names : anti_disassembly condition: any of them } - -rule invalid_dex : abnormal -{ - meta: - description = "invalid dex (parsing error)" - - condition: - is_dex and dex.invalid_dex == 1 -} diff --git a/apkid/rules/dex/anti-vm.yara b/apkid/rules/dex/anti-vm.yara index 6f65f8ef..98eaa2d7 100644 --- a/apkid/rules/dex/anti-vm.yara +++ b/apkid/rules/dex/anti-vm.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -28,8 +28,11 @@ import "dex" include "common.yara" -private rule uses_build_class +private rule uses_build_class : internal { + meta: + description = "References android.os.Build class" + strings: // Landroid/os/Build; $a = {00 12 4C 61 6E 64 72 6F 69 64 2F 6F 73 2F 42 75 69 6C 64 3B 00} @@ -38,8 +41,10 @@ private rule uses_build_class and $a } -private rule uses_debug_class +private rule uses_debug_class : internal { + meta: + description = "References android.os.Debug class" strings: // Landroid/os/Debug; @@ -49,8 +54,11 @@ private rule uses_debug_class and $a } -private rule uses_telephony_class +private rule uses_telephony_class : internal { + meta: + description = "References android.telephony.TelephonyManager class" + strings: // Landroid/telephony/TelephonyManager; $a = {00 24 4C 61 6E 64 72 6F 69 64 2F 74 65 6C 65 70 68 6F 6E 79 2F 54 diff --git a/apkid/rules/dex/common.yara b/apkid/rules/dex/common.yara index 74d469ba..b3873065 100644 --- a/apkid/rules/dex/common.yara +++ b/apkid/rules/dex/common.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -25,7 +25,7 @@ * **/ -private rule is_dex +private rule is_dex : internal { meta: description = "Resembles a DEX file" diff --git a/apkid/rules/dex/compilers.yara b/apkid/rules/dex/compilers.yara index d73dad8e..e43cb8f9 100644 --- a/apkid/rules/dex/compilers.yara +++ b/apkid/rules/dex/compilers.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -28,30 +28,39 @@ import "dex" include "common.yara" -private rule unsorted_string_table +private rule unsorted_string_pool : internal { + meta: + description = "String pool in non-standard order" + condition: /* * DEX format requires string IDs to be sorted according to the data at their offsets but the actual * ordering of the string pool is undefined. Dexlib (smali/apktool) 1.x sorts strings by class and - * proximity. DX sorts strings in the same order as the string table. + * proximity. DX sorts the string pool in the same order as the string table. * * Note: It's probably only necessary to check the first several strings. */ - for any i in (0..dex.header.string_ids_size - 1) : (dex.string_ids[i].offset + dex.string_ids[i].item_size + 1 < dex.string_ids[i + 1].offset) + for any i in (0..dex.header.string_ids_size - 1) : (dex.string_ids[i].offset + dex.string_ids[i].size + 1 < dex.string_ids[i + 1].offset) } -private rule dexlib2_map_type_order +private rule dexlib2_map_type_order : internal { + meta: + description = "dexlib2 map_list type order" + condition: /* * The map_list types are in different orders for DX, dexmerge, and dexlib (1 and 2 are the same) */ - dex.map_list.map_items[7].type == 0x2002 // TYPE_STRING_DATA_ITEM + dex.map_list.map_item[7].type == 0x2002 // TYPE_STRING_DATA_ITEM } -private rule null_interfaces +private rule null_interfaces : internal { + meta: + description = "null interfaces offset" + condition: /* * Dexlib2 adds a non-zero interfaces_offset to every class_def_item, even if the class doesn't implement an @@ -62,26 +71,32 @@ private rule null_interfaces for any i in (0..dex.header.class_defs_size) : (dex.class_defs[i].interfaces_offset > 0 and uint32(dex.class_defs[i].interfaces_offset) == 0) } -private rule dx_map_type_order +private rule dx_map_type_order : internal { + meta: + description = "dx map_list type order" + condition: /* * The map_list types are in different orders for DX, dexmerge, and dexlib (1 and 2 are the same) * DX order derrived from: http://osxr.org/android/source/dalvik/dx/src/com/android/dx/dex/file/DexFile.java#0111 */ - (dex.map_list.map_items[7].type == 0x1002 or // TYPE_ANNOTATION_SET_REF_LIST - dex.map_list.map_items[7].type == 0x1003 or // TYPE_ANNOTATION_SET_ITEM - dex.map_list.map_items[7].type == 0x2001) // TYPE_CODE_ITEM + (dex.map_list.map_item[7].type == 0x1002 or // TYPE_ANNOTATION_SET_REF_LIST + dex.map_list.map_item[7].type == 0x1003 or // TYPE_ANNOTATION_SET_ITEM + dex.map_list.map_item[7].type == 0x2001) // TYPE_CODE_ITEM } -private rule dexmerge_map_type_order +private rule dexmerge_map_type_order : internal { + meta: + description = "dexmerge map_list type order" + condition: /* * The map_list types are in different orders for DX, dexmerge, and dexlib (1 and 2 are the same) * DexMerge order derrived from: http://osxr.org/android/source/dalvik/dx/src/com/android/dx/merge/DexMerger.java#0111 */ - dex.map_list.map_items[7].type == 0x1000 // TYPE_MAP_LIST + dex.map_list.map_item[7].type == 0x1000 // TYPE_MAP_LIST } rule jack_4_12 : compiler @@ -135,9 +150,12 @@ rule jack_5x : compiler is_dex and $jack_emitter } -private rule has_jack_anon_methods +private rule has_jack_anon_methods : internal { - // https://calebfenton.github.io/2016/12/01/building-with-and-detecting-jack/ + meta: + description = "has Jack compiler anonymous methods" + url = "https://calebfenton.github.io/2016/12/01/building-with-and-detecting-jack/" + strings: $anon_set = {00 05 2D 73 65 74 30 00} // -set0 $anon_get = {00 05 2D 67 65 74 30 00} // -get0 @@ -147,8 +165,11 @@ private rule has_jack_anon_methods 2 of ($anon_*) } -private rule jack_emitter +private rule jack_emitter : internal { + meta: + description = "has Jack compiler emitter string" + strings: // "\0emitter: jack-?.?\0" $jack_emitter = {00 1? 65 6D 69 74 74 65 72 3A 20 6A 61 63 6B 2D ?? 2E [1-3] 00} @@ -157,9 +178,12 @@ private rule jack_emitter $jack_emitter or has_jack_anon_methods } -private rule has_javac_anon_methods +private rule has_javac_anon_methods : internal { - // https://calebfenton.github.io/2016/12/01/building-with-and-detecting-jack/ + meta: + description = "has Javac compiler anonymous methods" + url = "https://calebfenton.github.io/2016/12/01/building-with-and-detecting-jack/" + strings: $anon_set = {00 0A 61 63 63 65 73 73 24 30 30 32 00} // access$002 $anon_get = {00 0A 61 63 63 65 73 73 24 30 30 30 00} // access$000 @@ -188,7 +212,7 @@ rule dexlib1 : compiler description = "dexlib 1.x" condition: - unsorted_string_table + unsorted_string_pool } rule dexlib2 : compiler @@ -243,4 +267,3 @@ rule dexmerge : manipulator condition: dexmerge_map_type_order } - diff --git a/apkid/rules/dex/obfuscators.yara b/apkid/rules/dex/obfuscators.yara index 070ec43f..8925402e 100644 --- a/apkid/rules/dex/obfuscators.yara +++ b/apkid/rules/dex/obfuscators.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -110,13 +110,12 @@ rule bitwise_antiskid : obfuscator any of them } - rule arxan : obfuscator { meta: description = "Arxan" url = "https://www.arxan.com/products/application-protection-mobile/" - example = "7bd1139b5f860d48e0c35a3f117f980564f45c177a6ef480588b5b5c8165f47e" + sample = "7bd1139b5f860d48e0c35a3f117f980564f45c177a6ef480588b5b5c8165f47e" author = "Eduardo Novella" strings: @@ -140,13 +139,12 @@ rule arxan : obfuscator 6 of ($m*) } - rule arxan_multidex : obfuscator { meta: description = "Arxan (multidex)" url = "https://www.arxan.com/products/application-protection-mobile/" - example = "9b2a978a937293d6cb93439e0f819b4e044a3fad80dde92dec9b67e419278b5d" + sample = "9b2a978a937293d6cb93439e0f819b4e044a3fad80dde92dec9b67e419278b5d" author = "Eduardo Novella" strings: @@ -191,10 +189,10 @@ rule allatori_demo : obfuscator rule aamo_str_enc : obfuscator { meta: - description = "AAMO (String decryption function)" + description = "AAMO" author = "P0r0" url = "https://github.com/necst/aamo" - example1 = "c1ef860af0e168f924663630ed3b61920b474d0c8b10e2bde6bfd3769dbd31a8" + example = "c1ef860af0e168f924663630ed3b61920b474d0c8b10e2bde6bfd3769dbd31a8" example2 = "eb0d4e1ba2e880749594eb8739e65aa21b6f7b43798f04b6681065b396c15a78" strings: @@ -243,4 +241,3 @@ rule aamo_str_enc : obfuscator condition: 1 of ($opcodes*) and all of ($a, $b) } - diff --git a/apkid/rules/dex/packers.yara b/apkid/rules/dex/packers.yara index 054a0673..ab56a886 100644 --- a/apkid/rules/dex/packers.yara +++ b/apkid/rules/dex/packers.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -31,7 +31,7 @@ rule pangxie_dex : packer { meta: description = "PangXie" - example = "ea70a5b3f7996e9bfea2d5d99693195fdb9ce86385b7116fd08be84032d43d2c" + sample = "ea70a5b3f7996e9bfea2d5d99693195fdb9ce86385b7116fd08be84032d43d2c" strings: // Lcom/merry/wapper/WapperApplication; @@ -71,17 +71,14 @@ rule medusah_appsolid_dex : packer is_dex and $loader and $main_activity } - - rule apkguard_dex : packer { meta: description = "APKGuard" url = "http://apkguard.io/" - example = "d9c98fff427646883ecb457fc2e9d2a8914ba7a9ee194735e0a18f56baa26cca" + sample = "d9c98fff427646883ecb457fc2e9d2a8914ba7a9ee194735e0a18f56baa26cca" strings: - $attachBaseContextOpcodes = { 120b // const/4 v11, #int 0 // #0 6f20 0100 fe00 // invoke-super {v14, v15}, Landroid/app/Application;.attachBaseContext:(Landroid/content/Context;)V // method@0001 @@ -150,15 +147,12 @@ rule apkguard_dex : packer is_dex and $attachBaseContextOpcodes } - - rule cryptoshell_dex : packer { meta: description = "CryptoShell" url = "http://cryptoshell.io" - example = "d6745c1533b440c93f7bdfbb106470043b23aafdf91506c52332ed192d7b7003" - + sample = "d6745c1533b440c93f7bdfbb106470043b23aafdf91506c52332ed192d7b7003" strings: @@ -224,11 +218,28 @@ rule cryptoshell_dex : packer 0d08 // move-exception v8 6e10 ??00 0800 // invoke-virtual {v8}, Ljava/lang/Exception;.printStackTrace:()V // method@000f 28fb // goto 0073 // -0005 -} + } condition: is_dex and $attachBaseContextOpcodes and not apkguard_dex +} + +rule jar_pack01 : packer +{ + meta: + // Unknown name, made this one up + description = "jar_pack01" + sample = "faf1e85f878ea52a3b3fbb67126132b527f509586706f242f39b8c1fdb4a2065" + + strings: + $pre_jar = { 00 6F 6E 43 72 65 61 74 65 00 28 29 56 00 63 6F 6D 2F 76 } // .onCreate.()V.com/v + $jar_data = { 2E 6A 61 72 00 2F 64 61 74 61 2F 64 61 74 61 2F 00 2F } // .jar./data/data + $post_jar = { 2E 6A 61 72 00 77 00 6A 61 76 61 2F 75 74 69 6C 2F 4D 61 70 00 67 65 74 49 6E 74 00 } // .jar.w.java/util/Map.getInt. + + condition: + is_dex and + ($pre_jar and $jar_data and $post_jar) } diff --git a/apkid/rules/dex/protectors.yara b/apkid/rules/dex/protectors.yara index f4ac93b1..51db52e9 100644 --- a/apkid/rules/dex/protectors.yara +++ b/apkid/rules/dex/protectors.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -32,7 +32,7 @@ rule CNProtect_dex : protector // https://github.com/rednaga/APKiD/issues/52 meta: description = "CNProtect (anti-disassemble)" - example = "5bf6887871ce5f00348b1ec6886f9dd10b5f3f5b85d3d628cf21116548a3b37d" + sample = "5bf6887871ce5f00348b1ec6886f9dd10b5f3f5b85d3d628cf21116548a3b37d" strings: // code segment of the injected methods plus junk opcodes diff --git a/apkid/rules/elf/anti-vm.yara b/apkid/rules/elf/anti-vm.yara index 1b999023..5644c8e6 100644 --- a/apkid/rules/elf/anti-vm.yara +++ b/apkid/rules/elf/anti-vm.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -30,8 +30,8 @@ import "elf" rule check_qemu_entropy : anti_vm { meta: - // https://github.com/Fuzion24/AndroidHostileEnvironmentDetection/blob/master/app/jni/emudetect.c description = "Checks for QEMU entropy" + url = "https://github.com/Fuzion24/AndroidHostileEnvironmentDetection/blob/master/app/jni/emudetect.c" strings: $a = "atomicallyIncreasingGlobalVarThread" diff --git a/apkid/rules/elf/obfuscators.yara b/apkid/rules/elf/obfuscators.yara index b1029e5f..ea180018 100644 --- a/apkid/rules/elf/obfuscators.yara +++ b/apkid/rules/elf/obfuscators.yara @@ -29,13 +29,12 @@ import "elf" include "common.yara" - rule ollvm_v3_4 : obfuscator { meta: description = "Obfuscator-LLVM version 3.4" url = "https://github.com/obfuscator-llvm/obfuscator/wiki" - example = "cd16ad33bf203dbaa9add803a7a0740e3727e8e60c316d33206230ae5b985f25" + sample = "cd16ad33bf203dbaa9add803a7a0740e3727e8e60c316d33206230ae5b985f25" author = "Eduardo Novella" strings: @@ -47,13 +46,12 @@ rule ollvm_v3_4 : obfuscator all of them } - rule ollvm_v3_5 : obfuscator { meta: description = "Obfuscator-LLVM version 3.5" url = "https://github.com/obfuscator-llvm/obfuscator/wiki" - example = "664214969f1b94494a8fc0491407f4440032fc5c922eb0664293d0440c52dbe7" + sample = "664214969f1b94494a8fc0491407f4440032fc5c922eb0664293d0440c52dbe7" author = "Eduardo Novella" strings: @@ -65,13 +63,12 @@ rule ollvm_v3_5 : obfuscator all of them } - rule ollvm_v3_6_1 : obfuscator { meta: description = "Obfuscator-LLVM version 3.6.1" url = "https://github.com/obfuscator-llvm/obfuscator/wiki" - example = "d84b45856b5c95f7a6e96ab0461648f22ad29d1c34a8e85588dad3d89f829208" + sample = "d84b45856b5c95f7a6e96ab0461648f22ad29d1c34a8e85588dad3d89f829208" author = "Eduardo Novella" strings: @@ -83,13 +80,12 @@ rule ollvm_v3_6_1 : obfuscator all of them } - rule ollvm_v4_0 : obfuscator { meta: description = "Obfuscator-LLVM version 4.0" url = "https://github.com/obfuscator-llvm/obfuscator/wiki" - example = "aaba570388d0fe25df45480ecf894625be7affefaba24695d8c1528b974c00df" + sample = "aaba570388d0fe25df45480ecf894625be7affefaba24695d8c1528b974c00df" author = "Eduardo Novella" strings: @@ -101,13 +97,12 @@ rule ollvm_v4_0 : obfuscator all of them } - rule ollvm_v6_0_strenc : obfuscator { meta: description = "Obfuscator-LLVM version 6.0 (string encryption)" url = "https://github.com/obfuscator-llvm/obfuscator/wiki" - example = "f3a2e6c57def9a8b4730965dd66ca0f243689153139758c44718b8c5ef9c1d17" + sample = "f3a2e6c57def9a8b4730965dd66ca0f243689153139758c44718b8c5ef9c1d17" author = "Eduardo Novella" strings: @@ -121,7 +116,6 @@ rule ollvm_v6_0_strenc : obfuscator all of them } - rule ollvm_v6_0 : obfuscator { meta: @@ -139,7 +133,6 @@ rule ollvm_v6_0 : obfuscator all of them and not ollvm_v6_0_strenc } - rule ollvm : obfuscator { meta: @@ -162,7 +155,6 @@ rule ollvm : obfuscator not ollvm_v6_0_strenc } - rule firehash : obfuscator { meta: @@ -172,13 +164,13 @@ rule firehash : obfuscator // original : https://firehash.grayhash.com/static/sample/dodocrackme_original.apk // firehashed : https://firehash.grayhash.com/static/sample/dodocrackme_obfuscated.apk - example1 = "38e2170a5f272ecae97dddb0dac0c1f39f7f71a4639477764a9154557106dd94" + sample = "38e2170a5f272ecae97dddb0dac0c1f39f7f71a4639477764a9154557106dd94" // original : 6352f6d0cdc85a42de3ccfd9226dfec28280aa835227acc507043a4403b7e700 - example2 = "c98af9a777d9633559b7903e21b61b845f7e1766afa74ef85e3380f41265e6b5" + sample2 = "c98af9a777d9633559b7903e21b61b845f7e1766afa74ef85e3380f41265e6b5" // original : 727be6789e8f4f6eab66288f957b58800e47a4bacebacc0dd700e8f9a374f116 - example3 = "423dc9866d1c5f32cabfeb254030d83e11db4d807394a8ff09be47d8bfc38f18" + sample3 = "423dc9866d1c5f32cabfeb254030d83e11db4d807394a8ff09be47d8bfc38f18" strings: // Library below heuristic is found inside of is normally named "libaurorabridge.so" diff --git a/apkid/rules/elf/packers.yara b/apkid/rules/elf/packers.yara index 6be1c677..052c4ca7 100644 --- a/apkid/rules/elf/packers.yara +++ b/apkid/rules/elf/packers.yara @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 RedNaga. http://rednaga.io + * Copyright (C) 2018 RedNaga. https://rednaga.io * All rights reserved. Contact: rednaga@protonmail.com * * @@ -74,7 +74,8 @@ rule upx_sharedlib_unmodifed : packer and $upx in (filesize - 50 .. filesize) and upx_stub } -rule upx_elf_3_94 : packer { +rule upx_elf_3_94 : packer +{ meta: description = "UPX 3.94 (unmodified)" @@ -85,7 +86,8 @@ rule upx_elf_3_94 : packer { upx_unmodified and $copyright } -rule upx_elf_3_93 : packer { +rule upx_elf_3_93 : packer +{ meta: description = "UPX 3.93 (unmodified)" @@ -317,7 +319,7 @@ rule promon : packer meta: description = "Promon Shield" info = "https://promon.co/" - example = "6a3352f54d9f5199e4bf39687224e58df642d1d91f1d32b069acd4394a0c4fe0" + sample = "6a3352f54d9f5199e4bf39687224e58df642d1d91f1d32b069acd4394a0c4fe0" strings: $a = "libshield.so" @@ -333,5 +335,3 @@ rule promon : packer ($a and $b and $c and $d) and 2 of ($s*) } - - diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index bda09e33..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '2' - -services: - apkid: - build: - context: . - dockerfile: docker/Dockerfile - image: rednaga/apkid:v1 diff --git a/docker/apkid.sh b/docker/apkid.sh index c26daa4b..31d47257 100755 --- a/docker/apkid.sh +++ b/docker/apkid.sh @@ -1,8 +1,13 @@ #!/bin/sh # This will simply take the argument passed to it, -# parse the dirctory and bind it as a read-only mount point on the container -# and pass in the filename as the argument to apkid +# parse the directory and bind it as a read-only mount point on the container +# and pass in the filename as the argument to APKiD # # This can easily be set to an alias and added to your .profile or whatever -docker run --rm -v "`dirname $1`":/input:ro -i rednaga/apkid:v1 "`basename $1`"; +# This assumes file target is last argument! +TARGET="${@: -1}" +INPUT_DIR=$(cd $(dirname "$TARGET") && pwd -P) +INPUT_FILE=$(basename $TARGET) + +docker run --rm --volume "$INPUT_DIR":/input:ro -i rednaga:apkid "/input/$INPUT_FILE" "${@:0:$#}"; diff --git a/prep-release.py b/prep-release.py index 359d620b..3aea74c5 100755 --- a/prep-release.py +++ b/prep-release.py @@ -1,39 +1,69 @@ #!/usr/bin/env python -import fnmatch +""" + Copyright (C) 2018 RedNaga. https://rednaga.io + All rights reserved. Contact: rednaga@protonmail.com + + + This file is part of APKiD + + + Commercial License Usage + ------------------------ + Licensees holding valid commercial APKiD licenses may use this file + in accordance with the commercial license agreement provided with the + Software or, alternatively, in accordance with the terms contained in + a written agreement between you and RedNaga. + + + GNU General Public License Usage + -------------------------------- + Alternatively, this file may be used under the terms of the GNU General + Public License version 3.0 as published by the Free Software Foundation + and appearing in the file LICENSE.GPL included in the packaging of this + file. Please visit http://www.gnu.org/copyleft/gpl.html and review the + information to ensure the GNU General Public License version 3.0 + requirements will be met. +""" + import os +import sys from codecs import open -import pypandoc -import yara - -rules_dir = 'apkid/rules/' -compiled_rules_path = os.path.join(rules_dir, 'rules.yarc') +from apkid import rules -print("[*] Converting Markdown README to reStructuredText") -rst = pypandoc.convert_file('README.md', 'rst') -with open('README.rst', 'w+', encoding='utf-8') as f: - f.write(rst) -print("[*] Finished converting to README.rst ({} bytes)".format(len(rst))) -yara_files = {} -for root, dirnames, filenames in os.walk(rules_dir): - for filename in fnmatch.filter(filenames, '*.yara'): - path = os.path.join(root, filename) - yara_files[path] = path +def convert_readme(): + print("[*] Converting Markdown README to reStructuredText") + import pypandoc + rst = pypandoc.convert_file('README.md', 'rst') + with open('README.rst', 'w+', encoding='utf-8') as f: + f.write(rst) + print("[*] Finished converting to README.rst ({} bytes)".format(len(rst))) -print("[*] Compiling {} Yara rule files".format(len(yara_files))) -rules = yara.compile(filepaths=yara_files) -rules.save(compiled_rules_path) -count = 0 -for _ in rules: - count += 1 -print("[*] Saved {} rules to {}".format(count, compiled_rules_path)) +if __name__ == '__main__': + print("[*] Compiling Yara files") + rulez = rules.compile() + rules_count, rules_path = rules.save(rulez) + print("[*] Saved {} rules to {}".format(rules_count, rules_path)) -# print("[*] Registering ...") -# os.system("python setup.py register") + tag_counts = {} + for rule in rules.load(): + for t in rule.tags: + if t not in tag_counts: + tag_counts[t] = 1 + else: + tag_counts[t] += 1 + print("[*] Rule tag counts:") + for tag in sorted(tag_counts.keys()): + count = tag_counts[tag] + print(" |-> {}: {}".format(tag, count)) -# print("[*] Cleaning up ...") -# os.remove('README.rst') + if len(sys.argv) > 1: + if sys.argv[1] == 'register': + print("[*] Registering ...") + os.system("python setup.py register") + if sys.argv[1] == 'readme': + convert_readme() -print("[*] Done.") + print("[*] Finished prepping.") diff --git a/setup.py b/setup.py index 411959ed..16e1aad0 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def package_files(directory): install_requires = [ - 'yara-python==3.5.0.999', + 'yara-python==3.7.0.999', 'argparse', ] @@ -35,26 +35,28 @@ def package_files(directory): author_email='rednaga@protonmail.com', license=apkid.__license__, classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 'License :: Other/Proprietary License', 'Natural Language :: English', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Security', 'Topic :: Utilities', ], - keywords='android analysis reversing malware apk dex', + keywords='android analysis reversing malware apk dex dalvik', packages=find_packages('.', exclude=['docs', 'tests']), package_data={ 'rules': package_files('apkid/rules/'), }, include_package_data=True, install_requires=install_requires, - dependency_links=[ - 'https://github.com/rednaga/yara-python/zipball/master#egg=yara-python-3.5.0.999' - ], extras_require={ 'dev': [ 'pypandoc' diff --git a/tests/test_rules.py b/tests/test_rules.py new file mode 100644 index 00000000..ac05340c --- /dev/null +++ b/tests/test_rules.py @@ -0,0 +1,21 @@ +import warnings + +from apkid import rules + + +def test_rules_compile(): + rulez = rules.load() + assert rulez + + +def test_lint_rules(): + for r in rules.load(): + if len(r.tags) == 0: + warnings.warn("rule has no tags: {}".format(r.identifier), stacklevel=0) + + if 'description' not in r.meta: + warnings.warn("rule has no tags: {}".format(r.identifier), stacklevel=0) + + if ('packer' in r.tags or 'protector' in r.tags or 'obfuscator' in r.tags) \ + and 'sample' not in r.meta: + warnings.warn("rule has no reference sample: {}".format(r.identifier), stacklevel=0)