Merge pull request #438 from Johann-PLW/main

Added profile files in the CLI

README.md

@@ -61,9 +61,9 @@ $ python aleapp.py --help
 
 ## Contributing artifact plugins
 
-Each plugin is a Python source file which should be added to the `scripts/artifacts` folder which will be loaded dynamically each time ILEAPP is run.
+Each plugin is a Python source file which should be added to the `scripts/artifacts` folder which will be loaded dynamically each time ALEAPP is run.
 
-The plugin source file must contain a dictionary named `__artifacts_v2__` at the very beginning of the module, which defines the artifacts that the plugin processes. The keys in the `__artifacts_v2__` dictionary should be IDs for the artifact(s) which must be unique within ILEAPP. The values should be dictionaries containing the following keys:
+The plugin source file must contain a dictionary named `__artifacts_v2__` at the very beginning of the module, which defines the artifacts that the plugin processes. The keys in the `__artifacts_v2__` dictionary should be IDs for the artifact(s) which must be unique within ALEAPP. The values should be dictionaries containing the following keys:
 
 - `name`: The name of the artifact as a string.
 - `description`: A description of the artifact as a string.
@@ -110,7 +110,7 @@ __artifacts_v2__ = {
 The functions referenced as entry points in the `__artifacts__` dictionary must take the following arguments:
 
 * An iterable of the files found which are to be processed (as strings)
-* The path of ILEAPP's output folder(as a string)
+* The path of ALEAPP's output folder(as a string)
 * The seeker (of type FileSeekerBase) which found the files
 * A Boolean value indicating whether or not the plugin is expected to wrap text
 
@@ -121,7 +121,7 @@ def get_cool_data1(files_found, report_folder, seeker, wrap_text):
     pass  # do processing here
 ```
 
-Plugins are generally expected to provide output in ILEAPP's HTML output format, TSV, and optionally submit records to 
+Plugins are generally expected to provide output in ALEAPP's HTML output format, TSV, and optionally submit records to 
 the timeline. Functions for generating this output can be found in the `artifact_report` and `ilapfuncs` modules. 
 At a high level, an example might resemble:
 

aleapp.py

@@ -1,17 +1,19 @@
+import json
 import argparse
 import io
 import os.path
 import typing
 import plugin_loader
 import scripts.report as report
 import traceback
+
 from scripts.search_files import *
 from scripts.ilapfuncs import *
 from scripts.version_info import aleapp_version
 from time import process_time, gmtime, strftime, perf_counter
 
 def validate_args(args):
-    if args.artifact_paths:
+    if args.artifact_paths or args.create_profile:
         return  # Skip further validation if --artifact_paths is used
 
     # Ensure other arguments are provided
@@ -28,10 +30,88 @@ def validate_args(args):
     if not os.path.exists(args.output_path):
         raise argparse.ArgumentError(None, 'OUTPUT folder does not exist! Run the program again.')
 
+    if args.load_profile and not os.path.exists(args.load_profile):
+        raise argparse.ArgumentError(None, 'ALEAPP Profile file not found! Run the program again.')
+
     try:
         timezone = pytz.timezone(args.timezone)
     except pytz.UnknownTimeZoneError:
       raise argparse.ArgumentError(None, 'Unknown timezone! Run the program again.')
+
+
+def create_profile(available_plugins, path):
+    available_parsers = []
+    for parser_data in available_plugins:
+        if parser_data.name != 'usagestatsVersion':
+            available_parsers.append((parser_data.category, parser_data.name))
+
+    available_parsers.sort()
+    parsers_in_profile = {}
+
+    user_choice = ''
+    print('-' * 50)
+    print('Welcome to the ALEAPP Profile file creation\n')
+    instructions = 'You can type:\n'
+    instructions += '   - \'a\' to add or remove parsers in the profile file\n'
+    instructions += '   - \'l\' to display the list of all available parsers with their number\n'
+    instructions += '   - \'p\' to display the parsers added into the profile file\n'
+    instructions += '   - \'q\' to quit and save\n'
+    while not user_choice:
+        print(instructions)
+        user_choice = input('Please enter your choice: ').lower()
+        print()
+        if user_choice == "l":
+            print('Available parsers:')
+            for number, available_plugin in enumerate(available_parsers):
+                print(number + 1, available_plugin)
+            print()
+            user_choice = ''
+        elif user_choice == "p":
+            if parsers_in_profile:
+                for number, parser in parsers_in_profile.items():
+                    print(number, parser)
+                print()
+            else:
+                print('No parser added to the profile file\n')
+            user_choice = ''
+        elif user_choice == 'a':
+            parser_numbers = input('Enter the numbers of parsers, seperated by a comma, to add or remove in the profile file: ')
+            parser_numbers = parser_numbers.split(',')
+            parser_numbers = [parser_number.strip() for parser_number in parser_numbers]
+            for parser_number in parser_numbers:
+                if parser_number.isdigit():
+                    parser_number = int(parser_number)
+                    if parser_number > 0 and parser_number <= len(available_parsers):
+                        if parser_number not in parsers_in_profile:
+                            parser_to_add = available_parsers[parser_number - 1]
+                            parsers_in_profile[parser_number] = parser_to_add
+                            print(f'parser number {parser_number} {parser_to_add} was added')
+                        else:
+                            parser_to_remove = parsers_in_profile[parser_number]
+                            print(f'parser number {parser_number} {parser_to_remove} was removed')
+                            del parsers_in_profile[parser_number]
+                    else:
+                        print('Please enter the number of a parser!!!\n')
+            print()
+            user_choice = ''
+        elif user_choice == "q":
+            if parsers_in_profile:
+                parsers = [parser_info[1] for parser_info in parsers_in_profile.values()]
+                profile_filename = ''
+                while not profile_filename:
+                    profile_filename = input('Enter the name of the profile: ')
+                profile_filename += '.alprofile'
+                filename = os.path.join(path, profile_filename)
+                with open(filename, "wt", encoding="utf-8") as profile_file:
+                    json.dump({"leapp": "aleapp", "format_version": 1, "plugins": parsers}, profile_file)
+                print('\nProfile saved:', filename)
+            else:
+                print('No parser added. The profile file was not created.\n')
+            return
+        else:
+            print('Please enter a valid choice!!!\n')
+            user_choice = ''
+
 
 def main():
     parser = argparse.ArgumentParser(description='ALEAPP: Android Logs, Events, and Protobuf Parser.')
@@ -46,13 +126,25 @@ def main():
     parser.add_argument('-tz', '--timezone', required=False, action="store", default='UTC', type=str, help="Timezone name (e.g., 'America/New_York')")
     parser.add_argument('-w', '--wrap_text', required=False, action="store_false", default=True,
                         help='Do not wrap text for output of data files')
+    parser.add_argument('-l', '--load_profile', required=False, action="store", help="Path to ALEAPP Profile file (.alprofile).")
+    parser.add_argument('-c', '--create_profile', required=False, action="store",
+                        help=("Generate an ALEAPP Profile file (.alprofile) into the specified path. "
+                              "This argument is meant to be used alone, without any other arguments."))
     parser.add_argument('-p', '--artifact_paths', required=False, action="store_true",
                         help=("Generate a text file list of artifact paths. "
                               "This argument is meant to be used alone, without any other arguments."))
 
     loader = plugin_loader.PluginLoader()
+    available_plugins = list(loader.plugins)
+    profile_filename = None
+    # Move usagestatsVersion plugin to first position
+    usagestatsVersion_index = next((i for i, selected_plugin in enumerate(available_plugins) 
+                  if selected_plugin.name == 'usagestatsVersion'), -1)
+    if usagestatsVersion_index != -1:
+        available_plugins.insert(0, available_plugins.pop(usagestatsVersion_index))
 
-    print(f"Info: {len(loader)} plugins loaded.")
+    print(f"Info: {len(available_plugins)} plugins loaded.")
+    selected_plugins = available_plugins.copy()
 
     args = parser.parse_args()
 
@@ -77,6 +169,40 @@ def main():
         print('Artifact path list generation completed')
         return
 
+    if args.create_profile:
+        if os.path.isdir(args.create_profile):
+            create_profile(selected_plugins, args.create_profile)
+            return
+        else:
+            print('OUTPUT folder for storing ALEAPP Profile file does not exist!\nRun the program again.')
+            return
+
+    if args.load_profile:
+        profile_filename = args.load_profile
+        profile_load_error = None
+        with open(profile_filename, "rt", encoding="utf-8") as profile_file:
+            try:
+                profile = json.load(profile_file)
+            except json.JSONDecodeError as json_ex:
+                profile_load_error = f"File was not a valid profile file: {json_ex}"
+                print(profile_load_error)
+                return
+
+        if not profile_load_error:
+            if isinstance(profile, dict):
+                if profile.get("leapp") != "aleapp" or profile.get("format_version") != 1:
+                    profile_load_error = "File was not a valid profile file: incorrect LEAPP or version"
+                    print(profile_load_error)
+                    return
+                else:
+                    profile_plugins = set(profile.get("plugins", []))
+                    selected_plugins = [selected_plugin for selected_plugin in available_plugins 
+                                        if selected_plugin.name in profile_plugins or selected_plugin.name == 'usagestatsVersion']
+            else:
+                profile_load_error = "File was not a valid profile file: invalid format"
+                print(profile_load_error)
+                return
+
     input_path = args.input_path
     extracttype = args.t
     wrap_text = args.wrap_text
@@ -96,12 +222,12 @@ def main():
     except NameError:
         casedata = {}
 
-    crunch_artifacts(list(loader.plugins), extracttype, input_path, out_params, 1, wrap_text, loader, casedata, time_offset)
+    crunch_artifacts(selected_plugins, extracttype, input_path, out_params, 1, wrap_text, loader, casedata, time_offset, profile_filename)
 
 
 def crunch_artifacts(
         plugins: typing.Sequence[plugin_loader.PluginSpec], extracttype, input_path, out_params, ratio, wrap_text,
-        loader: plugin_loader.PluginLoader, casedata, time_offset):
+        loader: plugin_loader.PluginLoader, casedata, time_offset, profile_filename):
     start = process_time()
     start_wall = perf_counter()
 
@@ -111,7 +237,7 @@ def crunch_artifacts(
     logfunc(f'ALEAPP v{aleapp_version}: ALEAPP Logs, Events, and Protobuf Parser')
     logfunc('Objective: Triage Android Full System Extractions.')
     logfunc('By: Alexis Brignoni | @AlexisBrignoni | abrignoni.com')
-    logfunc('By: Yogesh Khatri   | @SwiftForensics | swiftforensics.com')
+    logfunc('By: Yogesh Khatri   | @SwiftForensics | swiftforensics.com\n')
     logdevinfo()
 
     seeker = None
@@ -137,7 +263,9 @@ def crunch_artifacts(
         return False
 
     # Now ready to run
-    logfunc(f'Artifact categories to parse: {str(len(plugins))}')
+    if profile_filename:
+        logfunc(f'Loaded profile: {profile_filename}')
+    logfunc(f'Artifact categories to parse: {len(plugins)}')
     logfunc(f'File/Directory selected: {input_path}')
     logfunc('\n--------------------------------------------------------------------------------------')
 

aleappGUI.py

@@ -1,13 +1,11 @@
 import json
-import pathlib
 import typing
 import aleapp
 import PySimpleGUI as sg
 import webbrowser
 import plugin_loader
 from scripts.ilapfuncs import *
 from scripts.version_info import aleapp_version
-from time import process_time, gmtime, strftime
 from scripts.search_files import *
 
 MODULE_START_INDEX = 1000
@@ -53,7 +51,7 @@ def ValidateInput(values, window):
 
 # initialize CheckBox control with module name   
 def CheckList(mtxt, lkey, mdstring, disable=False):
-    if mdstring == 'photosMetadata' or mdstring == 'journalStrings' or mdstring == 'walStrings': #items in the if are modules that take a long time to run. Deselects them by default.
+    if mdstring == 'walStrings': #items in the if are modules that take a long time to run. Deselects them by default.
         dstate = False
     else:
         dstate = True
@@ -120,6 +118,7 @@ def pickModules():
 # Create the Window
 window = sg.Window(f'ALEAPP version {aleapp_version}', layout)
 GuiWindow.progress_bar_handle = window['PROGRESSBAR']
+profile_filename = None
 
 # Event Loop to process "events" and get the "values" of the inputs
 while True:
@@ -171,7 +170,7 @@ def pickModules():
                         profile_load_error = "File was not a valid profile file: incorrect LEAPP or version"
                     else:
                         ticked = set(profile.get("plugins", []))
-                        ticked.add("lastbuild")  # always
+                        ticked.add("usagestatsVersion")  # always
                         for x in range(MODULE_START_INDEX, module_end_index):
                             if window[x].metadata in ticked:
                                 window[x].update(True)
@@ -184,6 +183,7 @@ def pickModules():
                 sg.popup(profile_load_error)
             else:
                 sg.popup(f"Loaded profile: {destination_path}")
+                profile_filename = destination_path
 
     if event == 'LOAD CASE DATA':
         destination_path = sg.popup_get_file(
@@ -218,21 +218,21 @@ def pickModules():
             input_path = values[0]
             output_folder = values[1]
 
-            # ios file system extractions contain paths > 260 char, which causes problems
+            # Android file system extractions contain paths > 260 char, which causes problems
             # This fixes the problem by prefixing \\?\ on each windows path.
             if is_platform_windows():
                 if input_path[1] == ':' and extracttype =='fs': input_path = '\\\\?\\' + input_path.replace('/', '\\')
                 if output_folder[1] == ':': output_folder = '\\\\?\\' + output_folder.replace('/', '\\')
 
             # re-create modules list based on user selection
-            # search_list = { 'lastBuild' : tosearch['lastBuild'] } # hardcode lastBuild as first item
-            search_list = [loader['usagestatsVersion']] # hardcode lastBuild as first item
+            # search_list = { 'usagestatsVersion' : tosearch['usagestatsVersion'] } # hardcode usagestatsVersion as first item
+            search_list = [loader['usagestatsVersion']] # hardcode usagestatsVersion as first item
 
             s_items = 0
             for x in range(MODULE_START_INDEX, module_end_index):
                 if window.FindElement(x).Get():
                     key = window[x].metadata
-                    if key in loader and key != 'lastbuild':
+                    if key in loader and key != 'usagestatsVersion':
                         search_list.append(loader[key])
                     s_items = s_items + 1 # for progress bar
 
@@ -255,7 +255,7 @@ def pickModules():
                 casedata = {}
 
             crunch_successful = aleapp.crunch_artifacts(
-                search_list, extracttype, input_path, out_params, len(loader)/s_items, wrap_text, loader, casedata, time_offset)
+                search_list, extracttype, input_path, out_params, len(loader)/s_items, wrap_text, loader, casedata, time_offset, profile_filename)
             if crunch_successful:
                 report_path = os.path.join(out_params.report_folder_base, 'index.html')
 

scripts/report.py

@@ -214,7 +214,7 @@
     },
     'DEVICE INFO': {
         'BUILD INFO': 'terminal',
-        'IOS SYSTEM VERSION': 'git-commit',
+        'ANDROID SYSTEM VERSION': 'git-commit',
         'PARTNER SETTINGS': 'settings',
         'SETTINGS_SECURE_': 'settings',
         'default': 'info',
@@ -496,10 +496,6 @@
         'ATTACHMENTS': 'paperclip',
         'CONTACTS': 'user',
     },
-    'IOS ATXDATASTORE': 'database',
-    'IOS BUILD': 'git-commit',
-    'IOS BUILD (ITUNES BACKUP)': 'git-commit',
-    'IOS SCREENS': 'maximize',
     'KEYBOARD': {
         'KEYBOARD APPLICATION USAGE': 'type',
         'KEYBOARD DYNAMIC LEXICON': 'type',