Merge pull request #456 from CybercentreCanada/patch/updater

Patch/updater update[dev]

README.md

@@ -230,3 +230,8 @@ could be at `/usr/share/perl5/INetSim/`. Restart INetSim with `sudo systemctl re
 #### CAPE-specific safelisted items
 The file at `al_config/system_safelist.yaml` contains suggested safelisted values that can be added to the Assemblyline system safelist
 either by copy-and-pasting directly to the text editor on the page `https://<Assemblyline Instance>/admin/tag_safelist` or through the [Assemblyline Client](https://github.com/CybercentreCanada/assemblyline_client).
+
+### Sources and prescript feature
+By default the CAPE updater fetch the rules from the community and base repository. They are known as source from the service standpoint. If you do not wish to load them or to remove the community rules this need to be edited in the manifest under the 'update_config-->sources'. 
+
+!Beta! There is also a feature to run Yara rules on the sample prior to the analysis which is called prescript. They will be used to dictate preconfiguration of the virtual machine before the analysis. Details are going to be given when the prescript detection feature is officially release in CAPE. In order to run rules via this feature, a given source will need to be added to the 'updater-->prescript_CAPE' section of the manifest.  

cape/cape_main.py

@@ -427,6 +427,7 @@ def _load_rules(self):
                         continue
 
                     for rule_path in rules.values():
+                        self.log.debug(f"Trying to load rule {rule_path}\n")
                         if rule_path.startswith(source_dir):
                             with open(rule_path) as fh:
                                 rule_content.append(fh.read())
@@ -524,16 +525,16 @@ def _general_flow(
                             if key.startswith("al_cape"):
                                 params = match.meta[key]
                                 action = key.replace("al_cape_", "")
-                                action = "".join(i for i in action if not i.isdigit())
+                                action = ''.join(i for i in action if not i.isdigit())
                                 if action.lower() in LIST_OF_VALID_ACTIONS:
                                     # The parameters in the rules need to be double encoded and escaped such as \ need to look like this \\\\ and ' become \\'
-                                    parsed_param = loads(params.replace("'", '"'))
+                                    parsed_param = loads(params.replace("'", "\""))
                                     option_passed += f" {action}"
                                     for param_key in ACTIONS_PARAMETERS[action]:
                                         if parsed_param[param_key] == "":
                                             parsed_param[param_key] = "None"
-                                        if isinstance(parsed_param[param_key], str) and '"' in parsed_param[param_key]:
-                                            parsed_param[param_key] = parsed_param[param_key].replace('"', '\\"')
+                                        if isinstance(parsed_param[param_key], str) and "\"" in parsed_param[param_key]:
+                                            parsed_param[param_key] = parsed_param[param_key].replace("\"", "\\\"")
                                         option_passed += f" {parsed_param[param_key]}"
                 else:
                     option_passed = ""

cape/update_server.py

@@ -57,47 +57,45 @@ def import_update(self, files_sha256, source_name: str, default_classification=c
         processed_files: set[str] = set()
         parser = Plyara()
         parser.STRING_ESCAPE_CHARS.add("r")
-        with tempfile.NamedTemporaryFile(mode="a+", suffix=source_name) as compiled_file:
-            # Aggregate files into one major source file
-            upload_list = []
-            yara_importer = YaraImporter(self.updater_type, self.client, logger=self.log)
-            for file, _ in files_sha256:
-                # File has already been processed before, skip it to avoid duplication of rules
-                if file in processed_files:
-                    continue
-
-                self.log.info(f"Processing file: {file}")
-
-                file_dirname = os.path.dirname(file)
-                processed_files.add(os.path.normpath(file))
+        upload_list = []
+        yara_importer = YaraImporter(self.updater_type, self.client, logger=self.log)
+        for file, _ in files_sha256:
+            # File has already been processed before, skip it to avoid duplication of rules
+            if file in processed_files:
+                continue
+
+            self.log.info(f"Processing file: {file}")
+
+            file_dirname = os.path.dirname(file)
+            processed_files.add(os.path.normpath(file))
+            try:
+                valid = validate_rule(file)
+            except Exception as e:
+                self.log.error(f"Error validating {file}: {e}")
+                raise e
+            if valid:
+                with open(file, "r", errors="surrogateescape") as f:
+                    f_lines = f.readlines()
+
+                temp_lines: list[str] = []
+                for _, f_line in enumerate(f_lines):
+                    if f_line.startswith("include"):
+                        lines, processed_files = replace_include(f_line, file_dirname, processed_files, self.log)
+                        temp_lines.extend(lines)
+                    else:
+                        temp_lines.append(f_line)
+
+                # guess the type of files that we have in the current file
+                # Try parsing the ruleset; on fail, move onto next set
                 try:
-                    valid = validate_rule(file)
+                    signatures: list[dict[str, Any]] = parser.parse_string("\n".join(temp_lines))
+                    upload_list.extend(signatures)
                 except Exception as e:
-                    self.log.error(f"Error validating {file}: {e}")
-                    raise e
-                if valid:
-                    with open(file, "r", errors="surrogateescape") as f:
-                        f_lines = f.readlines()
-
-                    temp_lines: list[str] = []
-                    for _, f_line in enumerate(f_lines):
-                        if f_line.startswith("include"):
-                            lines, processed_files = replace_include(f_line, file_dirname, processed_files, self.log)
-                            temp_lines.extend(lines)
-                        else:
-                            temp_lines.append(f_line)
-
-                    # guess the type of files that we have in the current file
-                    # Try parsing the ruleset; on fail, move onto next set
-                    try:
-                        signatures: list[dict[str, Any]] = parser.parse_string("\n".join(temp_lines))
-                        upload_list.extend(signatures)
-                    except Exception as e:
-                        self.log.error(f"Problem parsing {file}: {e}")
-                        continue
-            yara_importer._save_signatures(
-                signatures=upload_list, source=source_name, default_classification=default_classification
-            )
+                    self.log.error(f"Problem parsing {file}: {e}")
+                    continue
+        yara_importer._save_signatures(
+            signatures=upload_list, source=source_name, default_classification=default_classification
+        )
 
     def is_valid(self, file_path) -> bool:
         # Purpose:  Used to determine if the file associated is 'valid' to be processed as a signature