Merge pull request #33 from CybercentreCanada/feature_standardizeline…

…sstartcharacters

Feature standardizelinesstartcharacters

CCCS_YARA.yml

@@ -95,7 +95,7 @@ version:
    optional: Optional
    validator: valid_version
 
-minimum_yara:
+yara_version:
    description: 'Minimum version of Yara to run the rule'
    format: 'x.x'
    unique: Yes

yara-validator/validator_cfg.yml

@@ -6,8 +6,6 @@
 #    description: <description of the setting>
 #    value: <value of the setting>
 #
-# --NOTE--
-# 1. Currently only one setting to set the default behavior
 #
 
 ---
@@ -17,4 +15,12 @@ string_encoding:
                     ascii:  Will check if the file contains only ASCII characters
                     utf-8:  Will check if the file contains only UTF-8 characters
                     none:   Will perform no check'
-   value: utf-8
+   value: utf-8
+
+white_space_replacement:
+   description: 'Used to set the white space which will be searched for, what it will be replaced with and how many
+                 of those characters will be used'
+   value:
+      char_to_replace: '\t'
+      char_replacement: ' '
+      count_of_replaced: 4

yara-validator/yara_file_processor.py

@@ -9,9 +9,13 @@ class YaraFileProcessor:
     YaraFileProcessor class is used to process a given rule file and parse it into one or more YARA rules
     """
 
-    def __init__(self, rule_file):
+    def __init__(self, rule_file, char_to_replace, char_replacement, count_of_replaced):
         # Original rule file
         self.original_rule_file = rule_file
+        # Variables for the white space standardization
+        self.char_to_replace = char_to_replace.encode('utf-8').decode('unicode_escape')
+        self.char_replacement = char_replacement.encode('utf-8').decode('unicode_escape')
+        self.count_of_replaced = count_of_replaced
         # String representation to contain edits to the original rule
         self.edited_rule_string = ''
         # Array to contain the YARA rules
@@ -81,6 +85,43 @@ def __process_rule_representiations_to_array(self):
                 yara_rule = YaraRule(string_of_rule, plyara_rule)
                 self.yara_rules.append(yara_rule)
 
+    def __replace_for_each_one_to_many(self, line):
+        """
+        Takes a line, transforms it into a list, parses through the list looking for the self.char_to_replace character
+            and replaces each instance found with self.char_replacement * self.count_of_replaced
+        :param line: a line that starts with at least one self.char_to_replace_character
+        :return:
+        """
+        new_list = []
+        character_replace = [self.char_replacement] * self.count_of_replaced
+        line_as_list = list(line)
+        non_white_space_index = 0
+        for index, character in enumerate(line_as_list):
+            if re.match(self.char_to_replace, character):
+                new_list = new_list + character_replace
+            elif re.match(self.char_replacement, character):
+                new_list.append(character)
+            else:
+                non_white_space_index = index
+                break
+
+        new_list = new_list + line_as_list[non_white_space_index:]
+
+        newline = ''.join(new_list)
+        return newline
+
+    def __standardize_white_space(self, edited_rule_string):
+        """
+        Takes the edited_rule_string, scans the start of each line for the self.char_to_replace and passes any line
+            found to start with that character to __replace_for_each_one_to_many
+        :param edited_rule_string: the array of lines
+        :return:
+        """
+        regex_of_char_to_replace = '^' + '[' + self.char_to_replace + self.char_replacement + ']' + '+'
+        for index, line in enumerate(edited_rule_string):
+            if re.match(regex_of_char_to_replace, line):
+                edited_rule_string[index] = self.__replace_for_each_one_to_many(line)
+
     def strings_of_rules_to_original_file(self):
         """
         This rebuilds a rule string incorporating any changes from the rule return objects
@@ -98,6 +139,7 @@ def strings_of_rules_to_original_file(self):
                 edited_rule_string = edited_rule_string[0:rule.rule_plyara['start_line'] - 1]\
                                      + changed_rule_string + edited_rule_string[rule.rule_plyara['stop_line']:]
 
+        self.__standardize_white_space(edited_rule_string)
         edited_rule_string = '\n'.join(edited_rule_string)
         self.edited_rule_string = edited_rule_string
 

yara-validator/yara_validator.py

@@ -30,7 +30,10 @@
 AUTHOR = 'author'
 VALUE = 'value'
 STRING_ENCODING = 'string_encoding'
-
+WHITE_SPACE_REPLACEMENT = 'white_space_replacement'
+CHAR_TO_REPLACE = 'char_to_replace'
+CHAR_REPLACEMENT = 'char_replacement'
+COUNT_OF_REPLACED = 'count_of_replaced'
 
 def check_validator_cfg(validator_cfg):
     """
@@ -42,18 +45,53 @@ def check_validator_cfg(validator_cfg):
     if string_encoding is not None:
         potential_values = set(item.value for item in StringEncoding)
         if string_encoding not in potential_values:
-            print('{!r}: {!r} has an invalid parameter - {!r}'.format(VALIDATOR_CFG, STRING_ENCODING, string_encoding))
+            print('{!r}: {!r} has an invalid parameter - {!r}'.format(VALIDATOR_CFG,
+                                                                      STRING_ENCODING,
+                                                                      string_encoding))
+            exit(1)
+    else:
+        print('{!r}: {!r} has a missing parameter - string_encoding'.format(VALIDATOR_CFG,
+                                                                            STRING_ENCODING))
+        exit(1)
+
+    white_space_replacement_values = validator_cfg.get(WHITE_SPACE_REPLACEMENT).get(VALUE)
+    if white_space_replacement_values is not None:
+        char_to_replace = white_space_replacement_values.get(CHAR_TO_REPLACE).encode('utf-8').decode('unicode_escape')
+        if char_to_replace is None or not re.fullmatch('\s', char_to_replace):
+            print('{!r}: {!r} has an invalid parameter - {!r}'.format(VALIDATOR_CFG,
+                                                                      CHAR_TO_REPLACE,
+                                                                      char_to_replace))
+            exit(1)
+        else:
+            white_space_replacement_values[CHAR_TO_REPLACE] = char_to_replace
+
+        char_replacement = white_space_replacement_values.get(CHAR_REPLACEMENT)\
+                                                            .encode('utf-8').decode('unicode_escape')
+        if char_replacement is None or not re.fullmatch('\s', char_replacement):
+            print('{!r}: {!r} has an invalid parameter - {!r}'.format(VALIDATOR_CFG,
+                                                                      CHAR_REPLACEMENT,
+                                                                      char_replacement))
+            exit(1)
+        else:
+            white_space_replacement_values[CHAR_REPLACEMENT] = char_replacement
+
+        count_of_replaced = white_space_replacement_values.get(COUNT_OF_REPLACED)
+        if count_of_replaced is None or count_of_replaced <= 0:
+            print('{!r}: {!r} has an invalid parameter - {!r}'.format(VALIDATOR_CFG,
+                                                                      COUNT_OF_REPLACED,
+                                                                      count_of_replaced))
             exit(1)
     else:
-        print('{!r}: {!r} has a missing parameter - string_encoding'.format(VALIDATOR_CFG, STRING_ENCODING))
+        print('{!r}: {!r} has a missing parameter - string_encoding'.format(VALIDATOR_CFG,
+                                                                            WHITE_SPACE_REPLACEMENT))
         exit(1)
 
 
 def run_yara_validator(yara_file, generate_values=True):
     """
     This is the base function that should be called to validate a rule. It will take as an argument the file path,
-        create a YaraValidator object, parse that file with plyara and pass that parsed object and the string representation
-        of the yara file to YaraValidator.valadation
+        create a YaraValidator object, parse that file with plyara and pass that parsed object and the string
+        representation of the yara file to YaraValidator.valadation
     :param yara_file: The file variable passed in. Usually a string or Path variable
     :param generate_values: determine if the values the validator can generate should be generated or not, default True
     :return:
@@ -62,7 +100,10 @@ def run_yara_validator(yara_file, generate_values=True):
         validator_configuration = yaml.safe_load(config_file)
 
     check_validator_cfg(validator_configuration)
-    yara_file_processor = YaraFileProcessor(yara_file)
+    char_to_replace = validator_configuration.get(WHITE_SPACE_REPLACEMENT).get(VALUE).get(CHAR_TO_REPLACE)
+    char_replacement = validator_configuration.get(WHITE_SPACE_REPLACEMENT).get(VALUE).get(CHAR_REPLACEMENT)
+    count_of_replaced = validator_configuration.get(WHITE_SPACE_REPLACEMENT).get(VALUE).get(COUNT_OF_REPLACED)
+    yara_file_processor = YaraFileProcessor(yara_file, char_to_replace, char_replacement, count_of_replaced)
 
     # If there are any issues with the yara file read process exit out and return the error
     if yara_file_processor.return_file_error_state():