Add unique signatures generation to general parse results

logspec/errors/error.py

@@ -3,8 +3,8 @@
 # Copyright (C) 2024 Collabora Limited
 # Author: Ricardo Cañuelo <[email protected]>
 
-import hashlib
-import json
+from logspec.utils.utils import generate_signature
+
 
 class Error():
     def __init__(self):
@@ -34,8 +34,8 @@ def parse(self, text):
         return parse_ret
 
     def _generate_signature(self):
-        """Generates a hash string to uniquely identify this error,
-        based on a custom set of error fields.
+        """Uses utils.generate_signature() to generate a unique hash
+        string for this error, based on a custom set of error fields.
 
         This method is meant to be called after the parsing has been
         done.
@@ -48,5 +48,4 @@ def _generate_signature(self):
                     signature_dict[field] = val
             except AttributeError:
                 continue
-        signature_json = json.dumps(signature_dict, sort_keys=True, ensure_ascii=False)
-        self._signature = hashlib.sha1(signature_json.encode('utf-8')).hexdigest()
+        self._signature = generate_signature(signature_dict)

logspec/errors/test.py

@@ -12,3 +12,9 @@ class TestError(Error):
     def __init__(self):
         super().__init__()
         self.error_type = "test"
+
+    def _parse(self, text):
+        """Dummy parse function. The purpose of this is to keep the
+        caller code working if it calls parse() to generate the error
+        signature"""
+        pass

logspec/main.py

@@ -10,6 +10,7 @@
 import logspec.version
 from logspec.parser_loader import parser_loader
 from logspec.utils.defs import JsonSerialize, JsonSerializeDebug
+from logspec.utils.utils import update_dict, generate_signature
 
 
 def format_data_output(data, full=False):
@@ -46,9 +47,28 @@ def parse_log(log, start_state):
       The FSM data (dict) after the parsing is done.
     """
     state = start_state
-    data = {}
+    data = {
+        '_signature_fields': [],
+        '_states_summary': [],
+    }
     cumulative_errors = []
     log_start = 0
+
+    def _generate_signature(data_dict):
+        """Uses utils.generate_signature() to generate and return a
+        unique hash for the list of '_signature_fields' found in
+        data_dict, if any.  The returned signature can be used to
+        uniquely identify the conditions described by those fields.
+
+        Returns None if data_dict doesn't define any signature fields.
+        """
+        signature_dict = {}
+        if not data_dict.get('_signature_fields'):
+            return None
+        for field in data_dict['_signature_fields']:
+            signature_dict[field] = data_dict[field]
+        return generate_signature(signature_dict)
+
     while state:
         # The log fragment to parse is adjusted after every state
         # transition if the state function sets a `match_end' field in
@@ -63,14 +83,20 @@ def parse_log(log, start_state):
         logging.debug(f"State: {state}")
         state_data = state.run(log)
         state = state.transition()
+
+        # Update collected data with the data generated in this state
         if 'errors' in state_data:
             cumulative_errors.extend(state_data['errors'])
-        data.update(state_data)
+        state_summary = state_data.pop('_summary', None)
+        if state_summary:
+            data['_states_summary'].append(state_summary)
+        update_dict(data, state_data)
         if '_match_end' in data:
             log_start += data['_match_end']
             log = log[data['_match_end']:]
             data['_match_end'] = log_start
     data['errors'] = cumulative_errors
+    data['_signature'] = _generate_signature(data)
     return data
 
 

logspec/states/chromebook_boot.py

@@ -39,16 +39,23 @@ def detect_bootloader_start(text, start=None, end=None):
     ]
     if start or end:
         text = text[start:end]
-    data = {}
+    data = {
+        '_signature_fields': [
+            'bootloader.start',
+            'bootloader.id',
+        ],
+    }
     regex = '|'.join(tags)
     match = re.search(regex, text)
     if match:
         data['_match_end'] = match.end()
         data['bootloader.start'] = True
         data['bootloader.id'] = 'depthcharge'
+        data['_summary'] = "Depthcharge started"
     else:
         data['_match_end'] = end if end else len(text)
         data['bootloader.start'] = False
+        data['_summary'] = "Depthcharge start not found"
     return data
 
 

logspec/states/generic_boot.py

@@ -35,15 +35,22 @@ def detect_bootloader_end(text, start=None, end=None):
     ]
     if start or end:
         text = text[start:end]
-    data = {}
+    data = {
+        '_signature_fields': [
+            'bootloader.done',
+        ],
+    }
     regex = '|'.join(tags)
     match = re.search(regex, text)
     if match:
         data['_match_end'] = match.end() + start if start else match.end()
         data['bootloader.done'] = True
+        data['_summary'] = "Bootloader stage done, jump to kernel"
     else:
         data['_match_end'] = end if end else len(text)
         data['bootloader.done'] = False
+        data['_summary'] = ("Bootloader stage failed, inconclusive or "
+                            "couldn't detect handover to kernel")
     return data
 
 

logspec/states/linux_kernel.py

@@ -7,10 +7,22 @@
 from logspec.parser_classes import State
 from logspec.utils.linux_kernel_errors import find_kernel_error
 from logspec.parser_loader import register_state
+from logspec.utils.defs import *
 
 MODULE_NAME = 'linux_kernel'
 
 
+# Utility functions
+def _detect_kernel_start(text):
+    """Checks if the first line of text looks like the output of a Linux
+    kernel starting. Returns a Match object if it does, None if it
+    doesn't.
+    """
+    first_line_end = text.index('\n')
+    return re.match(fr'{LINUX_TIMESTAMP} .*',
+                    text[:first_line_end])
+
+
 # State functions
 
 def detect_linux_prompt(text, start=None, end=None):
@@ -34,15 +46,29 @@ def detect_linux_prompt(text, start=None, end=None):
     ]
     if start or end:
         text = text[start:end]
-    data = {}
+    data = {
+        '_signature_fields': [
+            'linux.boot.prompt',
+            'linux.boot.kernel_started',
+        ],
+    }
     regex = '|'.join(tags)
     match = re.search(regex, text)
     if match:
         data['_match_end'] = match.end() + start if start else match.end()
+        data['linux.boot.kernel_started'] = True
         data['linux.boot.prompt'] = True
+        data['_summary'] = "Linux boot prompt found"
     else:
-        data['_match_end'] = end if end else len(text)
         data['linux.boot.prompt'] = False
+        kernel_first_line_start = text.index('\n') + 1
+        if _detect_kernel_start(text[kernel_first_line_start:]):
+            data['linux.boot.kernel_started'] = True
+            data['_summary'] = "Linux boot prompt not found"
+        else:
+            data['linux.boot.kernel_started'] = False
+            data['_summary'] = "Kernel didn't start"
+        data['_match_end'] = end if end else len(text)
 
     # Check for linux-specific errors in the log. If the `done'
     # condition was found, search only before it. Otherwise search in

logspec/states/test_baseline.py

@@ -19,18 +19,24 @@ def detect_test_baseline(text, start=None, end=None):
     ]
     if start or end:
         text = text[start:end]
-    data = {}
+    data = {
+        '_signature_fields': [
+            'test.baseline.start',
+        ],
+    }
     regex = '|'.join(start_tags)
 
     # Check for test start
     match = re.search(regex, text)
     if not match:
         data['test.baseline.start'] = False
         data['_match_end'] = end if end else len(text)
+        data['_summary'] = "Baseline test not detected"
         return data
     test_start = match.end()
     test_end = None
     data['test.baseline.start'] = True
+    data['_summary'] = "Baseline test started"
 
     # Check for test end
     end_tags = [

logspec/utils/test_baseline_errors.py

@@ -17,6 +17,9 @@ def find_test_baseline_dmesg_error(text):
     error = TestError()
     error.error_type += ".baseline.dmesg"
     error.error_summary = match.group('message')
+    # Parsing on a generic TestError object simply generates a
+    # signature, we already did the parsing above
+    error.parse(text)
     return {
         'error': error,
         '_end': match.end(),

logspec/utils/utils.py

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+# Copyright (C) 2024 Collabora Limited
+# Author: Ricardo Cañuelo <[email protected]>
+
+import hashlib
+import json
+
+def update_dict(dest_dict, new_data):
+    """Updates dest_dict in place with the contents of dict
+    new_data. This is equivalent to dest_dict.update(new_data) except
+    that the nested lists in dest_dict are extended/appended if found in
+    new_data rather than replaced.
+    """
+    for k, v in new_data.items():
+        if k in dest_dict and isinstance(dest_dict[k], list):
+            if isinstance(v, list):
+                dest_dict[k].extend(v)
+            else:
+                dest_dict[k].append(v)
+        else:
+            dest_dict[k] = v
+
+
+def generate_signature(data_dict):
+    """Generates a hash string of the data_dict contents"""
+    signature_json = json.dumps(data_dict, sort_keys=True, ensure_ascii=False)
+    return hashlib.sha1(signature_json.encode('utf-8')).hexdigest()

tests/test_baseline.py

@@ -22,6 +22,7 @@
      {
          "bootloader.done": True,
          "errors": [],
+         "linux.boot.kernel_started": True,
          "linux.boot.prompt": True,
          "test.baseline.start": False,
      }),
@@ -32,6 +33,7 @@
      {
          "bootloader.done": True,
          "errors": [],
+         "linux.boot.kernel_started": True,
          "linux.boot.prompt": True,
          "test.baseline.start": True,
      }),
@@ -55,6 +57,7 @@
                  "error_type": "test.baseline.dmesg",
              }
          ],
+         "linux.boot.kernel_started": True,
          "linux.boot.prompt": True,
          "test.baseline.start": True,
      }),
@@ -146,6 +149,7 @@
                  "error_type": "test.baseline.dmesg",
              }
          ],
+         "linux.boot.kernel_started": True,
          "linux.boot.prompt": True,
          "test.baseline.start": True,
      }),

tests/test_linux_boot.py

@@ -77,6 +77,7 @@
                 "hardware": "BCM2835"
             },
         ],
+        "linux.boot.kernel_started": True,
         "linux.boot.prompt": False,
      }),
 
@@ -86,6 +87,7 @@
      {
          "bootloader.done": True,
          "errors": [],
+         "linux.boot.kernel_started": False,
          "linux.boot.prompt": False,
      }),
 
@@ -95,7 +97,8 @@
      {
         "bootloader.done": True,
         "errors": [],
-         "linux.boot.prompt": True,
+        "linux.boot.kernel_started": True,
+        "linux.boot.prompt": True,
      }),
 
     # Command-line prompt found, multiple errors found (WARNINGs and BUGs)
@@ -274,6 +277,7 @@
                 "modules": []
             }
         ],
+        "linux.boot.kernel_started": True,
         "linux.boot.prompt": True,
     }),
 
@@ -290,6 +294,7 @@
                  "location": "./include/linux/log2.h:57:13"
              }
          ],
+         "linux.boot.kernel_started": True,
          "linux.boot.prompt": True,
      }),
 ])