Allow adapters to include python package logging in dbt logs

.changes/unreleased/Features-20230915-123733.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: 'Allow adapters to include package logs in dbt standard logging '
+time: 2023-09-15T12:37:33.862862-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "7859"

core/dbt/events/adapter_endpoint.py

@@ -1,7 +1,8 @@
 import traceback
 from dataclasses import dataclass
-from dbt.events.functions import fire_event
+from dbt.events.functions import fire_event, EVENT_MANAGER
 from dbt.events.contextvars import get_node_info
+from dbt.events.event_handler import set_package_logging
 from dbt.events.types import (
     AdapterEventDebug,
     AdapterEventInfo,
@@ -56,3 +57,10 @@
             name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info()
         )
         fire_event(event)
+
+    @staticmethod
+    def set_adapter_dependency_log_level(package_name, level):
+        """By default dbt suppresses non-dbt package logs. This method allows
+        you to set the log level for a specific package.
+        """
+        set_package_logging(package_name, level, EVENT_MANAGER)

core/dbt/events/event_handler.py

@@ -0,0 +1,46 @@
+import logging
+from typing import Union
+
+from dbt.events.base_types import EventLevel
+from dbt.events.types import Note
+
+from dbt.events.eventmgr import IEventManager
+
+
+def log_level_to_event_level(log_level: int) -> EventLevel:
+    if log_level == logging.DEBUG:
+        return EventLevel.DEBUG
+    elif log_level == logging.INFO:
+        return EventLevel.INFO
+    elif log_level == logging.WARNING:
+        return EventLevel.WARN
+    elif log_level == logging.ERROR:
+        return EventLevel.ERROR
+    elif log_level == logging.CRITICAL:
+        return EventLevel.ERROR
+    else:
+        return EventLevel.DEBUG
+
+
+class DbtEventLoggingHandler(logging.Handler):
+    """A logging handler that wraps the EventManager
+    This allows non-dbt packages to log to the dbt event stream.
+    All logs are generated as "Note" events.
+    """
+
+    def __init__(self, event_manager: IEventManager, level=logging.NOTSET):
+        super().__init__(level)
+        self.event_manager = event_manager
+
+    def emit(self, record: logging.LogRecord):
+        note = Note(msg=record.getMessage())
+        level = log_level_to_event_level(record.levelno)
+        self.event_manager.fire_event(e=note, level=level)
+
+
+def set_package_logging(package_name: str, log_level: Union[str, int], event_mgr: IEventManager):
+    """Attach dbt's custom logging handler to the package's logger."""
+    log = logging.getLogger(package_name)
+    log.setLevel(log_level)
+    event_handler = DbtEventLoggingHandler(event_manager=event_mgr, level=log_level)
+    log.addHandler(event_handler)

core/dbt/events/functions.py

@@ -95,7 +95,6 @@ def _get_stdout_config(
     level: EventLevel,
     log_cache_events: bool,
 ) -> LoggerConfig:
-
     return LoggerConfig(
         name="stdout_log",
         level=level,
@@ -127,6 +126,7 @@ def _get_logfile_config(
     level: EventLevel,
     log_file_max_bytes: int,
 ) -> LoggerConfig:
+
     return LoggerConfig(
         name="file_log",
         line_format=line_format,

tests/unit/test_event_handler.py

@@ -0,0 +1,27 @@
+import logging
+
+from dbt.events.base_types import EventLevel
+from dbt.events.event_handler import DbtEventLoggingHandler
+from dbt.events.eventmgr import TestEventManager
+
+
+def test_event_logging_handler_emits_records_correctly():
+    event_manager = TestEventManager()
+    handler = DbtEventLoggingHandler(event_manager=event_manager)
+    log = logging.getLogger("test")
+    log.setLevel(logging.DEBUG)
+    log.addHandler(handler)
+
+    log.debug("test")
+    log.info("test")
+    log.warning("test")
+    log.error("test")
+    log.exception("test")
+    log.critical("test")
+    assert len(event_manager.event_history) == 6
+    assert event_manager.event_history[0][1] == EventLevel.DEBUG
+    assert event_manager.event_history[1][1] == EventLevel.INFO
+    assert event_manager.event_history[2][1] == EventLevel.WARN
+    assert event_manager.event_history[3][1] == EventLevel.ERROR
+    assert event_manager.event_history[4][1] == EventLevel.ERROR
+    assert event_manager.event_history[5][1] == EventLevel.ERROR