diff --git a/.changes/unreleased/Features-20240514-162052.yaml b/.changes/unreleased/Features-20240514-162052.yaml new file mode 100644 index 00000000..1503c479 --- /dev/null +++ b/.changes/unreleased/Features-20240514-162052.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support adding callbacks to the event manager +time: 2024-05-14T16:20:52.120336-07:00 +custom: + Author: QMalcolm + Issue: "131" diff --git a/.changes/unreleased/Features-20240515-161106.yaml b/.changes/unreleased/Features-20240515-161106.yaml new file mode 100644 index 00000000..ef4b7afb --- /dev/null +++ b/.changes/unreleased/Features-20240515-161106.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add a print event to support firing event to console in --quiet mode +time: 2024-05-15T16:11:06.815526-07:00 +custom: + Author: ChenyuLInx + Issue: "8756" diff --git a/dbt_common/__about__.py b/dbt_common/__about__.py index 8646006a..b2b60a55 100644 --- a/dbt_common/__about__.py +++ b/dbt_common/__about__.py @@ -1 +1 @@ -version = "1.0.4" +version = "1.1.0" diff --git a/dbt_common/events/base_types.py b/dbt_common/events/base_types.py index 2a90e78f..78b03682 100644 --- a/dbt_common/events/base_types.py +++ b/dbt_common/events/base_types.py @@ -6,14 +6,11 @@ from google.protobuf.json_format import ParseDict, MessageToDict, MessageToJson from google.protobuf.message import Message from dbt_common.events.helpers import get_json_string_utcnow -from typing import Optional +from typing import Callable, Optional from dbt_common.invocation import get_invocation_id -if sys.version_info >= (3, 8): - from typing import Protocol -else: - from typing_extensions import Protocol +from typing import Protocol # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -128,6 +125,9 @@ class EventMsg(Protocol): data: Message +TCallback = Callable[[EventMsg], None] + + def msg_from_base_event(event: BaseEvent, level: Optional[EventLevel] = None): msg_class_name = f"{type(event).__name__}Msg" msg_cls = getattr(event.PROTO_TYPES_MODULE, msg_class_name) diff --git a/dbt_common/events/event_manager.py b/dbt_common/events/event_manager.py index 96e61f6b..507588f3 100644 --- a/dbt_common/events/event_manager.py +++ b/dbt_common/events/event_manager.py @@ -1,15 +1,15 @@ import os import traceback -from typing import Callable, List, Optional, Protocol, Tuple +from typing import List, Optional, Protocol, Tuple -from dbt_common.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg +from dbt_common.events.base_types import BaseEvent, EventLevel, msg_from_base_event, TCallback from dbt_common.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat class EventManager: def __init__(self) -> None: self.loggers: List[_Logger] = [] - self.callbacks: List[Callable[[EventMsg], None]] = [] + self.callbacks: List[TCallback] = [] def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None: msg = msg_from_base_event(e, level=level) @@ -37,13 +37,16 @@ def add_logger(self, config: LoggerConfig) -> None: ) self.loggers.append(logger) + def add_callback(self, callback: TCallback) -> None: + self.callbacks.append(callback) + def flush(self) -> None: for logger in self.loggers: logger.flush() class IEventManager(Protocol): - callbacks: List[Callable[[EventMsg], None]] + callbacks: List[TCallback] loggers: List[_Logger] def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None: @@ -52,6 +55,9 @@ def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None: def add_logger(self, config: LoggerConfig) -> None: ... + def add_callback(self, callback: TCallback) -> None: + ... + class TestEventManager(IEventManager): __test__ = False diff --git a/dbt_common/events/event_manager_client.py b/dbt_common/events/event_manager_client.py index 1b674f6e..538d3199 100644 --- a/dbt_common/events/event_manager_client.py +++ b/dbt_common/events/event_manager_client.py @@ -1,6 +1,7 @@ # Since dbt-rpc does not do its own log setup, and since some events can # currently fire before logs can be configured by setup_event_logger(), we # create a default configuration with default settings and no file output. +from dbt_common.events.base_types import TCallback from dbt_common.events.event_manager import IEventManager, EventManager _EVENT_MANAGER: IEventManager = EventManager() @@ -16,6 +17,11 @@ def add_logger_to_manager(logger) -> None: _EVENT_MANAGER.add_logger(logger) +def add_callback_to_manager(callback: TCallback) -> None: + global _EVENT_MANAGER + _EVENT_MANAGER.add_callback(callback) + + def ctx_set_event_manager(event_manager: IEventManager) -> None: global _EVENT_MANAGER _EVENT_MANAGER = event_manager diff --git a/dbt_common/events/logger.py b/dbt_common/events/logger.py index f7a0a139..fae4d9b4 100644 --- a/dbt_common/events/logger.py +++ b/dbt_common/events/logger.py @@ -13,6 +13,13 @@ from dbt_common.events.format import timestamp_to_datetime_string from dbt_common.utils.encoding import ForgivingJSONEncoder +PRINT_EVENT_NAME = "PrintEvent" + + +def _is_print_event(msg: EventMsg) -> bool: + return msg.info.name == PRINT_EVENT_NAME + + # A Filter is a function which takes a BaseEvent and returns True if the event # should be logged, False otherwise. Filter = Callable[[EventMsg], bool] @@ -120,7 +127,14 @@ def create_line(self, msg: EventMsg) -> str: def write_line(self, msg: EventMsg): line = self.create_line(msg) if self._python_logger is not None: - send_to_logger(self._python_logger, msg.info.level, line) + # We send PrintEvent to logger as error so it goes to stdout + # when --quiet flag is set. + # --quiet flag will filter out all events lower than ERROR. + if _is_print_event(msg): + level = "error" + else: + level = msg.info.level + send_to_logger(self._python_logger, level, line) def flush(self): if self._python_logger is not None: @@ -138,8 +152,11 @@ def create_line(self, msg: EventMsg) -> str: return self.create_debug_line(msg) if self.use_debug_format else self.create_info_line(msg) def create_info_line(self, msg: EventMsg) -> str: - ts: str = datetime.utcnow().strftime("%H:%M:%S") scrubbed_msg: str = self.scrubber(msg.info.msg) # type: ignore + if _is_print_event(msg): + # PrintEvent is a special case, we don't want to add a timestamp + return scrubbed_msg + ts: str = datetime.utcnow().strftime("%H:%M:%S") return f"{self._get_color_tag()}{ts} {scrubbed_msg}" def create_debug_line(self, msg: EventMsg) -> str: diff --git a/dbt_common/events/types.proto b/dbt_common/events/types.proto index ad791315..3ca066d4 100644 --- a/dbt_common/events/types.proto +++ b/dbt_common/events/types.proto @@ -119,3 +119,13 @@ message NoteMsg { EventInfo info = 1; Note data = 2; } + +// Z052 +message PrintEvent { + string msg = 1; +} + +message PrintEventMsg { + EventInfo info = 1; + PrintEvent data = 2; +} diff --git a/dbt_common/events/types.py b/dbt_common/events/types.py index 3c4b02e0..b60f9e14 100644 --- a/dbt_common/events/types.py +++ b/dbt_common/events/types.py @@ -125,3 +125,14 @@ def code(self) -> str: def message(self) -> str: return self.msg + + +class PrintEvent(InfoLevel): + # Use this event to skip any formatting and just print a message + # This event will get to stdout even if the logger is set to ERROR + # This is to support commands that want --quiet option but also log something to stdout + def code(self) -> str: + return "Z052" + + def message(self) -> str: + return self.msg diff --git a/dbt_common/events/types_pb2.py b/dbt_common/events/types_pb2.py index 0bf1c693..2112273b 100644 --- a/dbt_common/events/types_pb2.py +++ b/dbt_common/events/types_pb2.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: types.proto +# Protobuf Python Version: 5.26.1 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -14,55 +15,59 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x0bproto_types\x1a\x1fgoogle/protobuf/timestamp.proto\"\x91\x02\n\tEventInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0b\n\x03msg\x18\x03 \x01(\t\x12\r\n\x05level\x18\x04 \x01(\t\x12\x15\n\rinvocation_id\x18\x05 \x01(\t\x12\x0b\n\x03pid\x18\x06 \x01(\x05\x12\x0e\n\x06thread\x18\x07 \x01(\t\x12&\n\x02ts\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x05\x65xtra\x18\t \x03(\x0b\x32!.proto_types.EventInfo.ExtraEntry\x12\x10\n\x08\x63\x61tegory\x18\n \x01(\t\x1a,\n\nExtraEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"6\n\x0eGenericMessage\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\"1\n\x11RetryExternalCall\x12\x0f\n\x07\x61ttempt\x18\x01 \x01(\x05\x12\x0b\n\x03max\x18\x02 \x01(\x05\"j\n\x14RetryExternalCallMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.RetryExternalCall\"#\n\x14RecordRetryException\x12\x0b\n\x03\x65xc\x18\x01 \x01(\t\"p\n\x17RecordRetryExceptionMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12/\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32!.proto_types.RecordRetryException\"@\n\x13SystemCouldNotWrite\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x0b\n\x03\x65xc\x18\x03 \x01(\t\"n\n\x16SystemCouldNotWriteMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.SystemCouldNotWrite\"!\n\x12SystemExecutingCmd\x12\x0b\n\x03\x63md\x18\x01 \x03(\t\"l\n\x15SystemExecutingCmdMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.SystemExecutingCmd\"\x1c\n\x0cSystemStdOut\x12\x0c\n\x04\x62msg\x18\x01 \x01(\t\"`\n\x0fSystemStdOutMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\'\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x19.proto_types.SystemStdOut\"\x1c\n\x0cSystemStdErr\x12\x0c\n\x04\x62msg\x18\x01 \x01(\t\"`\n\x0fSystemStdErrMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\'\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x19.proto_types.SystemStdErr\",\n\x16SystemReportReturnCode\x12\x12\n\nreturncode\x18\x01 \x01(\x05\"t\n\x19SystemReportReturnCodeMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\x31\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32#.proto_types.SystemReportReturnCode\"\x19\n\nFormatting\x12\x0b\n\x03msg\x18\x01 \x01(\t\"\\\n\rFormattingMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.proto_types.Formatting\"\x13\n\x04Note\x12\x0b\n\x03msg\x18\x01 \x01(\t\"P\n\x07NoteMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\x1f\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x11.proto_types.Noteb\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'types_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x0bproto_types\x1a\x1fgoogle/protobuf/timestamp.proto\"\x91\x02\n\tEventInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0b\n\x03msg\x18\x03 \x01(\t\x12\r\n\x05level\x18\x04 \x01(\t\x12\x15\n\rinvocation_id\x18\x05 \x01(\t\x12\x0b\n\x03pid\x18\x06 \x01(\x05\x12\x0e\n\x06thread\x18\x07 \x01(\t\x12&\n\x02ts\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x05\x65xtra\x18\t \x03(\x0b\x32!.proto_types.EventInfo.ExtraEntry\x12\x10\n\x08\x63\x61tegory\x18\n \x01(\t\x1a,\n\nExtraEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"6\n\x0eGenericMessage\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\"1\n\x11RetryExternalCall\x12\x0f\n\x07\x61ttempt\x18\x01 \x01(\x05\x12\x0b\n\x03max\x18\x02 \x01(\x05\"j\n\x14RetryExternalCallMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.RetryExternalCall\"#\n\x14RecordRetryException\x12\x0b\n\x03\x65xc\x18\x01 \x01(\t\"p\n\x17RecordRetryExceptionMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12/\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32!.proto_types.RecordRetryException\"@\n\x13SystemCouldNotWrite\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x0b\n\x03\x65xc\x18\x03 \x01(\t\"n\n\x16SystemCouldNotWriteMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.SystemCouldNotWrite\"!\n\x12SystemExecutingCmd\x12\x0b\n\x03\x63md\x18\x01 \x03(\t\"l\n\x15SystemExecutingCmdMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.SystemExecutingCmd\"\x1c\n\x0cSystemStdOut\x12\x0c\n\x04\x62msg\x18\x01 \x01(\t\"`\n\x0fSystemStdOutMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\'\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x19.proto_types.SystemStdOut\"\x1c\n\x0cSystemStdErr\x12\x0c\n\x04\x62msg\x18\x01 \x01(\t\"`\n\x0fSystemStdErrMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\'\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x19.proto_types.SystemStdErr\",\n\x16SystemReportReturnCode\x12\x12\n\nreturncode\x18\x01 \x01(\x05\"t\n\x19SystemReportReturnCodeMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\x31\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32#.proto_types.SystemReportReturnCode\"\x19\n\nFormatting\x12\x0b\n\x03msg\x18\x01 \x01(\t\"\\\n\rFormattingMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.proto_types.Formatting\"\x13\n\x04Note\x12\x0b\n\x03msg\x18\x01 \x01(\t\"P\n\x07NoteMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12\x1f\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x11.proto_types.Note\"\x19\n\nPrintEvent\x12\x0b\n\x03msg\x18\x01 \x01(\t\"\\\n\rPrintEventMsg\x12$\n\x04info\x18\x01 \x01(\x0b\x32\x16.proto_types.EventInfo\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.proto_types.PrintEventb\x06proto3') - DESCRIPTOR._options = None - _EVENTINFO_EXTRAENTRY._options = None - _EVENTINFO_EXTRAENTRY._serialized_options = b'8\001' - _EVENTINFO._serialized_start=62 - _EVENTINFO._serialized_end=335 - _EVENTINFO_EXTRAENTRY._serialized_start=291 - _EVENTINFO_EXTRAENTRY._serialized_end=335 - _GENERICMESSAGE._serialized_start=337 - _GENERICMESSAGE._serialized_end=391 - _RETRYEXTERNALCALL._serialized_start=393 - _RETRYEXTERNALCALL._serialized_end=442 - _RETRYEXTERNALCALLMSG._serialized_start=444 - _RETRYEXTERNALCALLMSG._serialized_end=550 - _RECORDRETRYEXCEPTION._serialized_start=552 - _RECORDRETRYEXCEPTION._serialized_end=587 - _RECORDRETRYEXCEPTIONMSG._serialized_start=589 - _RECORDRETRYEXCEPTIONMSG._serialized_end=701 - _SYSTEMCOULDNOTWRITE._serialized_start=703 - _SYSTEMCOULDNOTWRITE._serialized_end=767 - _SYSTEMCOULDNOTWRITEMSG._serialized_start=769 - _SYSTEMCOULDNOTWRITEMSG._serialized_end=879 - _SYSTEMEXECUTINGCMD._serialized_start=881 - _SYSTEMEXECUTINGCMD._serialized_end=914 - _SYSTEMEXECUTINGCMDMSG._serialized_start=916 - _SYSTEMEXECUTINGCMDMSG._serialized_end=1024 - _SYSTEMSTDOUT._serialized_start=1026 - _SYSTEMSTDOUT._serialized_end=1054 - _SYSTEMSTDOUTMSG._serialized_start=1056 - _SYSTEMSTDOUTMSG._serialized_end=1152 - _SYSTEMSTDERR._serialized_start=1154 - _SYSTEMSTDERR._serialized_end=1182 - _SYSTEMSTDERRMSG._serialized_start=1184 - _SYSTEMSTDERRMSG._serialized_end=1280 - _SYSTEMREPORTRETURNCODE._serialized_start=1282 - _SYSTEMREPORTRETURNCODE._serialized_end=1326 - _SYSTEMREPORTRETURNCODEMSG._serialized_start=1328 - _SYSTEMREPORTRETURNCODEMSG._serialized_end=1444 - _FORMATTING._serialized_start=1446 - _FORMATTING._serialized_end=1471 - _FORMATTINGMSG._serialized_start=1473 - _FORMATTINGMSG._serialized_end=1565 - _NOTE._serialized_start=1567 - _NOTE._serialized_end=1586 - _NOTEMSG._serialized_start=1588 - _NOTEMSG._serialized_end=1668 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'types_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_EVENTINFO_EXTRAENTRY']._loaded_options = None + _globals['_EVENTINFO_EXTRAENTRY']._serialized_options = b'8\001' + _globals['_EVENTINFO']._serialized_start=62 + _globals['_EVENTINFO']._serialized_end=335 + _globals['_EVENTINFO_EXTRAENTRY']._serialized_start=291 + _globals['_EVENTINFO_EXTRAENTRY']._serialized_end=335 + _globals['_GENERICMESSAGE']._serialized_start=337 + _globals['_GENERICMESSAGE']._serialized_end=391 + _globals['_RETRYEXTERNALCALL']._serialized_start=393 + _globals['_RETRYEXTERNALCALL']._serialized_end=442 + _globals['_RETRYEXTERNALCALLMSG']._serialized_start=444 + _globals['_RETRYEXTERNALCALLMSG']._serialized_end=550 + _globals['_RECORDRETRYEXCEPTION']._serialized_start=552 + _globals['_RECORDRETRYEXCEPTION']._serialized_end=587 + _globals['_RECORDRETRYEXCEPTIONMSG']._serialized_start=589 + _globals['_RECORDRETRYEXCEPTIONMSG']._serialized_end=701 + _globals['_SYSTEMCOULDNOTWRITE']._serialized_start=703 + _globals['_SYSTEMCOULDNOTWRITE']._serialized_end=767 + _globals['_SYSTEMCOULDNOTWRITEMSG']._serialized_start=769 + _globals['_SYSTEMCOULDNOTWRITEMSG']._serialized_end=879 + _globals['_SYSTEMEXECUTINGCMD']._serialized_start=881 + _globals['_SYSTEMEXECUTINGCMD']._serialized_end=914 + _globals['_SYSTEMEXECUTINGCMDMSG']._serialized_start=916 + _globals['_SYSTEMEXECUTINGCMDMSG']._serialized_end=1024 + _globals['_SYSTEMSTDOUT']._serialized_start=1026 + _globals['_SYSTEMSTDOUT']._serialized_end=1054 + _globals['_SYSTEMSTDOUTMSG']._serialized_start=1056 + _globals['_SYSTEMSTDOUTMSG']._serialized_end=1152 + _globals['_SYSTEMSTDERR']._serialized_start=1154 + _globals['_SYSTEMSTDERR']._serialized_end=1182 + _globals['_SYSTEMSTDERRMSG']._serialized_start=1184 + _globals['_SYSTEMSTDERRMSG']._serialized_end=1280 + _globals['_SYSTEMREPORTRETURNCODE']._serialized_start=1282 + _globals['_SYSTEMREPORTRETURNCODE']._serialized_end=1326 + _globals['_SYSTEMREPORTRETURNCODEMSG']._serialized_start=1328 + _globals['_SYSTEMREPORTRETURNCODEMSG']._serialized_end=1444 + _globals['_FORMATTING']._serialized_start=1446 + _globals['_FORMATTING']._serialized_end=1471 + _globals['_FORMATTINGMSG']._serialized_start=1473 + _globals['_FORMATTINGMSG']._serialized_end=1565 + _globals['_NOTE']._serialized_start=1567 + _globals['_NOTE']._serialized_end=1586 + _globals['_NOTEMSG']._serialized_start=1588 + _globals['_NOTEMSG']._serialized_end=1668 + _globals['_PRINTEVENT']._serialized_start=1670 + _globals['_PRINTEVENT']._serialized_end=1695 + _globals['_PRINTEVENTMSG']._serialized_start=1697 + _globals['_PRINTEVENTMSG']._serialized_end=1789 # @@protoc_insertion_point(module_scope) diff --git a/pyproject.toml b/pyproject.toml index 93524a2c..c1f4f281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ lint = [ ] test = [ "pytest>=7.3,<8.0", + "pytest-mock", "pytest-xdist>=3.2,<4.0", "pytest-cov>=4.1,<5.0", "hypothesis>=6.87,<7.0", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/events/test_logger.py b/tests/unit/events/test_logger.py new file mode 100644 index 00000000..9e8d9321 --- /dev/null +++ b/tests/unit/events/test_logger.py @@ -0,0 +1,40 @@ +import json +from dbt_common.events.logger import LoggerConfig, _TextLogger, _JsonLogger +from dbt_common.events.base_types import EventLevel, msg_from_base_event +from dbt_common.events.types import PrintEvent + + +def test_create_print_line(): + # No format, still fired even when error is the level + config = LoggerConfig(name="test_logger", level=EventLevel.ERROR) + logger = _TextLogger(config) + msg = msg_from_base_event(PrintEvent(msg="This is a print event")) + expected_line = "This is a print event" + actual_line = logger.create_line(msg) + assert actual_line == expected_line + + +def test_create_print_json(): + # JSON format still have event level being info + config = LoggerConfig(name="test_logger", level=EventLevel.ERROR) + logger = _JsonLogger(config) + msg = msg_from_base_event(PrintEvent(msg="This is a print event")) + expected_json = { + "data": {"msg": "This is a print event"}, + "info": { + "category": "", + "code": "Z052", + "extra": {}, + "level": "info", + "msg": "This is a print event", + "name": "PrintEvent", + "thread": "MainThread", + }, + } + actual_line = logger.create_line(msg) + actual_json = json.loads(actual_line) + assert "info" in actual_json + actual_json["info"].pop("invocation_id") + actual_json["info"].pop("ts") + actual_json["info"].pop("pid") + assert actual_json == expected_json diff --git a/tests/unit/test_event_manager.py b/tests/unit/test_event_manager.py new file mode 100644 index 00000000..3728b6bd --- /dev/null +++ b/tests/unit/test_event_manager.py @@ -0,0 +1,11 @@ +from dbt_common.events.event_manager import EventManager +from tests.unit.utils import EventCatcher + + +class TestEventManager: + def test_add_callback(self) -> None: + event_manager = EventManager() + assert len(event_manager.callbacks) == 0 + + event_manager.add_callback(EventCatcher().catch) + assert len(event_manager.callbacks) == 1 diff --git a/tests/unit/test_event_manager_client.py b/tests/unit/test_event_manager_client.py new file mode 100644 index 00000000..bd73d61a --- /dev/null +++ b/tests/unit/test_event_manager_client.py @@ -0,0 +1,15 @@ +from pytest_mock import MockerFixture + +from dbt_common.events.event_manager import EventManager +from dbt_common.events.event_manager_client import add_callback_to_manager, get_event_manager +from tests.unit.utils import EventCatcher + + +def test_add_callback_to_manager(mocker: MockerFixture) -> None: + # mock out the global event manager so the callback doesn't get added to all other tests + mocker.patch("dbt_common.events.event_manager_client._EVENT_MANAGER", EventManager()) + manager = get_event_manager() + assert len(manager.callbacks) == 0 + + add_callback_to_manager(EventCatcher().catch) + assert len(manager.callbacks) == 1 diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 3484cd97..800790d8 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -27,7 +27,7 @@ def get_all_subclasses(cls): InfoLevel, ErrorLevel, DynamicLevel, - ] and not subclass.__module__.startswith("test_"): + ] and not subclass.__module__.startswith("tests."): all_subclasses.append(subclass) all_subclasses.extend(get_all_subclasses(subclass)) return set(all_subclasses) @@ -72,6 +72,7 @@ class TestEventJSONSerialization: types.SystemReportReturnCode(returncode=0), types.Formatting(), types.Note(msg="This is a note."), + types.PrintEvent(msg="This is a print event."), ] def test_all_serializable(self): diff --git a/tests/unit/test_functions.py b/tests/unit/test_functions.py index fa240d1d..372b2bda 100644 --- a/tests/unit/test_functions.py +++ b/tests/unit/test_functions.py @@ -1,13 +1,13 @@ import pytest -from dataclasses import dataclass, field from dbt_common.events import functions -from dbt_common.events.base_types import EventLevel, EventMsg, WarnLevel +from dbt_common.events.base_types import EventLevel, WarnLevel from dbt_common.events.event_manager import EventManager from dbt_common.events.event_manager_client import ctx_set_event_manager from dbt_common.exceptions import EventCompilationError from dbt_common.helper_types import WarnErrorOptions -from typing import List, Set +from tests.unit.utils import EventCatcher +from typing import Set # Re-implementing `Note` event as a warn event for @@ -20,14 +20,6 @@ def message(self) -> str: return self.msg -@dataclass -class EventCatcher: - caught_events: List[EventMsg] = field(default_factory=list) - - def catch(self, event: EventMsg) -> None: - self.caught_events.append(event) - - @pytest.fixture(scope="function") def event_catcher() -> EventCatcher: return EventCatcher() diff --git a/tests/unit/test_semver.py b/tests/unit/test_semver.py new file mode 100644 index 00000000..ae48e592 --- /dev/null +++ b/tests/unit/test_semver.py @@ -0,0 +1,298 @@ +import itertools +import unittest +from typing import List + +from dbt_common.exceptions import VersionsNotCompatibleError +from dbt_common.semver import ( + UnboundedVersionSpecifier, + VersionRange, + VersionSpecifier, + filter_installable, + reduce_versions, + resolve_to_specific_version, + versions_compatible, +) + + +def semver_regex_versioning(versions: List[str]) -> bool: + for version_string in versions: + try: + VersionSpecifier.from_version_string(version_string) + except Exception: + return False + return True + + +def create_range(start_version_string, end_version_string): + start = UnboundedVersionSpecifier() + end = UnboundedVersionSpecifier() + + if start_version_string is not None: + start = VersionSpecifier.from_version_string(start_version_string) + + if end_version_string is not None: + end = VersionSpecifier.from_version_string(end_version_string) + + return VersionRange(start=start, end=end) + + +class TestSemver(unittest.TestCase): + def assertVersionSetResult(self, inputs, output_range): + expected = create_range(*output_range) + + for permutation in itertools.permutations(inputs): + self.assertEqual(reduce_versions(*permutation), expected) + + def assertInvalidVersionSet(self, inputs): + for permutation in itertools.permutations(inputs): + with self.assertRaises(VersionsNotCompatibleError): + reduce_versions(*permutation) + + def test__versions_compatible(self): + self.assertTrue(versions_compatible("0.0.1", "0.0.1")) + self.assertFalse(versions_compatible("0.0.1", "0.0.2")) + self.assertTrue(versions_compatible(">0.0.1", "0.0.2")) + self.assertFalse(versions_compatible("0.4.5a1", "0.4.5a2")) + + def test__semver_regex_versions(self): + self.assertTrue( + semver_regex_versioning( + [ + "0.0.4", + "1.2.3", + "10.20.30", + "1.1.2-prerelease+meta", + "1.1.2+meta", + "1.1.2+meta-valid", + "1.0.0-alpha", + "1.0.0-beta", + "1.0.0-alpha.beta", + "1.0.0-alpha.beta.1", + "1.0.0-alpha.1", + "1.0.0-alpha0.valid", + "1.0.0-alpha.0valid", + "1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay", + "1.0.0-rc.1+build.1", + "2.0.0-rc.1+build.123", + "1.2.3-beta", + "10.2.3-DEV-SNAPSHOT", + "1.2.3-SNAPSHOT-123", + "1.0.0", + "2.0.0", + "1.1.7", + "2.0.0+build.1848", + "2.0.1-alpha.1227", + "1.0.0-alpha+beta", + "1.2.3----RC-SNAPSHOT.12.9.1--.12+788", + "1.2.3----R-S.12.9.1--.12+meta", + "1.2.3----RC-SNAPSHOT.12.9.1--.12", + "1.0.0+0.build.1-rc.10000aaa-kk-0.1", + "99999999999999999999999.999999999999999999.99999999999999999", + "1.0.0-0A.is.legal", + ] + ) + ) + + self.assertFalse( + semver_regex_versioning( + [ + "1", + "1.2", + "1.2.3-0123", + "1.2.3-0123.0123", + "1.1.2+.123", + "+invalid", + "-invalid", + "-invalid+invalid", + "-invalid.01", + "alpha", + "alpha.beta", + "alpha.beta.1", + "alpha.1", + "alpha+beta", + "alpha_beta", + "alpha.", + "alpha..", + "beta", + "1.0.0-alpha_beta", + "-alpha.", + "1.0.0-alpha..", + "1.0.0-alpha..1", + "1.0.0-alpha...1", + "1.0.0-alpha....1", + "1.0.0-alpha.....1", + "1.0.0-alpha......1", + "1.0.0-alpha.......1", + "01.1.1", + "1.01.1", + "1.1.01", + "1.2", + "1.2.3.DEV", + "1.2-SNAPSHOT", + "1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788", + "1.2-RC-SNAPSHOT", + "-1.0.3-gamma+b7718", + "+justmeta", + "9.8.7+meta+meta", + "9.8.7-whatever+meta+meta", + "99999999999999999999999.999999999999999999.99999999999999999----RC-SNAPSHOT.12.09.1--------------------------------..12", + ] + ) + ) + + def test__reduce_versions(self): + self.assertVersionSetResult(["0.0.1", "0.0.1"], ["=0.0.1", "=0.0.1"]) + + self.assertVersionSetResult(["0.0.1"], ["=0.0.1", "=0.0.1"]) + + self.assertVersionSetResult([">0.0.1"], [">0.0.1", None]) + + self.assertVersionSetResult(["<0.0.1"], [None, "<0.0.1"]) + + self.assertVersionSetResult([">0.0.1", "0.0.2"], ["=0.0.2", "=0.0.2"]) + + self.assertVersionSetResult(["0.0.2", ">=0.0.2"], ["=0.0.2", "=0.0.2"]) + + self.assertVersionSetResult([">0.0.1", ">0.0.2", ">0.0.3"], [">0.0.3", None]) + + self.assertVersionSetResult([">0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"]) + + self.assertVersionSetResult([">0.0.1", "0.0.2", "<0.0.3"], ["=0.0.2", "=0.0.2"]) + + self.assertVersionSetResult([">0.0.1", ">=0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"]) + + self.assertVersionSetResult([">0.0.1", "<0.0.3", "<=0.0.3"], [">0.0.1", "<0.0.3"]) + + self.assertVersionSetResult([">0.0.1", ">0.0.2", "<0.0.3", "<0.0.4"], [">0.0.2", "<0.0.3"]) + + self.assertVersionSetResult(["<=0.0.3", ">=0.0.3"], [">=0.0.3", "<=0.0.3"]) + + self.assertInvalidVersionSet([">0.0.2", "0.0.1"]) + self.assertInvalidVersionSet([">0.0.2", "0.0.2"]) + self.assertInvalidVersionSet(["<0.0.2", "0.0.2"]) + self.assertInvalidVersionSet(["<0.0.2", ">0.0.3"]) + self.assertInvalidVersionSet(["<=0.0.3", ">0.0.3"]) + self.assertInvalidVersionSet(["<0.0.3", ">=0.0.3"]) + self.assertInvalidVersionSet(["<0.0.3", ">0.0.3"]) + + def test__resolve_to_specific_version(self): + self.assertEqual( + resolve_to_specific_version(create_range(">0.0.1", None), ["0.0.1", "0.0.2"]), "0.0.2" + ) + + self.assertEqual( + resolve_to_specific_version(create_range(">=0.0.2", None), ["0.0.1", "0.0.2"]), "0.0.2" + ) + + self.assertEqual( + resolve_to_specific_version(create_range(">=0.0.3", None), ["0.0.1", "0.0.2"]), None + ) + + self.assertEqual( + resolve_to_specific_version( + create_range(">=0.0.3", "<0.0.5"), ["0.0.3", "0.0.4", "0.0.5"] + ), + "0.0.4", + ) + + self.assertEqual( + resolve_to_specific_version( + create_range(None, "<=0.0.5"), ["0.0.3", "0.1.4", "0.0.5"] + ), + "0.0.5", + ) + + self.assertEqual( + resolve_to_specific_version( + create_range("=0.4.5a2", "=0.4.5a2"), ["0.4.5a1", "0.4.5a2"] + ), + "0.4.5a2", + ) + + self.assertEqual( + resolve_to_specific_version(create_range("=0.7.6", "=0.7.6"), ["0.7.6-b1", "0.7.6"]), + "0.7.6", + ) + + self.assertEqual( + resolve_to_specific_version( + create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"] + ), + "1.2.0a1", + ) + + self.assertEqual( + resolve_to_specific_version( + create_range(">=1.0.0", "<1.2.0"), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"] + ), + "1.1.0", + ) + + self.assertEqual( + resolve_to_specific_version( + create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"] + ), + "1.2.0", + ) + + self.assertEqual( + resolve_to_specific_version( + create_range(">=1.0.0", "<1.2.0"), + ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"], + ), + "1.1.0", + ) + + self.assertEqual( + resolve_to_specific_version( + # https://github.com/dbt-labs/dbt-core/issues/7039 + # 10 is greater than 9 + create_range(">0.9.0", "<0.10.0"), + ["0.9.0", "0.9.1", "0.10.0"], + ), + "0.9.1", + ) + + def test__filter_installable(self): + installable = filter_installable( + [ + "1.1.0", + "1.2.0a1", + "1.0.0", + "2.1.0-alpha", + "2.2.0asdf", + "2.1.0", + "2.2.0", + "2.2.0-fishtown-beta", + "2.2.0-2", + ], + install_prerelease=True, + ) + expected = [ + "1.0.0", + "1.1.0", + "1.2.0a1", + "2.1.0-alpha", + "2.1.0", + "2.2.0-2", + "2.2.0asdf", + "2.2.0-fishtown-beta", + "2.2.0", + ] + assert installable == expected + + installable = filter_installable( + [ + "1.1.0", + "1.2.0a1", + "1.0.0", + "2.1.0-alpha", + "2.2.0asdf", + "2.1.0", + "2.2.0", + "2.2.0-fishtown-beta", + ], + install_prerelease=False, + ) + expected = ["1.0.0", "1.1.0", "2.1.0", "2.2.0"] + assert installable == expected diff --git a/tests/unit/utils.py b/tests/unit/utils.py new file mode 100644 index 00000000..c79ecc54 --- /dev/null +++ b/tests/unit/utils.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass, field +from typing import List + +from dbt_common.events.base_types import EventMsg + + +@dataclass +class EventCatcher: + caught_events: List[EventMsg] = field(default_factory=list) + + def catch(self, event: EventMsg) -> None: + self.caught_events.append(event)