From e779d700f3505bdf3ad219afc9f5b556b8153676 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 13:35:59 -0700 Subject: [PATCH 001/347] First pass at an execution mode config --- morpheus/config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/morpheus/config.py b/morpheus/config.py index 074d1515aa..a665b41c9b 100644 --- a/morpheus/config.py +++ b/morpheus/config.py @@ -140,6 +140,11 @@ class PipelineModes(str, Enum): AE = "AE" +class ExecutionMode(str, Enum): + GPU = "GPU" + CPU = "CPU" + + class CppConfig: """ Allows setting whether C++ implementations should be used for Morpheus stages and messages. Defaults to True, @@ -200,6 +205,9 @@ class Config(ConfigBase): File corresponding to this Config. """ + # TODO: Store this as __execution_mode or move it to the CppConfig class + execution_mode: ExecutionMode = ExecutionMode.GPU + # Whether in Debug mode. debug: bool = False log_level: int = logging.WARN @@ -220,6 +228,10 @@ class Config(ConfigBase): ae: ConfigAutoEncoder = dataclasses.field(default=None) fil: ConfigFIL = dataclasses.field(default=None) + def __post_init__(self): + if self.execution_mode is ExecutionMode.CPU: + CppConfig.set_should_use_cpp(False) + def save(self, filename: str): """ Save Config to file. From 8c344f3422027c6044a82fe333820f46ed41f54c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 13:36:31 -0700 Subject: [PATCH 002/347] Sketching out execution modes in the register_stage wrapper --- morpheus/cli/register_stage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/morpheus/cli/register_stage.py b/morpheus/cli/register_stage.py index f1b47e67a9..b472200a6b 100644 --- a/morpheus/cli/register_stage.py +++ b/morpheus/cli/register_stage.py @@ -34,6 +34,7 @@ from morpheus.cli.utils import parse_enum from morpheus.cli.utils import prepare_command from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.utils.type_utils import _DecoratorType from morpheus.utils.type_utils import get_full_qualname @@ -248,6 +249,7 @@ def compute_option_name(stage_arg_name: str, rename_options: typing.Dict[str, st def register_stage(command_name: str = None, modes: typing.Sequence[PipelineModes] = None, + execute_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, ), ignore_args: typing.List[str] = None, command_args: dict = None, option_args: typing.Dict[str, dict] = None, From 288c9ab6b114d20b0b5ece5d852cb8df0bebfc68 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 13:38:55 -0700 Subject: [PATCH 003/347] Raise on incompatible execution modes --- morpheus/pipeline/pipeline.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/morpheus/pipeline/pipeline.py b/morpheus/pipeline/pipeline.py index 6f719e4d54..1377eeb64a 100644 --- a/morpheus/pipeline/pipeline.py +++ b/morpheus/pipeline/pipeline.py @@ -30,6 +30,8 @@ import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import from morpheus.config import Config +from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.utils.type_utils import pretty_print_type_name logger = logging.getLogger(__name__) @@ -60,6 +62,8 @@ class Pipeline(): """ def __init__(self, config: Config): + if config.execution_mode is ExecutionMode.CPU and CppConfig.get_should_use_cpp(): + raise RuntimeError("C++ mode requires GPU execution mode.") self._mutex = threading.RLock() From 095f35851ff7c920107382fac492a5da67a5836a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 13:39:11 -0700 Subject: [PATCH 004/347] Don't import cudf --- morpheus/utils/type_aliases.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/morpheus/utils/type_aliases.py b/morpheus/utils/type_aliases.py index cd394664e6..25b6f16b0b 100644 --- a/morpheus/utils/type_aliases.py +++ b/morpheus/utils/type_aliases.py @@ -17,7 +17,8 @@ import pandas as pd -import cudf +if typing.TYPE_CHECKING: + import cudf -DataFrameType = typing.Union[pd.DataFrame, cudf.DataFrame] -SeriesType = typing.Union[pd.Series, cudf.Series] +DataFrameType = typing.Union[pd.DataFrame, "cudf.DataFrame"] +SeriesType = typing.Union[pd.Series, "cudf.Series"] From d495ea3567e5ba92277ad7e7db6d380260e57784 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 14:05:50 -0700 Subject: [PATCH 005/347] WIP --- morpheus/io/deserializers.py | 14 ++++++++++---- morpheus/io/serializers.py | 10 +++++----- morpheus/io/utils.py | 10 +++++++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/morpheus/io/deserializers.py b/morpheus/io/deserializers.py index 31499b4359..42549ff66c 100644 --- a/morpheus/io/deserializers.py +++ b/morpheus/io/deserializers.py @@ -17,9 +17,11 @@ import io import typing +import numpy as np import pandas as pd -import cudf +if typing.TYPE_CHECKING: + import cudf from morpheus.common import FileTypes from morpheus.common import determine_file_type @@ -60,16 +62,20 @@ def _read_file_to_df_py(*, # Update with any args set by the user. User values overwrite defaults kwargs.update(parser_kwargs) - df_class = cudf if df_type == "cudf" else pd + if df_type == "cudf": + import cudf + df_class = cudf + else: + df_class = pd df = None if (mode == FileTypes.JSON): df = df_class.read_json(file_name, **kwargs) elif (mode == FileTypes.CSV): - df: pd.DataFrame = df_class.read_csv(file_name, **kwargs) + df: DataFrameType = df_class.read_csv(file_name, **kwargs) - if (len(df.columns) > 1 and df.columns[0] == "Unnamed: 0" and df.iloc[:, 0].dtype == cudf.dtype(int)): + if (len(df.columns) > 1 and df.columns[0] == "Unnamed: 0" and df.iloc[:, 0].dtype == np.dtype(int)): df.set_index("Unnamed: 0", drop=True, inplace=True) df.index.name = "" df.sort_index(inplace=True) diff --git a/morpheus/io/serializers.py b/morpheus/io/serializers.py index 8672383f9e..e67318b8ef 100644 --- a/morpheus/io/serializers.py +++ b/morpheus/io/serializers.py @@ -19,8 +19,6 @@ from io import IOBase from io import StringIO -import cudf - from morpheus.common import FileTypes from morpheus.common import determine_file_type from morpheus.common import write_df_to_file as write_df_to_file_cpp @@ -203,9 +201,11 @@ def write_df_to_file(df: DataFrameType, file_name: str, file_type: FileTypes = F Additional arguments forwarded to the underlying serialization function. Where the underlying serialization function is one of `write_df_to_file_cpp`, `df_to_stream_csv`, or `df_to_stream_json`. """ - if (CppConfig.get_should_use_cpp() and isinstance(df, cudf.DataFrame)): - # Use the C++ implementation - write_df_to_file_cpp(df=df, filename=file_name, file_type=file_type, **kwargs) + if (CppConfig.get_should_use_cpp()): + import cudf + if (isinstance(df, cudf.DataFrame)): + # Use the C++ implementation + return write_df_to_file_cpp(df=df, filename=file_name, file_type=file_type, **kwargs) mode = file_type diff --git a/morpheus/io/utils.py b/morpheus/io/utils.py index 9a20afb4d5..6aeabab3e8 100644 --- a/morpheus/io/utils.py +++ b/morpheus/io/utils.py @@ -15,10 +15,12 @@ """IO utilities.""" import logging +import typing import pandas as pd -import cudf +if typing.TYPE_CHECKING: + import cudf from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import SeriesType @@ -44,7 +46,7 @@ def filter_null_data(x: DataFrameType, column_name: str = "data") -> DataFrameTy return x[~x[column_name].isna()] -def cudf_string_cols_exceed_max_bytes(df: cudf.DataFrame, column_max_bytes: dict[str, int]) -> bool: +def cudf_string_cols_exceed_max_bytes(df: "cudf.DataFrame", column_max_bytes: dict[str, int]) -> bool: """ Checks a cudf DataFrame for string columns that exceed a maximum number of bytes and thus need to be truncated by calling `truncate_string_cols_by_bytes`. @@ -64,6 +66,7 @@ def cudf_string_cols_exceed_max_bytes(df: cudf.DataFrame, column_max_bytes: dict bool True if truncation is needed, False otherwise. """ + import cudf if not isinstance(df, cudf.DataFrame): raise ValueError("Expected cudf DataFrame") @@ -101,7 +104,7 @@ def truncate_string_cols_by_bytes(df: DataFrameType, """ performed_truncation = False - is_cudf = isinstance(df, cudf.DataFrame) + is_cudf = not isinstance(df, pd.DataFrame) for (col, max_bytes) in column_max_bytes.items(): series: SeriesType = df[col] @@ -124,6 +127,7 @@ def truncate_string_cols_by_bytes(df: DataFrameType, decoded_series = truncated_series.str.decode(encoding='utf-8', errors='ignore') if is_cudf: + import cudf df[col] = cudf.Series.from_pandas(decoded_series) else: df[col] = decoded_series From e99a96c332cf5af6319ef169af738569fb5ddd3f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 14:14:18 -0700 Subject: [PATCH 006/347] Determine the df type based on the execution mode --- morpheus/stages/input/file_source_stage.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/morpheus/stages/input/file_source_stage.py b/morpheus/stages/input/file_source_stage.py index 9b3551dce6..cd1801d21e 100644 --- a/morpheus/stages/input/file_source_stage.py +++ b/morpheus/stages/input/file_source_stage.py @@ -19,11 +19,10 @@ import mrc -# pylint: disable=morpheus-incorrect-lib-from-import -from morpheus._lib.messages import MessageMeta as CppMessageMeta from morpheus.cli import register_stage from morpheus.common import FileTypes from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta @@ -99,6 +98,11 @@ def __init__(self, self._iterative = iterative self._repeat_count = repeat + if c.execution_mode is ExecutionMode.GPU: + self._df_type = "cudf" + else: + self._df_type = "pandas" + @property def name(self) -> str: """Return the name of the stage""" @@ -140,14 +144,11 @@ def _generate_frames(self) -> typing.Iterable[MessageMeta]: filter_nulls=self._filter_null, filter_null_columns=self._filter_null_columns, parser_kwargs=self._parser_kwargs, - df_type="cudf", + df_type=self._df_type, ) for i in range(self._repeat_count): - if (self._build_cpp_node()): - x = CppMessageMeta(df) - else: - x = MessageMeta(df) + x = MessageMeta(df) # If we are looping, copy the object. Do this before we push the object in case it changes if (i + 1 < self._repeat_count): From ac9667d3cd35422784a835b26d1d3d2b448bb3e8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 14:42:00 -0700 Subject: [PATCH 007/347] First pass at supporting CPU only mode in the file and in memory source stages --- morpheus/stages/input/file_source_stage.py | 4 +++- morpheus/stages/input/in_memory_source_stage.py | 7 +++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/morpheus/stages/input/file_source_stage.py b/morpheus/stages/input/file_source_stage.py index cd1801d21e..2c47b2e57c 100644 --- a/morpheus/stages/input/file_source_stage.py +++ b/morpheus/stages/input/file_source_stage.py @@ -33,7 +33,9 @@ logger = logging.getLogger(__name__) -@register_stage("from-file", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER]) +@register_stage("from-file", + modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], + execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class FileSourceStage(PreallocatorMixin, SingleOutputSource): """ Load messages from a file. diff --git a/morpheus/stages/input/in_memory_source_stage.py b/morpheus/stages/input/in_memory_source_stage.py index 5109f61800..a88e78bd86 100644 --- a/morpheus/stages/input/in_memory_source_stage.py +++ b/morpheus/stages/input/in_memory_source_stage.py @@ -14,13 +14,12 @@ import typing -import cudf - from morpheus.config import Config from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.stage_schema import StageSchema from morpheus.stages.input.in_memory_data_generation_stage import InMemoryDataGenStage +from morpheus.utils.type_aliases import DataFrameType class InMemorySourceStage(PreallocatorMixin, InMemoryDataGenStage): @@ -31,13 +30,13 @@ class InMemorySourceStage(PreallocatorMixin, InMemoryDataGenStage): ---------- c : `morpheus.config.Config` Pipeline configuration instance. - dataframes : typing.List[cudf.DataFrame] + dataframes : list[DataFrameType] List of dataframes to emit wrapped in `MessageMeta` instances in order. repeat : int, default = 1, min = 1 Repeats the input dataset multiple times. Useful to extend small datasets for debugging. """ - def __init__(self, c: Config, dataframes: typing.List[cudf.DataFrame], repeat: int = 1): + def __init__(self, c: Config, dataframes: list[DataFrameType], repeat: int = 1): # Prepare a generator function based on the provided dataframes and repeat count self._dataframes = dataframes self._repeat_count = repeat From 4896370fe6c98c9c73b49d5fbd0e392e98db8e00 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 14:49:48 -0700 Subject: [PATCH 008/347] Support cpu mode in write to file --- morpheus/stages/output/write_to_file_stage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/morpheus/stages/output/write_to_file_stage.py b/morpheus/stages/output/write_to_file_stage.py index 46b7e5cec6..e644086ed7 100644 --- a/morpheus/stages/output/write_to_file_stage.py +++ b/morpheus/stages/output/write_to_file_stage.py @@ -18,17 +18,19 @@ import mrc import mrc.core.operators as ops -import morpheus._lib.stages as _stages from morpheus.cli.register_stage import register_stage from morpheus.common import FileTypes from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.controllers.write_to_file_controller import WriteToFileController from morpheus.messages import MessageMeta from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage -@register_stage("to-file", rename_options={"include_index_col": "--include-index-col"}) +@register_stage("to-file", + rename_options={"include_index_col": "--include-index-col"}, + execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class WriteToFileStage(PassThruTypeMixin, SinglePortStage): """ Write all messages to a file. @@ -92,6 +94,7 @@ def supports_cpp_node(self): def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: # Sink to file if (self._build_cpp_node()): + import morpheus._lib.stages as _stages to_file_node = _stages.WriteToFileStage(builder, self.unique_name, self._controller.output_file, From 772627a87aecae57f1d5d366df5b68d022b70475 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 14:50:08 -0700 Subject: [PATCH 009/347] Lazily import the C++ stage impls --- morpheus/stages/inference/triton_inference_stage.py | 2 +- morpheus/stages/input/kafka_source_stage.py | 2 +- morpheus/stages/postprocess/filter_detections_stage.py | 2 +- morpheus/stages/postprocess/serialize_stage.py | 2 +- morpheus/stages/preprocess/preprocess_fil_stage.py | 2 +- morpheus/stages/preprocess/preprocess_nlp_stage.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/morpheus/stages/inference/triton_inference_stage.py b/morpheus/stages/inference/triton_inference_stage.py index 26420c2f51..e8d51f5e6d 100644 --- a/morpheus/stages/inference/triton_inference_stage.py +++ b/morpheus/stages/inference/triton_inference_stage.py @@ -28,7 +28,6 @@ from tritonclient.utils import InferenceServerException from tritonclient.utils import triton_to_np_dtype -import morpheus._lib.stages as _stages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes @@ -782,6 +781,7 @@ def _get_inference_worker(self, inf_queue: ProducerConsumerQueue) -> TritonInfer needs_logits=self._needs_logits) def _get_cpp_inference_node(self, builder: mrc.Builder) -> mrc.SegmentObject: + import morpheus._lib.stages as _stages if self._schema.input_type == ControlMessage: return _stages.InferenceClientStageCM(builder, self.unique_name, diff --git a/morpheus/stages/input/kafka_source_stage.py b/morpheus/stages/input/kafka_source_stage.py index 8893110cf8..0c12c6db49 100644 --- a/morpheus/stages/input/kafka_source_stage.py +++ b/morpheus/stages/input/kafka_source_stage.py @@ -24,7 +24,6 @@ import cudf -import morpheus._lib.stages as _stages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes @@ -239,6 +238,7 @@ def _source_generator(self): def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: if (self._build_cpp_node()): + import morpheus._lib.stages as _stages source = _stages.KafkaSourceStage(builder, self.unique_name, self._max_batch_size, diff --git a/morpheus/stages/postprocess/filter_detections_stage.py b/morpheus/stages/postprocess/filter_detections_stage.py index 9cadb26290..68e36053b0 100644 --- a/morpheus/stages/postprocess/filter_detections_stage.py +++ b/morpheus/stages/postprocess/filter_detections_stage.py @@ -18,7 +18,6 @@ import mrc from mrc.core import operators as ops -import morpheus._lib.stages as _stages from morpheus.cli.register_stage import register_stage from morpheus.common import FilterSource from morpheus.config import Config @@ -118,6 +117,7 @@ def supports_cpp_node(self): def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: if self._build_cpp_node(): + import morpheus._lib.stages as _stages if (self._schema.input_type == ControlMessage): node = _stages.FilterDetectionsControlMessageStage(builder, self.unique_name, diff --git a/morpheus/stages/postprocess/serialize_stage.py b/morpheus/stages/postprocess/serialize_stage.py index 8262e1b4e1..c3a421e848 100644 --- a/morpheus/stages/postprocess/serialize_stage.py +++ b/morpheus/stages/postprocess/serialize_stage.py @@ -19,7 +19,6 @@ import mrc from mrc.core import operators as ops -import morpheus._lib.stages as _stages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.controllers.serialize_controller import SerializeController @@ -92,6 +91,7 @@ def supports_cpp_node(self): def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: if (self._build_cpp_node()): + import morpheus._lib.stages as _stages if (self._schema.input_type == ControlMessage): node = _stages.SerializeControlMessageStage(builder, self.unique_name, diff --git a/morpheus/stages/preprocess/preprocess_fil_stage.py b/morpheus/stages/preprocess/preprocess_fil_stage.py index 8ff369ebe7..312843fcc6 100644 --- a/morpheus/stages/preprocess/preprocess_fil_stage.py +++ b/morpheus/stages/preprocess/preprocess_fil_stage.py @@ -24,7 +24,6 @@ import cudf import morpheus._lib.messages as _messages -import morpheus._lib.stages as _stages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes @@ -172,6 +171,7 @@ def _get_preprocess_fn( return partial(PreprocessFILStage.pre_process_batch, fea_len=self._fea_length, fea_cols=self.features) def _get_preprocess_node(self, builder: mrc.Builder): + import morpheus._lib.stages as _stages if (self._use_control_message): return _stages.PreprocessFILControlMessageStage(builder, self.unique_name, self.features) diff --git a/morpheus/stages/preprocess/preprocess_nlp_stage.py b/morpheus/stages/preprocess/preprocess_nlp_stage.py index de610ab52c..c5d73c241d 100644 --- a/morpheus/stages/preprocess/preprocess_nlp_stage.py +++ b/morpheus/stages/preprocess/preprocess_nlp_stage.py @@ -25,7 +25,6 @@ import cudf import morpheus._lib.messages as _messages -import morpheus._lib.stages as _stages from morpheus.cli.register_stage import register_stage from morpheus.cli.utils import MorpheusRelativePath from morpheus.cli.utils import get_package_relative_file @@ -263,6 +262,7 @@ def _get_preprocess_fn( column=self._column) def _get_preprocess_node(self, builder: mrc.Builder): + import morpheus._lib.stages as _stages if (self._use_control_message): return _stages.PreprocessNLPControlMessageStage(builder, self.unique_name, From d91e6ddbdd06ee53906c204639243555f51746a9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 15:03:44 -0700 Subject: [PATCH 010/347] Add --use_cpu_only --- morpheus/cli/commands.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/morpheus/cli/commands.py b/morpheus/cli/commands.py index 3136e3e0c5..ce231a75fe 100644 --- a/morpheus/cli/commands.py +++ b/morpheus/cli/commands.py @@ -38,6 +38,7 @@ from morpheus.config import ConfigFIL from morpheus.config import ConfigOnnxToTRT from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.utils.logger import configure_logging @@ -288,6 +289,10 @@ def install(**kwargs): type=bool, help=("Whether or not to use C++ node and message types or to prefer python. " "Only use as a last resort if bugs are encountered")) +@click.option('--use_cpu_only', + default=False, + type=bool, + help=("Whether or not to run in CPU only mode, setting this to True will disable C++ mode.")) @click.option('--manual_seed', default=None, type=click.IntRange(min=1), @@ -297,7 +302,19 @@ def install(**kwargs): def run(ctx: click.Context, **kwargs): """Run subcommand, used for running a pipeline""" # Since the option isnt the same name as `should_use_cpp` anymore, manually set the value here. - CppConfig.set_should_use_cpp(kwargs.pop("use_cpp", CppConfig.get_should_use_cpp())) + + use_cpu_only = kwargs.pop("use_cpu_only") + use_cpu = kwargs.pop("use_cpp") + if use_cpu_only: + if use_cpu: + logger.warning("use_cpu_only is set to True, disabling C++ mode") + + CppConfig.set_should_use_cpp(False) + else: + CppConfig.set_should_use_cpp(use_cpu) + + config = get_config_from_ctx(ctx) + config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU manual_seed_val = kwargs.pop("manual_seed", None) if manual_seed_val is not None: From ab1ab7e967376e9fcfbf2ff635db329a7190b687 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 15:08:03 -0700 Subject: [PATCH 011/347] Make --use_cpu_only a flag --- morpheus/cli/commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/morpheus/cli/commands.py b/morpheus/cli/commands.py index ce231a75fe..45c45bef4d 100644 --- a/morpheus/cli/commands.py +++ b/morpheus/cli/commands.py @@ -292,6 +292,7 @@ def install(**kwargs): @click.option('--use_cpu_only', default=False, type=bool, + is_flag=True, help=("Whether or not to run in CPU only mode, setting this to True will disable C++ mode.")) @click.option('--manual_seed', default=None, From cb0c2f57b63ec36381eaa9e2b1d585360334e607 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 15:26:10 -0700 Subject: [PATCH 012/347] Test pipeline --- examples/cpu_only/run.py | 106 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 examples/cpu_only/run.py diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py new file mode 100644 index 0000000000..7353dd9cac --- /dev/null +++ b/examples/cpu_only/run.py @@ -0,0 +1,106 @@ +# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import pathlib +import typing + +import click + +from morpheus.cli.utils import get_log_levels +from morpheus.config import Config +from morpheus.config import CppConfig +from morpheus.config import ExecutionMode +from morpheus.messages import MessageMeta +from morpheus.pipeline.linear_pipeline import LinearPipeline +from morpheus.pipeline.stage_decorator import stage +from morpheus.stages.input.file_source_stage import FileSourceStage +from morpheus.stages.output.write_to_file_stage import WriteToFileStage +from morpheus.utils.logger import configure_logging +from morpheus.cli.utils import parse_log_level + +logger = logging.getLogger(f"morpheus.{__name__}") + + +@click.command() +@click.option('--use_cpu_only', + default=False, + type=bool, + is_flag=True, + help=("Whether or not to run in CPU only mode, setting this to True will disable C++ mode.")) +@click.option("--use_python", is_flag=True, default=False, show_default=True) +@click.option("--log_level", + default="DEBUG", + type=click.Choice(get_log_levels(), case_sensitive=False), + callback=parse_log_level, + show_default=True, + help="Specify the logging level to use.") +@click.option( + "--in_file", + help="Input file", + required=True, + type=click.Path(exists=True, readable=True), +) +@click.option( + "--out_file", + help="Output file", + type=click.Path(dir_okay=False), + default="output.csv", + required=True, +) +def run_pipeline(log_level: int, use_python: bool, use_cpu_only: bool, in_file: pathlib.Path, out_file: pathlib.Path): + # Enable the default logger + configure_logging(log_level=log_level) + + if use_cpu_only: + execution_mode = ExecutionMode.CPU + if not use_python: + logging.warning("C++ mode is disabled when running in CPU only mode.") + use_python = True + + else: + execution_mode = ExecutionMode.GPU + + CppConfig.set_should_use_cpp(not use_python) + + config = Config() + config.execution_mode = execution_mode + + pipeline = LinearPipeline(config) + + pipeline.set_source(FileSourceStage(config, filename=in_file)) + + @stage + def print_msg(msg: typing.Any) -> typing.Any: + log_msg = [f"Receive a message of type {type(msg)}"] + if isinstance(msg, MessageMeta): + log_msg.append(f"- df type: {type(msg.df)}") + + print(" ".join(log_msg)) + + return msg + + pipeline.add_stage(print_msg(config)) + pipeline.add_stage(WriteToFileStage(config, filename=out_file, overwrite=True)) + pipeline.build() + + logger.info("Running pipeline\tC++ mode = %s\texecution_mode = %s", + CppConfig.get_should_use_cpp(), + config.execution_mode) + + pipeline.run() + + +if __name__ == "__main__": + run_pipeline() From 0be76c9ae66b7d0a8a47594027473658e2f605e6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 15:37:50 -0700 Subject: [PATCH 013/347] Don't define a module config unless we are building a module --- .../stages/preprocess/deserialize_stage.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/morpheus/stages/preprocess/deserialize_stage.py b/morpheus/stages/preprocess/deserialize_stage.py index c8861cb4bb..b07685ec2d 100644 --- a/morpheus/stages/preprocess/deserialize_stage.py +++ b/morpheus/stages/preprocess/deserialize_stage.py @@ -20,6 +20,7 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta @@ -33,6 +34,7 @@ @register_stage("deserialize", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], + execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU), ignore_args=["message_type", "task_type", "task_payload"]) class DeserializeStage(MultiMessageStage): """ @@ -88,16 +90,6 @@ def __init__(self, else: raise ValueError(f"Invalid message type: {self._message_type}") - self._module_config = { - "ensure_sliceable_index": self._ensure_sliceable_index, - "message_type": "MultiMessage" if self._message_type is MultiMessage else "ControlMessage", - "task_type": self._task_type, - "task_payload": self._task_payload, - "batch_size": self._batch_size, - "max_concurrency": self._max_concurrent, - "should_log_timestamp": self._should_log_timestamps - } - @property def name(self) -> str: return "deserialize" @@ -134,8 +126,18 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> builder.make_edge(input_node, out_node) else: + module_config = { + "ensure_sliceable_index": self._ensure_sliceable_index, + "message_type": "MultiMessage" if self._message_type is MultiMessage else "ControlMessage", + "task_type": self._task_type, + "task_payload": self._task_payload, + "batch_size": self._batch_size, + "max_concurrency": self._max_concurrent, + "should_log_timestamp": self._should_log_timestamps + } + module_loader = DeserializeLoaderFactory.get_instance(module_name=f"deserialize_{self.unique_name}", - module_config=self._module_config) + module_config=module_config) module = module_loader.load(builder=builder) From 17e1a3c3d0d50aa6c885731c94367e8898e5ea29 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 15:40:54 -0700 Subject: [PATCH 014/347] Advertize cpu mode --- morpheus/stages/postprocess/serialize_stage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/morpheus/stages/postprocess/serialize_stage.py b/morpheus/stages/postprocess/serialize_stage.py index c3a421e848..d7fddbe7d6 100644 --- a/morpheus/stages/postprocess/serialize_stage.py +++ b/morpheus/stages/postprocess/serialize_stage.py @@ -21,6 +21,7 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.controllers.serialize_controller import SerializeController from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta @@ -31,7 +32,7 @@ logger = logging.getLogger(__name__) -@register_stage("serialize") +@register_stage("serialize", execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class SerializeStage(SinglePortStage): """ Includes & excludes columns from messages. From f9caeda73f4690baff3e21feb459e5779e91550b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 16:27:36 -0700 Subject: [PATCH 015/347] Perform lazy imports of cudf and lib morpheus --- morpheus/stages/inference/inference_stage.py | 12 ++-- morpheus/stages/llm/llm_engine_stage.py | 3 +- .../stages/preprocess/preprocess_fil_stage.py | 33 ++++----- .../stages/preprocess/preprocess_nlp_stage.py | 71 ++++++++++--------- 4 files changed, 60 insertions(+), 59 deletions(-) diff --git a/morpheus/stages/inference/inference_stage.py b/morpheus/stages/inference/inference_stage.py index ab12afe4d3..58394aa0f1 100644 --- a/morpheus/stages/inference/inference_stage.py +++ b/morpheus/stages/inference/inference_stage.py @@ -21,10 +21,6 @@ import mrc from mrc.core import operators as ops -import cudf - -# pylint: disable=morpheus-incorrect-lib-from-import -from morpheus._lib.messages import MessageMeta as CppMessageMeta from morpheus.config import Config from morpheus.messages import ControlMessage from morpheus.messages import InferenceMemoryNLP @@ -167,6 +163,9 @@ class InferenceStage(MultiMessageStage): def __init__(self, c: Config): super().__init__(c) + import cudf + self._cudf = cudf + self._fea_length = c.feature_length self._thread_count = c.num_threads @@ -280,11 +279,10 @@ def set_output_fut(resp: TensorMemory, inner_batch, batch_future: mrc.Future): for f in fut_list: f.result() - # TODO(Devin): This is a hack to support ControlMessage side channel. if (isinstance(_message, ControlMessage)): - _df = cudf.DataFrame(output_message.get_meta()) + _df = self._cudf.DataFrame(output_message.get_meta()) if (_df is not None and not _df.empty): - _message_meta = CppMessageMeta(df=_df) + _message_meta = MessageMeta(df=_df) _message.payload(_message_meta) response_tensors = output_message.tensors diff --git a/morpheus/stages/llm/llm_engine_stage.py b/morpheus/stages/llm/llm_engine_stage.py index 012681c3d4..157fbf06ef 100644 --- a/morpheus/stages/llm/llm_engine_stage.py +++ b/morpheus/stages/llm/llm_engine_stage.py @@ -17,7 +17,6 @@ import mrc -import morpheus._lib.llm as _llm from morpheus.config import Config from morpheus.llm import LLMEngine from morpheus.messages import ControlMessage @@ -66,7 +65,7 @@ def supports_cpp_node(self): return True def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: - + import morpheus._lib.llm as _llm node = _llm.LLMEngineStage(builder, self.unique_name, self._engine) node.launch_options.pe_count = 1 diff --git a/morpheus/stages/preprocess/preprocess_fil_stage.py b/morpheus/stages/preprocess/preprocess_fil_stage.py index 312843fcc6..edd6d5c87f 100644 --- a/morpheus/stages/preprocess/preprocess_fil_stage.py +++ b/morpheus/stages/preprocess/preprocess_fil_stage.py @@ -21,9 +21,6 @@ import numpy as np import pandas as pd -import cudf - -import morpheus._lib.messages as _messages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes @@ -52,6 +49,12 @@ class PreprocessFILStage(PreprocessBaseStage): def __init__(self, c: Config): super().__init__(c) + import morpheus._lib.messages as _messages + self._lib_messages = _messages + + import cudf + self._cudf = cudf + self._fea_length = c.feature_length self.features = c.fil.feature_columns @@ -65,8 +68,7 @@ def name(self) -> str: def supports_cpp_node(self): return True - @staticmethod - def pre_process_batch(x: typing.Union[MultiMessage, ControlMessage], fea_len: int, + def pre_process_batch(self, x: typing.Union[MultiMessage, ControlMessage], fea_len: int, fea_cols: typing.List[str]) -> typing.Union[MultiMessage, ControlMessage]: """ For FIL category usecases, this function performs pre-processing. @@ -87,16 +89,15 @@ def pre_process_batch(x: typing.Union[MultiMessage, ControlMessage], fea_len: in """ if isinstance(x, ControlMessage): - return PreprocessFILStage.process_control_message(x, fea_len, fea_cols) + return self.process_control_message(x, fea_len, fea_cols) if isinstance(x, MultiMessage): - return PreprocessFILStage.process_multi_message(x, fea_len, fea_cols) + return self.process_multi_message(x, fea_len, fea_cols) raise TypeError(f"Unsupported message type: {type(x)}") - @staticmethod - def process_control_message(x: ControlMessage, fea_len: int, fea_cols: typing.List[str]) -> ControlMessage: + def process_control_message(self, x: ControlMessage, fea_len: int, fea_cols: typing.List[str]) -> ControlMessage: try: - df: cudf.DataFrame = x.payload().get_data(fea_cols) + df: self._cudf.DataFrame = x.payload().get_data(fea_cols) except KeyError: logger.exception("Requested feature columns does not exist in the dataframe.", exc_info=True) raise @@ -112,7 +113,7 @@ def process_control_message(x: ControlMessage, fea_len: int, fea_cols: typing.Li df[col] = df[col].astype("float32") if (isinstance(df, pd.DataFrame)): - df = cudf.from_pandas(df) + df = self._cudf.from_pandas(df) # Convert the dataframe to cupy the same way cuml does data = cp.asarray(df.to_cupy()) @@ -124,11 +125,11 @@ def process_control_message(x: ControlMessage, fea_len: int, fea_cols: typing.Li seg_ids[:, 2] = fea_len - 1 # We need the C++ impl of TensorMemory until #1646 is resolved - x.tensors(_messages.TensorMemory(count=count, tensors={"input__0": data, "seq_ids": seg_ids})) + x.tensors(self._lib_messages.TensorMemory(count=count, tensors={"input__0": data, "seq_ids": seg_ids})) return x - @staticmethod - def process_multi_message(x: MultiMessage, fea_len: int, fea_cols: typing.List[str]) -> MultiInferenceFILMessage: + def process_multi_message(self, x: MultiMessage, fea_len: int, + fea_cols: typing.List[str]) -> MultiInferenceFILMessage: try: df = x.get_meta(fea_cols) except KeyError: @@ -146,7 +147,7 @@ def process_multi_message(x: MultiMessage, fea_len: int, fea_cols: typing.List[s df[col] = df[col].astype("float32") if (isinstance(df, pd.DataFrame)): - df = cudf.from_pandas(df) + df = self._cudf.from_pandas(df) # Convert the dataframe to cupy the same way cuml does data = cp.asarray(df.to_cupy()) @@ -168,7 +169,7 @@ def _get_preprocess_fn( self ) -> typing.Callable[[typing.Union[MultiMessage, ControlMessage]], typing.Union[MultiInferenceMessage, ControlMessage]]: - return partial(PreprocessFILStage.pre_process_batch, fea_len=self._fea_length, fea_cols=self.features) + return partial(self.pre_process_batch, fea_len=self._fea_length, fea_cols=self.features) def _get_preprocess_node(self, builder: mrc.Builder): import morpheus._lib.stages as _stages diff --git a/morpheus/stages/preprocess/preprocess_nlp_stage.py b/morpheus/stages/preprocess/preprocess_nlp_stage.py index c5d73c241d..21380c1c4b 100644 --- a/morpheus/stages/preprocess/preprocess_nlp_stage.py +++ b/morpheus/stages/preprocess/preprocess_nlp_stage.py @@ -22,9 +22,6 @@ import mrc import numpy as np -import cudf - -import morpheus._lib.messages as _messages from morpheus.cli.register_stage import register_stage from morpheus.cli.utils import MorpheusRelativePath from morpheus.cli.utils import get_package_relative_file @@ -116,6 +113,12 @@ def __init__(self, column: str = "data"): super().__init__(c) + import morpheus._lib.messages as _messages + self._lib_messages = _messages + + import cudf + self._cudf = cudf + self._column = column self._seq_length = c.feature_length self._vocab_hash_file = get_package_relative_file(vocab_hash_file) @@ -139,8 +142,8 @@ def name(self) -> str: def supports_cpp_node(self): return True - @staticmethod - def pre_process_batch(message: typing.Union[MultiMessage, ControlMessage], + def pre_process_batch(self, + message: typing.Union[MultiMessage, ControlMessage], vocab_hash_file: str, do_lower_case: bool, seq_len: int, @@ -160,28 +163,28 @@ def pre_process_batch(message: typing.Union[MultiMessage, ControlMessage], """ if isinstance(message, ControlMessage): - return PreprocessNLPStage.process_control_message(message, - vocab_hash_file, - do_lower_case, - seq_len, - stride, - truncation, - add_special_tokens, - column) + return self.process_control_message(message, + vocab_hash_file, + do_lower_case, + seq_len, + stride, + truncation, + add_special_tokens, + column) if isinstance(message, MultiMessage): - return PreprocessNLPStage.process_multi_message(message, - vocab_hash_file, - do_lower_case, - seq_len, - stride, - truncation, - add_special_tokens, - column) + return self.process_multi_message(message, + vocab_hash_file, + do_lower_case, + seq_len, + stride, + truncation, + add_special_tokens, + column) raise TypeError("Unsupported message type") - @staticmethod - def process_control_message(message: ControlMessage, + def process_control_message(self, + message: ControlMessage, vocab_hash_file: str, do_lower_case: bool, seq_len: int, @@ -191,7 +194,7 @@ def process_control_message(message: ControlMessage, column: str) -> ControlMessage: with message.payload().mutable_dataframe() as mdf: - text_series = cudf.Series(mdf[column]) + text_series = self._cudf.Series(mdf[column]) tokenized = tokenize_text_series(vocab_hash_file=vocab_hash_file, do_lower_case=do_lower_case, @@ -205,18 +208,18 @@ def process_control_message(message: ControlMessage, # We need the C++ impl of TensorMemory until #1646 is resolved message.tensors( - _messages.TensorMemory(count=tokenized.input_ids.shape[0], - tensors={ - "input_ids": tokenized.input_ids, - "input_mask": tokenized.input_mask, - "seq_ids": tokenized.segment_ids - })) + self._lib_messages.TensorMemory(count=tokenized.input_ids.shape[0], + tensors={ + "input_ids": tokenized.input_ids, + "input_mask": tokenized.input_mask, + "seq_ids": tokenized.segment_ids + })) message.set_metadata("inference_memory_params", {"inference_type": "nlp"}) return message - @staticmethod - def process_multi_message(message: MultiMessage, + def process_multi_message(self, + message: MultiMessage, vocab_hash_file: str, do_lower_case: bool, seq_len: int, @@ -225,7 +228,7 @@ def process_multi_message(message: MultiMessage, add_special_tokens: bool, column: str) -> MultiInferenceNLPMessage: # Existing logic for MultiMessage - text_ser = cudf.Series(message.get_meta(column)) + text_ser = self._cudf.Series(message.get_meta(column)) tokenized = tokenize_text_series(vocab_hash_file=vocab_hash_file, do_lower_case=do_lower_case, @@ -252,7 +255,7 @@ def _get_preprocess_fn( self ) -> typing.Callable[[typing.Union[MultiMessage, ControlMessage]], typing.Union[MultiInferenceMessage, ControlMessage]]: - return partial(PreprocessNLPStage.pre_process_batch, + return partial(self..pre_process_batch, vocab_hash_file=self._vocab_hash_file, do_lower_case=self._do_lower_case, stride=self._stride, From 5dfe9f4a587269f031d7cd8a7a6edc12ae814462 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 17:00:35 -0700 Subject: [PATCH 016/347] WIP --- morpheus/stages/input/arxiv_source.py | 14 ++++++--- .../databricks_deltalake_source_stage.py | 18 ++++++++--- .../stages/input/http_client_source_stage.py | 31 ++++++++++++------- .../stages/preprocess/preprocess_nlp_stage.py | 2 +- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/morpheus/stages/input/arxiv_source.py b/morpheus/stages/input/arxiv_source.py index c1ed77c0cb..dd85973012 100644 --- a/morpheus/stages/input/arxiv_source.py +++ b/morpheus/stages/input/arxiv_source.py @@ -20,10 +20,9 @@ import mrc.core.operators as ops import pandas as pd -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource @@ -40,7 +39,7 @@ "--file conda/environments/all_cuda-121_arch-x86_64.yaml --prune`") -@register_stage("from-arxiv") +@register_stage("from-arxiv", execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class ArxivSource(PreallocatorMixin, SingleOutputSource): """ Source stage that downloads PDFs from arxiv and converts them to dataframes. @@ -99,6 +98,10 @@ def __init__(self, self._stop_requested = False self._cache_dir = cache_dir + if c.execution_mode is ExecutionMode.GPU: + import cudf + self._cudf = cudf + @property def name(self) -> str: """Return the name of the stage""" @@ -196,4 +199,7 @@ def _splitting_pages(self, documents: list["Document"]): df.rename(columns=map_cols, inplace=True) - return MessageMeta(cudf.from_pandas(df)) + if self._config.execution_mode is ExecutionMode.GPU: + df = self._cudf.from_pandas(df) + + return MessageMeta(df) diff --git a/morpheus/stages/input/databricks_deltalake_source_stage.py b/morpheus/stages/input/databricks_deltalake_source_stage.py index 4d6304641a..0f172d19ef 100644 --- a/morpheus/stages/input/databricks_deltalake_source_stage.py +++ b/morpheus/stages/input/databricks_deltalake_source_stage.py @@ -16,10 +16,9 @@ import mrc -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages.message_meta import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource @@ -38,7 +37,7 @@ IMPORT_EXCEPTION = import_exc -@register_stage("from-databricks-deltalake") +@register_stage("from-databricks-deltalake", execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class DataBricksDeltaLakeSourceStage(PreallocatorMixin, SingleOutputSource): """ Source stage used to load messages from a DeltaLake table. @@ -77,6 +76,10 @@ def __init__(self, self.items_per_page = items_per_page self.offset = 0 + if config.execution_mode is ExecutionMode.GPU: + import cudf + self._cudf = cudf + @property def name(self) -> str: return "from-databricks-deltalake" @@ -104,7 +107,14 @@ def source_generator(self): str(self.offset), str(self.offset + self.items_per_page + 1)) self.offset += self.items_per_page + 1 - yield MessageMeta(df=cudf.from_pandas(df.toPandas().drop(["_id"], axis=1))) + + df = df.toPandas().drop(["_id"], axis=1) + + if self._config.execution_mode is ExecutionMode.GPU: + df = self._cudf.from_pandas(df) + + yield MessageMeta(df=df) + except Exception as e: logger.error( "Error occurred while reading data from \ diff --git a/morpheus/stages/input/http_client_source_stage.py b/morpheus/stages/input/http_client_source_stage.py index 4a101e0992..e283114fda 100644 --- a/morpheus/stages/input/http_client_source_stage.py +++ b/morpheus/stages/input/http_client_source_stage.py @@ -21,20 +21,22 @@ import mrc import requests -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema from morpheus.utils import http_utils +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) -@register_stage("from-http-client", ignore_args=["query_params", "headers", "**request_kwargs"]) +@register_stage("from-http-client", + execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU), + ignore_args=["query_params", "headers", "**request_kwargs"]) class HttpClientSourceStage(PreallocatorMixin, SingleOutputSource): """ Source stage that polls a remote HTTP server for incoming data. @@ -82,7 +84,8 @@ class HttpClientSourceStage(PreallocatorMixin, SingleOutputSource): Stops ingesting after emitting `stop_after` records (rows in the dataframe). Useful for testing. Disabled if `0` payload_to_df_fn : callable, default None A callable that takes the HTTP payload bytes as the first argument and the `lines` parameter is passed in as - the second argument and returns a cudf.DataFrame. If unset cudf.read_json is used. + the second argument and returns a DataFrame. If unset `cudf.read_json` is used in GPU mode and + `pandas.read_json` in CPU mode. **request_kwargs : dict Additional arguments to pass to the `requests.request` function. """ @@ -101,7 +104,7 @@ def __init__(self, max_retries: int = 10, lines: bool = False, stop_after: int = 0, - payload_to_df_fn: typing.Callable[[bytes, bool], cudf.DataFrame] = None, + payload_to_df_fn: typing.Callable[[bytes, bool], DataFrameType] = None, **request_kwargs): super().__init__(config) self._url = http_utils.prepare_url(url) @@ -139,9 +142,18 @@ def __init__(self, self._stop_after = stop_after self._lines = lines - self._payload_to_df_fn = payload_to_df_fn self._requst_kwargs = request_kwargs + if payload_to_df_fn is None: + if config.execution_mode == ExecutionMode.GPU: + import cudf + self._payload_to_df_fn = cudf.read_json + else: + import pandas + self._payload_to_df_fn = pandas.read_json + else: + self._payload_to_df_fn = payload_to_df_fn + @property def name(self) -> str: """Unique name of the stage""" @@ -154,16 +166,13 @@ def supports_cpp_node(self) -> bool: def compute_schema(self, schema: StageSchema): schema.output_schema.set_type(MessageMeta) - def _parse_response(self, response: requests.Response) -> typing.Union[cudf.DataFrame, None]: + def _parse_response(self, response: requests.Response) -> typing.Union[DataFrameType, None]: """ Returns a DataFrame parsed from the response payload. If the response payload is empty, then `None` is returned. """ payload = response.content - if self._payload_to_df_fn is not None: - return self._payload_to_df_fn(payload, self._lines) - - return cudf.read_json(payload, lines=self._lines, engine='cudf') + return self._payload_to_df_fn(payload, self._lines) def _generate_frames(self) -> typing.Iterator[MessageMeta]: # Running counter of the number of messages emitted by this source diff --git a/morpheus/stages/preprocess/preprocess_nlp_stage.py b/morpheus/stages/preprocess/preprocess_nlp_stage.py index 21380c1c4b..42dd4b64e4 100644 --- a/morpheus/stages/preprocess/preprocess_nlp_stage.py +++ b/morpheus/stages/preprocess/preprocess_nlp_stage.py @@ -255,7 +255,7 @@ def _get_preprocess_fn( self ) -> typing.Callable[[typing.Union[MultiMessage, ControlMessage]], typing.Union[MultiInferenceMessage, ControlMessage]]: - return partial(self..pre_process_batch, + return partial(self.pre_process_batch, vocab_hash_file=self._vocab_hash_file, do_lower_case=self._do_lower_case, stride=self._stride, From a279c5786767ac255588d27534d7eaedfffae473 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 17:05:08 -0700 Subject: [PATCH 017/347] Fix CR year --- examples/cpu_only/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index 7353dd9cac..bde5caa965 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import click from morpheus.cli.utils import get_log_levels +from morpheus.cli.utils import parse_log_level from morpheus.config import Config from morpheus.config import CppConfig from morpheus.config import ExecutionMode @@ -28,7 +29,6 @@ from morpheus.stages.input.file_source_stage import FileSourceStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage from morpheus.utils.logger import configure_logging -from morpheus.cli.utils import parse_log_level logger = logging.getLogger(f"morpheus.{__name__}") From 216e0d27da1d3b3a85db8f9aecf5420cb2fe4ee1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 12 Aug 2024 17:11:17 -0700 Subject: [PATCH 018/347] Add DeserializeStage & SerializeStage to test --- examples/cpu_only/run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index bde5caa965..dbbb25ce6b 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -28,6 +28,8 @@ from morpheus.pipeline.stage_decorator import stage from morpheus.stages.input.file_source_stage import FileSourceStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage +from morpheus.stages.postprocess.serialize_stage import SerializeStage +from morpheus.stages.preprocess.deserialize_stage import DeserializeStage from morpheus.utils.logger import configure_logging logger = logging.getLogger(f"morpheus.{__name__}") @@ -92,6 +94,9 @@ def print_msg(msg: typing.Any) -> typing.Any: return msg pipeline.add_stage(print_msg(config)) + + pipeline.add_stage(DeserializeStage(config)) + pipeline.add_stage(SerializeStage(config)) pipeline.add_stage(WriteToFileStage(config, filename=out_file, overwrite=True)) pipeline.build() From e8d97f2a566a4015225ad8cd17ad7e2e1937c511 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 13 Aug 2024 11:11:33 -0700 Subject: [PATCH 019/347] Rename execute_modes to execution_modes --- morpheus/cli/register_stage.py | 2 +- morpheus/stages/input/arxiv_source.py | 2 +- morpheus/stages/input/databricks_deltalake_source_stage.py | 2 +- morpheus/stages/input/file_source_stage.py | 2 +- morpheus/stages/input/http_client_source_stage.py | 2 +- morpheus/stages/output/write_to_file_stage.py | 2 +- morpheus/stages/postprocess/serialize_stage.py | 2 +- morpheus/stages/preprocess/deserialize_stage.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/morpheus/cli/register_stage.py b/morpheus/cli/register_stage.py index b472200a6b..48000a273f 100644 --- a/morpheus/cli/register_stage.py +++ b/morpheus/cli/register_stage.py @@ -249,7 +249,7 @@ def compute_option_name(stage_arg_name: str, rename_options: typing.Dict[str, st def register_stage(command_name: str = None, modes: typing.Sequence[PipelineModes] = None, - execute_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, ), + execution_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, ), ignore_args: typing.List[str] = None, command_args: dict = None, option_args: typing.Dict[str, dict] = None, diff --git a/morpheus/stages/input/arxiv_source.py b/morpheus/stages/input/arxiv_source.py index dd85973012..b05083d4f3 100644 --- a/morpheus/stages/input/arxiv_source.py +++ b/morpheus/stages/input/arxiv_source.py @@ -39,7 +39,7 @@ "--file conda/environments/all_cuda-121_arch-x86_64.yaml --prune`") -@register_stage("from-arxiv", execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("from-arxiv", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class ArxivSource(PreallocatorMixin, SingleOutputSource): """ Source stage that downloads PDFs from arxiv and converts them to dataframes. diff --git a/morpheus/stages/input/databricks_deltalake_source_stage.py b/morpheus/stages/input/databricks_deltalake_source_stage.py index 0f172d19ef..95d37b8caa 100644 --- a/morpheus/stages/input/databricks_deltalake_source_stage.py +++ b/morpheus/stages/input/databricks_deltalake_source_stage.py @@ -37,7 +37,7 @@ IMPORT_EXCEPTION = import_exc -@register_stage("from-databricks-deltalake", execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("from-databricks-deltalake", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class DataBricksDeltaLakeSourceStage(PreallocatorMixin, SingleOutputSource): """ Source stage used to load messages from a DeltaLake table. diff --git a/morpheus/stages/input/file_source_stage.py b/morpheus/stages/input/file_source_stage.py index 2c47b2e57c..548eceb385 100644 --- a/morpheus/stages/input/file_source_stage.py +++ b/morpheus/stages/input/file_source_stage.py @@ -35,7 +35,7 @@ @register_stage("from-file", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], - execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) + execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class FileSourceStage(PreallocatorMixin, SingleOutputSource): """ Load messages from a file. diff --git a/morpheus/stages/input/http_client_source_stage.py b/morpheus/stages/input/http_client_source_stage.py index e283114fda..cc7191ba43 100644 --- a/morpheus/stages/input/http_client_source_stage.py +++ b/morpheus/stages/input/http_client_source_stage.py @@ -35,7 +35,7 @@ @register_stage("from-http-client", - execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU), + execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), ignore_args=["query_params", "headers", "**request_kwargs"]) class HttpClientSourceStage(PreallocatorMixin, SingleOutputSource): """ diff --git a/morpheus/stages/output/write_to_file_stage.py b/morpheus/stages/output/write_to_file_stage.py index e644086ed7..967fedc338 100644 --- a/morpheus/stages/output/write_to_file_stage.py +++ b/morpheus/stages/output/write_to_file_stage.py @@ -30,7 +30,7 @@ @register_stage("to-file", rename_options={"include_index_col": "--include-index-col"}, - execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) + execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class WriteToFileStage(PassThruTypeMixin, SinglePortStage): """ Write all messages to a file. diff --git a/morpheus/stages/postprocess/serialize_stage.py b/morpheus/stages/postprocess/serialize_stage.py index d7fddbe7d6..c659e6b1e1 100644 --- a/morpheus/stages/postprocess/serialize_stage.py +++ b/morpheus/stages/postprocess/serialize_stage.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) -@register_stage("serialize", execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("serialize", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) class SerializeStage(SinglePortStage): """ Includes & excludes columns from messages. diff --git a/morpheus/stages/preprocess/deserialize_stage.py b/morpheus/stages/preprocess/deserialize_stage.py index b07685ec2d..8afbc2643c 100644 --- a/morpheus/stages/preprocess/deserialize_stage.py +++ b/morpheus/stages/preprocess/deserialize_stage.py @@ -34,7 +34,7 @@ @register_stage("deserialize", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], - execute_modes=(ExecutionMode.CPU, ExecutionMode.GPU), + execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), ignore_args=["message_type", "task_type", "task_payload"]) class DeserializeStage(MultiMessageStage): """ From 34b625c38c4d48f01376911dab1d70fca9cd3c81 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 13 Aug 2024 14:02:24 -0700 Subject: [PATCH 020/347] WIP --- morpheus/io/utils.py | 36 +++++++++++++++++++ morpheus/stages/input/file_source_stage.py | 2 ++ .../stages/input/http_client_source_stage.py | 13 +------ .../stages/input/http_server_source_stage.py | 33 ++++++++--------- .../stages/input/http_source_stage_base.py | 35 ++++++++++++++++++ morpheus/stages/input/kafka_source_stage.py | 10 ++++-- .../stages/output/compare_dataframe_stage.py | 7 ++-- .../stages/output/http_server_sink_stage.py | 16 +++++---- .../write_to_databricks_deltalake_stage.py | 5 ++- .../output/write_to_elasticsearch_stage.py | 5 ++- .../postprocess/generate_viz_frames_stage.py | 10 +++--- 11 files changed, 118 insertions(+), 54 deletions(-) create mode 100644 morpheus/stages/input/http_source_stage_base.py diff --git a/morpheus/io/utils.py b/morpheus/io/utils.py index 6aeabab3e8..ba6abcbfc5 100644 --- a/morpheus/io/utils.py +++ b/morpheus/io/utils.py @@ -14,7 +14,9 @@ # limitations under the License. """IO utilities.""" +import functools import logging +import types import typing import pandas as pd @@ -22,6 +24,8 @@ if typing.TYPE_CHECKING: import cudf +from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import SeriesType @@ -133,3 +137,35 @@ def truncate_string_cols_by_bytes(df: DataFrameType, df[col] = decoded_series return performed_truncation + + +def get_df_pkg(config: Config) -> types.ModuleType: + """ + Return the appropriate DataFrame package based on the execution mode. + """ + if config.execution_mode is ExecutionMode.GPU: + import cudf + return cudf + else: + return pd + + +def get_df_class(config: Config) -> type[DataFrameType]: + """ + Return the appropriate DataFrame class based on the execution mode. + """ + df_pkg = get_df_pkg(config) + return df_pkg.DataFrame + + +def get_json_reader(config: Config) -> typing.Callable[..., DataFrameType]: + """ + Return the appropriate JSON reader based on the execution mode. + """ + if config.execution_mode is ExecutionMode.GPU: + import cudf + reader = functools.partial(cudf.read_json, engine='cudf') + else: + reader = pd.read_json + + return reader diff --git a/morpheus/stages/input/file_source_stage.py b/morpheus/stages/input/file_source_stage.py index 548eceb385..260b3c530b 100644 --- a/morpheus/stages/input/file_source_stage.py +++ b/morpheus/stages/input/file_source_stage.py @@ -101,8 +101,10 @@ def __init__(self, self._repeat_count = repeat if c.execution_mode is ExecutionMode.GPU: + print("GPU MODE Using cudf") self._df_type = "cudf" else: + print("CPU MODE Using pandas") self._df_type = "pandas" @property diff --git a/morpheus/stages/input/http_client_source_stage.py b/morpheus/stages/input/http_client_source_stage.py index cc7191ba43..bb6595120b 100644 --- a/morpheus/stages/input/http_client_source_stage.py +++ b/morpheus/stages/input/http_client_source_stage.py @@ -144,15 +144,7 @@ def __init__(self, self._lines = lines self._requst_kwargs = request_kwargs - if payload_to_df_fn is None: - if config.execution_mode == ExecutionMode.GPU: - import cudf - self._payload_to_df_fn = cudf.read_json - else: - import pandas - self._payload_to_df_fn = pandas.read_json - else: - self._payload_to_df_fn = payload_to_df_fn + self._payload_to_df_fn = payload_to_df_fn or self._get_default_payload_to_df_fn(config) @property def name(self) -> str: @@ -163,9 +155,6 @@ def supports_cpp_node(self) -> bool: """Indicates whether or not this stage supports a C++ implementation""" return False - def compute_schema(self, schema: StageSchema): - schema.output_schema.set_type(MessageMeta) - def _parse_response(self, response: requests.Response) -> typing.Union[DataFrameType, None]: """ Returns a DataFrame parsed from the response payload. If the response payload is empty, then `None` is returned. diff --git a/morpheus/stages/input/http_server_source_stage.py b/morpheus/stages/input/http_server_source_stage.py index ed8d99612f..b0dcc3b620 100644 --- a/morpheus/stages/input/http_server_source_stage.py +++ b/morpheus/stages/input/http_server_source_stage.py @@ -22,18 +22,16 @@ import mrc -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages import MessageMeta -from morpheus.pipeline.preallocator_mixin import PreallocatorMixin -from morpheus.pipeline.single_output_source import SingleOutputSource -from morpheus.pipeline.stage_schema import StageSchema +from morpheus.stages.input.http_source_stage_base import HttpSourceStageBase from morpheus.utils.http_utils import HTTPMethod from morpheus.utils.http_utils import HttpParseResponse from morpheus.utils.http_utils import MimeTypes from morpheus.utils.producer_consumer_queue import Closed +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) @@ -41,8 +39,8 @@ HEALTH_SUPPORTED_METHODS = (HTTPMethod.GET, HTTPMethod.POST) -@register_stage("from-http") -class HttpServerSourceStage(PreallocatorMixin, SingleOutputSource): +@register_stage("from-http", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +class HttpServerSourceStage(HttpSourceStageBase): """ Source stage that starts an HTTP server and listens for incoming requests on a specified endpoint. @@ -80,7 +78,7 @@ class HttpServerSourceStage(PreallocatorMixin, SingleOutputSource): Stops ingesting after emitting `stop_after` records (rows in the dataframe). Useful for testing. Disabled if `0` payload_to_df_fn : callable, default None A callable that takes the HTTP payload string as the first argument and the `lines` parameter is passed in as - the second argument and returns a cudf.DataFrame. When supplied, the C++ implementation of this stage is + the second argument and returns a DataFrame. When supplied, the C++ implementation of this stage is disabled, and the Python impl is used. """ @@ -103,7 +101,7 @@ def __init__(self, request_timeout_secs: int = 30, lines: bool = False, stop_after: int = 0, - payload_to_df_fn: typing.Callable[[str, bool], cudf.DataFrame] = None): + payload_to_df_fn: typing.Callable[[str, bool], DataFrameType] = None): super().__init__(config) self._bind_address = bind_address self._port = port @@ -122,9 +120,11 @@ def __init__(self, self._request_timeout_secs = request_timeout_secs self._lines = lines self._stop_after = stop_after - self._payload_to_df_fn = payload_to_df_fn self._http_server = None + # Leave this as None so we can check if it's set later + self._payload_to_df_fn = payload_to_df_fn + # These are only used when C++ mode is disabled self._queue = None self._queue_size = 0 @@ -146,17 +146,9 @@ def supports_cpp_node(self) -> bool: """Indicates whether this stage supports C++ nodes.""" return True - def compute_schema(self, schema: StageSchema): - schema.output_schema.set_type(MessageMeta) - def _parse_payload(self, payload: str) -> HttpParseResponse: try: - if self._payload_to_df_fn is not None: - df = self._payload_to_df_fn(payload, self._lines) - else: - # engine='cudf' is needed when lines=False to avoid using pandas - df = cudf.read_json(payload, lines=self._lines, engine='cudf') - + df = self._payload_to_df_fn(payload, self._lines) except Exception as e: err_msg = "Error occurred converting HTTP payload to Dataframe" logger.error("%s: %s", err_msg, e) @@ -277,6 +269,9 @@ def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: lines=self._lines, stop_after=self._stop_after) else: + if self._payload_to_df_fn is None: + self._payload_to_df_fn = self._get_default_payload_to_df_fn(self._config) + node = builder.make_source(self.unique_name, self._generate_frames()) return node diff --git a/morpheus/stages/input/http_source_stage_base.py b/morpheus/stages/input/http_source_stage_base.py new file mode 100644 index 0000000000..e83a53f353 --- /dev/null +++ b/morpheus/stages/input/http_source_stage_base.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Base class for HTTP sources.""" + +import typing + +from morpheus.config import Config +from morpheus.io.utils import get_json_reader +from morpheus.messages import MessageMeta +from morpheus.pipeline.preallocator_mixin import PreallocatorMixin +from morpheus.pipeline.single_output_source import SingleOutputSource +from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_aliases import DataFrameType + + +class HttpSourceStageBase(PreallocatorMixin, SingleOutputSource): + + def _get_default_payload_to_df_fn(self, config: Config) -> typing.Callable[[str, bool], DataFrameType]: + reader = get_json_reader(config) + + return lambda payload, lines: reader(payload, lines=lines) + + def compute_schema(self, schema: StageSchema): + schema.output_schema.set_type(MessageMeta) diff --git a/morpheus/stages/input/kafka_source_stage.py b/morpheus/stages/input/kafka_source_stage.py index 0c12c6db49..c98170fa30 100644 --- a/morpheus/stages/input/kafka_source_stage.py +++ b/morpheus/stages/input/kafka_source_stage.py @@ -22,16 +22,16 @@ import mrc import pandas as pd -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes from morpheus.config import auto_determine_bootstrap +from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) @@ -130,6 +130,9 @@ def __init__(self, self._poll_interval = pd.Timedelta(poll_interval).total_seconds() self._started = False + # Defined lated if in CPU mode + self._json_reader: typing.Callable[..., DataFrameType] = None + self._records_emitted = 0 self._num_messages = 0 @@ -167,7 +170,7 @@ def _process_batch(self, consumer, batch): df = None try: buffer.seek(0) - df = cudf.io.read_json(buffer, engine='cudf', lines=True, orient='records') + df = self._json_reader(buffer, lines=True, orient='records') except Exception as e: logger.error("Error parsing payload into a dataframe : %s", e) finally: @@ -254,6 +257,7 @@ def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: # multiple threads source.launch_options.pe_count = self._max_concurrent else: + self._json_reader = get_json_reader(self._config) source = builder.make_source(self.unique_name, self._source_generator) return source diff --git a/morpheus/stages/output/compare_dataframe_stage.py b/morpheus/stages/output/compare_dataframe_stage.py index 86ae3dc6ce..ecd94563c5 100644 --- a/morpheus/stages/output/compare_dataframe_stage.py +++ b/morpheus/stages/output/compare_dataframe_stage.py @@ -21,8 +21,6 @@ import pandas as pd -import cudf - from morpheus.config import Config from morpheus.io.deserializers import read_file_to_df from morpheus.stages.output.in_memory_sink_stage import InMemorySinkStage @@ -74,8 +72,6 @@ def __init__(self, if isinstance(compare_df, str): compare_df = read_file_to_df(compare_df, df_type='pandas') - elif isinstance(compare_df, cudf.DataFrame): - compare_df = compare_df.to_pandas() elif isinstance(compare_df, list): tmp_dfs = [] for item in compare_df: @@ -83,6 +79,9 @@ def __init__(self, tmp_dfs.append(tmp_df) compare_df = pd.concat(tmp_dfs) compare_df.reset_index(inplace=True, drop=True) + elif not isinstance(compare_df, pd.DataFrame): + # assume it is a cudf DataFrame + compare_df = compare_df.to_pandas() self._compare_df = compare_df diff --git a/morpheus/stages/output/http_server_sink_stage.py b/morpheus/stages/output/http_server_sink_stage.py index aac5a93b99..9f8462263b 100644 --- a/morpheus/stages/output/http_server_sink_stage.py +++ b/morpheus/stages/output/http_server_sink_stage.py @@ -25,11 +25,11 @@ import pandas as pd from mrc.core import operators as ops -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.io import serializers +from morpheus.io.utils import get_df_pkg from morpheus.messages import MessageMeta from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @@ -41,7 +41,9 @@ logger = logging.getLogger(__name__) -@register_stage("to-http-server", ignore_args=["df_serializer_fn"]) +@register_stage("to-http-server", + execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), + ignore_args=["df_serializer_fn"]) class HttpServerSinkStage(PassThruTypeMixin, SinglePortStage): """ Sink stage that starts an HTTP server and listens for incoming requests on a specified endpoint. @@ -116,6 +118,8 @@ def __init__(self, self._df_serializer_fn = df_serializer_fn or self._default_df_serializer + self._df_pkg = get_df_pkg(config) + # FiberQueue doesn't have a way to check the size, nor does it have a way to check if it's empty without # attempting to perform a read. We'll keep track of the size ourselves. self._queue = queue.Queue(maxsize=max_queue_size or config.edge_buffer_size) @@ -201,10 +205,10 @@ def _request_handler(self, _: str) -> HttpParseResponse: body=err_msg) if (len(data_frames) > 0): - df = data_frames[0] if len(data_frames) > 1: - cat_fn = pd.concat if isinstance(df, pd.DataFrame) else cudf.concat - df = cat_fn(data_frames) + df = self._df_pkg.concat(data_frames) + else: + df = data_frames[0] return HttpParseResponse(status_code=HTTPStatus.OK.value, content_type=self._content_type, diff --git a/morpheus/stages/output/write_to_databricks_deltalake_stage.py b/morpheus/stages/output/write_to_databricks_deltalake_stage.py index 6b98ffeb92..53d028d987 100644 --- a/morpheus/stages/output/write_to_databricks_deltalake_stage.py +++ b/morpheus/stages/output/write_to_databricks_deltalake_stage.py @@ -19,8 +19,6 @@ import pandas as pd from mrc.core import operators as ops -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages import MessageMeta @@ -97,8 +95,9 @@ def write_to_deltalake(meta: MessageMeta): convert cudf to spark dataframe """ df = meta.copy_dataframe() - if isinstance(df, cudf.DataFrame): + if not isinstance(df, pd.DataFrame): df = df.to_pandas() + schema = self._extract_schema_from_pandas_dataframe(df) spark_df = self.spark.createDataFrame(df, schema=schema) spark_df.write \ diff --git a/morpheus/stages/output/write_to_elasticsearch_stage.py b/morpheus/stages/output/write_to_elasticsearch_stage.py index eede6926e8..f26948cf6a 100644 --- a/morpheus/stages/output/write_to_elasticsearch_stage.py +++ b/morpheus/stages/output/write_to_elasticsearch_stage.py @@ -18,10 +18,9 @@ import mrc import mrc.core.operators as ops +import pandas as pd import yaml -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.controllers.elasticsearch_controller import ElasticsearchController @@ -110,7 +109,7 @@ def on_data(meta: MessageMeta) -> MessageMeta: self._controller.refresh_client() df = meta.copy_dataframe() - if isinstance(df, cudf.DataFrame): + if not isinstance(df, pd.DataFrame): df = df.to_pandas() logger.debug("Converted cudf of size: %s to pandas dataframe.", len(df)) diff --git a/morpheus/stages/postprocess/generate_viz_frames_stage.py b/morpheus/stages/postprocess/generate_viz_frames_stage.py index b2d059666c..d50279e2e6 100644 --- a/morpheus/stages/postprocess/generate_viz_frames_stage.py +++ b/morpheus/stages/postprocess/generate_viz_frames_stage.py @@ -27,17 +27,17 @@ import websockets.legacy.server from websockets.server import serve -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes +from morpheus.io.utils import get_df_class from morpheus.messages import ControlMessage from morpheus.messages import MultiResponseMessage from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils.producer_consumer_queue import AsyncIOProducerConsumerQueue from morpheus.utils.producer_consumer_queue import Closed +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) @@ -82,6 +82,8 @@ def __init__(self, self._server_task: asyncio.Task = None self._server_close_event: asyncio.Event = None + self._df_class: type[DataFrameType] = get_df_class(c) + @property def name(self) -> str: return "gen_viz" @@ -146,7 +148,7 @@ def indent_data(y: str): except Exception: return y - if isinstance(df, cudf.DataFrame): + if not isinstance(df, pd.DataFrame): df = df.to_pandas() df["data"] = df["data"].apply(indent_data) @@ -291,7 +293,7 @@ def write_batch(x: MultiResponseMessage | ControlMessage): elif isinstance(x, ControlMessage): df = x.payload().get_data(columns) - out_df = cudf.DataFrame() + out_df = self._df_class() out_df["dt"] = (df["timestamp"] - time0).astype(np.int32) out_df["src"] = df["src_ip"].str.ip_to_int().astype(np.int32) From 589d14b7647f26052ca999113901fc79b886f1e8 Mon Sep 17 00:00:00 2001 From: Anuradha Karuppiah Date: Wed, 14 Aug 2024 21:18:58 +0000 Subject: [PATCH 021/347] Update location of setup and other morpheus files in VS settings This is a fixup to commit 4e3edb9653e4f that refactored the setup files Signed-off-by: Anuradha Karuppiah --- morpheus.code-workspace | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/morpheus.code-workspace b/morpheus.code-workspace index 9e25e5b414..5fb3a58375 100644 --- a/morpheus.code-workspace +++ b/morpheus.code-workspace @@ -46,7 +46,7 @@ // "from-kafka", "deserialize", "preprocess", - "--vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt", + "--vocab_hash_file=./python/morpheus/morpheus/data/bert-base-uncased-hash.txt", "--truncation=True", "--do_lower_case=True", "--add_special_tokens=False", @@ -79,7 +79,7 @@ "cwd": "${workspaceFolder}", "justMyCode": false, "name": "Python: Run Pipeline (NLP)", - "program": "${workspaceFolder}/morpheus/cli/run.py", + "program": "${workspaceFolder}/python/morpheus/morpheus/cli/run.py", "request": "launch", "subProcess": true, "type": "debugpy" @@ -135,7 +135,7 @@ "cwd": "${workspaceFolder}", "justMyCode": false, "name": "Python: Run Pipeline (FIL)", - "program": "${workspaceFolder}/morpheus/cli/run.py", + "program": "${workspaceFolder}/python/morpheus/morpheus/cli/run.py", "request": "launch", "subProcess": true, "type": "debugpy" @@ -149,7 +149,7 @@ "--model_max_batch_size=1024", "--use_cpp=False", "pipeline-ae", - "--columns_file=morpheus/data/columns_ae_cloudtrail.txt", + "--columns_file=python/morpheus/morpheus/data/columns_ae_cloudtrail.txt", "--userid_column_name=userIdentitysessionContextsessionIssueruserName", "--userid_filter=user123", "--timestamp_column_name=event_dt", @@ -197,7 +197,7 @@ "cwd": "${workspaceFolder}", "justMyCode": false, "name": "Python: Run Pipeline (AE)", - "program": "${workspaceFolder}/morpheus/cli/run.py", + "program": "${workspaceFolder}/python/morpheus/morpheus/cli/run.py", "request": "launch", "subProcess": true, "type": "debugpy" @@ -212,7 +212,7 @@ "--model_max_batch_size=32", // "--use_cpp=False", "pipeline-nlp", - "--labels_file=morpheus/data/labels_phishing.txt", + "--labels_file=python/morpheus/morpheus/data/labels_phishing.txt", "--model_seq_length=128", "from-file", "--filename=models/datasets/validation-data/phishing-email-validation-data.jsonlines", @@ -221,7 +221,7 @@ // "from-kafka", "deserialize", "preprocess", - "--vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt", + "--vocab_hash_file=./python/morpheus/morpheus/data/bert-base-uncased-hash.txt", "--truncation=True", // "--stride=", "--do_lower_case=True", @@ -262,7 +262,7 @@ "cwd": "${workspaceFolder}", "justMyCode": false, "name": "Python: Run Pipeline (NLP-Phishing)", - "program": "${workspaceFolder}/morpheus/cli/run.py", + "program": "${workspaceFolder}/python/morpheus/morpheus/cli/run.py", "request": "launch", "subProcess": true, "type": "debugpy" @@ -307,7 +307,7 @@ { "MIMode": "gdb", "args": [ - "./morpheus/cli.py", + "./python/morpheus/morpheus/cli.py", "--log_level=DEBUG", "run", "--num_threads=2", @@ -325,7 +325,7 @@ // "from-kafka", "deserialize", "preprocess", - "--vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt", + "--vocab_hash_file=./python/morpheus/morpheus/data/bert-base-uncased-hash.txt", "--truncation=True", "--do_lower_case=True", "--add_special_tokens=False", @@ -403,7 +403,7 @@ { "MIMode": "gdb", "args": [ - "./morpheus/cli.py", + "./python/morpheus/morpheus/cli.py", "--log_level=DEBUG", // "--debug", "run", @@ -496,7 +496,7 @@ { "MIMode": "gdb", "args": [ - "./morpheus/cli.py", + "./python/morpheus/morpheus/cli.py", "--log_level=DEBUG", "run", "--num_threads=1", @@ -600,7 +600,7 @@ "externalConsole": false, "miDebuggerPath": "gdb", "name": "Debug LLM C++ Tests", - "program": "${workspaceFolder}/build/morpheus/_lib/tests/test_llm.x", + "program": "${workspaceFolder}/build/python/morpheus/morpheus/_lib/tests/test_llm.x", "request": "launch", "setupCommands": [ { @@ -679,7 +679,7 @@ "files.trimFinalNewlines": true, "files.trimTrailingWhitespace": true, "flake8.args": [ - "--style=${workspaceFolder}/setup.cfg" + "--style=${workspaceFolder}/python/morpheus/setup.cfg" ], "pylint.args": [ "--rcfile=${workspaceFolder}/pyproject.toml", @@ -730,7 +730,7 @@ } ], "yapf.args": [ - "--style=${workspaceFolder}/setup.cfg" + "--style=${workspaceFolder}/python/morpheus/setup.cfg" ] } } From c8493b4ec83a0e5969dea0d7522ac7e1ef56c3ff Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 16 Aug 2024 16:07:46 -0700 Subject: [PATCH 022/347] WIP --- python/morpheus/morpheus/config.py | 2 +- python/morpheus/morpheus/io/utils.py | 10 +++---- python/morpheus/morpheus/pipeline/pipeline.py | 9 ++---- .../morpheus/stages/input/arxiv_source.py | 7 ++--- .../databricks_deltalake_source_stage.py | 7 ++--- .../stages/input/file_source_stage.py | 6 ++-- tests/conftest.py | 28 ++++++++++--------- tests/test_messages.py | 1 + 8 files changed, 32 insertions(+), 38 deletions(-) diff --git a/python/morpheus/morpheus/config.py b/python/morpheus/morpheus/config.py index a665b41c9b..6db7117ca0 100644 --- a/python/morpheus/morpheus/config.py +++ b/python/morpheus/morpheus/config.py @@ -229,7 +229,7 @@ class Config(ConfigBase): fil: ConfigFIL = dataclasses.field(default=None) def __post_init__(self): - if self.execution_mode is ExecutionMode.CPU: + if self.execution_mode == ExecutionMode.CPU: CppConfig.set_should_use_cpp(False) def save(self, filename: str): diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index ba6abcbfc5..21ef7de626 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -24,10 +24,8 @@ if typing.TYPE_CHECKING: import cudf -from morpheus.config import Config -from morpheus.config import ExecutionMode -from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import SeriesType +from morpheus.config import Config, ExecutionMode +from morpheus.utils.type_aliases import DataFrameType, SeriesType logger = logging.getLogger(__name__) @@ -143,7 +141,7 @@ def get_df_pkg(config: Config) -> types.ModuleType: """ Return the appropriate DataFrame package based on the execution mode. """ - if config.execution_mode is ExecutionMode.GPU: + if config.execution_mode == ExecutionMode.GPU: import cudf return cudf else: @@ -162,7 +160,7 @@ def get_json_reader(config: Config) -> typing.Callable[..., DataFrameType]: """ Return the appropriate JSON reader based on the execution mode. """ - if config.execution_mode is ExecutionMode.GPU: + if config.execution_mode == ExecutionMode.GPU: import cudf reader = functools.partial(cudf.read_json, engine='cudf') else: diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index 1377eeb64a..b749a18c37 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -19,8 +19,7 @@ import sys import threading import typing -from collections import OrderedDict -from collections import defaultdict +from collections import OrderedDict, defaultdict from enum import Enum from functools import partial @@ -29,9 +28,7 @@ from tqdm import tqdm import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import -from morpheus.config import Config -from morpheus.config import CppConfig -from morpheus.config import ExecutionMode +from morpheus.config import Config, CppConfig, ExecutionMode from morpheus.utils.type_utils import pretty_print_type_name logger = logging.getLogger(__name__) @@ -62,7 +59,7 @@ class Pipeline(): """ def __init__(self, config: Config): - if config.execution_mode is ExecutionMode.CPU and CppConfig.get_should_use_cpp(): + if config.execution_mode == ExecutionMode.CPU and CppConfig.get_should_use_cpp(): raise RuntimeError("C++ mode requires GPU execution mode.") self._mutex = threading.RLock() diff --git a/python/morpheus/morpheus/stages/input/arxiv_source.py b/python/morpheus/morpheus/stages/input/arxiv_source.py index 46f9998834..25eeb36e14 100644 --- a/python/morpheus/morpheus/stages/input/arxiv_source.py +++ b/python/morpheus/morpheus/stages/input/arxiv_source.py @@ -21,8 +21,7 @@ import pandas as pd from morpheus.cli.register_stage import register_stage -from morpheus.config import Config -from morpheus.config import ExecutionMode +from morpheus.config import Config, ExecutionMode from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource @@ -97,7 +96,7 @@ def __init__(self, self._total_chunks = 0 self._cache_dir = cache_dir - if c.execution_mode is ExecutionMode.GPU: + if c.execution_mode == ExecutionMode.GPU: import cudf self._cudf = cudf @@ -198,7 +197,7 @@ def _splitting_pages(self, documents: list["Document"]): df.rename(columns=map_cols, inplace=True) - if self._config.execution_mode is ExecutionMode.GPU: + if self._config.execution_mode == ExecutionMode.GPU: df = self._cudf.from_pandas(df) return MessageMeta(df) diff --git a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py index 95d37b8caa..630aa37f7c 100644 --- a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py +++ b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py @@ -17,8 +17,7 @@ import mrc from morpheus.cli.register_stage import register_stage -from morpheus.config import Config -from morpheus.config import ExecutionMode +from morpheus.config import Config, ExecutionMode from morpheus.messages.message_meta import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource @@ -76,7 +75,7 @@ def __init__(self, self.items_per_page = items_per_page self.offset = 0 - if config.execution_mode is ExecutionMode.GPU: + if config.execution_mode == ExecutionMode.GPU: import cudf self._cudf = cudf @@ -110,7 +109,7 @@ def source_generator(self): df = df.toPandas().drop(["_id"], axis=1) - if self._config.execution_mode is ExecutionMode.GPU: + if self._config.execution_mode == ExecutionMode.GPU: df = self._cudf.from_pandas(df) yield MessageMeta(df=df) diff --git a/python/morpheus/morpheus/stages/input/file_source_stage.py b/python/morpheus/morpheus/stages/input/file_source_stage.py index 260b3c530b..76b3d68102 100644 --- a/python/morpheus/morpheus/stages/input/file_source_stage.py +++ b/python/morpheus/morpheus/stages/input/file_source_stage.py @@ -21,9 +21,7 @@ from morpheus.cli import register_stage from morpheus.common import FileTypes -from morpheus.config import Config -from morpheus.config import ExecutionMode -from morpheus.config import PipelineModes +from morpheus.config import Config, ExecutionMode, PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin @@ -100,7 +98,7 @@ def __init__(self, self._iterative = iterative self._repeat_count = repeat - if c.execution_mode is ExecutionMode.GPU: + if c.execution_mode == ExecutionMode.GPU: print("GPU MODE Using cudf") self._df_type = "cudf" else: diff --git a/tests/conftest.py b/tests/conftest.py index 19b92df1b1..b7f01ee99a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,12 +30,14 @@ import pytest import requests - from _utils import import_or_skip +from _utils.kafka import \ + kafka_bootstrap_servers_fixture # noqa: F401 pylint:disable=unused-import +from _utils.kafka import \ + kafka_consumer_fixture # noqa: F401 pylint:disable=unused-import +from _utils.kafka import \ + kafka_topics_fixture # noqa: F401 pylint:disable=unused-import from _utils.kafka import _init_pytest_kafka -from _utils.kafka import kafka_bootstrap_servers_fixture # noqa: F401 pylint:disable=unused-import -from _utils.kafka import kafka_consumer_fixture # noqa: F401 pylint:disable=unused-import -from _utils.kafka import kafka_topics_fixture # noqa: F401 pylint:disable=unused-import # Don't let pylint complain about pytest fixtures # pylint: disable=redefined-outer-name,unused-argument @@ -43,9 +45,12 @@ (PYTEST_KAFKA_AVAIL, PYTEST_KAFKA_ERROR) = _init_pytest_kafka() if PYTEST_KAFKA_AVAIL: # Pull out the fixtures into this namespace - from _utils.kafka import _kafka_consumer # noqa: F401 pylint:disable=unused-import - from _utils.kafka import kafka_server # noqa: F401 pylint:disable=unused-import - from _utils.kafka import zookeeper_proc # noqa: F401 pylint:disable=unused-import + from _utils.kafka import \ + _kafka_consumer # noqa: F401 pylint:disable=unused-import + from _utils.kafka import \ + kafka_server # noqa: F401 pylint:disable=unused-import + from _utils.kafka import \ + zookeeper_proc # noqa: F401 pylint:disable=unused-import def pytest_addoption(parser: pytest.Parser): @@ -250,8 +255,7 @@ def config_only_cpp(): `@pytest.mark.usefixtures("config_only_cpp")` """ - from morpheus.config import Config - from morpheus.config import CppConfig + from morpheus.config import Config, CppConfig CppConfig.set_should_use_cpp(True) @@ -265,8 +269,7 @@ def config_no_cpp(): `@pytest.mark.usefixtures("config_no_cpp")` """ - from morpheus.config import Config - from morpheus.config import CppConfig + from morpheus.config import Config, CppConfig CppConfig.set_should_use_cpp(False) @@ -504,8 +507,7 @@ def reset_plugin_manger(): @pytest.fixture(scope="function") def reset_global_stage_registry(): - from morpheus.cli.stage_registry import GlobalStageRegistry - from morpheus.cli.stage_registry import StageRegistry + from morpheus.cli.stage_registry import GlobalStageRegistry, StageRegistry GlobalStageRegistry._global_registry = StageRegistry() yield diff --git a/tests/test_messages.py b/tests/test_messages.py index 6c37535293..22c7947188 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -167,6 +167,7 @@ def check_all_messages(should_be_cpp: bool, no_cpp_class: bool): ) +@pytest.mark.usefixtures("use_cpp") def test_constructor_cpp(): check_all_messages(morpheus.config.CppConfig.get_should_use_cpp(), False) From 00705928da059a352b1d7c5228c786f07f9a39d6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 16 Aug 2024 16:09:10 -0700 Subject: [PATCH 023/347] Move isort settings into pyproject.toml --- ci/scripts/common.sh | 1 + ci/scripts/python_checks.sh | 2 +- pyproject.toml | 34 ++++++++++++++++++++++++++++ python/morpheus/setup.cfg | 44 ------------------------------------- 4 files changed, 36 insertions(+), 45 deletions(-) diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh index 38f3b01526..258fe689c2 100644 --- a/ci/scripts/common.sh +++ b/ci/scripts/common.sh @@ -20,6 +20,7 @@ export MORPHEUS_ROOT=${MORPHEUS_ROOT:-"$(realpath ${SCRIPT_DIR}/../..)"} export PY_ROOT="${MORPHEUS_ROOT}" export PY_CFG="${PY_ROOT}/python/morpheus/setup.cfg" +export PROJ_TOML="${MORPHEUS_ROOT}/pyproject.toml" export PY_DIRS="${PY_ROOT} ci/scripts" # Determine the commits to compare against. If running in CI, these will be set. Otherwise, diff with main diff --git a/ci/scripts/python_checks.sh b/ci/scripts/python_checks.sh index 7455b2edfc..e7b9386e95 100755 --- a/ci/scripts/python_checks.sh +++ b/ci/scripts/python_checks.sh @@ -42,7 +42,7 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then done if [[ "${SKIP_ISORT}" == "" ]]; then - ISORT_OUTPUT=`python3 -m isort --settings-file ${PY_CFG} --filter-files --check-only ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` + ISORT_OUTPUT=`python3 -m isort --settings-file ${PROJ_TOML} --filter-files --check-only ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` ISORT_RETVAL=$? fi diff --git a/pyproject.toml b/pyproject.toml index 6d47ec2cb7..a2490e0456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -719,3 +719,37 @@ ignored-argument-names = "_.*|^ignored_|^unused_" # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] + + +# ===== isort Config ===== +[tool.isort] +line_length=120 +multi_line_output=3 +include_trailing_comma=true +force_grid_wrap=0 +combine_as_imports=true +order_by_type=true +force_single_line=true +src_paths = ["python", "tests"] +known_dask = ["dask", "distributed", "dask_cuda"] +known_rapids = ["nvstrings", "nvcategory", "nvtext", "cudf", "cuml", "cugraph", "dask_cudf"] +known_first_party = ["morpheus", "_utils"] +default_section = "THIRDPARTY" +sections= ["FUTURE", "STDLIB", "THIRDPARTY", "DASK", "RAPIDS", "FIRSTPARTY", "LOCALFOLDER"] +skip= [ + "__init__.py", + # Skip _version.py as it is auto-generated + "_version.py", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tmp", + ".tox", + ".venv", + "buck-out", + "build", + "dist", + "models", + "thirdparty" +] diff --git a/python/morpheus/setup.cfg b/python/morpheus/setup.cfg index 5a0f651f90..1c7bcb39fe 100644 --- a/python/morpheus/setup.cfg +++ b/python/morpheus/setup.cfg @@ -24,50 +24,6 @@ versionfile_build = morpheus/_version.py tag_prefix = v parentdir_prefix = morpheus- -# ===== isort Config ===== -[isort] -line_length=120 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -force_single_line=True -src_paths=morpheus,tests -known_dask= - dask - distributed - dask_cuda -known_rapids= - nvstrings - nvcategory - nvtext - cudf - cuml - cugraph - dask_cudf -known_first_party= - morpheus - _utils -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - __init__.py - # Skip _version.py as it is auto-generated - _version.py - .eggs - .git - .hg - .mypy_cache - .tmp - .tox - .venv - buck-out - build - dist - models - thirdparty - # ===== flake8 Config ===== [flake8] filename = *.py, *.pyx, *.pxd From 12a2f38ac2db8a51fd30be13229968c51fb35f18 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 16 Aug 2024 16:13:45 -0700 Subject: [PATCH 024/347] Fix imports --- tests/conftest.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b7f01ee99a..19b92df1b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,14 +30,12 @@ import pytest import requests + from _utils import import_or_skip -from _utils.kafka import \ - kafka_bootstrap_servers_fixture # noqa: F401 pylint:disable=unused-import -from _utils.kafka import \ - kafka_consumer_fixture # noqa: F401 pylint:disable=unused-import -from _utils.kafka import \ - kafka_topics_fixture # noqa: F401 pylint:disable=unused-import from _utils.kafka import _init_pytest_kafka +from _utils.kafka import kafka_bootstrap_servers_fixture # noqa: F401 pylint:disable=unused-import +from _utils.kafka import kafka_consumer_fixture # noqa: F401 pylint:disable=unused-import +from _utils.kafka import kafka_topics_fixture # noqa: F401 pylint:disable=unused-import # Don't let pylint complain about pytest fixtures # pylint: disable=redefined-outer-name,unused-argument @@ -45,12 +43,9 @@ (PYTEST_KAFKA_AVAIL, PYTEST_KAFKA_ERROR) = _init_pytest_kafka() if PYTEST_KAFKA_AVAIL: # Pull out the fixtures into this namespace - from _utils.kafka import \ - _kafka_consumer # noqa: F401 pylint:disable=unused-import - from _utils.kafka import \ - kafka_server # noqa: F401 pylint:disable=unused-import - from _utils.kafka import \ - zookeeper_proc # noqa: F401 pylint:disable=unused-import + from _utils.kafka import _kafka_consumer # noqa: F401 pylint:disable=unused-import + from _utils.kafka import kafka_server # noqa: F401 pylint:disable=unused-import + from _utils.kafka import zookeeper_proc # noqa: F401 pylint:disable=unused-import def pytest_addoption(parser: pytest.Parser): @@ -255,7 +250,8 @@ def config_only_cpp(): `@pytest.mark.usefixtures("config_only_cpp")` """ - from morpheus.config import Config, CppConfig + from morpheus.config import Config + from morpheus.config import CppConfig CppConfig.set_should_use_cpp(True) @@ -269,7 +265,8 @@ def config_no_cpp(): `@pytest.mark.usefixtures("config_no_cpp")` """ - from morpheus.config import Config, CppConfig + from morpheus.config import Config + from morpheus.config import CppConfig CppConfig.set_should_use_cpp(False) @@ -507,7 +504,8 @@ def reset_plugin_manger(): @pytest.fixture(scope="function") def reset_global_stage_registry(): - from morpheus.cli.stage_registry import GlobalStageRegistry, StageRegistry + from morpheus.cli.stage_registry import GlobalStageRegistry + from morpheus.cli.stage_registry import StageRegistry GlobalStageRegistry._global_registry = StageRegistry() yield From c1852400e9bc43c6c902af6754d64dc6adbcc567 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 16 Aug 2024 16:27:47 -0700 Subject: [PATCH 025/347] Revert base stage idea --- .../stages/input/http_client_source_stage.py | 10 +++++- .../stages/input/http_server_source_stage.py | 24 +++++++++---- .../stages/input/http_source_stage_base.py | 35 ------------------- 3 files changed, 27 insertions(+), 42 deletions(-) delete mode 100644 python/morpheus/morpheus/stages/input/http_source_stage_base.py diff --git a/python/morpheus/morpheus/stages/input/http_client_source_stage.py b/python/morpheus/morpheus/stages/input/http_client_source_stage.py index 29009fb66f..f05adea710 100644 --- a/python/morpheus/morpheus/stages/input/http_client_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_client_source_stage.py @@ -24,6 +24,7 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import ExecutionMode +from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource @@ -144,7 +145,11 @@ def __init__(self, self._lines = lines self._requst_kwargs = request_kwargs - self._payload_to_df_fn = payload_to_df_fn or self._get_default_payload_to_df_fn(config) + if payload_to_df_fn is not None: + self._payload_to_df_fn = payload_to_df_fn + else: + reader = get_json_reader(self._config) + self._payload_to_df_fn = lambda payload, lines: reader(payload, lines=lines) @property def name(self) -> str: @@ -155,6 +160,9 @@ def supports_cpp_node(self) -> bool: """Indicates whether or not this stage supports a C++ implementation""" return False + def compute_schema(self, schema: StageSchema): + schema.output_schema.set_type(MessageMeta) + def _parse_response(self, response: requests.Response) -> typing.Union[DataFrameType, None]: """ Returns a DataFrame parsed from the response payload. If the response payload is empty, then `None` is returned. diff --git a/python/morpheus/morpheus/stages/input/http_server_source_stage.py b/python/morpheus/morpheus/stages/input/http_server_source_stage.py index 344275caaa..80b7c17871 100644 --- a/python/morpheus/morpheus/stages/input/http_server_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_server_source_stage.py @@ -23,10 +23,16 @@ import mrc from morpheus.cli.register_stage import register_stage -from morpheus.config import Config, ExecutionMode +from morpheus.config import Config +from morpheus.config import ExecutionMode +from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta -from morpheus.stages.input.http_source_stage_base import HttpSourceStageBase -from morpheus.utils.http_utils import HTTPMethod, HttpParseResponse, MimeTypes +from morpheus.pipeline.preallocator_mixin import PreallocatorMixin +from morpheus.pipeline.single_output_source import SingleOutputSource +from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.http_utils import HTTPMethod +from morpheus.utils.http_utils import HttpParseResponse +from morpheus.utils.http_utils import MimeTypes from morpheus.utils.producer_consumer_queue import Closed from morpheus.utils.type_aliases import DataFrameType @@ -37,7 +43,7 @@ @register_stage("from-http", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) -class HttpServerSourceStage(HttpSourceStageBase): +class HttpServerSourceStage(PreallocatorMixin, SingleOutputSource): """ Source stage that starts an HTTP server and listens for incoming requests on a specified endpoint. @@ -143,6 +149,9 @@ def supports_cpp_node(self) -> bool: """Indicates whether this stage supports C++ nodes.""" return True + def compute_schema(self, schema: StageSchema): + schema.output_schema.set_type(MessageMeta) + def stop(self): """ Performs cleanup steps when pipeline is stopped. @@ -215,7 +224,9 @@ def _readiness_check(self, _: str) -> HttpParseResponse: body=err_msg) def _generate_frames(self) -> typing.Iterator[MessageMeta]: - from morpheus.common import FiberQueue, HttpEndpoint, HttpServer + from morpheus.common import FiberQueue + from morpheus.common import HttpEndpoint + from morpheus.common import HttpServer msg = HttpEndpoint(self._parse_payload, self._endpoint, self._method.name) live = HttpEndpoint(self._liveliness_check, self._live_endpoint, self._live_method.name) @@ -276,7 +287,8 @@ def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: stop_after=self._stop_after) else: if self._payload_to_df_fn is None: - self._payload_to_df_fn = self._get_default_payload_to_df_fn(self._config) + reader = get_json_reader(self._config) + self._payload_to_df_fn = lambda payload, lines: reader(payload, lines=lines) node = builder.make_source(self.unique_name, self._generate_frames()) diff --git a/python/morpheus/morpheus/stages/input/http_source_stage_base.py b/python/morpheus/morpheus/stages/input/http_source_stage_base.py deleted file mode 100644 index e83a53f353..0000000000 --- a/python/morpheus/morpheus/stages/input/http_source_stage_base.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Base class for HTTP sources.""" - -import typing - -from morpheus.config import Config -from morpheus.io.utils import get_json_reader -from morpheus.messages import MessageMeta -from morpheus.pipeline.preallocator_mixin import PreallocatorMixin -from morpheus.pipeline.single_output_source import SingleOutputSource -from morpheus.pipeline.stage_schema import StageSchema -from morpheus.utils.type_aliases import DataFrameType - - -class HttpSourceStageBase(PreallocatorMixin, SingleOutputSource): - - def _get_default_payload_to_df_fn(self, config: Config) -> typing.Callable[[str, bool], DataFrameType]: - reader = get_json_reader(config) - - return lambda payload, lines: reader(payload, lines=lines) - - def compute_schema(self, schema: StageSchema): - schema.output_schema.set_type(MessageMeta) From db04cc5180e181c70fed069f9695b686084ca286 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 16 Aug 2024 16:46:09 -0700 Subject: [PATCH 026/347] Fix http server python tests --- .../morpheus/stages/input/http_server_source_stage.py | 7 +++++-- tests/stages/test_http_server_source_stage.py | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/stages/input/http_server_source_stage.py b/python/morpheus/morpheus/stages/input/http_server_source_stage.py index 80b7c17871..cdd45f0112 100644 --- a/python/morpheus/morpheus/stages/input/http_server_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_server_source_stage.py @@ -267,6 +267,10 @@ def _generate_frames(self) -> typing.Iterator[MessageMeta]: if self._stop_after > 0 and self._records_emitted >= self._stop_after: self._processing = False + def _set_default_payload_to_df_fn(self): + reader = get_json_reader(self._config) + self._payload_to_df_fn = lambda payload, lines: reader(payload, lines=lines) + def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: if self._build_cpp_node() and self._payload_to_df_fn is None: import morpheus._lib.stages as _stages @@ -287,8 +291,7 @@ def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: stop_after=self._stop_after) else: if self._payload_to_df_fn is None: - reader = get_json_reader(self._config) - self._payload_to_df_fn = lambda payload, lines: reader(payload, lines=lines) + self._set_default_payload_to_df_fn() node = builder.make_source(self.unique_name, self._generate_frames()) diff --git a/tests/stages/test_http_server_source_stage.py b/tests/stages/test_http_server_source_stage.py index 01763d78db..15c8e585da 100644 --- a/tests/stages/test_http_server_source_stage.py +++ b/tests/stages/test_http_server_source_stage.py @@ -95,6 +95,9 @@ def test_generate_frames(config: Config, dataset_pandas: DatasetManager, lines: lines=lines, payload_to_df_fn=payload_to_df_fn) + if not use_payload_to_df_fn: + stage._set_default_payload_to_df_fn() + generate_frames = stage._generate_frames() msg_queue = queue.SimpleQueue() From 3740f67d236309f8a986277af8040be1d3bd320c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 19 Aug 2024 12:01:16 -0700 Subject: [PATCH 027/347] Lazily import pandas in type_aliases --- python/morpheus/morpheus/utils/type_aliases.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/morpheus/morpheus/utils/type_aliases.py b/python/morpheus/morpheus/utils/type_aliases.py index 25b6f16b0b..b9ed411744 100644 --- a/python/morpheus/morpheus/utils/type_aliases.py +++ b/python/morpheus/morpheus/utils/type_aliases.py @@ -15,10 +15,10 @@ import typing -import pandas as pd - if typing.TYPE_CHECKING: + import pandas + import cudf -DataFrameType = typing.Union[pd.DataFrame, "cudf.DataFrame"] -SeriesType = typing.Union[pd.Series, "cudf.Series"] +DataFrameType = typing.Union["pandas.DataFrame", "cudf.DataFrame"] +SeriesType = typing.Union["pandas.Series", "cudf.Series"] From ea64e614ed8bae10a84074d4b2cfed52059e9aa1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 19 Aug 2024 12:29:56 -0700 Subject: [PATCH 028/347] Add intermediate stage which adds a column --- examples/cpu_only/run.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index dbbb25ce6b..072ead86d4 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -23,6 +23,7 @@ from morpheus.config import Config from morpheus.config import CppConfig from morpheus.config import ExecutionMode +from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.pipeline.stage_decorator import stage @@ -95,7 +96,20 @@ def print_msg(msg: typing.Any) -> typing.Any: pipeline.add_stage(print_msg(config)) - pipeline.add_stage(DeserializeStage(config)) + # TODO: Remove if PR #1803 is merged first + pipeline.add_stage(DeserializeStage(config, message_type=ControlMessage)) + + @stage + def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") -> ControlMessage: + meta = msg.payload() + + with meta.mutable_dataframe() as df: + print(f"Received a ControlMessage with a dataframe of type {type(df)}") + df[total_column_name] = df.select_dtypes(include="number").sum(axis=1) + + return msg + + pipeline.add_stage(calculate_totals(config)) pipeline.add_stage(SerializeStage(config)) pipeline.add_stage(WriteToFileStage(config, filename=out_file, overwrite=True)) pipeline.build() From ee279225b31fcecbbe8068f05ce0a9bc23f2b3d5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 19 Aug 2024 15:05:49 -0700 Subject: [PATCH 029/347] Update the string to `ControlMessageType` logic to be performed in a case-insensitive manner. The reason for this is that `s_task_type_map` contains lower-cased keys, however the enum exported to Python uses upper-cased keys. We need a way to ensure this is consistent between Python and C++ impls. Optionally throw an error on an invalid task type, current code just ignores invalid types. Add helper method `to_task_type` to consolidate string to `ControlMessageType` logic. Remove old commented out code [no ci] --- .../include/morpheus/messages/control.hpp | 124 +----------------- .../morpheus/_lib/src/messages/control.cpp | 49 ++++--- 2 files changed, 32 insertions(+), 141 deletions(-) diff --git a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp index 2565bf4874..3b48833a00 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp @@ -39,128 +39,6 @@ enum class MORPHEUS_EXPORT ControlMessageType TRAINING }; -// class PayloadManager -// { -// public: -// /** -// * @brief Get the tensor object identified by `name` -// * -// * @param name -// * @return TensorObject& -// * @throws std::runtime_error If no tensor matching `name` exists -// */ -// TensorObject& get_tensor(const std::string& name) -// { -// return m_tensors->get_tensor(name); -// } - -// /** -// * @brief Get the tensor object identified by `name` -// * -// * @param name -// * @return const TensorObject& -// * @throws std::runtime_error If no tensor matching `name` exists -// */ -// const TensorObject& get_tensor(const std::string& name) const -// { -// return m_tensors->get_tensor(name); -// } - -// /** -// * @brief Set the tensor object identified by `name` -// * -// * @param name -// * @param tensor -// * @throws std::length_error If the number of rows in `tensor` does not match `count`. -// */ -// void set_tensor(const std::string& name, TensorObject&& tensor) -// { -// m_tensors->set_tensor(name, std::move(tensor)); -// } - -// /** -// * @brief Get a reference to the internal tensors map -// * -// * @return const TensorMap& -// */ -// const TensorMap& get_tensors() const -// { -// return m_tensors->get_tensors(); -// } - -// /** -// * @brief Set the tensors object -// * -// * @param tensors -// * @throws std::length_error If the number of rows in the `tensors` do not match `count`. -// */ -// void set_tensors(TensorMap&& tensors) -// { -// m_tensors->set_tensors(std::move(tensors)); -// } - -// /** -// * @brief Get the tensor object identified by `name` -// * -// * @param name -// * @return TensorObject& -// * @throws std::runtime_error If no tensor matching `name` exists -// */ -// TensorObject& get_column(const std::string& name) -// { -// return m_tensors->get_tensor(name); -// } - -// /** -// * @brief Get the tensor object identified by `name` -// * -// * @param name -// * @return const TensorObject& -// * @throws std::runtime_error If no tensor matching `name` exists -// */ -// const TensorObject& get_column(const std::string& name) const -// { -// return m_tensors->get_tensor(name); -// } - -// /** -// * @brief Set the tensor object identified by `name` -// * -// * @param name -// * @param tensor -// * @throws std::length_error If the number of rows in `tensor` does not match `count`. -// */ -// void set_column(const std::string& name, TensorObject&& tensor) -// { -// m_tensors->set_tensor(name, std::move(tensor)); -// } - -// /** -// * @brief Get a reference to the internal tensors map -// * -// * @return const TensorMap& -// */ -// TableInfo get_columns() const -// { -// return m_df->get_info(); -// } - -// /** -// * @brief Set the tensors object -// * -// * @param tensors -// * @throws std::length_error If the number of rows in the `tensors` do not match `count`. -// */ -// void set_columns(TableInfo&& tensors) -// { -// m_tensors->set_tensors(std::move(tensors)); -// } - -// private: -// std::shared_ptr m_df; -// std::shared_ptr m_tensors; -// }; - class MORPHEUS_EXPORT TensorMemory; // System-clock for better compatibility with pybind11/chrono @@ -368,6 +246,8 @@ class MORPHEUS_EXPORT ControlMessage static const std::string s_config_schema; // NOLINT static std::map s_task_type_map; // NOLINT + ControlMessageType to_task_type(const std::string& task_type, bool throw_on_error) const; + ControlMessageType m_cm_type{ControlMessageType::NONE}; std::shared_ptr m_payload{nullptr}; std::shared_ptr m_tensors{nullptr}; diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index ca23c5f9f8..8e1f0965ee 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -19,12 +19,13 @@ #include "morpheus/messages/meta.hpp" // for MessageMeta, MessageMetaInterfaceProxy -#include // for COMPACT_GOOGLE_LOG_INFO, LogMessage, VLOG -#include // for basic_json, json_ref, iter_impl, operator<< -#include // IWYU pragma: keep -#include // for cast, object::cast -#include // for object, none, dict, isinstance, list, str, value_error, generic_item -#include // for cast_from_pyobject +#include // for to_lower_copy +#include // for COMPACT_GOOGLE_LOG_INFO, LogMessage, VLOG +#include // for basic_json, json_ref, iter_impl, operator<< +#include // IWYU pragma: keep +#include // for cast, object::cast +#include // for object, none, dict, isinstance, list, str, value_error, generic_item +#include // for cast_from_pyobject #include // for optional, nullopt #include // for basic_ostream, operator<< @@ -40,6 +41,7 @@ namespace morpheus { const std::string ControlMessage::s_config_schema = R"()"; std::map ControlMessage::s_task_type_map{{"inference", ControlMessageType::INFERENCE}, + {"none", ControlMessageType::NONE}, {"training", ControlMessageType::TRAINING}}; ControlMessage::ControlMessage() : m_config({{"metadata", morpheus::utilities::json_t::object()}}), m_tasks({}) {} @@ -65,16 +67,16 @@ const morpheus::utilities::json_t& ControlMessage::config() const void ControlMessage::add_task(const std::string& task_type, const morpheus::utilities::json_t& task) { VLOG(20) << "Adding task of type " << task_type << " to control message" << task.dump(4); - auto _task_type = s_task_type_map.contains(task_type) ? s_task_type_map[task_type] : ControlMessageType::NONE; + auto _task_type = to_task_type(task_type, false); - if (this->task_type() == ControlMessageType::NONE) + if (_task_type != ControlMessageType::NONE and this->task_type() != _task_type) { - this->task_type(_task_type); + throw std::runtime_error("Cannot add inference and training tasks to the same control message"); } - if (_task_type != ControlMessageType::NONE and this->task_type() != _task_type) + if (this->task_type() == ControlMessageType::NONE) { - throw std::runtime_error("Cannot add inference and training tasks to the same control message"); + this->task_type(_task_type); } m_tasks[task_type].push_back(task); @@ -197,14 +199,7 @@ void ControlMessage::config(const morpheus::utilities::json_t& config) { if (config.contains("type")) { - auto task_type = config.at("type"); - auto _task_type = - s_task_type_map.contains(task_type) ? s_task_type_map.at(task_type) : ControlMessageType::NONE; - - if (this->task_type() == ControlMessageType::NONE) - { - this->task_type(_task_type); - } + this->task_type(to_task_type(config.at("type").get(), true)); } if (config.contains("tasks")) @@ -256,6 +251,22 @@ void ControlMessage::task_type(ControlMessageType type) m_cm_type = type; } +ControlMessageType ControlMessage::to_task_type(const std::string& task_type, bool throw_on_error) const +{ + auto lower_task_type = boost::to_lower_copy(task_type); + if (ControlMessage::s_task_type_map.contains(lower_task_type)) + { + return ControlMessage::s_task_type_map.at(lower_task_type); + } + + if (throw_on_error) + { + throw std::runtime_error("Invalid task type: " + task_type); + } + + return ControlMessageType::NONE; +} + /*** Proxy Implementations ***/ std::shared_ptr ControlMessageProxy::create(py::dict& config) { From 5d03a018aa2f5641270893c864799efd64f20911 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 07:43:01 -0700 Subject: [PATCH 030/347] First pass at a python impl for ControlMessage --- python/morpheus/morpheus/messages/__init__.py | 4 +- .../morpheus/messages/control_message.py | 168 ++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 python/morpheus/morpheus/messages/control_message.py diff --git a/python/morpheus/morpheus/messages/__init__.py b/python/morpheus/morpheus/messages/__init__.py index b10100cba7..0ed0b34b4c 100644 --- a/python/morpheus/morpheus/messages/__init__.py +++ b/python/morpheus/morpheus/messages/__init__.py @@ -18,7 +18,7 @@ # Import order is very important here. Import base classes before child ones # isort: off -from morpheus._lib.messages import ControlMessage +from morpheus._lib.messages import ControlMessageType from morpheus._lib.messages import DataLoaderRegistry from morpheus._lib.messages import RawPacketMessage from morpheus.messages.memory.tensor_memory import TensorMemory @@ -41,9 +41,11 @@ from morpheus.messages.multi_response_message import MultiResponseMessage from morpheus.messages.multi_response_message import MultiResponseProbsMessage from morpheus.messages.multi_tensor_message import MultiTensorMessage +from morpheus.messages.control_message import ControlMessage __all__ = [ "ControlMessage", + "ControlMessageType", "DataLoaderRegistry", "InferenceMemory", "InferenceMemoryAE", diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py new file mode 100644 index 0000000000..90763b7350 --- /dev/null +++ b/python/morpheus/morpheus/messages/control_message.py @@ -0,0 +1,168 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import logging +import re +import typing +from collections import defaultdict +from collections import deque +from datetime import datetime + +# Users of this module should import ControlMessageType from morpheus.messages, we can't do that here without causing a +# circular import error, instead we import it from the _lib module, we don't want to put `_messages.ControlMessageType` +# in the public API and confuse users +import morpheus._lib.messages as _messages +from morpheus._lib.messages import ControlMessageType # pylint: disable=morpheus-incorrect-lib-from-import +from morpheus.cli.utils import get_enum_keys +from morpheus.cli.utils import get_enum_members +from morpheus.messages.memory.tensor_memory import TensorMemory +from morpheus.messages.message_base import MessageBase +from morpheus.messages.message_meta import MessageMeta + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass(init=False) +class ControlMessage(MessageBase, cpp_class=_messages.ControlMessage): + + def __init__(self, config: dict = None): + super().__init__() + + self._payload: MessageMeta = None + self._tensors: TensorMemory = None + + self._tasks: dict[str, deque] = defaultdict(deque) + self._timestamps: dict[str, datetime] = {} + self._type: ControlMessageType = ControlMessageType.NONE + + self.config: dict = config + + @property + def config(self) -> dict: + return self._config + + @config.setter + def config(self, config: dict): + cm_type: str | ControlMessageType = config.get("type") + if cm_type is not None: + if isinstance(cm_type, str): + try: + cm_type = get_enum_members(ControlMessageType)[cm_type] + except KeyError as exc: + raise ValueError( + f"Invalid ControlMessageType: {cm_type}, supported types: {get_enum_keys(ControlMessageType)}" + ) from exc + + self._type = cm_type + + tasks = config.get("tasks") + if tasks is not None: + for task in tasks: + self.add_task(task["type"], task["properties"]) + + self._config = {"metadata": config.get("metadata", {}).copy()} + + def has_task(self, task_type: str) -> bool: + """ + Return True if the control message has at least one task of the given type + """ + # Using `get` to avoid creating an empty list if the task type is not present + tasks = self._tasks.get(task_type, []) + return len(tasks) > 0 + + def add_task(self, task_type: str, properties: dict): + if isinstance(task_type, str): + cm_type = get_enum_members(ControlMessageType).get(task_type, ControlMessageType.NONE) + if cm_type != ControlMessageType.NONE: + if self._type == ControlMessageType.NONE: + self._type = cm_type + elif self._type != cm_type: + raise ValueError("Cannot mix different types of tasks on the same control message") + + self._tasks[task_type].append(properties) + + def remove_task(self, task_type: str) -> dict: + tasks = self._tasks.get(task_type, []) + if len(tasks) == 0: + raise ValueError(f"No task of type {task_type} found") + + return tasks.popleft() + + def get_tasks(self) -> dict[str, deque]: + return self._tasks + + def set_metadata(self, key: str, value: typing.Any): + self._config["metadata"][key] = value + + def has_metadata(self, key: str) -> bool: + return key in self._config["metadata"] + + def get_metadata(self, key: str = None, fail_on_nonexist: bool = False) -> typing.Any | None: + # Not using `get` since `None` is a valid value + if key is None: + return self._config["metadata"] + + try: + return self._config["metadata"][key] + except KeyError: + if fail_on_nonexist: + raise + return None + + def list_metadata(self) -> list[str]: + return sorted(self._config["metadata"].keys()) + + def payload(self) -> MessageMeta | None: + return self._payload + + def set_payload(self, payload: MessageMeta): + self._payload = payload + + @property + def tensors(self) -> TensorMemory | None: + return self._tensors + + @tensors.setter + def tensors(self, tensors: TensorMemory): + self._tensors = tensors + + @property + def task_type(self) -> ControlMessageType: + return self._type + + @task_type.setter + def task_type(self, task_type: ControlMessageType): + self._type = task_type + + def set_timestamp(self, key: str, timestamp: datetime): + self._timestamps[key] = timestamp + + def get_timestamp(self, key: str, fail_if_nonexist: bool = False) -> datetime | None: + try: + return self._timestamps[key] + except KeyError: + if fail_if_nonexist: + raise + return None + + def filter_timestamp(self, regex_filter: str | re.Pattern) -> dict[str, datetime]: + if isinstance(regex_filter, str): + re_obj = re.compile(regex_filter) + elif isinstance(regex_filter, re.Pattern): + re_obj = regex_filter + else: + raise ValueError("regex_filter must be a string or a compiled regex object") + + return {key: value for key, value in self._timestamps.items() if re_obj.match(key)} From 7f7557dc989bbe06a6af0bfe439ecf374732dfea Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 07:43:56 -0700 Subject: [PATCH 031/347] Refactor logic to be a little more clean --- .../morpheus/_lib/src/messages/control.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index 8e1f0965ee..0cdf827375 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -69,14 +69,17 @@ void ControlMessage::add_task(const std::string& task_type, const morpheus::util VLOG(20) << "Adding task of type " << task_type << " to control message" << task.dump(4); auto _task_type = to_task_type(task_type, false); - if (_task_type != ControlMessageType::NONE and this->task_type() != _task_type) + if (_task_type != ControlMessageType::NONE) { - throw std::runtime_error("Cannot add inference and training tasks to the same control message"); - } - - if (this->task_type() == ControlMessageType::NONE) - { - this->task_type(_task_type); + auto current_task_type = this->task_type(); + if (current_task_type == ControlMessageType::NONE) + { + this->task_type(_task_type); + } + else if (current_task_type != _task_type) + { + throw std::runtime_error("Cannot mix different types of tasks on the same control message"); + } } m_tasks[task_type].push_back(task); From 13681ea215fd49098eca5084fd9a5c208961b40a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 08:02:50 -0700 Subject: [PATCH 032/347] Adjust python API for ControlMessage to match that of the bindings for the C++ impl --- .../morpheus/messages/control_message.py | 64 +++++++++---------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index 90763b7350..b179fb464e 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -47,32 +47,30 @@ def __init__(self, config: dict = None): self._timestamps: dict[str, datetime] = {} self._type: ControlMessageType = ControlMessageType.NONE - self.config: dict = config + self.config(config) - @property - def config(self) -> dict: - return self._config + def config(self, config: dict = None) -> dict: + if config is not None: + cm_type: str | ControlMessageType = config.get("type") + if cm_type is not None: + if isinstance(cm_type, str): + try: + cm_type = get_enum_members(ControlMessageType)[cm_type] + except KeyError as exc: + raise ValueError( + f"Invalid ControlMessageType: {cm_type}, supported types: {get_enum_keys(ControlMessageType)}" + ) from exc - @config.setter - def config(self, config: dict): - cm_type: str | ControlMessageType = config.get("type") - if cm_type is not None: - if isinstance(cm_type, str): - try: - cm_type = get_enum_members(ControlMessageType)[cm_type] - except KeyError as exc: - raise ValueError( - f"Invalid ControlMessageType: {cm_type}, supported types: {get_enum_keys(ControlMessageType)}" - ) from exc + self._type = cm_type - self._type = cm_type + tasks = config.get("tasks") + if tasks is not None: + for task in tasks: + self.add_task(task["type"], task["properties"]) - tasks = config.get("tasks") - if tasks is not None: - for task in tasks: - self.add_task(task["type"], task["properties"]) + self._config = {"metadata": config.get("metadata", {}).copy()} - self._config = {"metadata": config.get("metadata", {}).copy()} + return self._config def has_task(self, task_type: str) -> bool: """ @@ -124,28 +122,24 @@ def get_metadata(self, key: str = None, fail_on_nonexist: bool = False) -> typin def list_metadata(self) -> list[str]: return sorted(self._config["metadata"].keys()) - def payload(self) -> MessageMeta | None: + def payload(self, payload: MessageMeta = None) -> MessageMeta | None: + if payload is not None: + self._payload = payload + return self._payload - def set_payload(self, payload: MessageMeta): - self._payload = payload + def tensors(self, tensors: TensorMemory = None) -> TensorMemory | None: + if tensors is not None: + self._tensors = tensors - @property - def tensors(self) -> TensorMemory | None: return self._tensors - @tensors.setter - def tensors(self, tensors: TensorMemory): - self._tensors = tensors + def task_type(self, new_task_type: ControlMessageType = None) -> ControlMessageType: + if new_task_type is not None: + self._type = new_task_type - @property - def task_type(self) -> ControlMessageType: return self._type - @task_type.setter - def task_type(self, task_type: ControlMessageType): - self._type = task_type - def set_timestamp(self, key: str, timestamp: datetime): self._timestamps[key] = timestamp From aa13a0ee5ecbf8bbbdc7eb3692a9cc5499560931 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 08:46:35 -0700 Subject: [PATCH 033/347] Give _config an initial value, add a copy method --- .../morpheus/messages/control_message.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index b179fb464e..5bfa48ca73 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -40,6 +40,8 @@ class ControlMessage(MessageBase, cpp_class=_messages.ControlMessage): def __init__(self, config: dict = None): super().__init__() + self._config: dict = {"metadata": {}} + self._payload: MessageMeta = None self._tensors: TensorMemory = None @@ -72,6 +74,24 @@ def config(self, config: dict = None) -> dict: return self._config + def copy(self) -> "ControlMessage": + config = self._config.copy() + config["type"] = self.task_type().name + + tasks = [] + for (task_type, task_queue) in self.get_tasks().items(): + for task in task_queue: + tasks.append({"type": task_type, "properties": task}) + + config["tasks"] = tasks + + new_cm = ControlMessage(config) + new_cm._payload = self._payload + new_cm._tensors = self._tensors + new_cm._timestamps = self._timestamps.copy() + + return new_cm + def has_task(self, task_type: str) -> bool: """ Return True if the control message has at least one task of the given type From b449bd4d6ef418f14f76c04a4444b7b589bb9a60 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 08:57:38 -0700 Subject: [PATCH 034/347] Align to be closer to matching the C++ bindings --- .../morpheus/messages/control_message.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index 5bfa48ca73..dc78a77f3b 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -127,17 +127,21 @@ def set_metadata(self, key: str, value: typing.Any): def has_metadata(self, key: str) -> bool: return key in self._config["metadata"] - def get_metadata(self, key: str = None, fail_on_nonexist: bool = False) -> typing.Any | None: + def get_metadata(self, key: str = None, default_value: typing.Any = None) -> typing.Any: + """ + Return a given piece of metadata, if `key` is `None` return the entire metadata dictionary. + If `key` is not found, `default_value` is returned. + + :param key: The key of the metadata to retrieve, or None for all metadata + :param default_value: The value to return if the key is not found, ignored if `key` is None + :return: The value of the metadata key, or the entire metadata dictionary if `key` is None + """ + # Not using `get` since `None` is a valid value if key is None: return self._config["metadata"] - try: - return self._config["metadata"][key] - except KeyError: - if fail_on_nonexist: - raise - return None + return self._config["metadata"].get(key, default_value) def list_metadata(self) -> list[str]: return sorted(self._config["metadata"].keys()) @@ -171,12 +175,7 @@ def get_timestamp(self, key: str, fail_if_nonexist: bool = False) -> datetime | raise return None - def filter_timestamp(self, regex_filter: str | re.Pattern) -> dict[str, datetime]: - if isinstance(regex_filter, str): - re_obj = re.compile(regex_filter) - elif isinstance(regex_filter, re.Pattern): - re_obj = regex_filter - else: - raise ValueError("regex_filter must be a string or a compiled regex object") + def filter_timestamp(self, regex_filter: str) -> dict[str, datetime]: + re_obj = re.compile(regex_filter) return {key: value for key, value in self._timestamps.items() if re_obj.match(key)} From c9ce404825349be1a3f423b722dd81b38cef1b02 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 10:33:56 -0700 Subject: [PATCH 035/347] Implement copy constructor --- .../morpheus/messages/control_message.py | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index dc78a77f3b..dd4d393178 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -37,7 +37,7 @@ @dataclasses.dataclass(init=False) class ControlMessage(MessageBase, cpp_class=_messages.ControlMessage): - def __init__(self, config: dict = None): + def __init__(self, config_or_message: typing.Union["ControlMessage", dict] = None): super().__init__() self._config: dict = {"metadata": {}} @@ -49,7 +49,15 @@ def __init__(self, config: dict = None): self._timestamps: dict[str, datetime] = {} self._type: ControlMessageType = ControlMessageType.NONE - self.config(config) + if isinstance(config_or_message, dict): + self.config(config_or_message) + elif isinstance(config_or_message, ControlMessage): + self._copy_impl(config_or_message, self) + elif config_or_message is not None: + raise ValueError(f"Invalid argument type {type(config_or_message)}, value must be a dict or ControlMessage") + + def copy(self) -> "ControlMessage": + return self._copy_impl(self) def config(self, config: dict = None) -> dict: if config is not None: @@ -74,24 +82,6 @@ def config(self, config: dict = None) -> dict: return self._config - def copy(self) -> "ControlMessage": - config = self._config.copy() - config["type"] = self.task_type().name - - tasks = [] - for (task_type, task_queue) in self.get_tasks().items(): - for task in task_queue: - tasks.append({"type": task_type, "properties": task}) - - config["tasks"] = tasks - - new_cm = ControlMessage(config) - new_cm._payload = self._payload - new_cm._tensors = self._tensors - new_cm._timestamps = self._timestamps.copy() - - return new_cm - def has_task(self, task_type: str) -> bool: """ Return True if the control message has at least one task of the given type @@ -100,7 +90,7 @@ def has_task(self, task_type: str) -> bool: tasks = self._tasks.get(task_type, []) return len(tasks) > 0 - def add_task(self, task_type: str, properties: dict): + def add_task(self, task_type: str, task: dict): if isinstance(task_type, str): cm_type = get_enum_members(ControlMessageType).get(task_type, ControlMessageType.NONE) if cm_type != ControlMessageType.NONE: @@ -109,7 +99,7 @@ def add_task(self, task_type: str, properties: dict): elif self._type != cm_type: raise ValueError("Cannot mix different types of tasks on the same control message") - self._tasks[task_type].append(properties) + self._tasks[task_type].append(task) def remove_task(self, task_type: str) -> dict: tasks = self._tasks.get(task_type, []) @@ -179,3 +169,25 @@ def filter_timestamp(self, regex_filter: str) -> dict[str, datetime]: re_obj = re.compile(regex_filter) return {key: value for key, value in self._timestamps.items() if re_obj.match(key)} + + @classmethod + def _copy_impl(cls, src: "ControlMessage", dst: "ControlMessage" = None) -> "ControlMessage": + config = src.config().copy() + config["type"] = src.task_type().name + + tasks = [] + for (task_type, task_queue) in src.get_tasks().items(): + for task in task_queue: + tasks.append({"type": task_type, "properties": task}) + + config["tasks"] = tasks + + if dst is None: + dst = cls() + + dst.config(config) + dst.payload(src.payload()) + dst.tensors(src.tensors()) + dst._timestamps = src._timestamps.copy() + + return dst From a4d205e3a1187946929cf4b5ee3349557fb56d29 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 12:23:26 -0700 Subject: [PATCH 036/347] Add support for constructing a ControlMessage from the python instance of a control message --- .../include/morpheus/messages/control.hpp | 8 ++-- .../morpheus/_lib/messages/__init__.pyi | 2 +- .../morpheus/_lib/messages/module.cpp | 2 +- .../morpheus/_lib/src/messages/control.cpp | 46 +++++++++++++++++-- .../morpheus/messages/control_message.py | 18 ++++++-- 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp index 3b48833a00..fa0390e11e 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp @@ -261,11 +261,13 @@ class MORPHEUS_EXPORT ControlMessage struct MORPHEUS_EXPORT ControlMessageProxy { /** - * @brief Creates a new ControlMessage instance from a configuration dictionary. - * @param config A pybind11::dict representing the configuration for the ControlMessage. + * @brief Creates a new ControlMessage instance from either a Python instance of a ControlMessage or a configuration + * dictionary. + * @param config_or_message Either a Python instance of a ControlMessage or a dict representing the configuration + * for the ControlMessage. * @return A shared_ptr to a newly created ControlMessage instance. */ - static std::shared_ptr create(pybind11::dict& config); + static std::shared_ptr create(pybind11::object& config_or_message); /** * @brief Creates a new ControlMessage instance as a copy of an existing one. diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi index f4ef520152..7a5ac78488 100644 --- a/python/morpheus/morpheus/_lib/messages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/messages/__init__.pyi @@ -43,7 +43,7 @@ class ControlMessage(): @typing.overload def __init__(self, arg0: ControlMessage) -> None: ... @typing.overload - def __init__(self, arg0: dict) -> None: ... + def __init__(self, arg0: object) -> None: ... def add_task(self, task_type: str, task: object) -> None: ... @typing.overload def config(self) -> object: ... diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index 270e52d0a5..60eddddabe 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -404,7 +404,7 @@ PYBIND11_MODULE(messages, _module) py::class_>(_module, "ControlMessage") .def(py::init<>()) - .def(py::init(py::overload_cast(&ControlMessageProxy::create))) + .def(py::init(py::overload_cast(&ControlMessageProxy::create))) .def(py::init(py::overload_cast>(&ControlMessageProxy::create))) .def("add_task", &ControlMessage::add_task, py::arg("task_type"), py::arg("task")) .def( diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index 0cdf827375..350d4a3ad3 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -17,7 +17,8 @@ #include "morpheus/messages/control.hpp" -#include "morpheus/messages/meta.hpp" // for MessageMeta, MessageMetaInterfaceProxy +#include "morpheus/messages/memory/tensor_memory.hpp" // for TensorMemory, TensorMemoryInterfaceProxy +#include "morpheus/messages/meta.hpp" // for MessageMeta, MessageMetaInterfaceProxy #include // for to_lower_copy #include // for COMPACT_GOOGLE_LOG_INFO, LogMessage, VLOG @@ -25,6 +26,7 @@ #include // IWYU pragma: keep #include // for cast, object::cast #include // for object, none, dict, isinstance, list, str, value_error, generic_item +#include // for casting to std::map #include // for cast_from_pyobject #include // for optional, nullopt @@ -271,9 +273,47 @@ ControlMessageType ControlMessage::to_task_type(const std::string& task_type, bo } /*** Proxy Implementations ***/ -std::shared_ptr ControlMessageProxy::create(py::dict& config) +std::shared_ptr ControlMessageProxy::create(py::object& config_or_message) { - return std::make_shared(mrc::pymrc::cast_from_pyobject(config)); + if (config_or_message.is_none()) + { + return std::make_shared(); + } + + if (py::isinstance(config_or_message)) + { + return std::make_shared(mrc::pymrc::cast_from_pyobject(config_or_message)); + } + + // Assume we received a Python instance of the ControlMessage object + py::dict config = config_or_message.attr("_export_config")(); + auto cm = std::make_shared(mrc::pymrc::cast_from_pyobject(config)); + + auto py_meta = config_or_message.attr("payload")(); + if (!py_meta.is_none()) + { + cm->payload(MessageMetaInterfaceProxy::init_python_meta(py_meta)); + } + + auto py_tensors = config_or_message.attr("tensors")(); + if (!py_tensors.is_none()) + { + auto count = py_tensors.attr("count").cast(); + auto py_tensors_map = py_tensors.attr("get_tensors")(); + cm->tensors(TensorMemoryInterfaceProxy::init(count, py_tensors_map)); + } + + auto py_timestamps = config_or_message.attr("_timestamps"); + if (!py_timestamps.is_none()) + { + auto timestamps_map = py_timestamps.cast>(); + for (const auto& t : timestamps_map) + { + cm->set_timestamp(t.first, t.second); + } + } + + return cm; } std::shared_ptr ControlMessageProxy::create(std::shared_ptr other) diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index dd4d393178..70b598e101 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -170,18 +170,26 @@ def filter_timestamp(self, regex_filter: str) -> dict[str, datetime]: return {key: value for key, value in self._timestamps.items() if re_obj.match(key)} - @classmethod - def _copy_impl(cls, src: "ControlMessage", dst: "ControlMessage" = None) -> "ControlMessage": - config = src.config().copy() - config["type"] = src.task_type().name + def _export_config(self) -> dict: + # Unfortunately there is no parity between the `config` object that the constructor accepts and the value + # returned by the `config` method. This method returns a config object that can be used to create a new instance + # with the same task type and tasks. + config = self.config().copy() + config["type"] = self.task_type().name tasks = [] - for (task_type, task_queue) in src.get_tasks().items(): + for (task_type, task_queue) in self.get_tasks().items(): for task in task_queue: tasks.append({"type": task_type, "properties": task}) config["tasks"] = tasks + return config + + @classmethod + def _copy_impl(cls, src: "ControlMessage", dst: "ControlMessage" = None) -> "ControlMessage": + config = src._export_config() + if dst is None: dst = cls() From 82a6259a1e3b0753ae87bbea548f0ebe131d4eab Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 12:57:36 -0700 Subject: [PATCH 037/347] Add a python->cpp cast function to support using the LLMEngine stage in python mode --- .../morpheus/stages/llm/llm_engine_stage.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/morpheus/morpheus/stages/llm/llm_engine_stage.py b/python/morpheus/morpheus/stages/llm/llm_engine_stage.py index 157fbf06ef..0338a2be4d 100644 --- a/python/morpheus/morpheus/stages/llm/llm_engine_stage.py +++ b/python/morpheus/morpheus/stages/llm/llm_engine_stage.py @@ -12,12 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import logging +import types import typing import mrc +from mrc.core import operators as ops from morpheus.config import Config +from morpheus.config import CppConfig from morpheus.llm import LLMEngine from morpheus.messages import ControlMessage from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin @@ -64,11 +68,28 @@ def supports_cpp_node(self): """Indicates whether this stage supports a C++ node.""" return True + def _cast_control_message(self, message: ControlMessage, *, cpp_messages_lib: types.ModuleType) -> ControlMessage: + """ + LLMEngineStage does not contain a Python implementation, however it is capable of running in Python/cpu-only + mode. This method is needed to cast the Python ControlMessage to a C++ ControlMessage. + + This is different than casting from the Python bindings for the C++ ControlMessage to a C++ ControlMessage. + """ + return cpp_messages_lib.ControlMessage(message) + def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: import morpheus._lib.llm as _llm node = _llm.LLMEngineStage(builder, self.unique_name, self._engine) node.launch_options.pe_count = 1 + if not CppConfig.get_should_use_cpp(): + import morpheus._lib.messages as _messages + cast_fn = functools.partial(self._cast_control_message, cpp_messages_lib=_messages) + pre_node = builder.make_node(f"{self.unique_name}-pre-cast", ops.map(cast_fn)) + builder.make_edge(input_node, pre_node) + + input_node = pre_node + builder.make_edge(input_node, node) return node From 4a1ca0b722172d408f20563690258a3d3e5335a2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 13:26:10 -0700 Subject: [PATCH 038/347] Change order of overloads giving the shared_ptr cast priotity --- python/morpheus/morpheus/_lib/messages/module.cpp | 2 +- python/morpheus/morpheus/_lib/src/messages/control.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index 60eddddabe..a8568de1df 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -404,8 +404,8 @@ PYBIND11_MODULE(messages, _module) py::class_>(_module, "ControlMessage") .def(py::init<>()) - .def(py::init(py::overload_cast(&ControlMessageProxy::create))) .def(py::init(py::overload_cast>(&ControlMessageProxy::create))) + .def(py::init(py::overload_cast(&ControlMessageProxy::create))) .def("add_task", &ControlMessage::add_task, py::arg("task_type"), py::arg("task")) .def( "config", py::overload_cast(&ControlMessage::config), py::arg("config")) diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index 350d4a3ad3..547a845ebe 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -285,7 +285,8 @@ std::shared_ptr ControlMessageProxy::create(py::object& config_o return std::make_shared(mrc::pymrc::cast_from_pyobject(config_or_message)); } - // Assume we received a Python instance of the ControlMessage object + // Assume we received an instance of the Python impl of ControlMessage object, as a Python bound instance of the C++ + // impl of the ControlMessage class would have invoked the shared_ptr overload of the create method py::dict config = config_or_message.attr("_export_config")(); auto cm = std::make_shared(mrc::pymrc::cast_from_pyobject(config)); From 5b87f411ce59358916f6cf30ed1fb283c0428e96 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 13:26:38 -0700 Subject: [PATCH 039/347] Only call to_pandas if we don't already have a pandas df --- .../morpheus/morpheus/stages/postprocess/timeseries_stage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/stages/postprocess/timeseries_stage.py b/python/morpheus/morpheus/stages/postprocess/timeseries_stage.py index 5005114df3..5d19cfe415 100644 --- a/python/morpheus/morpheus/stages/postprocess/timeseries_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/timeseries_stage.py @@ -366,7 +366,9 @@ def _calc_timeseries(self, x: MultiResponseMessage | ControlMessage, is_complete if isinstance(x, MultiResponseMessage): new_timedata = x.get_meta([self._timestamp_col]) elif isinstance(x, ControlMessage): - new_timedata = x.payload().get_data([self._timestamp_col]).to_pandas() + new_timedata = x.payload().get_data([self._timestamp_col]) + if not isinstance(new_timedata, pd.DataFrame): + new_timedata = new_timedata.to_pandas() # Save this message event times in the event list. Ensure the values are always sorted self._timeseries_data = pd.concat([self._timeseries_data, new_timedata]).sort_index() From 356c0c91470364ae81087b9a4e2257e22f97a444 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 13:28:54 -0700 Subject: [PATCH 040/347] Don't use the C++ impl of TensorMemory directly --- tests/stages/test_filter_detections_stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/stages/test_filter_detections_stage.py b/tests/stages/test_filter_detections_stage.py index 4f9cb43cfe..73633f2f99 100644 --- a/tests/stages/test_filter_detections_stage.py +++ b/tests/stages/test_filter_detections_stage.py @@ -17,11 +17,11 @@ import cupy as cp import pytest -import morpheus._lib.messages as _messages from morpheus.common import FilterSource from morpheus.messages import ControlMessage from morpheus.messages import MultiResponseMessage from morpheus.messages import ResponseMemory +from morpheus.messages import TensorMemory from morpheus.messages.message_meta import MessageMeta from morpheus.stages.postprocess.filter_detections_stage import FilterDetectionsStage @@ -37,7 +37,7 @@ def _make_control_message(df, probs): df_ = df[0:len(probs)] cm = ControlMessage() cm.payload(MessageMeta(df_)) - cm.tensors(_messages.TensorMemory(count=len(df_), tensors={'probs': probs})) + cm.tensors(TensorMemory(count=len(df_), tensors={'probs': probs})) return cm From 0c425603e72b98e3ba31c6b1b80943df70235e22 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 13:58:09 -0700 Subject: [PATCH 041/347] Add a 'supported_execution_modes' to the base stage class which defaults to GPU only, create two mixins: 'CpuOnlyMixin' and 'GpuAndCpuMixin' which should cover most use-cases to avoid the need for stage authors to re-implement the 'supported_execution_modes' method --- .../pipeline/execution_mode_mixins.py | 46 +++++++++++++++++++ .../morpheus/morpheus/pipeline/stage_base.py | 20 ++++++++ 2 files changed, 66 insertions(+) create mode 100644 python/morpheus/morpheus/pipeline/execution_mode_mixins.py diff --git a/python/morpheus/morpheus/pipeline/execution_mode_mixins.py b/python/morpheus/morpheus/pipeline/execution_mode_mixins.py new file mode 100644 index 0000000000..6bef674d8d --- /dev/null +++ b/python/morpheus/morpheus/pipeline/execution_mode_mixins.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Mixins to indicate which execution modes are supported for a given stage. These mixins should be used for any stage +that needs to support execution modes other than the default GPU mode, and the supported execution modes do not change +based upon configuration or runtime conditions. +""" + +from abc import ABC + +from morpheus.config import ExecutionMode + + +class CpuOnlyMixin(ABC): + """ + Mixin intented to be added to stages which support only CPU execution modes. + """ + + def supported_execution_modes(self) -> tuple[ExecutionMode]: + """ + Returns a tuple of supported execution modes of this stage. + """ + return (ExecutionMode.CPU, ) + + +class GpuAndCpuMixin(ABC): + """ + Mixin intented to be added to stages which support both GPU and CPU execution modes. + """ + + def supported_execution_modes(self) -> tuple[ExecutionMode]: + """ + Returns a tuple of supported execution modes of this stage. + """ + return (ExecutionMode.GPU, ExecutionMode.CPU) diff --git a/python/morpheus/morpheus/pipeline/stage_base.py b/python/morpheus/morpheus/pipeline/stage_base.py index 290ed83992..5be1905ab7 100644 --- a/python/morpheus/morpheus/pipeline/stage_base.py +++ b/python/morpheus/morpheus/pipeline/stage_base.py @@ -27,6 +27,7 @@ import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import from morpheus.config import Config from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.utils.atomic_integer import AtomicInteger from morpheus.utils.type_utils import _DecoratorType @@ -285,6 +286,19 @@ def supports_cpp_node(self): # return False pass + def supported_execution_modes(self) -> tuple[ExecutionMode]: + """ + Returns a tuple of supported execution modes of this stage. By default this returns `(ExecutionMode.GPU,)`. + Subclasses can override this method to specify different execution modes. + + For most stages the values will be static, and this can be accomplished by making use of either the + `CpuOnlyMixin` or `GpuAndCpuMixin` mixins. + + However, complex stages may choose to make this decision at runtime, in which case this method should be + overridden. directly within the stage class. + """ + return (ExecutionMode.GPU, ) + def _build_cpp_node(self): """ Specifies whether to build a C++ node. Only should be called during the build phase. @@ -347,6 +361,12 @@ def can_build(self, check_ports=False) -> bool: def _pre_build(self, do_propagate: bool = True): assert not self.is_built, "build called prior to _pre_build" assert not self.is_pre_built, "Can only pre-build stages once!" + + # Check the execution mode + if (self._config.execution_mode not in self.supported_execution_modes()): + raise ValueError(f"Stage {self.name} does not support execution mode {self._config.execution_mode}") + + # Perform schema validation schema = _pipeline.StageSchema(self) self._pre_compute_schema(schema) self.compute_schema(schema) From fe56dc60418bc1bf53cec4c4f2f87666a15eaff6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 14:14:42 -0700 Subject: [PATCH 042/347] Lint fixes --- python/morpheus/morpheus/io/deserializers.py | 3 --- python/morpheus/morpheus/io/serializers.py | 7 +++++-- python/morpheus/morpheus/io/utils.py | 6 ++++-- python/morpheus/morpheus/messages/control_message.py | 4 ++-- python/morpheus/morpheus/pipeline/pipeline.py | 7 +++++-- python/morpheus/morpheus/stages/input/arxiv_source.py | 3 ++- .../stages/input/databricks_deltalake_source_stage.py | 3 ++- python/morpheus/morpheus/stages/input/file_source_stage.py | 4 +++- .../morpheus/stages/output/http_server_sink_stage.py | 1 - 9 files changed, 23 insertions(+), 15 deletions(-) diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index 42549ff66c..488fd9f172 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -20,9 +20,6 @@ import numpy as np import pandas as pd -if typing.TYPE_CHECKING: - import cudf - from morpheus.common import FileTypes from morpheus.common import determine_file_type from morpheus.common import read_file_to_df as read_file_to_df_cpp diff --git a/python/morpheus/morpheus/io/serializers.py b/python/morpheus/morpheus/io/serializers.py index 75eef4767c..f6c9e4d4f3 100644 --- a/python/morpheus/morpheus/io/serializers.py +++ b/python/morpheus/morpheus/io/serializers.py @@ -15,9 +15,12 @@ """DataFrame serializers.""" import typing -from io import BytesIO, IOBase, StringIO +from io import BytesIO +from io import IOBase +from io import StringIO -from morpheus.common import FileTypes, determine_file_type +from morpheus.common import FileTypes +from morpheus.common import determine_file_type from morpheus.common import write_df_to_file as write_df_to_file_cpp from morpheus.config import CppConfig from morpheus.utils.type_aliases import DataFrameType diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index 21ef7de626..5054c78569 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -24,8 +24,10 @@ if typing.TYPE_CHECKING: import cudf -from morpheus.config import Config, ExecutionMode -from morpheus.utils.type_aliases import DataFrameType, SeriesType +from morpheus.config import Config +from morpheus.config import ExecutionMode +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import SeriesType logger = logging.getLogger(__name__) diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index 70b598e101..1096815984 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -67,9 +67,9 @@ def config(self, config: dict = None) -> dict: try: cm_type = get_enum_members(ControlMessageType)[cm_type] except KeyError as exc: + enum_names = ", ".join(get_enum_keys(ControlMessageType)) raise ValueError( - f"Invalid ControlMessageType: {cm_type}, supported types: {get_enum_keys(ControlMessageType)}" - ) from exc + f"Invalid ControlMessageType: {cm_type}, supported types: {enum_names}") from exc self._type = cm_type diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index b749a18c37..b134694fad 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -19,7 +19,8 @@ import sys import threading import typing -from collections import OrderedDict, defaultdict +from collections import OrderedDict +from collections import defaultdict from enum import Enum from functools import partial @@ -28,7 +29,9 @@ from tqdm import tqdm import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import -from morpheus.config import Config, CppConfig, ExecutionMode +from morpheus.config import Config +from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.utils.type_utils import pretty_print_type_name logger = logging.getLogger(__name__) diff --git a/python/morpheus/morpheus/stages/input/arxiv_source.py b/python/morpheus/morpheus/stages/input/arxiv_source.py index 25eeb36e14..1b87ad750e 100644 --- a/python/morpheus/morpheus/stages/input/arxiv_source.py +++ b/python/morpheus/morpheus/stages/input/arxiv_source.py @@ -21,7 +21,8 @@ import pandas as pd from morpheus.cli.register_stage import register_stage -from morpheus.config import Config, ExecutionMode +from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource diff --git a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py index 630aa37f7c..4d21ac8aac 100644 --- a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py +++ b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py @@ -17,7 +17,8 @@ import mrc from morpheus.cli.register_stage import register_stage -from morpheus.config import Config, ExecutionMode +from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages.message_meta import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource diff --git a/python/morpheus/morpheus/stages/input/file_source_stage.py b/python/morpheus/morpheus/stages/input/file_source_stage.py index 76b3d68102..eeeac937ab 100644 --- a/python/morpheus/morpheus/stages/input/file_source_stage.py +++ b/python/morpheus/morpheus/stages/input/file_source_stage.py @@ -21,7 +21,9 @@ from morpheus.cli import register_stage from morpheus.common import FileTypes -from morpheus.config import Config, ExecutionMode, PipelineModes +from morpheus.config import Config +from morpheus.config import ExecutionMode +from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta from morpheus.pipeline.preallocator_mixin import PreallocatorMixin diff --git a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py index 9f8462263b..bca6207e8e 100644 --- a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py @@ -22,7 +22,6 @@ from io import StringIO import mrc -import pandas as pd from mrc.core import operators as ops from morpheus.cli.register_stage import register_stage From 22c17766c5952b4ffcc98eadaf21ba5a3667c4f5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 14:17:27 -0700 Subject: [PATCH 043/347] Lint fixes --- python/morpheus/morpheus/io/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index 5054c78569..6cfdd9ae58 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -21,14 +21,14 @@ import pandas as pd -if typing.TYPE_CHECKING: - import cudf - from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import SeriesType +if typing.TYPE_CHECKING: + import cudf + logger = logging.getLogger(__name__) @@ -146,8 +146,8 @@ def get_df_pkg(config: Config) -> types.ModuleType: if config.execution_mode == ExecutionMode.GPU: import cudf return cudf - else: - return pd + + return pd def get_df_class(config: Config) -> type[DataFrameType]: From 0538825fe782774dfa8b75a4f11f3e53b6ec47a4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 14:49:25 -0700 Subject: [PATCH 044/347] Add the ability to 'freeze' the config, making it immutable, this will be invoked when the config is passed to a pipeline or stage --- python/morpheus/morpheus/config.py | 18 ++++++++++++++++++ python/morpheus/morpheus/pipeline/pipeline.py | 1 + .../morpheus/morpheus/pipeline/stage_base.py | 1 + 3 files changed, 20 insertions(+) diff --git a/python/morpheus/morpheus/config.py b/python/morpheus/morpheus/config.py index 6db7117ca0..a88535a415 100644 --- a/python/morpheus/morpheus/config.py +++ b/python/morpheus/morpheus/config.py @@ -227,11 +227,29 @@ class Config(ConfigBase): ae: ConfigAutoEncoder = dataclasses.field(default=None) fil: ConfigFIL = dataclasses.field(default=None) + frozen: bool = False def __post_init__(self): if self.execution_mode == ExecutionMode.CPU: CppConfig.set_should_use_cpp(False) + def freeze(self): + """ + Freeze the Config object, making it immutable. This method will be invoked when the config object is passed to + a pipeline or stage for the first time. + + Calling `freeze` on a frozen instance will not have any effect. + """ + if not self.frozen: + self.frozen = True + + def __setattr__(self, name, value): + # Since __frozen is defined in the __post_init__, the attribute won't exist in the __init__ method. + if self.frozen: + raise dataclasses.FrozenInstanceError("Cannot modify frozen Config object.") + + super().__setattr__(name, value) + def save(self, filename: str): """ Save Config to file. diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index b134694fad..9d4e3ecd8b 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -62,6 +62,7 @@ class Pipeline(): """ def __init__(self, config: Config): + config.freeze() if config.execution_mode == ExecutionMode.CPU and CppConfig.get_should_use_cpp(): raise RuntimeError("C++ mode requires GPU execution mode.") diff --git a/python/morpheus/morpheus/pipeline/stage_base.py b/python/morpheus/morpheus/pipeline/stage_base.py index 5be1905ab7..065883ff33 100644 --- a/python/morpheus/morpheus/pipeline/stage_base.py +++ b/python/morpheus/morpheus/pipeline/stage_base.py @@ -85,6 +85,7 @@ class StageBase(ABC, collections.abc.Hashable): def __init__(self, config: Config): # Save the config + config.freeze() self._config = config self._id = StageBase.__ID_COUNTER.get_and_inc() From b99a829a5688253a29bc275bb74f9ed4b8984c9a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 15:47:42 -0700 Subject: [PATCH 045/347] Add tests for fozen functionality --- tests/test_config.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index a1517ad355..9b526d2980 100755 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -16,8 +16,11 @@ import json import os +from dataclasses import FrozenInstanceError from unittest import mock +import pytest + import morpheus import morpheus.config from _utils import assert_path_exists @@ -104,3 +107,47 @@ def test_to_string(config): conf_str = config.to_string() assert isinstance(conf_str, str) assert isinstance(json.loads(conf_str), dict) + + +def test_frozen(config: morpheus.config.Config): + assert not config.frozen + + # Ensure that it is safe to call freeze() multiple times + for _ in range(2): + config.freeze() + assert config.frozen + + +@pytest.mark.parametrize('use_attr', [False, True]) +def test_frozen_immutible(config: morpheus.config.Config, use_attr: bool): + """ + Test for the freeze functionality. + + There are currently two ways to bypass the freeze functionality: + 1. By accessing the __dict__ attribute of the Config object. + 2. Modifying any of the mutable objects in the Config object (ex: `config.class_labels.append('new_label')`). + """ + assert not config.frozen + + # ensure that we can set some attributes + config.feature_length = 45 + + # freeze the config, freezing the config via the attribute or method should have the same effect, the only + # difference is that it is safe to call freeze() multiple times, while setting the attribute will raise an exception + # just like attempting to set any other attribute on a frozen object + if use_attr: + config.frozen = True + else: + config.freeze() + + assert config.frozen + + with pytest.raises(FrozenInstanceError): + config.feature_length = 100 + + # ensure setattr also raises an exception + with pytest.raises(FrozenInstanceError): + setattr(config, 'feature_length', 100) + + # ensure the config still has the original value + assert config.feature_length == 45 From 20eaf53d6e73f7850c6920056fe9c12441748f60 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 16:03:06 -0700 Subject: [PATCH 046/347] Remove out of date comment --- python/morpheus/morpheus/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/morpheus/morpheus/config.py b/python/morpheus/morpheus/config.py index a88535a415..7d1b6ca9f6 100644 --- a/python/morpheus/morpheus/config.py +++ b/python/morpheus/morpheus/config.py @@ -204,8 +204,6 @@ class Config(ConfigBase): log_config_file : str File corresponding to this Config. """ - - # TODO: Store this as __execution_mode or move it to the CppConfig class execution_mode: ExecutionMode = ExecutionMode.GPU # Whether in Debug mode. From 0dc4554a388b16193dd544180427fbcdc8f7dbe3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 16:30:42 -0700 Subject: [PATCH 047/347] Type aliases for cupy/numpy arrays --- python/morpheus/morpheus/utils/type_aliases.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/morpheus/morpheus/utils/type_aliases.py b/python/morpheus/morpheus/utils/type_aliases.py index b9ed411744..e94ccd5abb 100644 --- a/python/morpheus/morpheus/utils/type_aliases.py +++ b/python/morpheus/morpheus/utils/type_aliases.py @@ -16,9 +16,14 @@ import typing if typing.TYPE_CHECKING: + import cupy + import numpy import pandas import cudf DataFrameType = typing.Union["pandas.DataFrame", "cudf.DataFrame"] SeriesType = typing.Union["pandas.Series", "cudf.Series"] + +NDArrayType = typing.Union["numpy.ndarray", "cupy.ndarray"] +TensorMapType = dict[str, NDArrayType] From f2b3115ff7c0afd58a1a87a7a40f8ac3020ce48f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 16:32:45 -0700 Subject: [PATCH 048/347] Avoid top-level cudf and cupy imports, remove cudf work-arounds --- .../morpheus/messages/message_meta.py | 56 +++---------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/python/morpheus/morpheus/messages/message_meta.py b/python/morpheus/morpheus/messages/message_meta.py index ecf542b553..d2fd18480e 100644 --- a/python/morpheus/morpheus/messages/message_meta.py +++ b/python/morpheus/morpheus/messages/message_meta.py @@ -18,15 +18,13 @@ import typing import warnings -import cupy as cp import numpy as np import pandas as pd -import cudf - import morpheus._lib.messages as _messages from morpheus.messages.message_base import MessageBase from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import SeriesType logger = logging.getLogger(__name__) @@ -49,7 +47,7 @@ class MutableTableCtxMgr: def __init__(self, meta) -> None: self.__dict__['__meta'] = meta - def __enter__(self) -> pd.DataFrame: + def __enter__(self) -> DataFrameType: meta = self.__dict__['__meta'] meta._mutex.acquire() return meta._df @@ -206,7 +204,7 @@ def get_meta_range(self, idx = self._df.index[mess_offset:mess_offset + message_count] - if (isinstance(idx, cudf.RangeIndex)): + if (isinstance(idx, pd.RangeIndex)): idx = slice(idx.start, idx.stop - 1, idx.step) if (columns is None): @@ -216,15 +214,15 @@ def get_meta_range(self, return self._df.loc[idx, columns] @typing.overload - def get_data(self) -> cudf.DataFrame: + def get_data(self) -> DataFrameType: ... @typing.overload - def get_data(self, columns: str) -> cudf.Series: + def get_data(self, columns: str) -> SeriesType: ... @typing.overload - def get_data(self, columns: typing.List[str]) -> cudf.DataFrame: + def get_data(self, columns: typing.List[str]) -> DataFrameType: ... def get_data(self, columns: typing.Union[None, str, typing.List[str]] = None): @@ -277,10 +275,6 @@ def set_data(self, columns: typing.Union[None, str, typing.List[str]], value): # First try to set the values on just our slice if the columns exist column_indexer = self._get_col_indexers(df, columns=columns) - # Check if the value is a cupy array and we have a pandas dataframe, convert to numpy - if (isinstance(value, cp.ndarray) and isinstance(df, pd.DataFrame)): - value = value.get() - # Check to see if we are adding a column. If so, we need to use df.loc instead of df.iloc if (-1 not in column_indexer): @@ -299,35 +293,8 @@ def set_data(self, columns: typing.Union[None, str, typing.List[str]], value): # Columns should never be empty if we get here assert columns is not None - # cudf is really bad at adding new columns - if (isinstance(df, cudf.DataFrame)): - - # TODO(morpheus#1487): This logic no longer works in CUDF 24.04. - # We should find a way to reinable the no-dropped-index path as - # that should be more performant than dropping the index. - # # saved_index = None - - # # # Check to see if we can use slices - # # if (not (df.index.is_unique and - # # (df.index.is_monotonic_increasing or df.index.is_monotonic_decreasing))): - # # # Save the index and reset - # # saved_index = df.index - # # df.reset_index(drop=True, inplace=True) - - # # # Perform the update via slices - # # df.loc[df.index[row_indexer], columns] = value - - # # # Reset the index if we changed it - # # if (saved_index is not None): - # # df.set_index(saved_index, inplace=True) - - saved_index = df.index - df.reset_index(drop=True, inplace=True) - df.loc[df.index[:], columns] = value - df.set_index(saved_index, inplace=True) - else: - # Now set the slice - df.loc[:, columns] = value + # Now set the slice + df.loc[:, columns] = value def get_slice(self, start, stop): """ @@ -350,12 +317,7 @@ def get_slice(self, start, stop): return MessageMeta(df.iloc[start:stop]) def _ranges_to_mask(self, df, ranges): - if isinstance(df, cudf.DataFrame): - zeros_fn = cp.zeros - else: - zeros_fn = np.zeros - - mask = zeros_fn(len(df), bool) + mask = np.zeros(len(df), bool) for range_ in ranges: mask[range_[0]:range_[1]] = True From d1ce6c643e0fa291d0501c076c2f0b7217109af9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 16:33:11 -0700 Subject: [PATCH 049/347] Avoid cupy import, update python impl to use numpy --- .../morpheus/messages/memory/tensor_memory.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/python/morpheus/morpheus/messages/memory/tensor_memory.py b/python/morpheus/morpheus/messages/memory/tensor_memory.py index 103240b15f..fc34ded920 100644 --- a/python/morpheus/morpheus/messages/memory/tensor_memory.py +++ b/python/morpheus/morpheus/messages/memory/tensor_memory.py @@ -16,10 +16,12 @@ import dataclasses import typing -import cupy as cp +import numpy as np import morpheus._lib.messages as _messages from morpheus.messages.message_base import MessageData +from morpheus.utils.type_aliases import NDArrayType +from morpheus.utils.type_aliases import TensorMapType @dataclasses.dataclass(init=False) @@ -37,9 +39,9 @@ class TensorMemory(MessageData, cpp_class=_messages.TensorMemory): """ count: int - tensors: typing.Dict[str, cp.ndarray] = dataclasses.field(repr=False) + tensors: TensorMapType = dataclasses.field(repr=False) - def __init__(self, *, count: int = None, tensors: typing.Dict[str, cp.ndarray] = None): + def __init__(self, *, count: int = None, tensors: TensorMapType = None): self.count = count @@ -50,11 +52,11 @@ def __init__(self, *, count: int = None, tensors: typing.Dict[str, cp.ndarray] = self._tensors = tensors - def _check_tensors(self, tensors: typing.Dict[str, cp.ndarray]): + def _check_tensors(self, tensors: TensorMapType): for tensor in tensors.values(): self._check_tensor(tensor) - def _check_tensor(self, tensor: cp.ndarray): + def _check_tensor(self, tensor: NDArrayType): if (tensor.shape[0] != self.count): class_name = type(self).__name__ raise ValueError( @@ -96,11 +98,11 @@ def get_tensors(self): Returns ------- - typing.Dict[str, cp.ndarray] + TensorMapType """ return self._tensors - def set_tensors(self, tensors: typing.Dict[str, cp.ndarray]): + def set_tensors(self, tensors: TensorMapType): """ Overwrite the tensors stored by this instance. If the length of the tensors has changed, then the `count` property should also be updated. @@ -158,7 +160,7 @@ def _get_tensor_prop(self, name: str): except KeyError as e: raise AttributeError from e - def set_tensor(self, name: str, tensor: cp.ndarray): + def set_tensor(self, name: str, tensor: NDArrayType): """ Update the tensor identified by `name`. @@ -175,6 +177,6 @@ def set_tensor(self, name: str, tensor: cp.ndarray): If the number of rows in `tensor` does not match `count` """ # Ensure that we have 2D array here (`ensure_2d` inserts the wrong axis) - reshaped_tensor = tensor if tensor.ndim == 2 else cp.reshape(tensor, (tensor.shape[0], -1)) + reshaped_tensor = tensor if tensor.ndim == 2 else np.reshape(tensor, (tensor.shape[0], -1)) self._check_tensor(reshaped_tensor) self._tensors[name] = reshaped_tensor From 940574333ff64138f9a6ac2760474803eb803386 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 16:37:22 -0700 Subject: [PATCH 050/347] Avoid direct usage of cupy --- .../messages/memory/inference_memory.py | 33 +++++++++---------- .../messages/memory/response_memory.py | 13 ++++---- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/python/morpheus/morpheus/messages/memory/inference_memory.py b/python/morpheus/morpheus/messages/memory/inference_memory.py index 9bdc7b6503..9a11e58788 100644 --- a/python/morpheus/morpheus/messages/memory/inference_memory.py +++ b/python/morpheus/morpheus/messages/memory/inference_memory.py @@ -15,11 +15,10 @@ import dataclasses -import cupy as cp - import morpheus._lib.messages as _messages from morpheus.messages.data_class_prop import DataClassProp from morpheus.messages.memory.tensor_memory import TensorMemory +from morpheus.utils.type_aliases import NDArrayType @dataclasses.dataclass(init=False) @@ -50,7 +49,7 @@ def get_input(self, name: str): """ return self.get_tensor(name) - def set_input(self, name: str, tensor: cp.ndarray): + def set_input(self, name: str, tensor: NDArrayType): """ Update the input tensor identified by `name`. Alias for `InferenceMemory.set_tensor` @@ -81,14 +80,14 @@ class InferenceMemoryNLP(InferenceMemory, cpp_class=_messages.InferenceMemoryNLP inputs than messages (i.e., if some messages get broken into multiple inference requests). """ - input_ids: dataclasses.InitVar[cp.ndarray] = DataClassProp(InferenceMemory._get_tensor_prop, - InferenceMemory.set_input) - input_mask: dataclasses.InitVar[cp.ndarray] = DataClassProp(InferenceMemory._get_tensor_prop, + input_ids: dataclasses.InitVar[NDArrayType] = DataClassProp(InferenceMemory._get_tensor_prop, InferenceMemory.set_input) - seq_ids: dataclasses.InitVar[cp.ndarray] = DataClassProp(InferenceMemory._get_tensor_prop, - InferenceMemory.set_input) + input_mask: dataclasses.InitVar[NDArrayType] = DataClassProp(InferenceMemory._get_tensor_prop, + InferenceMemory.set_input) + seq_ids: dataclasses.InitVar[NDArrayType] = DataClassProp(InferenceMemory._get_tensor_prop, + InferenceMemory.set_input) - def __init__(self, *, count: int, input_ids: cp.ndarray, input_mask: cp.ndarray, seq_ids: cp.ndarray): + def __init__(self, *, count: int, input_ids: NDArrayType, input_mask: NDArrayType, seq_ids: NDArrayType): super().__init__(count=count, tensors={'input_ids': input_ids, 'input_mask': input_mask, 'seq_ids': seq_ids}) @@ -107,12 +106,12 @@ class InferenceMemoryFIL(InferenceMemory, cpp_class=_messages.InferenceMemoryFIL inputs than messages (i.e., if some messages get broken into multiple inference requests). """ - input__0: dataclasses.InitVar[cp.ndarray] = DataClassProp(InferenceMemory._get_tensor_prop, + input__0: dataclasses.InitVar[NDArrayType] = DataClassProp(InferenceMemory._get_tensor_prop, + InferenceMemory.set_input) + seq_ids: dataclasses.InitVar[NDArrayType] = DataClassProp(InferenceMemory._get_tensor_prop, InferenceMemory.set_input) - seq_ids: dataclasses.InitVar[cp.ndarray] = DataClassProp(InferenceMemory._get_tensor_prop, - InferenceMemory.set_input) - def __init__(self, *, count: int, input__0: cp.ndarray, seq_ids: cp.ndarray): + def __init__(self, *, count: int, input__0: NDArrayType, seq_ids: NDArrayType): super().__init__(count=count, tensors={'input__0': input__0, 'seq_ids': seq_ids}) @@ -130,9 +129,9 @@ class InferenceMemoryAE(InferenceMemory, cpp_class=None): inputs than messages (i.e., if some messages get broken into multiple inference requests). """ - input: dataclasses.InitVar[cp.ndarray] = DataClassProp(InferenceMemory._get_tensor_prop, InferenceMemory.set_input) - seq_ids: dataclasses.InitVar[cp.ndarray] = DataClassProp(InferenceMemory._get_tensor_prop, - InferenceMemory.set_input) + input: dataclasses.InitVar[NDArrayType] = DataClassProp(InferenceMemory._get_tensor_prop, InferenceMemory.set_input) + seq_ids: dataclasses.InitVar[NDArrayType] = DataClassProp(InferenceMemory._get_tensor_prop, + InferenceMemory.set_input) - def __init__(self, *, count: int, inputs: cp.ndarray, seq_ids: cp.ndarray): + def __init__(self, *, count: int, inputs: NDArrayType, seq_ids: NDArrayType): super().__init__(count=count, tensors={'input': inputs, 'seq_ids': seq_ids}) diff --git a/python/morpheus/morpheus/messages/memory/response_memory.py b/python/morpheus/morpheus/messages/memory/response_memory.py index eb4318f928..ffce421c7f 100644 --- a/python/morpheus/morpheus/messages/memory/response_memory.py +++ b/python/morpheus/morpheus/messages/memory/response_memory.py @@ -16,12 +16,11 @@ import dataclasses import logging -import cupy as cp - import morpheus._lib.messages as _messages from morpheus.messages.data_class_prop import DataClassProp from morpheus.messages.memory.tensor_memory import TensorMemory from morpheus.utils import logger as morpheus_logger +from morpheus.utils.type_aliases import NDArrayType logger = logging.getLogger(__name__) @@ -56,7 +55,7 @@ def get_output(self, name: str): """ return self.get_tensor(name) - def set_output(self, name: str, tensor: cp.ndarray): + def set_output(self, name: str, tensor: NDArrayType): """ Update the output tensor identified by `name`. Alias for `ResponseMemory.set_tensor` @@ -85,9 +84,9 @@ class ResponseMemoryProbs(ResponseMemory, cpp_class=_messages.ResponseMemoryProb probs : cupy.ndarray Probabilities tensor """ - probs: dataclasses.InitVar[cp.ndarray] = DataClassProp(ResponseMemory._get_tensor_prop, ResponseMemory.set_output) + probs: dataclasses.InitVar[NDArrayType] = DataClassProp(ResponseMemory._get_tensor_prop, ResponseMemory.set_output) - def __init__(self, *, count: int, probs: cp.ndarray): + def __init__(self, *, count: int, probs: NDArrayType): super().__init__(count=count, tensors={'probs': probs}) @@ -108,9 +107,9 @@ class ResponseMemoryAE(ResponseMemory, cpp_class=None): Explainability Dataframe, for each feature a column will exist with a name in the form of: `{feature}_z_loss` containing the loss z-score along with `max_abs_z` and `mean_abs_z` columns """ - probs: dataclasses.InitVar[cp.ndarray] = DataClassProp(ResponseMemory._get_tensor_prop, ResponseMemory.set_output) + probs: dataclasses.InitVar[NDArrayType] = DataClassProp(ResponseMemory._get_tensor_prop, ResponseMemory.set_output) user_id = "" explain_df = None - def __init__(self, *, count: int, probs: cp.ndarray): + def __init__(self, *, count: int, probs: NDArrayType): super().__init__(count=count, tensors={'probs': probs}) From eae1bffec9cb5f375f1e4e965d85a159e9a54365 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 Aug 2024 17:15:52 -0700 Subject: [PATCH 051/347] Remove using cudf+Python as a supported test combination --- tests/conftest.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 19b92df1b1..41c5384af9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -183,10 +183,15 @@ def should_filter_test(item: pytest.Item): use_cpp = item.get_closest_marker("use_cpp") use_pandas = item.get_closest_marker("use_pandas") + use_cudf = item.get_closest_marker("use_cudf") + use_python = item.get_closest_marker("use_python") if (use_cpp and use_pandas): return False + if (use_cudf and use_python): + return False + return True # Now delete tests with incompatible markers @@ -896,8 +901,8 @@ def test_something(dataset: DatasetManager): A test that requests this fixture will parameterize on the type of DataFrame returned by the DatasetManager. If a test requests both this fixture and the `use_cpp` fixture, or indirectly via the `config` fixture, then the test will parameterize over both df_type:[cudf, pandas] and use_cpp[True, False]. However it will remove the - df_type=pandas & use_cpp=True combinations as this will cause an unsupported usage of Pandas dataframes with the - C++ implementation of message classes. + df_type=pandas & use_cpp=True and df_type=cudf & use_cpp=False combinations as this will cause an unsupported usage + of Pandas dataframes with the C++ implementation of message classes, and cuDF with CPU-only implementations. This behavior can also be overridden by using the `use_cudf`, `use_pandas`, `use_cpp` or `use_pandas` marks ex: ``` @@ -905,13 +910,15 @@ def test_something(dataset: DatasetManager): @pytest.mark.use_cpp def test something(dataset: DatasetManager): ... - # This test will run once for each dataframe type, with C++ disabled both times + # This test will run once for with pandas and C++ disabled @pytest.mark.use_python - import sysdf dataframes both times + def test something(dataset: DatasetManager): + ... + # This test will run once with C++ mode enabled, using cudf dataframes @pytest.mark.use_cudf def test something(use_cpp: bool, dataset: DatasetManager): ... - # This test will run only once + # This test creates an incompatible combination and will raise a RuntimeError without being executed @pytest.mark.use_cudf @pytest.mark.use_python def test something(dataset: DatasetManager): From de31ff79755e2069c8f679eb809b1b38fdbb9180 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 09:16:40 -0700 Subject: [PATCH 052/347] Add comment about questionable cpu support for the AE pipeline --- python/morpheus/morpheus/cli/commands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/morpheus/morpheus/cli/commands.py b/python/morpheus/morpheus/cli/commands.py index 45c45bef4d..c5da90cb74 100644 --- a/python/morpheus/morpheus/cli/commands.py +++ b/python/morpheus/morpheus/cli/commands.py @@ -537,9 +537,11 @@ def pipeline_ae(ctx: click.Context, **kwargs): config = get_config_from_ctx(ctx) config.mode = PipelineModes.AE + # TODO: Need to determine if we can enable C++ for AE pipelines, or if we can get this working in CPUY mode if CppConfig.get_should_use_cpp(): logger.warning("C++ is disabled for AutoEncoder pipelines at this time.") CppConfig.set_should_use_cpp(False) + config.execution_mode = ExecutionMode.CPU config.ae = ConfigAutoEncoder() config.ae.userid_column_name = kwargs["userid_column_name"] From a2d3a1e9ba8cb111ba7836e3e17193528e972cc6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 09:18:47 -0700 Subject: [PATCH 053/347] Rather than raising an exception, log a warning and set the appropriate value for C++ mode, this may need to be re-evaluated --- python/morpheus/morpheus/pipeline/pipeline.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index 9d4e3ecd8b..d167273b32 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -63,8 +63,15 @@ class Pipeline(): def __init__(self, config: Config): config.freeze() + + # Ensure we have a valid configuration if config.execution_mode == ExecutionMode.CPU and CppConfig.get_should_use_cpp(): - raise RuntimeError("C++ mode requires GPU execution mode.") + logger.warning("CPU execution mode requires disabling C++ execution.") + CppConfig.set_should_use_cpp(False) + + elif config.execution_mode == ExecutionMode.GPU and not CppConfig.get_should_use_cpp(): + logger.warning("GPU mode requires C++ execution mode.") + CppConfig.set_should_use_cpp(True) self._mutex = threading.RLock() From 9ad465fee3bd0c14913725c232aee2efbd017754 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 09:19:27 -0700 Subject: [PATCH 054/347] Set exexcution mode based on the C++/Python mode --- tests/conftest.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 41c5384af9..a7278db60a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -212,10 +212,7 @@ def pytest_runtest_teardown(item, nextitem): reset_logging(logger_name=None) # Reset the root logger as well -# This fixture will be used by all tests. -@pytest.fixture(scope="function", autouse=True) -def _set_use_cpp(request: pytest.FixtureRequest): - +def _get_use_cpp(request: pytest.FixtureRequest) -> bool: do_use_cpp: bool = True # Check for the param if this was indirectly set @@ -233,6 +230,14 @@ def _set_use_cpp(request: pytest.FixtureRequest): # This will default to True or follow use_cpp do_use_cpp = not use_python + return do_use_cpp + + +# This fixture will be used by all tests. +@pytest.fixture(scope="function", autouse=True) +def _set_use_cpp(request: pytest.FixtureRequest): + do_use_cpp = _get_use_cpp(request) + from morpheus.config import CppConfig CppConfig.set_should_use_cpp(do_use_cpp) @@ -272,10 +277,13 @@ def config_no_cpp(): from morpheus.config import Config from morpheus.config import CppConfig + from morpheus.config import ExecutionMode CppConfig.set_should_use_cpp(False) + config = Config() + config.execution_mode = ExecutionMode.CPU - yield Config() + yield config @pytest.fixture(scope="function") @@ -317,8 +325,12 @@ def my_python_test(config: Config): """ from morpheus.config import Config + from morpheus.config import ExecutionMode + config = Config() + if not use_cpp: + config.execution_mode = ExecutionMode.CPU - yield Config() + yield config @pytest.fixture(scope="function") From 07ea414fd906fb6c47fce218f3e9ab07472859e5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 09:20:12 -0700 Subject: [PATCH 055/347] Not sure, this looks like an old bug --- tests/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1f578e5990..718562154a 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -218,8 +218,8 @@ def test_pipeline_ae(self, config, callback_values): assert config.num_threads == 12 assert isinstance(config.ae, ConfigAutoEncoder) - config.ae.userid_column_name = "user_col" - config.ae.userid_filter = "user321" + assert config.ae.userid_column_name == "user_col" + assert config.ae.userid_filter == "user321" expected_columns = load_labels_file(os.path.join(TEST_DIRS.data_dir, 'columns_ae_cloudtrail.txt')) assert config.ae.feature_columns == expected_columns From dfe349aa67cab73dc1256a025e1510bb8cd1d824 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 09:21:02 -0700 Subject: [PATCH 056/347] Fix mixing of pandas and cudf issue --- tests/test_filter_detections_stage_pipe.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_filter_detections_stage_pipe.py b/tests/test_filter_detections_stage_pipe.py index 15e36bd244..fa4fa0dea7 100755 --- a/tests/test_filter_detections_stage_pipe.py +++ b/tests/test_filter_detections_stage_pipe.py @@ -95,13 +95,13 @@ def _test_filter_detections_stage_multi_segment_pipe(config: Config, dataset_pan def _test_filter_detections_control_message_stage_multi_segment_pipe(config: Config, - dataset_pandas: DatasetManager, + dataset: DatasetManager, copy: bool = True): threshold = 0.75 - input_df = dataset_pandas["filter_probs.csv"] + input_df = dataset["filter_probs.csv"] pipe = LinearPipeline(config) - pipe.set_source(InMemorySourceStage(config, [cudf.DataFrame(input_df)])) + pipe.set_source(InMemorySourceStage(config, [input_df])) pipe.add_segment_boundary(MessageMeta) pipe.add_stage(DeserializeStage(config, message_type=ControlMessage)) pipe.add_segment_boundary(data_type=ControlMessage) @@ -111,8 +111,7 @@ def _test_filter_detections_control_message_stage_multi_segment_pipe(config: Con pipe.add_segment_boundary(ControlMessage) pipe.add_stage(SerializeStage(config)) pipe.add_segment_boundary(MessageMeta) - comp_stage = pipe.add_stage( - CompareDataFrameStage(config, build_expected(dataset_pandas["filter_probs.csv"], threshold))) + comp_stage = pipe.add_stage(CompareDataFrameStage(config, build_expected(dataset["filter_probs.csv"], threshold))) pipe.run() assert_results(comp_stage.get_results()) @@ -139,6 +138,6 @@ def test_filter_detections_stage_multi_segment_pipe(config: Config, dataset_pand @pytest.mark.parametrize('do_copy', [True, False]) def test_filter_detections_control_message_stage_multi_segment_pipe(config: Config, - dataset_pandas: DatasetManager, + dataset: DatasetManager, do_copy: bool): - return _test_filter_detections_control_message_stage_multi_segment_pipe(config, dataset_pandas, do_copy) + return _test_filter_detections_control_message_stage_multi_segment_pipe(config, dataset, do_copy) From 7edc244e76c8ee002d9c164a7d74d5c079dead1c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 11:31:37 -0700 Subject: [PATCH 057/347] WIP --- .../morpheus/morpheus/pipeline/stage_base.py | 6 ++ .../morpheus/stages/input/arxiv_source.py | 3 +- .../databricks_deltalake_source_stage.py | 3 +- .../stages/input/file_source_stage.py | 3 +- .../stages/input/http_client_source_stage.py | 3 +- .../stages/input/http_server_source_stage.py | 3 +- .../stages/output/http_server_sink_stage.py | 3 +- .../stages/output/write_to_file_stage.py | 3 +- .../stages/postprocess/serialize_stage.py | 3 +- .../stages/preprocess/deserialize_stage.py | 3 +- tests/pipeline/test_execution_mode_mixins.py | 62 +++++++++++++++++++ tests/stages/test_file_source_stage.py | 31 ++++++++++ 12 files changed, 117 insertions(+), 9 deletions(-) create mode 100755 tests/pipeline/test_execution_mode_mixins.py create mode 100755 tests/stages/test_file_source_stage.py diff --git a/python/morpheus/morpheus/pipeline/stage_base.py b/python/morpheus/morpheus/pipeline/stage_base.py index 065883ff33..2659651c85 100644 --- a/python/morpheus/morpheus/pipeline/stage_base.py +++ b/python/morpheus/morpheus/pipeline/stage_base.py @@ -84,6 +84,12 @@ class StageBase(ABC, collections.abc.Hashable): _schema: _pipeline.StageSchema def __init__(self, config: Config): + # Check the execution mode + if (config.execution_mode not in self.supported_execution_modes()): + supported_modes = ", ".join(str(x) for x in self.supported_execution_modes()) + raise RuntimeError(f"Unsupported execution mode {config.execution_mode} for stage {self.name}, " + f"supported exexution modes are {supported_modes}") + # Save the config config.freeze() self._config = config diff --git a/python/morpheus/morpheus/stages/input/arxiv_source.py b/python/morpheus/morpheus/stages/input/arxiv_source.py index 1b87ad750e..385e8be2a5 100644 --- a/python/morpheus/morpheus/stages/input/arxiv_source.py +++ b/python/morpheus/morpheus/stages/input/arxiv_source.py @@ -24,6 +24,7 @@ from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema @@ -40,7 +41,7 @@ @register_stage("from-arxiv", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) -class ArxivSource(PreallocatorMixin, SingleOutputSource): +class ArxivSource(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage that downloads PDFs from arxiv and converts them to dataframes. diff --git a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py index 4d21ac8aac..77a63a2165 100644 --- a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py +++ b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py @@ -20,6 +20,7 @@ from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.messages.message_meta import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema @@ -38,7 +39,7 @@ @register_stage("from-databricks-deltalake", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) -class DataBricksDeltaLakeSourceStage(PreallocatorMixin, SingleOutputSource): +class DataBricksDeltaLakeSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage used to load messages from a DeltaLake table. diff --git a/python/morpheus/morpheus/stages/input/file_source_stage.py b/python/morpheus/morpheus/stages/input/file_source_stage.py index eeeac937ab..ad3c5d75c0 100644 --- a/python/morpheus/morpheus/stages/input/file_source_stage.py +++ b/python/morpheus/morpheus/stages/input/file_source_stage.py @@ -26,6 +26,7 @@ from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema @@ -36,7 +37,7 @@ @register_stage("from-file", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) -class FileSourceStage(PreallocatorMixin, SingleOutputSource): +class FileSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Load messages from a file. diff --git a/python/morpheus/morpheus/stages/input/http_client_source_stage.py b/python/morpheus/morpheus/stages/input/http_client_source_stage.py index f05adea710..727b304ddb 100644 --- a/python/morpheus/morpheus/stages/input/http_client_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_client_source_stage.py @@ -26,6 +26,7 @@ from morpheus.config import ExecutionMode from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema @@ -38,7 +39,7 @@ @register_stage("from-http-client", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), ignore_args=["query_params", "headers", "**request_kwargs"]) -class HttpClientSourceStage(PreallocatorMixin, SingleOutputSource): +class HttpClientSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage that polls a remote HTTP server for incoming data. diff --git a/python/morpheus/morpheus/stages/input/http_server_source_stage.py b/python/morpheus/morpheus/stages/input/http_server_source_stage.py index cdd45f0112..3db97cac29 100644 --- a/python/morpheus/morpheus/stages/input/http_server_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_server_source_stage.py @@ -27,6 +27,7 @@ from morpheus.config import ExecutionMode from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema @@ -43,7 +44,7 @@ @register_stage("from-http", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) -class HttpServerSourceStage(PreallocatorMixin, SingleOutputSource): +class HttpServerSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage that starts an HTTP server and listens for incoming requests on a specified endpoint. diff --git a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py index bca6207e8e..ca5ac3c1d5 100644 --- a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py @@ -30,6 +30,7 @@ from morpheus.io import serializers from morpheus.io.utils import get_df_pkg from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils.http_utils import HTTPMethod @@ -43,7 +44,7 @@ @register_stage("to-http-server", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), ignore_args=["df_serializer_fn"]) -class HttpServerSinkStage(PassThruTypeMixin, SinglePortStage): +class HttpServerSinkStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Sink stage that starts an HTTP server and listens for incoming requests on a specified endpoint. diff --git a/python/morpheus/morpheus/stages/output/write_to_file_stage.py b/python/morpheus/morpheus/stages/output/write_to_file_stage.py index 967fedc338..e3f9b06cbf 100644 --- a/python/morpheus/morpheus/stages/output/write_to_file_stage.py +++ b/python/morpheus/morpheus/stages/output/write_to_file_stage.py @@ -24,6 +24,7 @@ from morpheus.config import ExecutionMode from morpheus.controllers.write_to_file_controller import WriteToFileController from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @@ -31,7 +32,7 @@ @register_stage("to-file", rename_options={"include_index_col": "--include-index-col"}, execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) -class WriteToFileStage(PassThruTypeMixin, SinglePortStage): +class WriteToFileStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Write all messages to a file. diff --git a/python/morpheus/morpheus/stages/postprocess/serialize_stage.py b/python/morpheus/morpheus/stages/postprocess/serialize_stage.py index c659e6b1e1..0372cd83ca 100644 --- a/python/morpheus/morpheus/stages/postprocess/serialize_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/serialize_stage.py @@ -26,6 +26,7 @@ from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema @@ -33,7 +34,7 @@ @register_stage("serialize", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) -class SerializeStage(SinglePortStage): +class SerializeStage(GpuAndCpuMixin, SinglePortStage): """ Includes & excludes columns from messages. diff --git a/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py b/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py index 8afbc2643c..f97d9bc2ec 100644 --- a/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py @@ -26,6 +26,7 @@ from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.modules.preprocess.deserialize import DeserializeLoaderFactory +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.multi_message_stage import MultiMessageStage from morpheus.pipeline.stage_schema import StageSchema @@ -36,7 +37,7 @@ modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), ignore_args=["message_type", "task_type", "task_payload"]) -class DeserializeStage(MultiMessageStage): +class DeserializeStage(GpuAndCpuMixin, MultiMessageStage): """ Messages are logically partitioned based on the pipeline config's `pipeline_batch_size` parameter. diff --git a/tests/pipeline/test_execution_mode_mixins.py b/tests/pipeline/test_execution_mode_mixins.py new file mode 100755 index 0000000000..090db3c002 --- /dev/null +++ b/tests/pipeline/test_execution_mode_mixins.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from _utils.stages.conv_msg import ConvMsg +from morpheus.config import Config +from morpheus.config import CppConfig +from morpheus.config import ExecutionMode +from morpheus.pipeline.execution_mode_mixins import CpuOnlyMixin +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin + + +class CpuOnlyStage(CpuOnlyMixin, ConvMsg): + pass + + +class GpuAndCpuStage(GpuAndCpuMixin, ConvMsg): + pass + + +@pytest.mark.parametrize("stage_cls, expected_modes", + [(ConvMsg, {ExecutionMode.GPU}), (CpuOnlyStage, {ExecutionMode.CPU}), + (GpuAndCpuStage, {ExecutionMode.GPU, ExecutionMode.CPU})]) +def test_execution_mode_mixins(stage_cls: type[ConvMsg], expected_modes: set): + # intentionally not using the config fixture so that we can set the execution mode and avoid iterating over + # python/C++ execution modes + config = Config() + if ExecutionMode.CPU in expected_modes: + config.execution_mode = ExecutionMode.CPU + else: + config.execution_mode = ExecutionMode.GPU + + stage = stage_cls(config) + assert set(stage.supported_execution_modes()) == expected_modes + + +@pytest.mark.parametrize("stage_cls", [ConvMsg, CpuOnlyStage]) +def test_unsupported_mode_error(stage_cls: type[ConvMsg]): + # intentionally not using the config fixture so that we can set the execution mode and avoid iterating over + # python/C++ execution modes + config = Config() + if issubclass(stage_cls, CpuOnlyMixin): + config.execution_mode = ExecutionMode.GPU + else: + config.execution_mode = ExecutionMode.CPU + + with pytest.raises(RuntimeError, match="Unsupported execution mode"): + stage_cls(config) diff --git a/tests/stages/test_file_source_stage.py b/tests/stages/test_file_source_stage.py new file mode 100755 index 0000000000..19d2dacd51 --- /dev/null +++ b/tests/stages/test_file_source_stage.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from _utils import TEST_DIRS +from morpheus.config import Config +from morpheus.config import ExecutionMode +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin +from morpheus.stages.input.file_source_stage import FileSourceStage + + +def test_execution_modes(config: Config): + assert issubclass(FileSourceStage, GpuAndCpuMixin) + stage = FileSourceStage(config, filename=os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv")) + + # we don't care about the order of the execution modes + assert set(stage.supported_execution_modes()) == {ExecutionMode.GPU, ExecutionMode.CPU} From 3fd975558bdaa4b2302ed239dcd35e11884bde1b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 12:39:32 -0700 Subject: [PATCH 058/347] Support execution_modes argument in stage and source decorators. Remove stage constructor test, this would require subclasses to set the modes prior to calling the parent constructor, instead depend on the pre-build check --- .../morpheus/morpheus/pipeline/stage_base.py | 10 +- .../morpheus/pipeline/stage_decorator.py | 78 ++++++++---- tests/pipeline/test_execution_mode_mixins.py | 62 ---------- tests/pipeline/test_execution_modes.py | 114 ++++++++++++++++++ 4 files changed, 172 insertions(+), 92 deletions(-) delete mode 100755 tests/pipeline/test_execution_mode_mixins.py create mode 100755 tests/pipeline/test_execution_modes.py diff --git a/python/morpheus/morpheus/pipeline/stage_base.py b/python/morpheus/morpheus/pipeline/stage_base.py index 2659651c85..444bb6304b 100644 --- a/python/morpheus/morpheus/pipeline/stage_base.py +++ b/python/morpheus/morpheus/pipeline/stage_base.py @@ -84,12 +84,6 @@ class StageBase(ABC, collections.abc.Hashable): _schema: _pipeline.StageSchema def __init__(self, config: Config): - # Check the execution mode - if (config.execution_mode not in self.supported_execution_modes()): - supported_modes = ", ".join(str(x) for x in self.supported_execution_modes()) - raise RuntimeError(f"Unsupported execution mode {config.execution_mode} for stage {self.name}, " - f"supported exexution modes are {supported_modes}") - # Save the config config.freeze() self._config = config @@ -371,7 +365,9 @@ def _pre_build(self, do_propagate: bool = True): # Check the execution mode if (self._config.execution_mode not in self.supported_execution_modes()): - raise ValueError(f"Stage {self.name} does not support execution mode {self._config.execution_mode}") + supported_modes = ", ".join(str(x) for x in self.supported_execution_modes()) + raise RuntimeError(f"Unsupported execution mode {self._config.execution_mode} for stage {self.name}, " + f"supported exexution modes are {supported_modes}") # Perform schema validation schema = _pipeline.StageSchema(self) diff --git a/python/morpheus/morpheus/pipeline/stage_decorator.py b/python/morpheus/morpheus/pipeline/stage_decorator.py index fa7bc86feb..4324dd7c92 100644 --- a/python/morpheus/morpheus/pipeline/stage_decorator.py +++ b/python/morpheus/morpheus/pipeline/stage_decorator.py @@ -27,6 +27,7 @@ import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import from morpheus.common import TypeId from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage @@ -94,7 +95,13 @@ class WrappedFunctionSourceStage(_pipeline.SingleOutputSource): Function to use for computing the schema of the stage. """ - def __init__(self, config: Config, *, name: str, gen_fn: GeneratorType, compute_schema_fn: ComputeSchemaType): + def __init__(self, + config: Config, + *, + name: str, + gen_fn: GeneratorType, + compute_schema_fn: ComputeSchemaType, + execution_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, )): super().__init__(config) # collections.abc.Generator is a subclass of collections.abc.Iterator if not inspect.isgeneratorfunction(gen_fn): @@ -103,6 +110,7 @@ def __init__(self, config: Config, *, name: str, gen_fn: GeneratorType, compute_ self._name = name self._gen_fn = gen_fn self._compute_schema_fn = compute_schema_fn + self._supported_execution_modes = execution_modes @property def name(self) -> str: @@ -114,6 +122,12 @@ def supports_cpp_node(self) -> bool: def compute_schema(self, schema: _pipeline.StageSchema): self._compute_schema_fn(schema) + def supported_execution_modes(self) -> tuple[ExecutionMode]: + """ + Returns a tuple of supported execution modes of this stage. + """ + return self._supported_execution_modes + def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: return builder.make_source(self.unique_name, self._gen_fn) @@ -143,7 +157,8 @@ def source( gen_fn: GeneratorType = None, *, name: str = None, - compute_schema_fn: ComputeSchemaType = None + compute_schema_fn: ComputeSchemaType = None, + execution_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, ) ) -> typing.Callable[typing.Concatenate[Config, _P], WrappedFunctionSourceStage]: """ Decorator for wrapping a function as a source stage. The function must be a generator method, and provide a @@ -167,7 +182,10 @@ def source( >>> pipe.set_source(source_gen(config, dataframes=[df])) """ if gen_fn is None: - return functools.partial(source, name=name, compute_schema_fn=compute_schema_fn) + return functools.partial(source, + name=name, + compute_schema_fn=compute_schema_fn, + execution_modes=execution_modes) # Use wraps to ensure user's don't lose their function name and docstrinsgs, however we do want to override the # annotations to reflect that the returned function requires a config and returns a stage @@ -208,12 +226,14 @@ def compute_schema_fn_inner(schema: _pipeline.StageSchema): return PreAllocatedWrappedFunctionStage(config=config, name=name, gen_fn=bound_gen_fn, - compute_schema_fn=compute_schema_fn) + compute_schema_fn=compute_schema_fn, + execution_modes=execution_modes) return WrappedFunctionSourceStage(config=config, name=name, gen_fn=bound_gen_fn, - compute_schema_fn=compute_schema_fn) + compute_schema_fn=compute_schema_fn, + execution_modes=execution_modes) return wrapper @@ -242,16 +262,15 @@ class WrappedFunctionStage(_pipeline.SinglePortStage): by the `PreAllocatedWrappedFunctionStage` to ensure the DataFrame has the needed columns allocated. """ - def __init__( - self, - config: Config, - *, - name: str = None, - on_data_fn: typing.Callable, - accept_type: type, - compute_schema_fn: ComputeSchemaType, - needed_columns: dict[str, TypeId] = None, - ): + def __init__(self, + config: Config, + *, + name: str = None, + on_data_fn: typing.Callable, + accept_type: type, + compute_schema_fn: ComputeSchemaType, + needed_columns: dict[str, TypeId] = None, + execution_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, )): super().__init__(config) self._name = name self._on_data_fn = on_data_fn @@ -261,6 +280,8 @@ def __init__( if needed_columns is not None: self._needed_columns.update(needed_columns) + self._supported_execution_modes = execution_modes + @property def name(self) -> str: return self._name @@ -274,6 +295,12 @@ def supports_cpp_node(self) -> bool: def compute_schema(self, schema: _pipeline.StageSchema): self._compute_schema_fn(schema) + def supported_execution_modes(self) -> tuple[ExecutionMode]: + """ + Returns a tuple of supported execution modes of this stage. + """ + return self._supported_execution_modes + def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: node = builder.make_node(self.unique_name, ops.map(self._on_data_fn)) builder.make_edge(input_node, node) @@ -284,12 +311,15 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> DecoratedStageType = typing.Callable[typing.Concatenate[Config, _P], WrappedFunctionStage] -def stage(on_data_fn: typing.Callable[typing.Concatenate[_InputT, _P], _OutputT] = None, - *, - name: str = None, - accept_type: type = None, - compute_schema_fn: ComputeSchemaType = None, - needed_columns: dict[str, TypeId] = None) -> DecoratedStageType: +def stage( + on_data_fn: typing.Callable[typing.Concatenate[_InputT, _P], _OutputT] = None, + *, + name: str = None, + accept_type: type = None, + compute_schema_fn: ComputeSchemaType = None, + needed_columns: dict[str, TypeId] = None, + execution_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, ) +) -> DecoratedStageType: """ Decorator for wrapping a function as a stage. The function must receive at least one argument, the first argument must be the incoming message, and must return a value. @@ -325,7 +355,8 @@ def stage(on_data_fn: typing.Callable[typing.Concatenate[_InputT, _P], _OutputT] name=name, accept_type=accept_type, compute_schema_fn=compute_schema_fn, - needed_columns=needed_columns) + needed_columns=needed_columns, + execution_modes=execution_modes) # Use wraps to ensure user's don't lose their function name and docstrinsgs, however we do want to override the # annotations to reflect that the returned function requires a config and returns a stage @@ -376,6 +407,7 @@ def compute_schema_fn_inner(schema: _pipeline.StageSchema): on_data_fn=bound_on_data_fn, accept_type=accept_type, compute_schema_fn=compute_schema_fn, - needed_columns=needed_columns) + needed_columns=needed_columns, + execution_modes=execution_modes) return wrapper diff --git a/tests/pipeline/test_execution_mode_mixins.py b/tests/pipeline/test_execution_mode_mixins.py deleted file mode 100755 index 090db3c002..0000000000 --- a/tests/pipeline/test_execution_mode_mixins.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from _utils.stages.conv_msg import ConvMsg -from morpheus.config import Config -from morpheus.config import CppConfig -from morpheus.config import ExecutionMode -from morpheus.pipeline.execution_mode_mixins import CpuOnlyMixin -from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin - - -class CpuOnlyStage(CpuOnlyMixin, ConvMsg): - pass - - -class GpuAndCpuStage(GpuAndCpuMixin, ConvMsg): - pass - - -@pytest.mark.parametrize("stage_cls, expected_modes", - [(ConvMsg, {ExecutionMode.GPU}), (CpuOnlyStage, {ExecutionMode.CPU}), - (GpuAndCpuStage, {ExecutionMode.GPU, ExecutionMode.CPU})]) -def test_execution_mode_mixins(stage_cls: type[ConvMsg], expected_modes: set): - # intentionally not using the config fixture so that we can set the execution mode and avoid iterating over - # python/C++ execution modes - config = Config() - if ExecutionMode.CPU in expected_modes: - config.execution_mode = ExecutionMode.CPU - else: - config.execution_mode = ExecutionMode.GPU - - stage = stage_cls(config) - assert set(stage.supported_execution_modes()) == expected_modes - - -@pytest.mark.parametrize("stage_cls", [ConvMsg, CpuOnlyStage]) -def test_unsupported_mode_error(stage_cls: type[ConvMsg]): - # intentionally not using the config fixture so that we can set the execution mode and avoid iterating over - # python/C++ execution modes - config = Config() - if issubclass(stage_cls, CpuOnlyMixin): - config.execution_mode = ExecutionMode.GPU - else: - config.execution_mode = ExecutionMode.CPU - - with pytest.raises(RuntimeError, match="Unsupported execution mode"): - stage_cls(config) diff --git a/tests/pipeline/test_execution_modes.py b/tests/pipeline/test_execution_modes.py new file mode 100755 index 0000000000..7d1ef2e81b --- /dev/null +++ b/tests/pipeline/test_execution_modes.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections.abc +import typing + +import pytest + +from _utils.stages.conv_msg import ConvMsg +from morpheus.config import Config +from morpheus.config import ExecutionMode +from morpheus.pipeline.execution_mode_mixins import CpuOnlyMixin +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin +from morpheus.pipeline.stage_decorator import source +from morpheus.pipeline.stage_decorator import stage + + +@source +def gpu_only_source() -> collections.abc.Iterator[int]: + for i in range(10): + yield i + + +@source(execution_modes=(ExecutionMode.CPU, )) +def cpu_only_source() -> collections.abc.Iterator[int]: + for i in range(10): + yield i + + +@source(execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +def gpu_cpu_source() -> collections.abc.Iterator[int]: + for i in range(10): + yield i + + +@stage +def gpu_only_stage(message: typing.Any) -> typing.Any: + return message + + +@stage(execution_modes=(ExecutionMode.CPU, )) +def cpu_only_stage(message: typing.Any) -> typing.Any: + return message + + +@stage(execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +def gpu_cpu_stage(message: typing.Any) -> typing.Any: + return message + + +class CpuOnlyStage(CpuOnlyMixin, ConvMsg): + pass + + +class GpuAndCpuStage(GpuAndCpuMixin, ConvMsg): + pass + + +@pytest.mark.parametrize("stage_cls, expected_modes", + [ + (ConvMsg, {ExecutionMode.GPU}), + (CpuOnlyStage, {ExecutionMode.CPU}), + (GpuAndCpuStage, {ExecutionMode.GPU, ExecutionMode.CPU}), + (gpu_only_source, {ExecutionMode.GPU}), + (cpu_only_source, {ExecutionMode.CPU}), + (gpu_cpu_source, {ExecutionMode.GPU, ExecutionMode.CPU}), + (gpu_only_stage, {ExecutionMode.GPU}), + (cpu_only_stage, {ExecutionMode.CPU}), + (gpu_cpu_stage, {ExecutionMode.GPU, ExecutionMode.CPU}), + ]) +def test_execution_mode_mixins(stage_cls: type[ConvMsg], expected_modes: set): + # intentionally not using the config fixture so that we can set the execution mode and avoid iterating over + # python/C++ execution modes + config = Config() + if ExecutionMode.CPU in expected_modes: + config.execution_mode = ExecutionMode.CPU + else: + config.execution_mode = ExecutionMode.GPU + + stage = stage_cls(config) + assert set(stage.supported_execution_modes()) == expected_modes + + +@pytest.mark.parametrize("stage_cls, execution_mode", + [ + (ConvMsg, ExecutionMode.CPU), + (gpu_only_source, ExecutionMode.CPU), + (gpu_only_stage, ExecutionMode.CPU), + (CpuOnlyStage, ExecutionMode.GPU), + (cpu_only_source, ExecutionMode.GPU), + (cpu_only_stage, ExecutionMode.GPU), + ]) +def test_unsupported_mode_error(stage_cls: type[ConvMsg], execution_mode: ExecutionMode): + # intentionally not using the config fixture so that we can set the execution mode and avoid iterating over + # python/C++ execution modes + config = Config() + config.execution_mode = execution_mode + + with pytest.raises(RuntimeError, match="Unsupported execution mode"): + stage = stage_cls(config) + stage._pre_build(do_propagate=False) From 5f5ad6f908f8b5424ed29badd331a6b41be64bc1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 12:51:00 -0700 Subject: [PATCH 059/347] Remove usage of cudf and cupy from python impls --- .../morpheus/messages/multi_message.py | 63 ++++--------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/python/morpheus/morpheus/messages/multi_message.py b/python/morpheus/morpheus/messages/multi_message.py index 44e1bb6cba..9f17b38ba2 100644 --- a/python/morpheus/morpheus/messages/multi_message.py +++ b/python/morpheus/morpheus/messages/multi_message.py @@ -17,15 +17,14 @@ import inspect import typing -import cupy as cp import numpy as np -import pandas as pd - -import cudf import morpheus._lib.messages as _messages from morpheus.messages.message_base import MessageData from morpheus.messages.message_meta import MessageMeta +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import NDArrayType +from morpheus.utils.type_aliases import SeriesType # Needed to provide the return type of `@classmethod` Self = typing.TypeVar("Self", bound="MultiMessage") @@ -167,15 +166,15 @@ def get_meta_column_names(self) -> list[str]: return self.meta.get_column_names() @typing.overload - def get_meta(self) -> cudf.DataFrame: + def get_meta(self) -> DataFrameType: ... @typing.overload - def get_meta(self, columns: str) -> cudf.Series: + def get_meta(self, columns: str) -> SeriesType: ... @typing.overload - def get_meta(self, columns: typing.List[str]) -> cudf.DataFrame: + def get_meta(self, columns: typing.List[str]) -> DataFrameType: ... def get_meta(self, columns: typing.Union[None, str, typing.List[str]] = None): @@ -245,10 +244,6 @@ def set_meta(self, columns: typing.Union[None, str, typing.List[str]], value): # First try to set the values on just our slice if the columns exist row_indexer, column_indexer = self._get_indexers(df, columns=columns) - # Check if the value is a cupy array and we have a pandas dataframe, convert to numpy - if (isinstance(value, cp.ndarray) and isinstance(df, pd.DataFrame)): - value = value.get() - # Check to see if we are adding a column. If so, we need to use df.loc instead of df.iloc if (-1 not in column_indexer): @@ -267,38 +262,11 @@ def set_meta(self, columns: typing.Union[None, str, typing.List[str]], value): # Columns should never be empty if we get here assert columns is not None - # cudf is really bad at adding new columns - if (isinstance(df, cudf.DataFrame)): - - # TODO(morpheus#1487): This logic no longer works in CUDF 24.04. - # We should find a way to reinable the no-dropped-index path as - # that should be more performant than dropping the index. - # # saved_index = None - - # # # Check to see if we can use slices - # # if (not (df.index.is_unique and - # # (df.index.is_monotonic_increasing or df.index.is_monotonic_decreasing))): - # # # Save the index and reset - # # saved_index = df.index - # # df.reset_index(drop=True, inplace=True) + # Need to determine the boolean mask to use indexes with df.loc + row_mask = self._ranges_to_mask(df, [(self.mess_offset, self.mess_offset + self.mess_count)]) - # # # Perform the update via slices - # # df.loc[df.index[row_indexer], columns] = value - - # # # Reset the index if we changed it - # # if (saved_index is not None): - # # df.set_index(saved_index, inplace=True) - - saved_index = df.index - df.reset_index(drop=True, inplace=True) - df.loc[df.index[row_indexer], columns] = value - df.set_index(saved_index, inplace=True) - else: - # Need to determine the boolean mask to use indexes with df.loc - row_mask = self._ranges_to_mask(df, [(self.mess_offset, self.mess_offset + self.mess_count)]) - - # Now set the slice - df.loc[row_mask, columns] = value + # Now set the slice + df.loc[row_mask, columns] = value def get_slice(self, start, stop): """ @@ -325,21 +293,14 @@ def get_slice(self, start, stop): return self.from_message(self, meta=self.meta, mess_offset=offset, mess_count=count) def _ranges_to_mask(self, df, ranges): - if isinstance(df, cudf.DataFrame): - zeros_fn = cp.zeros - else: - zeros_fn = np.zeros - - mask = zeros_fn(len(df), bool) + mask = np.zeros(len(df), bool) for range_ in ranges: mask[range_[0]:range_[1]] = True return mask - def copy_meta_ranges(self, - ranges: typing.List[typing.Tuple[int, int]], - mask: typing.Union[None, cp.ndarray, np.ndarray] = None): + def copy_meta_ranges(self, ranges: typing.List[typing.Tuple[int, int]], mask: NDArrayType = None): """ Perform a copy of the underlying dataframe for the given `ranges` of rows. From ec1c4a22d7158611b7ec8faa2f940b77ae99d8f7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 14:04:54 -0700 Subject: [PATCH 060/347] Do not call CudfHelper::load() automatically on import for messages and llm, for llm we plan to support cpu-only, for messages we need to be able to perform the import in cpu mode --- python/morpheus/morpheus/_lib/llm/module.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/morpheus/morpheus/_lib/llm/module.cpp b/python/morpheus/morpheus/_lib/llm/module.cpp index 8a23f559dc..7bd1fbdf8c 100644 --- a/python/morpheus/morpheus/_lib/llm/module.cpp +++ b/python/morpheus/morpheus/_lib/llm/module.cpp @@ -68,9 +68,6 @@ PYBIND11_MODULE(llm, _module) )pbdoc"; - // Load the cudf helpers - CudfHelper::load(); - // Import the mrc coro module mrc::pymrc::import(_module, "mrc.core.coro"); From 8f3356ccb61d4812e743ada4b7d26d8c3c2804fa Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 14:57:22 -0700 Subject: [PATCH 061/347] WIP --- python/morpheus/morpheus/_lib/common/module.cpp | 5 +---- python/morpheus/morpheus/_lib/messages/module.cpp | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/python/morpheus/morpheus/_lib/common/module.cpp b/python/morpheus/morpheus/_lib/common/module.cpp index b21ad3cfe2..b3ae6d5493 100644 --- a/python/morpheus/morpheus/_lib/common/module.cpp +++ b/python/morpheus/morpheus/_lib/common/module.cpp @@ -1,4 +1,4 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * @@ -58,9 +58,6 @@ PYBIND11_MODULE(common, _module) :toctree: _generate )pbdoc"; - // Load the cudf helpers - CudfHelper::load(); - LoaderRegistry::register_factory_fn( "file", [](nlohmann::json config) { diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index a8568de1df..a52879d924 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -134,10 +134,7 @@ PYBIND11_MODULE(messages, _module) )pbdoc"; - // Load the cudf helpers - CudfHelper::load(); - - mrc::pymrc::import(_module, "cupy"); + mrc::pymrc::import(_module, "cupy"); // It should be safe to import cupy in CPU only mode mrc::pymrc::import(_module, "morpheus._lib.common"); // Required for SegmentObject From 5b5db254237fd111f495bd55c0e33ca225e585e5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 14:56:38 -0700 Subject: [PATCH 062/347] Don't automatically call CudfHelper::load() on imports, instead expose it to python and call it during pipeline build --- python/morpheus/morpheus/_lib/common/__init__.pyi | 3 +++ python/morpheus/morpheus/_lib/common/module.cpp | 4 ++++ python/morpheus/morpheus/_lib/messages/module.cpp | 4 ++++ python/morpheus/morpheus/_lib/stages/module.cpp | 3 --- python/morpheus/morpheus/pipeline/pipeline.py | 3 +++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi index 7e11e81ccd..bba480d220 100644 --- a/python/morpheus/morpheus/_lib/common/__init__.pyi +++ b/python/morpheus/morpheus/_lib/common/__init__.pyi @@ -18,6 +18,7 @@ __all__ = [ "Tensor", "TypeId", "determine_file_type", + "load_cudf_helper", "read_file_to_df", "typeid_is_fully_supported", "typeid_to_numpy_str", @@ -202,6 +203,8 @@ def determine_file_type(filename: os.PathLike) -> FileTypes: @typing.overload def determine_file_type(filename: str) -> FileTypes: pass +def load_cudf_helper() -> None: + pass def read_file_to_df(filename: str, file_type: FileTypes = FileTypes.Auto) -> object: pass def typeid_is_fully_supported(arg0: TypeId) -> bool: diff --git a/python/morpheus/morpheus/_lib/common/module.cpp b/python/morpheus/morpheus/_lib/common/module.cpp index b3ae6d5493..54f0cc850e 100644 --- a/python/morpheus/morpheus/_lib/common/module.cpp +++ b/python/morpheus/morpheus/_lib/common/module.cpp @@ -143,6 +143,10 @@ PYBIND11_MODULE(common, _module) py::arg("filename"), py::arg("file_type") = FileTypes::Auto); + _module.def("load_cudf_helper", [] { + CudfHelper::load(); + }); + py::enum_( _module, "FilterSource", "Enum to indicate which source the FilterDetectionsStage should operate on.") .value("Auto", FilterSource::Auto) diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index a52879d924..bfcd840812 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -134,7 +134,11 @@ PYBIND11_MODULE(messages, _module) )pbdoc"; +<<<<<<< HEAD mrc::pymrc::import(_module, "cupy"); // It should be safe to import cupy in CPU only mode +======= + mrc::pymrc::import(_module, "cupy"); +>>>>>>> 57a126676 (Don't automatically call CudfHelper::load() on imports, instead expose it to python and call it during pipeline build) mrc::pymrc::import(_module, "morpheus._lib.common"); // Required for SegmentObject diff --git a/python/morpheus/morpheus/_lib/stages/module.cpp b/python/morpheus/morpheus/_lib/stages/module.cpp index dc6537fa7d..bdb55979cf 100644 --- a/python/morpheus/morpheus/_lib/stages/module.cpp +++ b/python/morpheus/morpheus/_lib/stages/module.cpp @@ -67,9 +67,6 @@ PYBIND11_MODULE(stages, _module) )pbdoc"; - // Load the cudf helpers - CudfHelper::load(); - // Make sure to load mrc.core.segment to get ObjectProperties mrc::pymrc::import(_module, "mrc.core.segment"); diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index d167273b32..ce3d97cdf7 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -303,6 +303,9 @@ def build(self): assert self._state == PipelineState.INITIALIZED, "Pipeline can only be built once!" assert len(self._sources) > 0, "Pipeline must have a source stage" + from morpheus._lib import common as _common + _common.load_cudf_helper() + self._pre_build() logger.info("====Registering Pipeline====") From 239e77ed4cc694951b42392c2f4469ca81d89469 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 15:04:48 -0700 Subject: [PATCH 063/347] Fix merge error --- python/morpheus/morpheus/_lib/messages/module.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index bfcd840812..a52879d924 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -134,11 +134,7 @@ PYBIND11_MODULE(messages, _module) )pbdoc"; -<<<<<<< HEAD mrc::pymrc::import(_module, "cupy"); // It should be safe to import cupy in CPU only mode -======= - mrc::pymrc::import(_module, "cupy"); ->>>>>>> 57a126676 (Don't automatically call CudfHelper::load() on imports, instead expose it to python and call it during pipeline build) mrc::pymrc::import(_module, "morpheus._lib.common"); // Required for SegmentObject From 6f57635a4faa35d40b96637fdbbeb360acaa2124 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 15:06:50 -0700 Subject: [PATCH 064/347] Only load cudf helpers if we are in GPU mode --- python/morpheus/morpheus/pipeline/pipeline.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index ce3d97cdf7..f249b1b349 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -103,6 +103,8 @@ def __init__(self, config: Config): # Future that allows post_start to propagate exceptions back to pipeline self._post_start_future: asyncio.Future = None + self._execution_mode = config.execution_mode + @property def state(self) -> PipelineState: return self._state @@ -303,8 +305,10 @@ def build(self): assert self._state == PipelineState.INITIALIZED, "Pipeline can only be built once!" assert len(self._sources) > 0, "Pipeline must have a source stage" - from morpheus._lib import common as _common - _common.load_cudf_helper() + if (self._execution_mode == ExecutionMode.GPU): + # Load the cudf helper + from morpheus._lib import common as _common + _common.load_cudf_helper() self._pre_build() From 1f723b21469b3f08ae506edc585a11c0e45fe093 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 15:22:14 -0700 Subject: [PATCH 065/347] Ensure all fields are copied in copy constructor --- python/morpheus/morpheus/_lib/src/messages/control.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index 547a845ebe..ecedcdfdcf 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -57,8 +57,14 @@ ControlMessage::ControlMessage(const morpheus::utilities::json_t& _config) : ControlMessage::ControlMessage(const ControlMessage& other) { + m_cm_type = other.m_cm_type; + m_payload = other.m_payload; + m_tensors = other.m_tensors; + m_config = other.m_config; m_tasks = other.m_tasks; + + m_timestamps = other.m_timestamps; } const morpheus::utilities::json_t& ControlMessage::config() const From 2bb739d6a310c78e697cf486aa9cbcbd356420f6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 15:41:17 -0700 Subject: [PATCH 066/347] Add comment --- python/morpheus/morpheus/stages/inference/inference_stage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/morpheus/morpheus/stages/inference/inference_stage.py b/python/morpheus/morpheus/stages/inference/inference_stage.py index 58394aa0f1..a00e4741b4 100644 --- a/python/morpheus/morpheus/stages/inference/inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/inference_stage.py @@ -163,6 +163,7 @@ class InferenceStage(MultiMessageStage): def __init__(self, c: Config): super().__init__(c) + # GPU only stage, assuming all messages are cuDF/CuPy based import cudf self._cudf = cudf From 4cbaffd111a87f26785d6101d26eb1c74a5e030f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 16:30:33 -0700 Subject: [PATCH 067/347] _get_preprocess_fn and _get_preprocess_node are now optional methods --- .../preprocess/preprocess_base_stage.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py index f115e38053..cf35449773 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py @@ -22,6 +22,7 @@ from morpheus.config import Config from morpheus.messages import ControlMessage +from morpheus.messages import MessageBase from morpheus.messages import MultiInferenceMessage from morpheus.messages import MultiMessage from morpheus.pipeline.multi_message_stage import MultiMessageStage @@ -42,10 +43,12 @@ class PreprocessBaseStage(MultiMessageStage): def __init__(self, c: Config): super().__init__(c) - self._preprocess_fn = None self._should_log_timestamps = True self._use_control_message = False + # only used when not using control message + self._fallback_output_type: type[MessageBase] = None + def accepted_types(self) -> typing.Tuple: """ Returns accepted input types for this stage. @@ -61,33 +64,32 @@ def compute_schema(self, schema: StageSchema): if (schema.input_type == ControlMessage): self._use_control_message = True out_type = ControlMessage - self._preprocess_fn = self._get_preprocess_fn() else: self._use_control_message = False - self._preprocess_fn = self._get_preprocess_fn() - preproc_sig = inspect.signature(self._preprocess_fn) - # If the innerfunction returns a type annotation, update the output type - if (preproc_sig.return_annotation - and typing_utils.issubtype(preproc_sig.return_annotation, MultiInferenceMessage)): - out_type = preproc_sig.return_annotation + out_type = self._fallback_output_type schema.output_schema.set_type(out_type) - @abstractmethod - def _get_preprocess_fn(self) -> typing.Callable[[MultiMessage], MultiInferenceMessage]: - pass + def _get_preprocess_fn( + self) -> typing.Callable[[ControlMessage | MultiMessage], ControlMessage | MultiInferenceMessage]: + """ + This method should be implemented by any subclasses with a Python implementation. + """ + raise NotImplementedError("No Python implementation provided by this stage") - @abstractmethod def _get_preprocess_node(self, builder: mrc.Builder) -> mrc.SegmentObject: - pass + """ + This method should be implemented by any subclasses with a C++ implementation. + """ + raise NotImplementedError("No Python implementation provided by this stage") def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: - assert self._preprocess_fn is not None, "Preprocess function not set" if self._build_cpp_node(): node = self._get_preprocess_node(builder) node.launch_options.pe_count = self._config.num_threads else: - node = builder.make_node(self.unique_name, ops.map(self._preprocess_fn)) + preprocess_fn = self._get_preprocess_fn() + node = builder.make_node(self.unique_name, ops.map(preprocess_fn)) builder.make_edge(input_node, node) From 9145c922dee5e0885040e6e21a3a12ea2b04a82c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 16:31:03 -0700 Subject: [PATCH 068/347] Remove unused python impls --- .../stages/preprocess/preprocess_ae_stage.py | 4 +- .../stages/preprocess/preprocess_fil_stage.py | 113 +----------- .../stages/preprocess/preprocess_nlp_stage.py | 170 +----------------- 3 files changed, 5 insertions(+), 282 deletions(-) diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py index 1cf7263d7c..3892d99bbc 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py @@ -51,6 +51,7 @@ def __init__(self, c: Config): self._fea_length = c.feature_length self._feature_columns = c.ae.feature_columns + self._fallback_output_type = MultiInferenceAEMessage @property def name(self) -> str: @@ -159,6 +160,3 @@ def _get_preprocess_fn( return partial(PreprocessAEStage.pre_process_batch, fea_len=self._fea_length, feature_columns=self._feature_columns) - - def _get_preprocess_node(self, builder: mrc.Builder): - raise NotImplementedError("No C++ node for AE") diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py index edd6d5c87f..bfe9535c55 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py @@ -16,7 +16,6 @@ import typing from functools import partial -import cupy as cp import mrc import numpy as np import pandas as pd @@ -29,6 +28,7 @@ from morpheus.messages import MultiInferenceFILMessage from morpheus.messages import MultiInferenceMessage from morpheus.messages import MultiMessage +from morpheus.messages import TensorMemory from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage logger = logging.getLogger(__name__) @@ -49,18 +49,14 @@ class PreprocessFILStage(PreprocessBaseStage): def __init__(self, c: Config): super().__init__(c) - import morpheus._lib.messages as _messages - self._lib_messages = _messages - - import cudf - self._cudf = cudf - self._fea_length = c.feature_length self.features = c.fil.feature_columns assert self._fea_length == len(self.features), \ f"Number of features in preprocessing {len(self.features)}, does not match configuration {self._fea_length}" + self._fallback_output_type = MultiInferenceFILMessage + @property def name(self) -> str: return "preprocess-fil" @@ -68,109 +64,6 @@ def name(self) -> str: def supports_cpp_node(self): return True - def pre_process_batch(self, x: typing.Union[MultiMessage, ControlMessage], fea_len: int, - fea_cols: typing.List[str]) -> typing.Union[MultiMessage, ControlMessage]: - """ - For FIL category usecases, this function performs pre-processing. - - Parameters - ---------- - x : `morpheus.pipeline.messages.MultiMessage` - Input rows received from Deserialized stage. - fea_len : int - Number features are being used in the inference. - fea_cols : typing.Tuple[str] - List of columns that are used as features. - - Returns - ------- - `morpheus.pipeline.messages.MultiInferenceFILMessage` - FIL inference message. - - """ - if isinstance(x, ControlMessage): - return self.process_control_message(x, fea_len, fea_cols) - if isinstance(x, MultiMessage): - return self.process_multi_message(x, fea_len, fea_cols) - raise TypeError(f"Unsupported message type: {type(x)}") - - def process_control_message(self, x: ControlMessage, fea_len: int, fea_cols: typing.List[str]) -> ControlMessage: - - try: - df: self._cudf.DataFrame = x.payload().get_data(fea_cols) - except KeyError: - logger.exception("Requested feature columns does not exist in the dataframe.", exc_info=True) - raise - - # Extract just the numbers from each feature col. Not great to operate on x.meta.df here but the operations will - # only happen once. - for col in fea_cols: - if (df[col].dtype == np.dtype(str) or df[col].dtype == np.dtype(object)): - # If the column is a string, parse the number - df[col] = df[col].str.extract(r"(\d+)", expand=False).astype("float32") - elif (df[col].dtype != np.float32): - # Convert to float32 - df[col] = df[col].astype("float32") - - if (isinstance(df, pd.DataFrame)): - df = self._cudf.from_pandas(df) - - # Convert the dataframe to cupy the same way cuml does - data = cp.asarray(df.to_cupy()) - - count = data.shape[0] - - seg_ids = cp.zeros((count, 3), dtype=cp.uint32) - seg_ids[:, 0] = cp.arange(0, count, dtype=cp.uint32) - seg_ids[:, 2] = fea_len - 1 - - # We need the C++ impl of TensorMemory until #1646 is resolved - x.tensors(self._lib_messages.TensorMemory(count=count, tensors={"input__0": data, "seq_ids": seg_ids})) - return x - - def process_multi_message(self, x: MultiMessage, fea_len: int, - fea_cols: typing.List[str]) -> MultiInferenceFILMessage: - try: - df = x.get_meta(fea_cols) - except KeyError: - logger.exception("Requested feature columns does not exist in the dataframe.", exc_info=True) - raise - - # Extract just the numbers from each feature col. Not great to operate on x.meta.df here but the operations will - # only happen once. - for col in fea_cols: - if (df[col].dtype == np.dtype(str) or df[col].dtype == np.dtype(object)): - # If the column is a string, parse the number - df[col] = df[col].str.extract(r"(\d+)", expand=False).astype("float32") - elif (df[col].dtype != np.float32): - # Convert to float32 - df[col] = df[col].astype("float32") - - if (isinstance(df, pd.DataFrame)): - df = self._cudf.from_pandas(df) - - # Convert the dataframe to cupy the same way cuml does - data = cp.asarray(df.to_cupy()) - - count = data.shape[0] - - seg_ids = cp.zeros((count, 3), dtype=cp.uint32) - seg_ids[:, 0] = cp.arange(x.mess_offset, x.mess_offset + count, dtype=cp.uint32) - seg_ids[:, 2] = fea_len - 1 - - # Create the inference memory. Keep in mind count here could be > than input count - memory = InferenceMemoryFIL(count=count, input__0=data, seq_ids=seg_ids) - - infer_message = MultiInferenceFILMessage.from_message(x, memory=memory) - - return infer_message - - def _get_preprocess_fn( - self - ) -> typing.Callable[[typing.Union[MultiMessage, ControlMessage]], - typing.Union[MultiInferenceMessage, ControlMessage]]: - return partial(self.pre_process_batch, fea_len=self._fea_length, fea_cols=self.features) - def _get_preprocess_node(self, builder: mrc.Builder): import morpheus._lib.stages as _stages if (self._use_control_message): diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py index 42dd4b64e4..b0bd5417db 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py @@ -12,62 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 -import json import logging -import typing -from functools import partial -import cupy as cp import mrc -import numpy as np from morpheus.cli.register_stage import register_stage from morpheus.cli.utils import MorpheusRelativePath from morpheus.cli.utils import get_package_relative_file from morpheus.config import Config from morpheus.config import PipelineModes -from morpheus.messages import ControlMessage -from morpheus.messages import InferenceMemoryNLP -from morpheus.messages import MultiInferenceMessage from morpheus.messages import MultiInferenceNLPMessage -from morpheus.messages import MultiMessage from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage -from morpheus.utils.cudf_subword_helper import tokenize_text_series logger = logging.getLogger(__name__) -def cupyarray_to_base64(cupy_array): - array_bytes = cupy_array.get().tobytes() - array_shape = cupy_array.shape - array_dtype = str(cupy_array.dtype) - - # Create a dictionary to store bytes, shape, and dtype - encoded_dict = {'bytes': base64.b64encode(array_bytes).decode("utf-8"), 'shape': array_shape, 'dtype': array_dtype} - - # Convert dictionary to JSON string for storage - return json.dumps(encoded_dict) - - -def base64_to_cupyarray(base64_str): - # Convert JSON string back to dictionary - encoded_dict = json.loads(base64_str) - - # Extract bytes, shape, and dtype - array_bytes = base64.b64decode(encoded_dict['bytes']) - array_shape = tuple(encoded_dict['shape']) - array_dtype = encoded_dict['dtype'] - - # Convert bytes back to a NumPy array and reshape - np_array = np.frombuffer(array_bytes, dtype=array_dtype).reshape(array_shape) - - # Convert NumPy array to CuPy array - cp_array = cp.array(np_array) - - return cp_array - - @register_stage( "preprocess", modes=[PipelineModes.NLP], @@ -113,12 +72,6 @@ def __init__(self, column: str = "data"): super().__init__(c) - import morpheus._lib.messages as _messages - self._lib_messages = _messages - - import cudf - self._cudf = cudf - self._column = column self._seq_length = c.feature_length self._vocab_hash_file = get_package_relative_file(vocab_hash_file) @@ -134,6 +87,7 @@ def __init__(self, self._truncation = truncation self._do_lower_case = do_lower_case self._add_special_tokens = add_special_tokens + self._fallback_output_type = MultiInferenceNLPMessage @property def name(self) -> str: @@ -142,128 +96,6 @@ def name(self) -> str: def supports_cpp_node(self): return True - def pre_process_batch(self, - message: typing.Union[MultiMessage, ControlMessage], - vocab_hash_file: str, - do_lower_case: bool, - seq_len: int, - stride: int, - truncation: bool, - add_special_tokens: bool, - column: str) -> typing.Union[MultiInferenceNLPMessage, ControlMessage]: - """ - For NLP category use cases, this function performs pre-processing. - - [parameters are the same as the original function] - - Returns - ------- - `morpheus.pipeline.messages.MultiInferenceNLPMessage` - NLP inference message. - - """ - if isinstance(message, ControlMessage): - return self.process_control_message(message, - vocab_hash_file, - do_lower_case, - seq_len, - stride, - truncation, - add_special_tokens, - column) - if isinstance(message, MultiMessage): - return self.process_multi_message(message, - vocab_hash_file, - do_lower_case, - seq_len, - stride, - truncation, - add_special_tokens, - column) - - raise TypeError("Unsupported message type") - - def process_control_message(self, - message: ControlMessage, - vocab_hash_file: str, - do_lower_case: bool, - seq_len: int, - stride: int, - truncation: bool, - add_special_tokens: bool, - column: str) -> ControlMessage: - - with message.payload().mutable_dataframe() as mdf: - text_series = self._cudf.Series(mdf[column]) - - tokenized = tokenize_text_series(vocab_hash_file=vocab_hash_file, - do_lower_case=do_lower_case, - text_ser=text_series, - seq_len=seq_len, - stride=stride, - truncation=truncation, - add_special_tokens=add_special_tokens) - - del text_series - - # We need the C++ impl of TensorMemory until #1646 is resolved - message.tensors( - self._lib_messages.TensorMemory(count=tokenized.input_ids.shape[0], - tensors={ - "input_ids": tokenized.input_ids, - "input_mask": tokenized.input_mask, - "seq_ids": tokenized.segment_ids - })) - - message.set_metadata("inference_memory_params", {"inference_type": "nlp"}) - return message - - def process_multi_message(self, - message: MultiMessage, - vocab_hash_file: str, - do_lower_case: bool, - seq_len: int, - stride: int, - truncation: bool, - add_special_tokens: bool, - column: str) -> MultiInferenceNLPMessage: - # Existing logic for MultiMessage - text_ser = self._cudf.Series(message.get_meta(column)) - - tokenized = tokenize_text_series(vocab_hash_file=vocab_hash_file, - do_lower_case=do_lower_case, - text_ser=text_ser, - seq_len=seq_len, - stride=stride, - truncation=truncation, - add_special_tokens=add_special_tokens) - del text_ser - - seg_ids = tokenized.segment_ids - seg_ids[:, 0] = seg_ids[:, 0] + message.mess_offset - - memory = InferenceMemoryNLP(count=tokenized.input_ids.shape[0], - input_ids=tokenized.input_ids, - input_mask=tokenized.input_mask, - seq_ids=seg_ids) - - infer_message = MultiInferenceNLPMessage.from_message(message, memory=memory) - - return infer_message - - def _get_preprocess_fn( - self - ) -> typing.Callable[[typing.Union[MultiMessage, ControlMessage]], - typing.Union[MultiInferenceMessage, ControlMessage]]: - return partial(self.pre_process_batch, - vocab_hash_file=self._vocab_hash_file, - do_lower_case=self._do_lower_case, - stride=self._stride, - seq_len=self._seq_length, - truncation=self._truncation, - add_special_tokens=self._add_special_tokens, - column=self._column) - def _get_preprocess_node(self, builder: mrc.Builder): import morpheus._lib.stages as _stages if (self._use_control_message): From ded3472ddf67e7ef714ed5242468bd17d4dc5e73 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 16:37:08 -0700 Subject: [PATCH 069/347] WIP --- .../morpheus/stages/preprocess/drop_null_stage.py | 12 ++++++------ .../stages/preprocess/group_by_column_stage.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py b/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py index 697cce089a..a0aac5deb5 100644 --- a/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py @@ -19,14 +19,14 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config -from morpheus.config import PipelineModes from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage -@register_stage("dropna", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER]) -class DropNullStage(PassThruTypeMixin, SinglePortStage): +@register_stage("dropna") +class DropNullStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Drop null data entries from a DataFrame. @@ -69,9 +69,9 @@ def supports_cpp_node(self): def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: - def on_next(x: MessageMeta): - - y = MessageMeta(x.df[~x.df[self._column].isna()]) + def on_next(msg: MessageMeta): + df = msg.copy_dataframe() + y = MessageMeta(df[~df[self._column].isna()]) return y diff --git a/python/morpheus/morpheus/stages/preprocess/group_by_column_stage.py b/python/morpheus/morpheus/stages/preprocess/group_by_column_stage.py index d69504dd27..e31f151068 100644 --- a/python/morpheus/morpheus/stages/preprocess/group_by_column_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/group_by_column_stage.py @@ -17,11 +17,12 @@ from morpheus.config import Config from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage -class GroupByColumnStage(PassThruTypeMixin, SinglePortStage): +class GroupByColumnStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Group the incoming message by a column in the DataFrame. From c94ae2e50dbddea4bc0801e32a59ad65e423cdc0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 16:57:27 -0700 Subject: [PATCH 070/347] Limit cudf imports --- python/morpheus/morpheus/utils/concat_df.py | 4 +--- python/morpheus/morpheus/utils/module_utils.py | 17 ++++++++++------- .../morpheus/utils/schema_transforms.py | 17 ++++++++++------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/python/morpheus/morpheus/utils/concat_df.py b/python/morpheus/morpheus/utils/concat_df.py index 8b25f3f08c..b3ec2fbc7b 100644 --- a/python/morpheus/morpheus/utils/concat_df.py +++ b/python/morpheus/morpheus/utils/concat_df.py @@ -16,8 +16,6 @@ import pandas as pd -import cudf - from morpheus.messages import ControlMessage from morpheus.messages import MessageBase from morpheus.messages import MultiMessage @@ -46,7 +44,7 @@ def concat_dataframes(messages: typing.List[MessageBase]) -> pd.DataFrame: else: df = x.df - if isinstance(df, cudf.DataFrame): + if not isinstance(df, pd.DataFrame): df = df.to_pandas() all_meta.append(df) diff --git a/python/morpheus/morpheus/utils/module_utils.py b/python/morpheus/morpheus/utils/module_utils.py index f1aca63334..f756a65cf4 100644 --- a/python/morpheus/morpheus/utils/module_utils.py +++ b/python/morpheus/morpheus/utils/module_utils.py @@ -24,8 +24,6 @@ import pandas as pd from pydantic import BaseModel -import cudf - from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) @@ -190,9 +188,9 @@ def merge_dictionaries(primary_dict, secondary_dict): } -def to_period_approximation(data_df: DataFrameType, period: str): +def to_period_approximation(data_df: DataFrameType, period: str) -> DataFrameType: """ - This function converts a cudf dataframe to a period approximation. + This function converts a dataframe to a period approximation. Parameters ---------- @@ -203,7 +201,7 @@ def to_period_approximation(data_df: DataFrameType, period: str): Returns ------- - cudf.DataFrame + DataFrame Period approximation of the input cudf/pandas dataframe. """ @@ -216,8 +214,13 @@ def to_period_approximation(data_df: DataFrameType, period: str): strptime_format = period_to_strptime[period] - df_mod = cudf if isinstance(data_df, cudf.DataFrame) else pd - data_df["period"] = df_mod.to_datetime(data_df["ts"].dt.strftime(strptime_format) + '-1', + if isinstance(data_df, pd.DataFrame): + df_pkg = pd + else: + import cudf + df_pkg = cudf + + data_df["period"] = df_pkg.to_datetime(data_df["ts"].dt.strftime(strptime_format) + '-1', format=f"{strptime_format}-%w") return data_df diff --git a/python/morpheus/morpheus/utils/schema_transforms.py b/python/morpheus/morpheus/utils/schema_transforms.py index 1cf8b65183..787fd2c441 100644 --- a/python/morpheus/morpheus/utils/schema_transforms.py +++ b/python/morpheus/morpheus/utils/schema_transforms.py @@ -17,9 +17,11 @@ import pandas as pd -import cudf - from morpheus.utils.column_info import DataFrameInputSchema +from morpheus.utils.type_aliases import DataFrameType + +if typing.TYPE_CHECKING: + import cudf logger = logging.getLogger(__name__) @@ -34,16 +36,16 @@ def process_dataframe( @typing.overload def process_dataframe( - df_in: cudf.DataFrame, + df_in: "cudf.DataFrame", input_schema: DataFrameInputSchema, -) -> cudf.DataFrame: +) -> "cudf.DataFrame": ... def process_dataframe( - df_in: typing.Union[pd.DataFrame, cudf.DataFrame], + df_in: DataFrameType, input_schema: DataFrameInputSchema, -) -> typing.Union[pd.DataFrame, cudf.DataFrame]: +) -> DataFrameType: """ Applies column transformations to the input dataframe as defined by the `input_schema`. @@ -73,7 +75,7 @@ def process_dataframe( output_df = pd.DataFrame() convert_to_cudf = False - if (isinstance(df_in, cudf.DataFrame)): + if (not isinstance(df_in, pd.DataFrame)): df_in = df_in.to_pandas() convert_to_cudf = True @@ -95,6 +97,7 @@ def process_dataframe( output_df[match_columns] = df_in[match_columns] if (convert_to_cudf): + import cudf return cudf.from_pandas(output_df) return output_df From 4bc97d7a42d7c994f63e14ad341ec8f67e988c6b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 Aug 2024 17:00:52 -0700 Subject: [PATCH 071/347] avoid cudf imports --- python/morpheus/morpheus/pipeline/stage_decorator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/morpheus/morpheus/pipeline/stage_decorator.py b/python/morpheus/morpheus/pipeline/stage_decorator.py index 4324dd7c92..df4e16ffa4 100644 --- a/python/morpheus/morpheus/pipeline/stage_decorator.py +++ b/python/morpheus/morpheus/pipeline/stage_decorator.py @@ -22,8 +22,6 @@ import pandas as pd from mrc.core import operators as ops -import cudf - import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import from morpheus.common import TypeId from morpheus.config import Config @@ -220,8 +218,13 @@ def compute_schema_fn_inner(schema: _pipeline.StageSchema): bound_gen_fn = functools.partial(gen_fn, **kwargs) + pre_allocation_output_types = [pd.DataFrame, MessageMeta, MultiMessage] + if config.execution_mode == ExecutionMode.GPU: + import cudf + pre_allocation_output_types.append(cudf.DataFrame) + # If the return type supports pre-allocation we use the pre-allocating source - if return_type in (pd.DataFrame, cudf.DataFrame, MessageMeta, MultiMessage): + if return_type in pre_allocation_output_types: return PreAllocatedWrappedFunctionStage(config=config, name=name, From 8d5600c84ae693625a3a194de399ff8e5dc02d42 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 10:39:44 -0700 Subject: [PATCH 072/347] Remove top-level cudf import --- .../morpheus/pipeline/preallocator_mixin.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/python/morpheus/morpheus/pipeline/preallocator_mixin.py b/python/morpheus/morpheus/pipeline/preallocator_mixin.py index acec20b9c7..3627bbf5cf 100644 --- a/python/morpheus/morpheus/pipeline/preallocator_mixin.py +++ b/python/morpheus/morpheus/pipeline/preallocator_mixin.py @@ -17,18 +17,16 @@ from abc import ABC from collections import OrderedDict -import cupy as cp import mrc import numpy as np import pandas as pd from mrc.core import operators as ops -import cudf - from morpheus.common import TypeId from morpheus.common import typeid_is_fully_supported from morpheus.common import typeid_to_numpy_str from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage @@ -60,7 +58,9 @@ def set_needed_columns(self, needed_columns: OrderedDict): def _preallocate_df(self, df: DataFrameType) -> DataFrameType: missing_columns = [col for col in self._needed_columns.keys() if col not in df.columns] if len(missing_columns) > 0: - if isinstance(df, cudf.DataFrame): + if not isinstance(df, pd.DataFrame): + # assume cudf.DataFrame + import cupy as cp alloc_func = cp.zeros else: alloc_func = np.zeros @@ -123,12 +123,18 @@ def _post_build_single(self, builder: mrc.Builder, out_node: mrc.SegmentObject) node = builder.make_node(node_name, ops.map(self._preallocate_meta)) else: node = builder.make_node(node_name, ops.map(self._preallocate_multi)) - elif issubclass(out_type, (cudf.DataFrame, pd.DataFrame)): - node = builder.make_node(node_name, ops.map(self._preallocate_df)) else: - msg = ("Additional columns were requested to be inserted into the Dataframe, but the output type " - f"{pretty_type} isn't a supported type") - raise RuntimeError(msg) + supported_df_types = [pd.DataFrame] + if self._config.execution_mode == ExecutionMode.GPU: + import cudf + supported_df_types.append(cudf.DataFrame) + + if issubclass(out_type, tuple(supported_df_types)): + node = builder.make_node(node_name, ops.map(self._preallocate_df)) + else: + msg = ("Additional columns were requested to be inserted into the Dataframe, but the output type " + f"{pretty_type} isn't a supported type") + raise RuntimeError(msg) builder.make_edge(out_node, node) out_node = node From b3a0d85a866a9773ce3db4fb44218c48eb89a098 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 11:43:56 -0700 Subject: [PATCH 073/347] Remove top-level cudf import --- python/morpheus/morpheus/utils/column_info.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/python/morpheus/morpheus/utils/column_info.py b/python/morpheus/morpheus/utils/column_info.py index eaef229666..d751dd6bb0 100644 --- a/python/morpheus/morpheus/utils/column_info.py +++ b/python/morpheus/morpheus/utils/column_info.py @@ -22,7 +22,7 @@ import pandas as pd -import cudf +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(f"morpheus.{__name__}") @@ -30,7 +30,7 @@ # Note(Devin): Proxying this for backwards compatibility. Had to move the primary definition to avoid circular imports. -def process_dataframe(df_in: typing.Union[pd.DataFrame, cudf.DataFrame], input_schema) -> pd.DataFrame: +def process_dataframe(df_in: DataFrameType, input_schema) -> pd.DataFrame: """ Processes a dataframe according to the given schema. @@ -83,7 +83,7 @@ def create_increment_col(df: pd.DataFrame, """ # Ensure we are pandas for this - if (isinstance(df, cudf.DataFrame)): + if (not isinstance(df, pd.DataFrame)): df = df.to_pandas() time_col = df[timestamp_column].fillna(pd.to_datetime(DEFAULT_DATE)) @@ -595,16 +595,16 @@ class PreparedDFInfo: Attributes ---------- - df : typing.Union[pd.DataFrame, cudf.DataFrame] + df : DataFrameType The prepared DataFrame. - columns_to_preserve : typing.List[str] + columns_to_preserve : list[str] A list of column names that are to be preserved. """ - df: typing.Union[pd.DataFrame, cudf.DataFrame] - columns_to_preserve: typing.List[str] + df: DataFrameType + columns_to_preserve: list[str] -def _json_flatten(df_input: typing.Union[pd.DataFrame, cudf.DataFrame], +def _json_flatten(df_input: DataFrameType, input_columns: dict[str, str], json_cols: list[str], preserve_re: re.Pattern = None): @@ -614,7 +614,7 @@ def _json_flatten(df_input: typing.Union[pd.DataFrame, cudf.DataFrame], Parameters ---------- - df_input : typing.Union[pd.DataFrame, cudf.DataFrame] + df_input : DataFrameType DataFrame to process. input_columns : dict[str, str] The final input columns that are needed for processing. All other columns will be removed @@ -625,7 +625,7 @@ def _json_flatten(df_input: typing.Union[pd.DataFrame, cudf.DataFrame], Returns ------- - typing.Union[pd.DataFrame, cudf.DataFrame] + DataFrameType The processed DataFrame. """ @@ -642,7 +642,7 @@ def _json_flatten(df_input: typing.Union[pd.DataFrame, cudf.DataFrame], if (not df_input.columns.intersection(json_cols).empty): convert_to_cudf = False - if (isinstance(df_input, cudf.DataFrame)): + if (not isinstance(df_input, pd.DataFrame)): convert_to_cudf = True df_input = df_input.to_pandas() @@ -673,6 +673,7 @@ def _json_flatten(df_input: typing.Union[pd.DataFrame, cudf.DataFrame], df_input = pd.concat([df_input[columns_to_keep]] + json_normalized, axis=1) if (convert_to_cudf): + import cudf df_input = cudf.from_pandas(df_input).reset_index(drop=True) # Remove all columns that are not in the input columns list. Ensure the correct types From 118846cce623a0772a2a7b7c44552a53f3c3526d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 12:28:39 -0700 Subject: [PATCH 074/347] Add load_cudf_helper to morpheus.common --- python/morpheus/morpheus/common/__init__.py | 2 ++ python/morpheus/morpheus/pipeline/pipeline.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/common/__init__.py b/python/morpheus/morpheus/common/__init__.py index 5935348143..97f441c1fb 100644 --- a/python/morpheus/morpheus/common/__init__.py +++ b/python/morpheus/morpheus/common/__init__.py @@ -23,6 +23,7 @@ from morpheus._lib.common import Tensor from morpheus._lib.common import TypeId from morpheus._lib.common import determine_file_type +from morpheus._lib.common import load_cudf_helper from morpheus._lib.common import read_file_to_df from morpheus._lib.common import typeid_is_fully_supported from morpheus._lib.common import typeid_to_numpy_str @@ -35,6 +36,7 @@ "FilterSource", "HttpEndpoint", "HttpServer", + "load_cudf_helper", "read_file_to_df", "Tensor", "typeid_is_fully_supported", diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index f249b1b349..2bfe9b9bd8 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -29,6 +29,7 @@ from tqdm import tqdm import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import +from morpheus.common import load_cudf_helper from morpheus.config import Config from morpheus.config import CppConfig from morpheus.config import ExecutionMode @@ -307,8 +308,7 @@ def build(self): if (self._execution_mode == ExecutionMode.GPU): # Load the cudf helper - from morpheus._lib import common as _common - _common.load_cudf_helper() + load_cudf_helper() self._pre_build() From 54d4dce527a4d9ae7c7b2e840dff172f4eceade8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 13:18:00 -0700 Subject: [PATCH 075/347] Add a mutex to prevent concurrent loads, although the GIL prevents concurrent byte-code execution, there is still a chance of context switching inside of the import_morpheus___lib__cudf_helpers --- .../morpheus/_lib/src/utilities/cudf_util.cpp | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp index fbc86ad0d2..554edacca9 100644 --- a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp +++ b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp @@ -25,8 +25,10 @@ #include #include +#include // for atomic #include // for getenv #include +#include #include // Needed for logging #include // for move /* @@ -36,19 +38,37 @@ */ #include "cudf_helpers_api.h" +namespace { +/* + * We want to prevent calling import_morpheus___lib__cudf_helpers() concurrently which is the purpose of the mutex. + * Although it is safe to call import_morpheus___lib__cudf_helpers() multiple times, we would still like to avoid it + * along with the call to std::getenv(), the atomic bool provides a cheap way to check if the library has been loaded. + */ +std::atomic g_cudf_helpers_loaded = false; +std::mutex g_cudf_helpers_load_mutex; +} // namespace + namespace morpheus { void CudfHelper::load() { - // Avoid loading cudf_helpers if we are in a sphinx build - if (std::getenv("MORPHEUS_IN_SPHINX_BUILD") == nullptr) + if (!g_cudf_helpers_loaded) { - if (import_morpheus___lib__cudf_helpers() != 0) + // Avoid loading cudf_helpers if we are in a sphinx build + if (std::getenv("MORPHEUS_IN_SPHINX_BUILD") == nullptr) { - pybind11::error_already_set ex; - - LOG(ERROR) << "Could not load cudf_helpers library: " << ex.what(); - throw ex; + std::lock_guard guard(g_cudf_helpers_load_mutex); + if (import_morpheus___lib__cudf_helpers() != 0) + { + pybind11::error_already_set ex; + + LOG(ERROR) << "Could not load cudf_helpers library: " << ex.what(); + throw ex; + } + else + { + g_cudf_helpers_loaded = true; + } } } } From 6f802a253f972851d24f59e14c1a0915cb554f42 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 13:18:21 -0700 Subject: [PATCH 076/347] Invoke CudfHelper::load --- python/morpheus/morpheus/_lib/src/io/deserializers.cpp | 1 + python/morpheus/morpheus/_lib/src/io/serializers.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/python/morpheus/morpheus/_lib/src/io/deserializers.cpp b/python/morpheus/morpheus/_lib/src/io/deserializers.cpp index 032cffd57b..8bd4ef5282 100644 --- a/python/morpheus/morpheus/_lib/src/io/deserializers.cpp +++ b/python/morpheus/morpheus/_lib/src/io/deserializers.cpp @@ -98,6 +98,7 @@ cudf::io::table_with_metadata load_table_from_file(const std::string& filename, pybind11::object read_file_to_df(const std::string& filename, FileTypes file_type) { + CudfHelper::load(); auto table = load_table_from_file(filename, file_type); int index_col_count = prepare_df_index(table); diff --git a/python/morpheus/morpheus/_lib/src/io/serializers.cpp b/python/morpheus/morpheus/_lib/src/io/serializers.cpp index 54234f1592..c0f5814305 100644 --- a/python/morpheus/morpheus/_lib/src/io/serializers.cpp +++ b/python/morpheus/morpheus/_lib/src/io/serializers.cpp @@ -252,6 +252,7 @@ void SerializersProxy::write_df_to_file(pybind11::object df, FileTypes file_type, const py::kwargs& kwargs) { + CudfHelper::load(); if (file_type == FileTypes::Auto) { file_type = determine_file_type(filename); // throws if it is unable to determine the type From 72577b5542ba46607c239d629ce07c6ac7b1238e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 13:59:27 -0700 Subject: [PATCH 077/347] Move get_df_class and get_df_pkg to type_utils.py, add new is_cudf_type loosely based on a method of the same name from dask-sql --- python/morpheus/morpheus/io/utils.py | 19 ----- .../stages/output/http_server_sink_stage.py | 2 +- .../postprocess/generate_viz_frames_stage.py | 2 +- python/morpheus/morpheus/utils/type_utils.py | 33 ++++++++ tests/utils/test_type_utils.py | 80 +++++++++++++++++++ 5 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 tests/utils/test_type_utils.py diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index 6cfdd9ae58..c62cb8f1b9 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -139,25 +139,6 @@ def truncate_string_cols_by_bytes(df: DataFrameType, return performed_truncation -def get_df_pkg(config: Config) -> types.ModuleType: - """ - Return the appropriate DataFrame package based on the execution mode. - """ - if config.execution_mode == ExecutionMode.GPU: - import cudf - return cudf - - return pd - - -def get_df_class(config: Config) -> type[DataFrameType]: - """ - Return the appropriate DataFrame class based on the execution mode. - """ - df_pkg = get_df_pkg(config) - return df_pkg.DataFrame - - def get_json_reader(config: Config) -> typing.Callable[..., DataFrameType]: """ Return the appropriate JSON reader based on the execution mode. diff --git a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py index ca5ac3c1d5..ab94ff0706 100644 --- a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py @@ -28,7 +28,6 @@ from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.io import serializers -from morpheus.io.utils import get_df_pkg from morpheus.messages import MessageMeta from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin @@ -37,6 +36,7 @@ from morpheus.utils.http_utils import HttpParseResponse from morpheus.utils.http_utils import MimeTypes from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) diff --git a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py index d50279e2e6..d0df7203db 100644 --- a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py @@ -30,7 +30,6 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes -from morpheus.io.utils import get_df_class from morpheus.messages import ControlMessage from morpheus.messages import MultiResponseMessage from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin @@ -38,6 +37,7 @@ from morpheus.utils.producer_consumer_queue import AsyncIOProducerConsumerQueue from morpheus.utils.producer_consumer_queue import Closed from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import get_df_class logger = logging.getLogger(__name__) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index a3aefdde8d..2e3bbccdcc 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -17,6 +17,13 @@ import typing from collections import defaultdict +import pandas as pd + +from morpheus.config import Config +from morpheus.config import ExecutionMode +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import SeriesType + # pylint: disable=invalid-name T_co = typing.TypeVar("T_co", covariant=True) @@ -162,3 +169,29 @@ def get_full_qualname(klass: type) -> str: if module == '__builtin__': return klass.__qualname__ return module + '.' + klass.__qualname__ + + +def get_df_pkg(config: Config) -> types.ModuleType: + """ + Return the appropriate DataFrame package based on the execution mode. + """ + if config.execution_mode == ExecutionMode.GPU: + import cudf + return cudf + + return pd + + +def get_df_class(config: Config) -> type[DataFrameType]: + """ + Return the appropriate DataFrame class based on the execution mode. + """ + df_pkg = get_df_pkg(config) + return df_pkg.DataFrame + + +def is_cudf_type(obj: typing.Any) -> bool: + """ + Check if a given object (DataFrame, Series, RangeIndex etc...) is a cuDF type. + """ + return "cudf" in str(type(obj)) diff --git a/tests/utils/test_type_utils.py b/tests/utils/test_type_utils.py new file mode 100644 index 0000000000..361672a21d --- /dev/null +++ b/tests/utils/test_type_utils.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing +from unittest.mock import MagicMock + +import pandas as pd +import pytest + +import cudf + +from morpheus.config import Config +from morpheus.config import ExecutionMode +from morpheus.utils.type_utils import get_df_class +from morpheus.utils.type_utils import get_df_pkg +from morpheus.utils.type_utils import is_cudf_type + + +def test_get_df_class(config: Config): + if config.execution_mode == ExecutionMode.GPU: + expected = cudf.DataFrame + else: + expected = pd.DataFrame + + assert get_df_class(config) is expected + + +def test_get_df_pkg(config: Config): + if config.execution_mode == ExecutionMode.GPU: + expected = cudf + else: + expected = pd + + assert get_df_pkg(config) is expected + + +@pytest.mark.parametrize( + "obj, expected", + [ + (cudf.DataFrame(), True), + (cudf.Series(), True), + (cudf.Index([]), True), + (cudf.RangeIndex(0), True), + (pd.DataFrame(), False), + (pd.Series(), False), + (pd.Index([]), False), + (pd.RangeIndex(0), False), + (None, False), + (0, False), + ("test", False), + ], + ids=[ + "cudf.DataFrame", + "cudf.Series", + "cudf.Index", + "cudf.RangeIndex", + "pd.DataFrame", + "pd.Series", + "pd.Index", + "pd.RangeIndex", + "None", + "int", + "str" + ], +) +def test_is_cudf_type(obj: typing.Any, expected: bool): + assert is_cudf_type(obj) == expected From 069bc89cf5cfda67371d1915daa90ae46d8c7d1a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 14:03:14 -0700 Subject: [PATCH 078/347] WIP --- python/morpheus/morpheus/io/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index c62cb8f1b9..b5d68b4abe 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -16,7 +16,6 @@ import functools import logging -import types import typing import pandas as pd @@ -25,6 +24,7 @@ from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import SeriesType +from morpheus.utils.type_utils import is_cudf_type if typing.TYPE_CHECKING: import cudf @@ -108,7 +108,7 @@ def truncate_string_cols_by_bytes(df: DataFrameType, """ performed_truncation = False - is_cudf = not isinstance(df, pd.DataFrame) + is_cudf = is_cudf_type(df) for (col, max_bytes) in column_max_bytes.items(): series: SeriesType = df[col] From ffcbdf871e27f55a4c2882a9e8ca053e81c625a2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 14:32:37 -0700 Subject: [PATCH 079/347] Add type for df literal --- python/morpheus/morpheus/utils/type_aliases.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/morpheus/morpheus/utils/type_aliases.py b/python/morpheus/morpheus/utils/type_aliases.py index e94ccd5abb..9c2c5e4a70 100644 --- a/python/morpheus/morpheus/utils/type_aliases.py +++ b/python/morpheus/morpheus/utils/type_aliases.py @@ -22,6 +22,7 @@ import cudf +DataFrameTypeStr = typing.Literal["cudf", "pandas"] DataFrameType = typing.Union["pandas.DataFrame", "cudf.DataFrame"] SeriesType = typing.Union["pandas.Series", "cudf.Series"] From 090860a8dc3849de94d7c9823e57a9f9e72f561d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 14:33:27 -0700 Subject: [PATCH 080/347] Add new df_type_str_to_pkg helper method --- python/morpheus/morpheus/utils/type_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 2e3bbccdcc..1f906da835 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -22,7 +22,7 @@ from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import SeriesType +from morpheus.utils.type_aliases import DataFrameTypeStr # pylint: disable=invalid-name T_co = typing.TypeVar("T_co", covariant=True) @@ -171,6 +171,17 @@ def get_full_qualname(klass: type) -> str: return module + '.' + klass.__qualname__ +def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: + """ + Return the appropriate DataFrame package based on the DataFrame type string. + """ + if df_type_str == "cudf": + import cudf + return cudf + + return pd + + def get_df_pkg(config: Config) -> types.ModuleType: """ Return the appropriate DataFrame package based on the execution mode. From 434d3afbc216dc60620112889a68ad6612cda2b2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 15:09:44 -0700 Subject: [PATCH 081/347] Type utils cleanups --- .../morpheus/controllers/rss_controller.py | 9 ++-- python/morpheus/morpheus/io/deserializers.py | 39 +++++++------- python/morpheus/morpheus/io/utils.py | 15 ++++-- .../stages/input/http_client_source_stage.py | 2 +- .../stages/input/http_server_source_stage.py | 2 +- .../stages/input/kafka_source_stage.py | 2 +- .../stages/output/http_server_sink_stage.py | 2 +- .../postprocess/generate_viz_frames_stage.py | 2 +- python/morpheus/morpheus/utils/type_utils.py | 40 +++++++++++--- tests/io/test_io_utils.py | 16 ++++++ tests/utils/test_type_utils.py | 52 +++++++++++++------ 11 files changed, 129 insertions(+), 52 deletions(-) diff --git a/python/morpheus/morpheus/controllers/rss_controller.py b/python/morpheus/morpheus/controllers/rss_controller.py index 13ed6b438c..60b9c39220 100644 --- a/python/morpheus/morpheus/controllers/rss_controller.py +++ b/python/morpheus/morpheus/controllers/rss_controller.py @@ -23,12 +23,13 @@ from datetime import timedelta from urllib.parse import urlparse +import pandas as pd import requests import requests_cache -import cudf - from morpheus.messages import MessageMeta +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import DataFrameTypeStr logger = logging.getLogger(__name__) @@ -104,7 +105,8 @@ def __init__(self, strip_markup: bool = False, stop_after: int = 0, interval_secs: float = 600, - should_stop_fn: Callable[[], bool] = None): + should_stop_fn: Callable[[], bool] = None, + df_type: DataFrameTypeStr = "cudf"): if IMPORT_EXCEPTION is not None: raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION @@ -140,6 +142,7 @@ def __init__(self, self._run_indefinitely = run_indefinitely self._interval_secs = interval_secs self._interval_td = timedelta(seconds=self._interval_secs) + self._df_type = df_type self._enable_cache = enable_cache diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index 488fd9f172..7b0709c2ce 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -25,14 +25,17 @@ from morpheus.common import read_file_to_df as read_file_to_df_cpp from morpheus.config import CppConfig from morpheus.io.utils import filter_null_data +from morpheus.io.utils import get_json_reader from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_utils import df_type_str_to_pkg def _read_file_to_df_py(*, file_name: typing.Union[str, io.IOBase], file_type: FileTypes, parser_kwargs: dict, - df_type: typing.Literal["cudf", "pandas"]) -> DataFrameType: + df_type: DataFrameTypeStr) -> DataFrameType: if (parser_kwargs is None): parser_kwargs = {} @@ -59,29 +62,27 @@ def _read_file_to_df_py(*, # Update with any args set by the user. User values overwrite defaults kwargs.update(parser_kwargs) - if df_type == "cudf": - import cudf - df_class = cudf - else: - df_class = pd - df = None if (mode == FileTypes.JSON): - df = df_class.read_json(file_name, **kwargs) + reader = get_json_reader(df_type) + df = reader(file_name, **kwargs) + + else: + df_class = df_type_str_to_pkg(df_type) - elif (mode == FileTypes.CSV): - df: DataFrameType = df_class.read_csv(file_name, **kwargs) + if (mode == FileTypes.CSV): + df: DataFrameType = df_class.read_csv(file_name, **kwargs) - if (len(df.columns) > 1 and df.columns[0] == "Unnamed: 0" and df.iloc[:, 0].dtype == np.dtype(int)): - df.set_index("Unnamed: 0", drop=True, inplace=True) - df.index.name = "" - df.sort_index(inplace=True) + if (len(df.columns) > 1 and df.columns[0] == "Unnamed: 0" and df.iloc[:, 0].dtype == np.dtype(int)): + df.set_index("Unnamed: 0", drop=True, inplace=True) + df.index.name = "" + df.sort_index(inplace=True) - elif (mode == FileTypes.PARQUET): - df = df_class.read_parquet(file_name, **kwargs) + elif (mode == FileTypes.PARQUET): + df = df_class.read_parquet(file_name, **kwargs) - else: - assert False, f"Unsupported file type mode: {mode}" + else: + assert False, f"Unsupported file type mode: {mode}" assert df is not None @@ -93,7 +94,7 @@ def read_file_to_df(file_name: typing.Union[str, io.IOBase], parser_kwargs: dict = None, filter_nulls: bool = True, filter_null_columns: list[str] | str = 'data', - df_type: typing.Literal["cudf", "pandas"] = "pandas") -> DataFrameType: + df_type: DataFrameTypeStr = "pandas") -> DataFrameType: """ Reads a file into a dataframe and performs any of the necessary cleanup. diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index b5d68b4abe..c3cb5d61fb 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -20,10 +20,11 @@ import pandas as pd -from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import DataFrameTypeStr from morpheus.utils.type_aliases import SeriesType +from morpheus.utils.type_utils import df_type_str_to_exec_mode from morpheus.utils.type_utils import is_cudf_type if typing.TYPE_CHECKING: @@ -139,11 +140,19 @@ def truncate_string_cols_by_bytes(df: DataFrameType, return performed_truncation -def get_json_reader(config: Config) -> typing.Callable[..., DataFrameType]: +@typing.overload +def get_json_reader(df_type_str: DataFrameTypeStr) -> typing.Callable[..., DataFrameType]: + ... + + +def get_json_reader(execution_mode: ExecutionMode) -> typing.Callable[..., DataFrameType]: """ Return the appropriate JSON reader based on the execution mode. """ - if config.execution_mode == ExecutionMode.GPU: + if not isinstance(execution_mode, ExecutionMode): + execution_mode = df_type_str_to_exec_mode(execution_mode) + + if (execution_mode == ExecutionMode.GPU): import cudf reader = functools.partial(cudf.read_json, engine='cudf') else: diff --git a/python/morpheus/morpheus/stages/input/http_client_source_stage.py b/python/morpheus/morpheus/stages/input/http_client_source_stage.py index 727b304ddb..8309ba5921 100644 --- a/python/morpheus/morpheus/stages/input/http_client_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_client_source_stage.py @@ -149,7 +149,7 @@ def __init__(self, if payload_to_df_fn is not None: self._payload_to_df_fn = payload_to_df_fn else: - reader = get_json_reader(self._config) + reader = get_json_reader(self._config.execution_mode) self._payload_to_df_fn = lambda payload, lines: reader(payload, lines=lines) @property diff --git a/python/morpheus/morpheus/stages/input/http_server_source_stage.py b/python/morpheus/morpheus/stages/input/http_server_source_stage.py index 3db97cac29..2d2afe53e4 100644 --- a/python/morpheus/morpheus/stages/input/http_server_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_server_source_stage.py @@ -269,7 +269,7 @@ def _generate_frames(self) -> typing.Iterator[MessageMeta]: self._processing = False def _set_default_payload_to_df_fn(self): - reader = get_json_reader(self._config) + reader = get_json_reader(self._config.execution_mode) self._payload_to_df_fn = lambda payload, lines: reader(payload, lines=lines) def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: diff --git a/python/morpheus/morpheus/stages/input/kafka_source_stage.py b/python/morpheus/morpheus/stages/input/kafka_source_stage.py index bd28ad6117..1d85cd8480 100644 --- a/python/morpheus/morpheus/stages/input/kafka_source_stage.py +++ b/python/morpheus/morpheus/stages/input/kafka_source_stage.py @@ -244,7 +244,7 @@ def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: # multiple threads source.launch_options.pe_count = self._max_concurrent else: - self._json_reader = get_json_reader(self._config) + self._json_reader = get_json_reader(self._config.execution_mode) source = builder.make_source(self.unique_name, self._source_generator) return source diff --git a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py index ab94ff0706..b02de333f8 100644 --- a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py @@ -118,7 +118,7 @@ def __init__(self, self._df_serializer_fn = df_serializer_fn or self._default_df_serializer - self._df_pkg = get_df_pkg(config) + self._df_pkg = get_df_pkg(config.execution_mode) # FiberQueue doesn't have a way to check the size, nor does it have a way to check if it's empty without # attempting to perform a read. We'll keep track of the size ourselves. diff --git a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py index d0df7203db..6bbf942c5d 100644 --- a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py @@ -82,7 +82,7 @@ def __init__(self, self._server_task: asyncio.Task = None self._server_close_event: asyncio.Event = None - self._df_class: type[DataFrameType] = get_df_class(c) + self._df_class: type[DataFrameType] = get_df_class(c.execution_mode) @property def name(self) -> str: diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 1f906da835..1d28444815 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -19,7 +19,6 @@ import pandas as pd -from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameTypeStr @@ -171,6 +170,19 @@ def get_full_qualname(klass: type) -> str: return module + '.' + klass.__qualname__ +def df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr) -> ExecutionMode: + """ + Return the appropriate execution mode based on the DataFrame type string. + """ + if df_type_str == "cudf": + return ExecutionMode.GPU + elif df_type_str == "pandas": + return ExecutionMode.CPU + + valid_values = ", ".join(typing.get_args(DataFrameTypeStr)) + raise ValueError(f"Invalid DataFrame type string: {df_type_str}, valid values are: {valid_values}") + + def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: """ Return the appropriate DataFrame package based on the DataFrame type string. @@ -178,26 +190,42 @@ def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: if df_type_str == "cudf": import cudf return cudf + elif df_type_str == "pandas": + return pd + + valid_values = ", ".join(typing.get_args(DataFrameTypeStr)) + raise ValueError(f"Invalid DataFrame type string: {df_type_str}, valid values are: {valid_values}") - return pd +@typing.overload +def get_df_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: + ... -def get_df_pkg(config: Config) -> types.ModuleType: + +def get_df_pkg(execution_mode: ExecutionMode) -> types.ModuleType: """ Return the appropriate DataFrame package based on the execution mode. """ - if config.execution_mode == ExecutionMode.GPU: + if not isinstance(execution_mode, ExecutionMode): + execution_mode = df_type_str_to_exec_mode(execution_mode) + + if execution_mode == ExecutionMode.GPU: import cudf return cudf return pd -def get_df_class(config: Config) -> type[DataFrameType]: +@typing.overload +def get_df_class(df_type_str: DataFrameTypeStr) -> type[DataFrameType]: + ... + + +def get_df_class(execution_mode: ExecutionMode) -> type[DataFrameType]: """ Return the appropriate DataFrame class based on the execution mode. """ - df_pkg = get_df_pkg(config) + df_pkg = get_df_pkg(execution_mode) return df_pkg.DataFrame diff --git a/tests/io/test_io_utils.py b/tests/io/test_io_utils.py index 1ad46b75cb..84655c2055 100755 --- a/tests/io/test_io_utils.py +++ b/tests/io/test_io_utils.py @@ -14,15 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing from collections.abc import Callable +import pandas as pd import pytest import cudf from _utils.dataset_manager import DatasetManager +from morpheus.config import ExecutionMode from morpheus.io import utils as io_utils from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import DataFrameTypeStr MULTI_BYTE_STRINGS = ["ñäμɛ", "Moρφέας", "taç"] @@ -132,3 +136,15 @@ def test_truncate_string_cols_by_bytes(dataset: DatasetManager, assert isinstance(df, expected_df_class) dataset.assert_df_equal(df, expected_df) + + +@pytest.mark.parametrize("mode, expected", + [(ExecutionMode.GPU, cudf.read_json), (ExecutionMode.CPU, pd.read_json), + ("cudf", cudf.read_json), ("pandas", pd.read_json)]) +def test_get_json_reader(mode: typing.Union[ExecutionMode, DataFrameTypeStr], expected: Callable[..., DataFrameType]): + reader = io_utils.get_json_reader(mode) + if hasattr(reader, "func"): + # Unwrap partial + reader = reader.func + + assert reader is expected diff --git a/tests/utils/test_type_utils.py b/tests/utils/test_type_utils.py index 361672a21d..cdcae86747 100644 --- a/tests/utils/test_type_utils.py +++ b/tests/utils/test_type_utils.py @@ -14,37 +14,34 @@ # See the License for the specific language governing permissions and # limitations under the License. +import types import typing -from unittest.mock import MagicMock import pandas as pd import pytest import cudf -from morpheus.config import Config from morpheus.config import ExecutionMode +from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_utils import df_type_str_to_exec_mode +from morpheus.utils.type_utils import df_type_str_to_pkg from morpheus.utils.type_utils import get_df_class from morpheus.utils.type_utils import get_df_pkg from morpheus.utils.type_utils import is_cudf_type -def test_get_df_class(config: Config): - if config.execution_mode == ExecutionMode.GPU: - expected = cudf.DataFrame - else: - expected = pd.DataFrame +@pytest.mark.parametrize("mode, expected", + [(ExecutionMode.GPU, cudf.DataFrame), (ExecutionMode.CPU, pd.DataFrame), + ("cudf", cudf.DataFrame), ("pandas", pd.DataFrame)]) +def test_get_df_class(mode: typing.Union[ExecutionMode, DataFrameTypeStr], expected: types.ModuleType): + assert get_df_class(mode) is expected - assert get_df_class(config) is expected - -def test_get_df_pkg(config: Config): - if config.execution_mode == ExecutionMode.GPU: - expected = cudf - else: - expected = pd - - assert get_df_pkg(config) is expected +@pytest.mark.parametrize("mode, expected", [(ExecutionMode.GPU, cudf), (ExecutionMode.CPU, pd), ("cudf", cudf), + ("pandas", pd)]) +def test_get_df_pkg(mode: typing.Union[ExecutionMode, DataFrameTypeStr], expected: types.ModuleType): + assert get_df_pkg(mode) is expected @pytest.mark.parametrize( @@ -78,3 +75,26 @@ def test_get_df_pkg(config: Config): ) def test_is_cudf_type(obj: typing.Any, expected: bool): assert is_cudf_type(obj) == expected + + +@pytest.mark.parametrize("df_type_str, expected", [("cudf", cudf), ("pandas", pd)], ids=["cudf", "pandas"]) +def test_df_type_str_to_pkg(df_type_str: DataFrameTypeStr, expected: types.ModuleType): + assert df_type_str_to_pkg(df_type_str) is expected + + +@pytest.mark.parametrize("invalid_type_str", ["invalid", "cuDF", "Pandas"]) +def test_df_type_str_to_pkg_invalid(invalid_type_str: typing.Any): + with pytest.raises(ValueError, match="Invalid DataFrame type string"): + df_type_str_to_pkg(invalid_type_str) + + +@pytest.mark.parametrize("df_type_str, expected", [("cudf", ExecutionMode.GPU), ("pandas", ExecutionMode.CPU)], + ids=["cudf", "pandas"]) +def test_df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr, expected: ExecutionMode): + assert df_type_str_to_exec_mode(df_type_str) == expected + + +@pytest.mark.parametrize("invalid_type_str", ["invalid", "cuDF", "Pandas"]) +def test_df_type_str_to_exec_mode_invalid(invalid_type_str: typing.Any): + with pytest.raises(ValueError, match="Invalid DataFrame type string"): + df_type_str_to_exec_mode(invalid_type_str) From 8ac41a4e3969f1a2729cd2ff09b5f10608dfabea Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 15:41:05 -0700 Subject: [PATCH 082/347] Add exec_mode_to_df_type_str --- python/morpheus/morpheus/utils/type_utils.py | 7 +++++++ tests/utils/test_type_utils.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 1d28444815..4122db9664 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -183,6 +183,13 @@ def df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr) -> ExecutionMode: raise ValueError(f"Invalid DataFrame type string: {df_type_str}, valid values are: {valid_values}") +def exec_mode_to_df_type_str(execution_mode: ExecutionMode) -> DataFrameTypeStr: + if execution_mode == ExecutionMode.GPU: + return "cudf" + + return "pandas" + + def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: """ Return the appropriate DataFrame package based on the DataFrame type string. diff --git a/tests/utils/test_type_utils.py b/tests/utils/test_type_utils.py index cdcae86747..1a01e2df67 100644 --- a/tests/utils/test_type_utils.py +++ b/tests/utils/test_type_utils.py @@ -26,6 +26,7 @@ from morpheus.utils.type_aliases import DataFrameTypeStr from morpheus.utils.type_utils import df_type_str_to_exec_mode from morpheus.utils.type_utils import df_type_str_to_pkg +from morpheus.utils.type_utils import exec_mode_to_df_type_str from morpheus.utils.type_utils import get_df_class from morpheus.utils.type_utils import get_df_pkg from morpheus.utils.type_utils import is_cudf_type @@ -98,3 +99,9 @@ def test_df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr, expected: Execu def test_df_type_str_to_exec_mode_invalid(invalid_type_str: typing.Any): with pytest.raises(ValueError, match="Invalid DataFrame type string"): df_type_str_to_exec_mode(invalid_type_str) + + +@pytest.mark.parametrize("exec_mode, expected", [(ExecutionMode.GPU, "cudf"), (ExecutionMode.CPU, "pandas")], + ids=["GPU", "CPU"]) +def test_exec_mode_to_df_type_str(exec_mode: ExecutionMode, expected: DataFrameTypeStr): + assert exec_mode_to_df_type_str(exec_mode) == expected From 6e155a99b3633129f9e6298ee1c2a2629cbe5c90 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 16:13:18 -0700 Subject: [PATCH 083/347] WIP - avoid cudf imports --- .../morpheus/controllers/rss_controller.py | 10 ++++----- .../morpheus/parsers/windows_event_parser.py | 21 ++++++++++-------- python/morpheus/morpheus/parsers/zeek.py | 15 ++++++++----- .../morpheus/service/vdb/faiss_vdb_service.py | 22 ++++++------------- .../service/vdb/milvus_vector_db_service.py | 9 ++++---- .../morpheus/service/vdb/vector_db_service.py | 19 ++++++---------- .../morpheus/stages/input/rss_source_stage.py | 4 +++- python/morpheus/morpheus/utils/type_utils.py | 11 ++++++++++ tests/test_windows_event_parser.py | 1 + 9 files changed, 60 insertions(+), 52 deletions(-) diff --git a/python/morpheus/morpheus/controllers/rss_controller.py b/python/morpheus/morpheus/controllers/rss_controller.py index 60b9c39220..b856b9a86c 100644 --- a/python/morpheus/morpheus/controllers/rss_controller.py +++ b/python/morpheus/morpheus/controllers/rss_controller.py @@ -23,13 +23,13 @@ from datetime import timedelta from urllib.parse import urlparse -import pandas as pd import requests import requests_cache from morpheus.messages import MessageMeta from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_utils import get_df_class logger = logging.getLogger(__name__) @@ -142,7 +142,7 @@ def __init__(self, self._run_indefinitely = run_indefinitely self._interval_secs = interval_secs self._interval_td = timedelta(seconds=self._interval_secs) - self._df_type = df_type + self._df_class: type[DataFrameType] = get_df_class(df_type) self._enable_cache = enable_cache @@ -351,7 +351,7 @@ def fetch_dataframes(self): Yeilds ------ - cudf.DataFrame + DataFrameType A DataFrame containing feed entry data. Raises @@ -376,14 +376,14 @@ def fetch_dataframes(self): entry_accumulator.append(entry) if self._batch_size > 0 and len(entry_accumulator) >= self._batch_size: - yield cudf.DataFrame(entry_accumulator) + yield self._df_class(entry_accumulator) entry_accumulator.clear() self._previous_entries = current_entries # Yield any remaining entries. if entry_accumulator: - yield cudf.DataFrame(entry_accumulator) + yield self._df_class(entry_accumulator) else: logger.debug("No new entries found.") diff --git a/python/morpheus/morpheus/parsers/windows_event_parser.py b/python/morpheus/morpheus/parsers/windows_event_parser.py index 475c4a405d..8b62c2cf0e 100644 --- a/python/morpheus/morpheus/parsers/windows_event_parser.py +++ b/python/morpheus/morpheus/parsers/windows_event_parser.py @@ -16,10 +16,11 @@ import os import typing -import cudf - import morpheus from morpheus.parsers.event_parser import EventParser +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import SeriesType +from morpheus.utils.type_utils import get_df_pkg_from_obj log = logging.getLogger(__name__) @@ -41,17 +42,17 @@ def __init__(self, interested_eventcodes=None): self._event_regex = self._load_regex_yaml(regex_filepath) EventParser.__init__(self, self.get_columns(), self.EVENT_NAME) - def parse(self, text: cudf.Series) -> cudf.Series: + def parse(self, text: SeriesType) -> DataFrameType: """Parses the Windows raw event. Parameters ---------- - text : cudf.Series + text : SeriesType Raw event log text to be parsed Returns ------- - cudf.DataFrame + DataFrameType Parsed logs dataframe """ # Clean raw data to be consistent. @@ -65,23 +66,25 @@ def parse(self, text: cudf.Series) -> cudf.Series: temp = self.parse_raw_event(input_chunk, self._event_regex[eventcode]) if not temp.empty: output_chunks.append(temp) - parsed_dataframe = cudf.concat(output_chunks) + + df_pkg = get_df_pkg_from_obj(text) + parsed_dataframe = df_pkg.concat(output_chunks) # Replace null values with empty. parsed_dataframe = parsed_dataframe.fillna("") return parsed_dataframe - def clean_raw_data(self, text: cudf.Series) -> cudf.Series: + def clean_raw_data(self, text: SeriesType) -> SeriesType: """ Lower casing and replacing escape characters. Parameters ---------- - text : cudf.Series + text : SeriesType Raw event log text to be clean Returns ------- - cudf.Series + SeriesType Clean raw event log text """ text = (text.str.lower().str.replace("\\\\t", "").str.replace("\\\\r", "").str.replace("\\\\n", "|")) diff --git a/python/morpheus/morpheus/parsers/zeek.py b/python/morpheus/morpheus/parsers/zeek.py index 44ef464e1f..ac9a4a24b8 100644 --- a/python/morpheus/morpheus/parsers/zeek.py +++ b/python/morpheus/morpheus/parsers/zeek.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import cudf +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_utils import get_df_pkg TYPE_DICT = { "bool": "bool", @@ -36,7 +38,7 @@ } -def parse(filepath: str) -> cudf.DataFrame: +def parse(filepath: str, df_type: DataFrameTypeStr = "cudf") -> DataFrameType: """ Parse Zeek log file and return cuDF dataframe. Uses header comments to get column names/types and configure parser. @@ -45,20 +47,23 @@ def parse(filepath: str) -> cudf.DataFrame: ---------- filepath : str File path of Zeek log file + df_type : DataFrameTypeStr, default 'cudf' + Type of dataframe to return. Either 'cudf' or 'pandas' Returns ------- - cudf.DataFrame + DataFrameType Parsed Zeek log dataframe """ - header_gdf = cudf.read_csv(filepath, names=["line"], nrows=8) + df_pkg = get_df_pkg(df_type) + header_gdf = df_pkg.read_csv(filepath, names=["line"], nrows=8) lines_gdf = header_gdf["line"].str.split() column_names = lines_gdf.iloc[6][1:] column_types = lines_gdf.iloc[7][1:] column_dtypes = list(map(lambda x: TYPE_DICT.get(x, "str"), column_types)) - log_gdf = cudf.read_csv( + log_gdf = df_pkg.read_csv( filepath, delimiter="\t", dtype=column_dtypes, diff --git a/python/morpheus/morpheus/service/vdb/faiss_vdb_service.py b/python/morpheus/morpheus/service/vdb/faiss_vdb_service.py index 36cf754252..f641e20aa5 100644 --- a/python/morpheus/morpheus/service/vdb/faiss_vdb_service.py +++ b/python/morpheus/morpheus/service/vdb/faiss_vdb_service.py @@ -17,12 +17,9 @@ import time import typing -import pandas as pd - -import cudf - from morpheus.service.vdb.vector_db_service import VectorDBResourceService from morpheus.service.vdb.vector_db_service import VectorDBService +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) @@ -81,13 +78,13 @@ def insert(self, data: list[list] | list[dict], **kwargs) -> dict: """ raise NotImplementedError("Insert operation is not supported in FAISS") - def insert_dataframe(self, df: typing.Union[cudf.DataFrame, pd.DataFrame], **kwargs) -> dict: + def insert_dataframe(self, df: DataFrameType, **kwargs) -> dict: """ Insert a dataframe entires into the vector database. Parameters ---------- - df : typing.Union[cudf.DataFrame, pd.DataFrame] + df : DataFrameType Dataframe to be inserted into the collection. **kwargs Extra keyword arguments specific to the vector database implementation. @@ -368,11 +365,7 @@ def create(self, name: str, overwrite: bool = False, **kwargs): """ raise NotImplementedError("create operation is not supported in FAISS") - def create_from_dataframe(self, - name: str, - df: typing.Union[cudf.DataFrame, pd.DataFrame], - overwrite: bool = False, - **kwargs) -> None: + def create_from_dataframe(self, name: str, df: DataFrameType, overwrite: bool = False, **kwargs) -> None: """ Create collections in the vector database. @@ -380,7 +373,7 @@ def create_from_dataframe(self, ---------- name : str Name of the collection. - df : Union[cudf.DataFrame, pd.DataFrame] + df : DataFrameType The dataframe to create the collection from. overwrite : bool, optional Whether to overwrite the collection if it already exists. Default is False. @@ -416,8 +409,7 @@ def insert(self, name: str, data: list[list] | list[dict], **kwargs) -> dict[str raise NotImplementedError("create_from_dataframe operation is not supported in FAISS") - def insert_dataframe(self, name: str, df: typing.Union[cudf.DataFrame, pd.DataFrame], - **kwargs) -> dict[str, typing.Any]: + def insert_dataframe(self, name: str, df: DataFrameType, **kwargs) -> dict[str, typing.Any]: """ Converts dataframe to rows and insert to the vector database. @@ -425,7 +417,7 @@ def insert_dataframe(self, name: str, df: typing.Union[cudf.DataFrame, pd.DataFr ---------- name : str Name of the collection to be inserted. - df : typing.Union[cudf.DataFrame, pd.DataFrame] + df : DataFrameType Dataframe to be inserted in the collection. **kwargs Additional keyword arguments containing collection configuration. diff --git a/python/morpheus/morpheus/service/vdb/milvus_vector_db_service.py b/python/morpheus/morpheus/service/vdb/milvus_vector_db_service.py index 09c68f15cd..34ed795b28 100644 --- a/python/morpheus/morpheus/service/vdb/milvus_vector_db_service.py +++ b/python/morpheus/morpheus/service/vdb/milvus_vector_db_service.py @@ -20,13 +20,12 @@ import typing from functools import wraps -import cudf - from morpheus.io.utils import cudf_string_cols_exceed_max_bytes from morpheus.io.utils import truncate_string_cols_by_bytes from morpheus.service.vdb.vector_db_service import VectorDBResourceService from morpheus.service.vdb.vector_db_service import VectorDBService from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import is_cudf_type logger = logging.getLogger(__name__) @@ -327,7 +326,7 @@ def insert_dataframe(self, df: DataFrameType, **kwargs: dict[str, typing.Any]) - logger.info("Skipped checking 'None' in the field: %s, with datatype: %s", field_name, dtype) needs_truncate = self._truncate_long_strings - if needs_truncate and isinstance(df, cudf.DataFrame): + if needs_truncate and is_cudf_type(df): # Cudf specific optimization, we can avoid a costly call to truncate_string_cols_by_bytes if all of the # string columns are already below the max length needs_truncate = cudf_string_cols_exceed_max_bytes(df, self._fields_max_length) @@ -336,7 +335,7 @@ def insert_dataframe(self, df: DataFrameType, **kwargs: dict[str, typing.Any]) - column_names = [field.name for field in self._fields if not field.auto_id] collection_df = df[column_names] - if isinstance(collection_df, cudf.DataFrame): + if is_cudf_type(collection_df): collection_df = collection_df.to_pandas() if needs_truncate: @@ -728,7 +727,7 @@ def _build_schema_conf(self, df: DataFrameType) -> list[dict]: # Always add a primary key fields.append({"name": "pk", "dtype": pymilvus.DataType.INT64, "is_primary": True, "auto_id": True}) - if isinstance(df, cudf.DataFrame): + if is_cudf_type(df): df = df.to_pandas() # Loop over all of the columns of the first row and build the schema diff --git a/python/morpheus/morpheus/service/vdb/vector_db_service.py b/python/morpheus/morpheus/service/vdb/vector_db_service.py index 8f2d346f55..bbf0439028 100644 --- a/python/morpheus/morpheus/service/vdb/vector_db_service.py +++ b/python/morpheus/morpheus/service/vdb/vector_db_service.py @@ -17,9 +17,7 @@ from abc import ABC from abc import abstractmethod -import pandas as pd - -import cudf +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) @@ -50,13 +48,13 @@ def insert(self, data: list[list] | list[dict], **kwargs: dict[str, typing.Any]) pass @abstractmethod - def insert_dataframe(self, df: typing.Union[cudf.DataFrame, pd.DataFrame], **kwargs: dict[str, typing.Any]) -> dict: + def insert_dataframe(self, df: DataFrameType, **kwargs: dict[str, typing.Any]) -> dict: """ Insert a dataframe into the vector database. Parameters ---------- - df : typing.Union[cudf.DataFrame, pd.DataFrame] + df : DataFrameType Dataframe to be inserted into the resource. **kwargs : dict[str, typing.Any] Extra keyword arguments specific to the vector database implementation. @@ -241,10 +239,7 @@ def insert(self, name: str, data: list[list] | list[dict], **kwargs: dict[str, t pass @abstractmethod - def insert_dataframe(self, - name: str, - df: typing.Union[cudf.DataFrame, pd.DataFrame], - **kwargs: dict[str, typing.Any]) -> dict: + def insert_dataframe(self, name: str, df: DataFrameType, **kwargs: dict[str, typing.Any]) -> dict: """ Converts dataframe to rows and insert into the vector database resource. @@ -252,7 +247,7 @@ def insert_dataframe(self, ---------- name : str Name of the resource to be inserted. - df : typing.Union[cudf.DataFrame, pd.DataFrame] + df : DataFrameType Dataframe to be inserted. **kwargs : dict[str, typing.Any] Additional keyword arguments containing collection configuration. @@ -391,7 +386,7 @@ def create(self, name: str, overwrite: bool = False, **kwargs: dict[str, typing. @abstractmethod def create_from_dataframe(self, name: str, - df: typing.Union[cudf.DataFrame, pd.DataFrame], + df: DataFrameType, overwrite: bool = False, **kwargs: dict[str, typing.Any]) -> None: """ @@ -401,7 +396,7 @@ def create_from_dataframe(self, ---------- name : str Name of the resource. - df : Union[cudf.DataFrame, pd.DataFrame] + df : DataFrameType The dataframe to create the resource from. overwrite : bool, optional Whether to overwrite the resource if it already exists. Default is False. diff --git a/python/morpheus/morpheus/stages/input/rss_source_stage.py b/python/morpheus/morpheus/stages/input/rss_source_stage.py index c9d9d01ac3..f548f54716 100644 --- a/python/morpheus/morpheus/stages/input/rss_source_stage.py +++ b/python/morpheus/morpheus/stages/input/rss_source_stage.py @@ -23,6 +23,7 @@ from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_utils import exec_mode_to_df_type_str logger = logging.getLogger(__name__) @@ -82,7 +83,8 @@ def __init__(self, strip_markup=strip_markup, stop_after=stop_after, interval_secs=interval_secs, - should_stop_fn=self.is_stop_requested) + should_stop_fn=self.is_stop_requested, + df_type=exec_mode_to_df_type_str(c.execution_mode)) @property def name(self) -> str: diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 4122db9664..52c17cb5e8 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -241,3 +241,14 @@ def is_cudf_type(obj: typing.Any) -> bool: Check if a given object (DataFrame, Series, RangeIndex etc...) is a cuDF type. """ return "cudf" in str(type(obj)) + + +def get_df_pkg_from_obj(obj: typing.Any) -> types.ModuleType: + """ + Return the appropriate DataFrame package based on the DataFrame object. + """ + if is_cudf_type(obj): + import cudf + return cudf + + return pd diff --git a/tests/test_windows_event_parser.py b/tests/test_windows_event_parser.py index c90207612e..f287abfcd2 100644 --- a/tests/test_windows_event_parser.py +++ b/tests/test_windows_event_parser.py @@ -630,6 +630,7 @@ def test_windows_event_parser(): test_logs = fh.readlines() test_input = cudf.Series(test_logs) test_output_df = wep.parse(test_input) + for parsed_rec in test_output_df.to_records(): eventcode = parsed_rec["eventcode"] validate_func = VALIDATE_DICT.get(eventcode, unknown_record_type) From 6fbc2d9ab17d43c771fc71bb4bd476f3246c5ef4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 Aug 2024 16:57:24 -0700 Subject: [PATCH 084/347] Avoid cudf import, add type hints, cache the suffix df on a per-df-type basis --- .../morpheus/morpheus/parsers/url_parser.py | 70 ++++++++++++------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/python/morpheus/morpheus/parsers/url_parser.py b/python/morpheus/morpheus/parsers/url_parser.py index bf3077a601..88a8b56ddf 100644 --- a/python/morpheus/morpheus/parsers/url_parser.py +++ b/python/morpheus/morpheus/parsers/url_parser.py @@ -13,21 +13,15 @@ # limitations under the License. import os - -import cudf +import types import morpheus +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import SeriesType +from morpheus.utils.type_utils import get_df_pkg_from_obj +from morpheus.utils.type_utils import is_cudf_type - -def _load_suffix_file(): - suffix_list_path = os.path.join(morpheus.DATA_DIR, "public_suffix_list.dat") - # Read suffix list csv file - suffix_df = cudf.io.csv.read_csv(suffix_list_path, names=["suffix"], header=None, dtype=["str"]) - suffix_df = suffix_df[suffix_df["suffix"].str.contains("^[^//]+$")] - return suffix_df - - -_SUFFIX_DF = _load_suffix_file() +_SUFFIX_DF_CACHE = {} _ALLOWED_OUTPUT_COLS = { "hostname", "subdomain", @@ -36,7 +30,24 @@ def _load_suffix_file(): } -def _handle_unknown_suffix(unknown_suffix_df, col_dict): +def _get_suffix_df(df_pkg: types.ModuleType) -> DataFrameType: + suffix_df = _SUFFIX_DF_CACHE.get(df_pkg) + if suffix_df is None: + suffix_list_path = os.path.join(morpheus.DATA_DIR, "public_suffix_list.dat") + # Read suffix list csv file, ignore comments and empty lines. + suffix_df = df_pkg.read_csv(suffix_list_path, + names=["suffix"], + header=None, + dtype={'suffix': "str"}, + comment='/', + skip_blank_lines=True) + suffix_df = suffix_df[suffix_df["suffix"].str.contains("^[^//]+$")] + _SUFFIX_DF_CACHE[df_pkg] = suffix_df + + return suffix_df + + +def _handle_unknown_suffix(unknown_suffix_df: DataFrameType, col_dict: dict[str, bool]) -> DataFrameType: if col_dict["hostname"]: unknown_suffix_df = unknown_suffix_df[["idx", "tld0"]] unknown_suffix_df = unknown_suffix_df.rename(columns={"tld0": "hostname"}) @@ -53,7 +64,8 @@ def _handle_unknown_suffix(unknown_suffix_df, col_dict): return unknown_suffix_df -def _extract_tld(input_df, suffix_df, col_len, col_dict): +def _extract_tld(input_df: DataFrameType, suffix_df: DataFrameType, col_len: int, + col_dict: dict[str, bool]) -> DataFrameType: tmp_dfs = [] # Left join on single column dataframe does not provide expected results hence adding dummy column. suffix_df["dummy"] = "" @@ -109,12 +121,14 @@ def _extract_tld(input_df, suffix_df, col_len, col_dict): tmp_dfs.append(unknown_suffix_df) else: continue + # Concat all temporary output dataframes - output_df = cudf.concat(tmp_dfs) + df_pkg = get_df_pkg_from_obj(input_df) + output_df = df_pkg.concat(tmp_dfs) return output_df -def _create_col_dict(allowed_output_cols, req_cols): +def _create_col_dict(allowed_output_cols: set[str], req_cols: set[str]) -> dict[str, bool]: """Creates dictionary to apply check condition while extracting tld. """ col_dict = {col: True for col in allowed_output_cols} @@ -124,7 +138,7 @@ def _create_col_dict(allowed_output_cols, req_cols): return col_dict -def _verify_req_cols(req_cols, allowed_output_cols): +def _verify_req_cols(req_cols: set[str], allowed_output_cols: set[str]) -> set[str]: """Verify user requested columns against allowed output columns. """ if req_cols is not None: @@ -135,7 +149,7 @@ def _verify_req_cols(req_cols, allowed_output_cols): return req_cols -def _generate_tld_cols(hostname_split_df, hostnames, col_len): +def _generate_tld_cols(hostname_split_df: DataFrameType, hostnames: SeriesType, col_len: int) -> DataFrameType: hostname_split_df = hostname_split_df.fillna("") hostname_split_df["tld" + str(col_len)] = hostname_split_df[col_len] # Add all other elements of hostname_split_df @@ -147,25 +161,25 @@ def _generate_tld_cols(hostname_split_df, hostnames, col_len): return hostname_split_df -def _extract_hostnames(urls): +def _extract_hostnames(urls: SeriesType) -> SeriesType: hostnames = urls.str.extract("([\\w]+[\\.].*[^/]|[\\-\\w]+[\\.].*[^/])")[0].str.extract("([\\w\\.\\-]+)")[0] return hostnames -def parse(urls, req_cols=None): +def parse(urls: SeriesType, req_cols: set[str] = None) -> DataFrameType: """ Extract hostname, domain, subdomain and suffix from URLs. Parameters ---------- - urls : cudf.Series + urls : SeriesType URLs to be parsed. req_cols : typing.Set[str] Selected columns to extract. Can be subset of (hostname, domain, subdomain and suffix). Returns ------- - cudf.DataFrame + DataFrameType Parsed dataframe with selected columns to extract. Examples @@ -196,6 +210,7 @@ def parse(urls, req_cols=None): 2 github com 3 pydata org """ + df_pkg = get_df_pkg_from_obj(urls) req_cols = _verify_req_cols(req_cols, _ALLOWED_OUTPUT_COLS) col_dict = _create_col_dict(req_cols, _ALLOWED_OUTPUT_COLS) hostnames = _extract_hostnames(urls) @@ -203,14 +218,21 @@ def parse(urls, req_cols=None): del urls hostname_split_ser = hostnames.str.findall("([^.]+)") hostname_split_df = hostname_split_ser.to_frame() - hostname_split_df = cudf.DataFrame(hostname_split_df[0].to_arrow().to_pylist()) + + if is_cudf_type(hostname_split_df): + hostname_split_df = df_pkg.DataFrame(hostname_split_df[0].to_arrow().to_pylist()) + else: + hostname_split_df = df_pkg.DataFrame(hostname_split_df[0].to_list()) + col_len = len(hostname_split_df.columns) - 1 hostname_split_df = _generate_tld_cols(hostname_split_df, hostnames, col_len) # remove hostnames since they are available in hostname_split_df del hostnames # Assign input index to idx column. hostname_split_df["idx"] = url_index - output_df = _extract_tld(hostname_split_df, _SUFFIX_DF, col_len, col_dict) + + suffix_df = _get_suffix_df(df_pkg) + output_df = _extract_tld(hostname_split_df, suffix_df, col_len, col_dict) # Sort index based on given input index order. output_df = output_df.sort_values("idx", ascending=True) # Drop temp columns. From 58a0e3bda036533b9ae4f4868f71d273c1cf0c3c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 23 Aug 2024 11:57:40 -0700 Subject: [PATCH 085/347] Avoid top-level cudf import, support pandas, consolidate netmask and hostmask --- python/morpheus/morpheus/parsers/ip.py | 227 ++++++++++++++----------- 1 file changed, 124 insertions(+), 103 deletions(-) diff --git a/python/morpheus/morpheus/parsers/ip.py b/python/morpheus/morpheus/parsers/ip.py index 814d46f9dd..8c80f09bec 100644 --- a/python/morpheus/morpheus/parsers/ip.py +++ b/python/morpheus/morpheus/parsers/ip.py @@ -12,24 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ipaddress + import numpy as np +import pandas as pd -import cudf +from morpheus.utils.type_aliases import SeriesType +from morpheus.utils.type_utils import get_df_pkg_from_obj +from morpheus.utils.type_utils import is_cudf_type -def ip_to_int(values): +def ip_to_int(values: SeriesType) -> SeriesType: """ Convert string column of IP addresses to integer values. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - values : cudf.Series + values : SeriesType IPv4 addresses to be converted Returns ------- - rtype : cudf.Series + rtype : SeriesType Integer representations of IP addresses Examples @@ -41,22 +46,26 @@ def ip_to_int(values): 1 167772161 dtype: int64 """ - return cudf.Series(values.str.ip2int()) + if (is_cudf_type(values)): + return values.str.ip2int() + + # Pandas does not have an ip2int method + return values.apply(lambda x: int(ipaddress.IPv4Address(x))) -def int_to_ip(values): +def int_to_ip(values: SeriesType) -> SeriesType: """ Convert integer column to IP addresses. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - values : cudf.Series + values : SeriesType Integer representations of IP addresses Returns ------- - rtype : cudf.Series + rtype : SeriesType IPv4 addresses Examples @@ -68,22 +77,27 @@ def int_to_ip(values): 1 10.0.0.1 dtype: object """ - return cudf.Series(values._column.int2ip()) + if (is_cudf_type(values)): + import cudf + return cudf.Series(values._column.int2ip()) + + # Pandas does not have an int2ip method + return values.apply(lambda x: str(ipaddress.IPv4Address(x))) -def is_ip(ips: str): +def is_ip(ips: SeriesType) -> SeriesType: """ Indicates whether each address is an ip string. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -95,23 +109,26 @@ def is_ip(ips: str): 1 False dtype: bool """ + if (is_cudf_type(ips)): + return ips.str.isipv4() + is_ip_regex = r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$" - return ips.str.match(is_ip_regex) + return ips.str.fullmatch(is_ip_regex) -def is_reserved(ips): +def is_reserved(ips: SeriesType) -> SeriesType: """ Indicates whether each address is reserved. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -129,19 +146,19 @@ def is_reserved(ips): return ips.str.match(reserved_ipv4_regex) -def is_loopback(ips): +def is_loopback(ips: SeriesType) -> SeriesType: """ Indicates whether each address is loopback. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -159,19 +176,19 @@ def is_loopback(ips): return ips.str.match(loopback_ipv4_regex) -def is_link_local(ips): +def is_link_local(ips: SeriesType) -> SeriesType: """ Indicates whether each address is link local. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -189,19 +206,19 @@ def is_link_local(ips): return ips.str.match(link_local_ipv4_regex) -def is_unspecified(ips): +def is_unspecified(ips: SeriesType) -> SeriesType: """ Indicates whether each address is unspecified. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -217,19 +234,19 @@ def is_unspecified(ips): return ips.str.match(unspecified_regex) -def is_multicast(ips): +def is_multicast(ips: SeriesType) -> SeriesType: """ Indicates whether each address is multicast. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -247,19 +264,19 @@ def is_multicast(ips): return ips.str.match(is_multicast_ipv4_regex) -def is_private(ips): +def is_private(ips: SeriesType) -> SeriesType: """ Indicates whether each address is private. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -290,19 +307,19 @@ def is_private(ips): return ips.str.match(private_regex) -def is_global(ips): +def is_global(ips: SeriesType) -> SeriesType: """ Indicates whether each address is global. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked Returns ------- - rtype : cudf.Series + rtype : SeriesType Boolean values true or false Examples @@ -323,7 +340,7 @@ def is_global(ips): return result -def _netmask_kernel(idx, out1, out2, out3, out4, kwarg1): +def _mask_kernel(idx, out1, out2, out3, out4, kwarg1): for i, _ in enumerate(idx): out1[i] = int(kwarg1 / 16777216) % 256 out2[i] = int(kwarg1 / 65536) % 256 @@ -331,21 +348,52 @@ def _netmask_kernel(idx, out1, out2, out3, out4, kwarg1): out4[i] = int(kwarg1) % 256 -def netmask(ips, prefixlen=16): +def _mask_pandas(df_cols: tuple[int], mask: int, series_name: str) -> pd.Series: + outputs = [int(mask / 16777216) % 256, int(mask / 65536) % 256, int(mask / 256) % 256, int(mask) % 256] + return pd.Series([df_cols.idx, ".".join(map(str, outputs))], index=["idx", series_name]) + + +def _compute_mask_impl(ips: SeriesType, mask: int, series_name: str) -> SeriesType: + df_pkg = get_df_pkg_from_obj(ips) + if is_cudf_type(ips): + df = df_pkg.DataFrame() + df["idx"] = ips.index + x = df.apply_rows( + _mask_kernel, + incols=["idx"], + outcols={ + "out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64 + }, + kwargs={"kwarg1": mask}, + ) + + out1 = x["out1"].astype(str) + out2 = x["out2"].astype(str) + out3 = x["out3"].astype(str) + out4 = x["out4"].astype(str) + df[series_name] = out1.str.cat(out2, sep=".").str.cat(out3, sep=".").str.cat(out4, sep=".") + else: + df = df_pkg.DataFrame({"idx": ips.index}) + df = df.apply(_mask_pandas, axis=1, args=(mask, series_name)) + + return df[series_name] + + +def netmask(ips: SeriesType, prefixlen: int = 16) -> SeriesType: """ Compute a column of netmasks for a column of IP addresses. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked prefixlen: int Length of the network prefix, in bits, for IPv4 addresses Returns ------- - rtype : cudf.Series + rtype : SeriesType Netmask ouput from set of IP address @@ -360,48 +408,24 @@ def netmask(ips, prefixlen=16): """ all_ones = (2**32) - 1 mask_int = all_ones ^ (all_ones >> prefixlen) - df = cudf.DataFrame() - df["idx"] = ips.index - x = df.apply_rows( - _netmask_kernel, - incols=["idx"], - outcols={ - "out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64 - }, - kwargs={"kwarg1": mask_int}, - ) - - out1 = x["out1"].astype(str) - out2 = x["out2"].astype(str) - out3 = x["out3"].astype(str) - out4 = x["out4"].astype(str) - df["net_mask"] = out1.str.cat(out2, sep=".").str.cat(out3, sep=".").str.cat(out4, sep=".") - return df["net_mask"] - - -def _hostmask_kernel(idx, out1, out2, out3, out4, kwarg1): - for i, _ in enumerate(idx): - out1[i] = int(kwarg1 / 16777216) % 256 - out2[i] = int(kwarg1 / 65536) % 256 - out3[i] = int(kwarg1 / 256) % 256 - out4[i] = int(kwarg1) % 256 + return _compute_mask_impl(ips, mask_int, "net_mask") -def hostmask(ips, prefixlen=16): +def hostmask(ips: SeriesType, prefixlen: int = 16) -> SeriesType: """ Compute a column of hostmasks for a column of IP addresses. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked prefixlen: integer Length of the network prefix, in bits, for IPv4 addresses Returns ------- - rtype : cudf.Series + rtype : SeriesType Hostmask ouput from set of IP address Examples @@ -415,24 +439,10 @@ def hostmask(ips, prefixlen=16): """ all_ones = (2**32) - 1 host_mask_int = int(all_ones ^ (all_ones >> prefixlen)) ^ all_ones - df = cudf.DataFrame() - df["idx"] = ips.index - x = df.apply_rows(_hostmask_kernel, - incols=["idx"], - outcols={ - "out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64 - }, - kwargs={"kwarg1": host_mask_int}) - - out1 = x["out1"].astype(str) - out2 = x["out2"].astype(str) - out3 = x["out3"].astype(str) - out4 = x["out4"].astype(str) - df["hostmask"] = out1.str.cat(out2, sep=".").str.cat(out3, sep=".").str.cat(out4, sep=".") - return df["hostmask"] - - -def _mask_kernel(masked_ip_int, out1, out2, out3, out4, kwarg1): # pylint: disable=unused-argument + return _compute_mask_impl(ips, host_mask_int, "hostmask") + + +def _mask_series_kernel(masked_ip_int, out1, out2, out3, out4, kwarg1): # pylint: disable=unused-argument for i, ipnum in enumerate(masked_ip_int): out1[i] = int(ipnum / 16777216) % 256 out2[i] = int(ipnum / 65536) % 256 @@ -440,21 +450,25 @@ def _mask_kernel(masked_ip_int, out1, out2, out3, out4, kwarg1): # pylint: disa out4[i] = int(ipnum) % 256 -def mask(ips, masks): +def _mask_series_pandas(df_cols: tuple[int], mask_series_name: str, output_series_name: str) -> pd.Series: + return _mask_pandas(df_cols, df_cols[mask_series_name], output_series_name) + + +def mask(ips: SeriesType, masks: SeriesType) -> SeriesType: """ Apply a mask to a column of IP addresses. **Addresses must be IPv4. IPv6 not yet supported.** Parameters ---------- - ips : cudf.Series + ips : SeriesType IPv4 addresses to be checked - masks: cudf.Series + masks: SeriesType The host or subnet masks to be applied Returns ------- - rtype : cudf.Series + rtype : SeriesType Masked IP address from list of IPs Examples @@ -468,21 +482,28 @@ def mask(ips, masks): 1 10.0.0.0 Name: mask, dtype: object """ - df = cudf.DataFrame() - df["int_mask"] = masks.str.ip2int() - df["int_ip"] = ips.str.ip2int() + df_pkg = get_df_pkg_from_obj(ips) + + df = df_pkg.DataFrame() + df["int_mask"] = ip_to_int(masks) + df["int_ip"] = ip_to_int(ips) df["masked_ip_int"] = df["int_mask"] & df["int_ip"] - x = df.apply_rows(_mask_kernel, - incols=["masked_ip_int"], - outcols={ - "out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64 - }, - kwargs={"kwarg1": 0}) - - out1 = x["out1"].astype(str) - out2 = x["out2"].astype(str) - out3 = x["out3"].astype(str) - out4 = x["out4"].astype(str) - df["mask"] = out1.str.cat(out2, sep=".").str.cat(out3, sep=".").str.cat(out4, sep=".") + if (is_cudf_type(df)): + x = df.apply_rows(_mask_series_kernel, + incols=["masked_ip_int"], + outcols={ + "out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64 + }, + kwargs={"kwarg1": 0}) + + out1 = x["out1"].astype(str) + out2 = x["out2"].astype(str) + out3 = x["out3"].astype(str) + out4 = x["out4"].astype(str) + df["mask"] = out1.str.cat(out2, sep=".").str.cat(out3, sep=".").str.cat(out4, sep=".") + else: + df["idx"] = ips.index + df = df.apply(_mask_series_pandas, axis=1, args=("masked_ip_int", "mask")) + return df["mask"] From cfbf27233929b5ee136c65217bb600f191b904ce Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 23 Aug 2024 12:26:38 -0700 Subject: [PATCH 086/347] Avoid cudf imports --- .../modules/output/write_to_vector_db.py | 33 ++++++++++--------- .../morpheus/morpheus/parsers/event_parser.py | 26 ++++++++------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/python/morpheus/morpheus/modules/output/write_to_vector_db.py b/python/morpheus/morpheus/modules/output/write_to_vector_db.py index 9e1fd4d11a..9715c1fdcd 100644 --- a/python/morpheus/morpheus/modules/output/write_to_vector_db.py +++ b/python/morpheus/morpheus/modules/output/write_to_vector_db.py @@ -22,8 +22,6 @@ from mrc.core import operators as ops from pydantic import ValidationError -import cudf - from morpheus.messages import ControlMessage from morpheus.messages import MultiMessage from morpheus.messages import MultiResponseMessage @@ -35,6 +33,8 @@ from morpheus.utils.module_ids import WRITE_TO_VECTOR_DB from morpheus.utils.module_utils import ModuleLoaderFactory from morpheus.utils.module_utils import register_module +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import get_df_pkg_from_obj logger = logging.getLogger(__name__) @@ -73,7 +73,7 @@ def preprocess_vdb_resources(service, recreate: bool, resource_schemas: dict): class AccumulationStats: msg_count: int last_insert_time: float - data: list[cudf.DataFrame] + data: list[DataFrameType] @register_module(WRITE_TO_VECTOR_DB, MORPHEUS_MODULE_NAMESPACE) @@ -143,15 +143,18 @@ def _write_to_vector_db(builder: mrc.Builder): def on_completed(): final_df_references = [] - # Pushing remaining messages - for key, accum_stats in accumulator_dict.items(): - try: - if accum_stats.data: - merged_df = cudf.concat(accum_stats.data) - service.insert_dataframe(name=key, df=merged_df) - final_df_references.append(accum_stats.data) - except Exception as e: - logger.error("Unable to upload dataframe entries to vector database: %s", e) + if len(accumulator_dict): + df_pkg = get_df_pkg_from_obj(next(iter(accumulator_dict.values()))) + + # Pushing remaining messages + for key, accum_stats in accumulator_dict.items(): + try: + if accum_stats.data: + merged_df = df_pkg.concat(accum_stats.data) + service.insert_dataframe(name=key, df=merged_df) + final_df_references.append(accum_stats.data) + except Exception as e: + logger.error("Unable to upload dataframe entries to vector database: %s", e) # Close vector database service connection service.close() @@ -183,9 +186,6 @@ def on_data(msg: typing.Union[ControlMessage, MultiResponseMessage, MultiMessage df, msg_resource_target = extract_df(msg) if df is not None and not df.empty: - if (not isinstance(df, cudf.DataFrame)): - df = cudf.DataFrame(df) - df_size = len(df) current_time = time.time() @@ -210,7 +210,8 @@ def on_data(msg: typing.Union[ControlMessage, MultiResponseMessage, MultiMessage (current_time - accum_stats.last_insert_time) >= write_time_interval): if accum_stats.data: - merged_df = cudf.concat(accum_stats.data) + df_pkg = get_df_pkg_from_obj(accum_stats.data) + merged_df = df_pkg.concat(accum_stats.data) # pylint: disable=not-a-mapping service.insert_dataframe(name=key, df=merged_df, **resource_kwargs) diff --git a/python/morpheus/morpheus/parsers/event_parser.py b/python/morpheus/morpheus/parsers/event_parser.py index 998232c130..179ba25218 100644 --- a/python/morpheus/morpheus/parsers/event_parser.py +++ b/python/morpheus/morpheus/parsers/event_parser.py @@ -14,13 +14,14 @@ """Abstract class for all event log parsers.""" import logging -import typing from abc import ABC from abc import abstractmethod import yaml -import cudf +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import SeriesType +from morpheus.utils.type_utils import get_df_pkg_from_obj log = logging.getLogger(__name__) @@ -31,13 +32,13 @@ class EventParser(ABC): Parameters ---------- - columns: typing.Set[str] + columns: set[str] Event column names event_name: str Event name """ - def __init__(self, columns: typing.Set[str], event_name: str): + def __init__(self, columns: set[str], event_name: str): self._columns = columns self._event_name = event_name @@ -48,7 +49,7 @@ def columns(self): Returns ------- - typing.Set[str] + set[str] Event column names """ return self._columns @@ -66,7 +67,7 @@ def event_name(self): return self._event_name @abstractmethod - def parse(self, text: cudf.Series) -> cudf.Series: + def parse(self, text: SeriesType) -> SeriesType: """ Abstract method 'parse' triggers the parsing functionality. Subclasses are required to implement and execute any parsing pre-processing steps. @@ -74,25 +75,26 @@ def parse(self, text: cudf.Series) -> cudf.Series: log.info("Begin parsing of dataframe") pass - def parse_raw_event(self, text: cudf.Series, event_regex: typing.Dict[str, any]) -> cudf.DataFrame: + def parse_raw_event(self, text: SeriesType, event_regex: dict[str, str]) -> DataFrameType: """ Processes parsing of a specific type of raw event records received as a dataframe. Parameters ---------- - text : cudf.Series + text : SeriesType Raw event log text to be parsed. - event_regex: typing.Dict[str, any] + event_regex: typing.Dict[str, str] Required regular expressions for a given event type. Returns ------- - cudf.DataFrame + DataFrameType Parsed logs dataframe """ log.debug("Parsing raw events. Event type: %s", self.event_name) - parsed_gdf = cudf.DataFrame({col: [""] for col in self.columns}) + df_pkg = get_df_pkg_from_obj(text) + parsed_gdf = df_pkg.DataFrame({col: [""] for col in self.columns}) parsed_gdf = parsed_gdf[:0] event_specific_columns = event_regex.keys() # Applies regex pattern for each expected output column to raw data @@ -109,7 +111,7 @@ def parse_raw_event(self, text: cudf.Series, event_regex: typing.Dict[str, any]) return parsed_gdf - def _load_regex_yaml(self, yaml_file) -> typing.Dict[str, any]: + def _load_regex_yaml(self, yaml_file) -> dict[str, str]: """Returns a dictionary of event regexes contained in the given yaml file.""" with open(yaml_file, encoding='UTF-8') as yaml_file_h: regex_dict = yaml.safe_load(yaml_file_h) From f87c108c1a3e058628f8f17dea336dc072a43e1d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 23 Aug 2024 12:42:30 -0700 Subject: [PATCH 087/347] Actually check if GPU libs were loaded --- examples/cpu_only/run.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index 072ead86d4..dc9cac1846 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -14,6 +14,7 @@ import logging import pathlib +import sys import typing import click @@ -84,7 +85,7 @@ def run_pipeline(log_level: int, use_python: bool, use_cpu_only: bool, in_file: pipeline.set_source(FileSourceStage(config, filename=in_file)) - @stage + @stage(execution_modes=(execution_mode, )) def print_msg(msg: typing.Any) -> typing.Any: log_msg = [f"Receive a message of type {type(msg)}"] if isinstance(msg, MessageMeta): @@ -99,7 +100,7 @@ def print_msg(msg: typing.Any) -> typing.Any: # TODO: Remove if PR #1803 is merged first pipeline.add_stage(DeserializeStage(config, message_type=ControlMessage)) - @stage + @stage(execution_modes=(execution_mode, )) def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") -> ControlMessage: meta = msg.payload() @@ -120,6 +121,20 @@ def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") - pipeline.run() + known_gpu_packages = ['cudf', 'cupy'] + known_gpu_packages_loaded = [pkg in sys.modules.keys() for pkg in known_gpu_packages] + + if any(known_gpu_packages_loaded): + for (i, pkg) in enumerate(known_gpu_packages): + if known_gpu_packages_loaded[i]: + msg = f"{pkg} is loaded" + if use_cpu_only: + logger.error(msg) + else: + logger.info(msg) + else: + logger.infp("No GPU packages loaded") + if __name__ == "__main__": run_pipeline() From 96ee3810cbcf5648ab2d67e0fc9ff3066364a0d4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 23 Aug 2024 12:44:04 -0700 Subject: [PATCH 088/347] WIP --- examples/cpu_only/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index dc9cac1846..1b0171b6d5 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -121,7 +121,7 @@ def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") - pipeline.run() - known_gpu_packages = ['cudf', 'cupy'] + known_gpu_packages = ['cudf', 'cuml', 'cupy', 'tensorrt', 'torch'] known_gpu_packages_loaded = [pkg in sys.modules.keys() for pkg in known_gpu_packages] if any(known_gpu_packages_loaded): From cd655b45431cb190d4963d360c752c21c3b4df9b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 23 Aug 2024 13:08:27 -0700 Subject: [PATCH 089/347] Don't import AutoEncoder as this imports torch, and we only need this for a type-hint --- .../morpheus/controllers/mlflow_model_writer_controller.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py b/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py index 582c98b5ad..f5099f65db 100644 --- a/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py +++ b/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py @@ -34,7 +34,9 @@ import cudf from morpheus.messages.multi_ae_message import MultiAEMessage -from morpheus.models.dfencoder import AutoEncoder + +if typing.TYPE_CHECKING: + from morpheus.models.dfencoder import AutoEncoder logger = logging.getLogger(__name__) @@ -220,7 +222,7 @@ def on_data(self, message: MultiAEMessage): user = message.meta.user_id - model: AutoEncoder = message.model + model: "AutoEncoder" = message.model model_path = "dfencoder" reg_model_name = self.user_id_to_model(user_id=user) From 6ec45692c7d5bec98ca2410c1de672e1514dc7b6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 23 Aug 2024 13:09:47 -0700 Subject: [PATCH 090/347] Avoid cudf import --- .../morpheus/controllers/mlflow_model_writer_controller.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py b/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py index f5099f65db..cc1190e53a 100644 --- a/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py +++ b/python/morpheus/morpheus/controllers/mlflow_model_writer_controller.py @@ -31,9 +31,8 @@ from mlflow.types.utils import _infer_pandas_column from mlflow.types.utils import _infer_schema -import cudf - from morpheus.messages.multi_ae_message import MultiAEMessage +from morpheus.utils.type_utils import is_cudf_type if typing.TYPE_CHECKING: from morpheus.models.dfencoder import AutoEncoder @@ -270,7 +269,7 @@ def on_data(self, message: MultiAEMessage): # prepare_df to show the actual inputs to the model (any extra are discarded) input_df = message.get_meta().iloc[0:1] - if isinstance(input_df, cudf.DataFrame): + if is_cudf_type(input_df): input_df = input_df.to_pandas() prepared_df = model.prepare_df(input_df) From 3a0695e3f917c7aef7410daab112de803099174e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 23 Aug 2024 13:23:10 -0700 Subject: [PATCH 091/347] Avoid top level cudf imports in modules and controllers imported by modules, in order for the module registry to be populated all modules need to be imported if any modules are to be used --- .../controllers/file_to_df_controller.py | 6 ++--- .../morpheus/modules/payload_batcher.py | 22 ++++++++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/python/morpheus/morpheus/controllers/file_to_df_controller.py b/python/morpheus/morpheus/controllers/file_to_df_controller.py index c5b7e9a6c2..686184463c 100644 --- a/python/morpheus/morpheus/controllers/file_to_df_controller.py +++ b/python/morpheus/morpheus/controllers/file_to_df_controller.py @@ -24,8 +24,6 @@ import fsspec import pandas as pd -import cudf - from morpheus.common import FileTypes from morpheus.io.deserializers import read_file_to_df from morpheus.utils.column_info import DataFrameInputSchema @@ -125,7 +123,7 @@ def __init__(self, self._downloader = Downloader() def _get_or_create_dataframe_from_batch( - self, file_object_batch: typing.Tuple[fsspec.core.OpenFiles, int]) -> typing.Tuple[cudf.DataFrame, bool]: + self, file_object_batch: typing.Tuple[fsspec.core.OpenFiles, int]) -> typing.Tuple[pd.DataFrame, bool]: if (not file_object_batch): raise RuntimeError("No file objects to process") @@ -204,7 +202,7 @@ def convert_to_dataframe(self, file_object_batch: typing.Tuple[fsspec.core.OpenF Returns ------- - cudf.DataFrame + pd.DataFrame The resulting DataFrame. """ diff --git a/python/morpheus/morpheus/modules/payload_batcher.py b/python/morpheus/morpheus/modules/payload_batcher.py index ca62a252bd..3ba8ad5c14 100644 --- a/python/morpheus/morpheus/modules/payload_batcher.py +++ b/python/morpheus/morpheus/modules/payload_batcher.py @@ -19,8 +19,6 @@ import mrc from mrc.core import operators as ops -import cudf - from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.utils.control_message_utils import cm_default_failure_context_manager @@ -28,6 +26,9 @@ from morpheus.utils.module_ids import MORPHEUS_MODULE_NAMESPACE from morpheus.utils.module_ids import PAYLOAD_BATCHER from morpheus.utils.module_utils import register_module +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import get_df_pkg_from_obj +from morpheus.utils.type_utils import is_cudf_type logger = logging.getLogger(__name__) @@ -103,7 +104,7 @@ def payload_batcher(builder: mrc.Builder): @cm_skip_processing_if_failed @cm_default_failure_context_manager(raise_on_failure=raise_on_failure) - def on_next(control_message: ControlMessage) -> typing.List[ControlMessage]: + def on_next(control_message: ControlMessage) -> list[ControlMessage]: nonlocal disable_max_batch_size message_meta = control_message.payload() @@ -119,7 +120,7 @@ def on_next(control_message: ControlMessage) -> typing.List[ControlMessage]: return control_messages - def _batch_dataframe(df: cudf.DataFrame) -> typing.List[cudf.DataFrame]: + def _batch_dataframe(df: DataFrameType) -> list[DataFrameType]: nonlocal max_batch_size dfm_length = len(df) @@ -131,7 +132,7 @@ def _batch_dataframe(df: cudf.DataFrame) -> typing.List[cudf.DataFrame]: dfs = [df.iloc[i * max_batch_size:(i + 1) * max_batch_size] for i in range(num_batches)] return dfs - def _batch_dataframe_by_group(df: cudf.DataFrame) -> typing.List[cudf.DataFrame]: + def _batch_dataframe_by_group(df: DataFrameType) -> list[DataFrameType]: nonlocal max_batch_size nonlocal group_by_columns nonlocal timestamp_column_name @@ -143,9 +144,14 @@ def _batch_dataframe_by_group(df: cudf.DataFrame) -> typing.List[cudf.DataFrame] if has_timestamp_column: # Apply timestamp pattern and group by the formatted timestamp column - df[period_column] = cudf.to_datetime(df[timestamp_column_name], format=timestamp_pattern) - # Period object conversion is not supported in cudf - df[period_column] = df[period_column].to_pandas().dt.to_period(period).astype('str') + df_pkg = get_df_pkg_from_obj(df) + period_series = df_pkg.to_datetime(df[timestamp_column_name], format=timestamp_pattern) + + if is_cudf_type(df): + # Period object conversion is not supported in cudf + period_series = period_series.to_pandas() + + df[period_column] = period_series.dt.to_period(period).astype('str') if len(group_by_columns) == 1: # Avoid warning from cudf regardning an upcoming change of behavior when applying a groupby to a single From 85864641e88f353137a6502b0452a42238a42693 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 09:35:16 -0700 Subject: [PATCH 092/347] Add a helper method to determine if a given object is a dataframe --- python/morpheus/morpheus/utils/type_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 52c17cb5e8..0fc7f0e35a 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -252,3 +252,11 @@ def get_df_pkg_from_obj(obj: typing.Any) -> types.ModuleType: return cudf return pd + + +def is_dataframe(obj: typing.Any) -> bool: + """ + Check if a given object is a pandas or cudf DataFrame. + """ + df_pkg = get_df_pkg_from_obj(obj) + return isinstance(obj, df_pkg.DataFrame) From d63940c38cc9fdedc97c703f6661cf4125e688e6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 09:35:46 -0700 Subject: [PATCH 093/347] Allow MonitorStage and TriggerStage to run in CPU mode --- examples/cpu_only/run.py | 8 ++++++++ .../morpheus/controllers/monitor_controller.py | 18 ++++++++++-------- .../morpheus/stages/general/monitor_stage.py | 3 ++- .../morpheus/stages/general/trigger_stage.py | 3 ++- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index 1b0171b6d5..2d23a20da5 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -28,6 +28,8 @@ from morpheus.messages import MessageMeta from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.pipeline.stage_decorator import stage +from morpheus.stages.general.monitor_stage import MonitorStage +from morpheus.stages.general.trigger_stage import TriggerStage from morpheus.stages.input.file_source_stage import FileSourceStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage from morpheus.stages.postprocess.serialize_stage import SerializeStage @@ -85,6 +87,10 @@ def run_pipeline(log_level: int, use_python: bool, use_cpu_only: bool, in_file: pipeline.set_source(FileSourceStage(config, filename=in_file)) + pipeline.add_stage(MonitorStage(config, description="source")) + + pipeline.add_stage(TriggerStage(config)) + @stage(execution_modes=(execution_mode, )) def print_msg(msg: typing.Any) -> typing.Any: log_msg = [f"Receive a message of type {type(msg)}"] @@ -100,6 +106,8 @@ def print_msg(msg: typing.Any) -> typing.Any: # TODO: Remove if PR #1803 is merged first pipeline.add_stage(DeserializeStage(config, message_type=ControlMessage)) + pipeline.add_stage(MonitorStage(config, description="deserialize")) + @stage(execution_modes=(execution_mode, )) def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") -> ControlMessage: meta = msg.payload() diff --git a/python/morpheus/morpheus/controllers/monitor_controller.py b/python/morpheus/morpheus/controllers/monitor_controller.py index aeb28adff1..2a86e716ac 100644 --- a/python/morpheus/morpheus/controllers/monitor_controller.py +++ b/python/morpheus/morpheus/controllers/monitor_controller.py @@ -19,16 +19,18 @@ import fsspec from tqdm import tqdm -import cudf - from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.utils.logger import LogLevels from morpheus.utils.monitor_utils import MorpheusTqdm +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import is_dataframe logger = logging.getLogger(__name__) +SupportedTypes = typing.Union[DataFrameType, MultiMessage, MessageMeta, ControlMessage, typing.List] + class MonitorController: """ @@ -126,19 +128,19 @@ def refresh_progress(self, _): """ self._progress.refresh() - def progress_sink(self, x: typing.Union[cudf.DataFrame, MultiMessage, MessageMeta, ControlMessage, typing.List]): + def progress_sink(self, x: SupportedTypes): """ Receives a message and determines the count of the message. The progress bar is displayed and the progress is updated. Parameters ---------- - x: typing.Union[cudf.DataFrame, MultiMessage, MessageMeta, ControlMessage, typing.List] + x: SupportedTypes Message that determines the count of the message Returns ------- - x: typing.Union[cudf.DataFrame, MultiMessage, MessageMeta, ControlMessage, typing.List] + x: SupportedTypes """ @@ -159,14 +161,14 @@ def progress_sink(self, x: typing.Union[cudf.DataFrame, MultiMessage, MessageMet return x - def auto_count_fn(self, x: typing.Union[cudf.DataFrame, MultiMessage, MessageMeta, ControlMessage, typing.List]): + def auto_count_fn(self, x: SupportedTypes): """ This is a helper function that is used to determine the count of messages received by the monitor. Parameters ---------- - x: typing.Union[cudf.DataFrame, MultiMessage, MessageMeta, ControlMessage, typing.List] + x: SupportedTypes Message that determines the count of the message Returns @@ -184,7 +186,7 @@ def auto_count_fn(self, x: typing.Union[cudf.DataFrame, MultiMessage, MessageMet if (isinstance(x, list) and len(x) == 0): return None - if (isinstance(x, cudf.DataFrame)): + if (is_dataframe(x)): return lambda y: len(y.index) if (isinstance(x, MultiMessage)): diff --git a/python/morpheus/morpheus/stages/general/monitor_stage.py b/python/morpheus/morpheus/stages/general/monitor_stage.py index cc3a96f33f..821fe729bd 100644 --- a/python/morpheus/morpheus/stages/general/monitor_stage.py +++ b/python/morpheus/morpheus/stages/general/monitor_stage.py @@ -22,6 +22,7 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.controllers.monitor_controller import MonitorController +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils.logger import LogLevels @@ -30,7 +31,7 @@ @register_stage("monitor", ignore_args=["determine_count_fn"]) -class MonitorStage(PassThruTypeMixin, SinglePortStage): +class MonitorStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Display throughput numbers at a specific point in the pipeline. diff --git a/python/morpheus/morpheus/stages/general/trigger_stage.py b/python/morpheus/morpheus/stages/general/trigger_stage.py index b8b754d910..3164a84b64 100644 --- a/python/morpheus/morpheus/stages/general/trigger_stage.py +++ b/python/morpheus/morpheus/stages/general/trigger_stage.py @@ -19,6 +19,7 @@ from mrc.core import operators as ops from morpheus.cli.register_stage import register_stage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @@ -26,7 +27,7 @@ @register_stage("trigger") -class TriggerStage(PassThruTypeMixin, SinglePortStage): +class TriggerStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Buffer data until the previous stage has completed. From 360130b2ab66cac252053cacfbe4a8a272023b0f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 09:43:53 -0700 Subject: [PATCH 094/347] Optionally avoid GPU settings in manual_seed --- python/morpheus/morpheus/utils/seed.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/python/morpheus/morpheus/utils/seed.py b/python/morpheus/morpheus/utils/seed.py index 84106fc5ae..529a1fbf9f 100644 --- a/python/morpheus/morpheus/utils/seed.py +++ b/python/morpheus/morpheus/utils/seed.py @@ -20,17 +20,26 @@ import torch -def manual_seed(seed: int): +def manual_seed(seed: int, cpu_only: bool = False): """ - Manually see the random number generators for the stdlib, PyTorch, NumPy and CuPy + Manually see the random number generators for the Python standard lib, PyTorch, NumPy and CuPy + + Parameters + ---------- + seed : int + The seed value to use + cpu_only : bool, default = False + When set to True, CuPy and CUDA specific PyTorch settings are not set. """ random.seed(seed) np.random.seed(seed) - cp.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) # the "all" refers to all GPUs - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True + if not cpu_only: + cp.random.seed(seed) + + torch.cuda.manual_seed_all(seed) # the "all" refers to all GPUs + + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True From ff45b02976a353bed1cb0af35e0e845c49833c1d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 10:15:09 -0700 Subject: [PATCH 095/347] Update to run in CPU only mode: --- .../controllers/filter_detections_controller.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/python/morpheus/morpheus/controllers/filter_detections_controller.py b/python/morpheus/morpheus/controllers/filter_detections_controller.py index 167bef64fb..145d240ace 100644 --- a/python/morpheus/morpheus/controllers/filter_detections_controller.py +++ b/python/morpheus/morpheus/controllers/filter_detections_controller.py @@ -15,7 +15,6 @@ import logging import typing -import cupy as cp import numpy as np import typing_utils @@ -23,6 +22,7 @@ from morpheus.messages import ControlMessage from morpheus.messages import MultiMessage from morpheus.messages import MultiResponseMessage +from morpheus.utils.type_aliases import NDArrayType logger = logging.getLogger(__name__) @@ -67,7 +67,7 @@ def field_name(self): """ return self._field_name - def _find_detections(self, x: MultiMessage | ControlMessage) -> typing.Union[cp.ndarray, np.ndarray]: + def _find_detections(self, x: MultiMessage | ControlMessage) -> NDArrayType: # Determine the filter source if isinstance(x, MultiMessage): if self._filter_source == FilterSource.TENSOR: @@ -80,11 +80,6 @@ def _find_detections(self, x: MultiMessage | ControlMessage) -> typing.Union[cp. else: filter_source = x.payload().get_data(self._field_name).values - if (isinstance(filter_source, np.ndarray)): - array_mod = np - else: - array_mod = cp - # Get per row detections detections = (filter_source > self._threshold) @@ -92,9 +87,9 @@ def _find_detections(self, x: MultiMessage | ControlMessage) -> typing.Union[cp. detections = detections.any(axis=1) # Surround in False to ensure we get an even number of pairs - detections = array_mod.concatenate([array_mod.array([False]), detections, array_mod.array([False])]) + detections = np.concatenate([np.array([False]), detections, np.array([False])]) - return array_mod.where(detections[1:] != detections[:-1])[0].reshape((-1, 2)) + return np.where(detections[1:] != detections[:-1])[0].reshape((-1, 2)) def filter_copy(self, x: MultiMessage | ControlMessage) -> MultiMessage | ControlMessage: """ From 9c7182648dce128dd7933f7c696be16b08732712 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 10:21:01 -0700 Subject: [PATCH 096/347] Add helper method for returning cupy/numpy --- python/morpheus/morpheus/utils/type_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 0fc7f0e35a..17bb66b35e 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -17,6 +17,7 @@ import typing from collections import defaultdict +import numpy as np import pandas as pd from morpheus.config import ExecutionMode @@ -223,6 +224,18 @@ def get_df_pkg(execution_mode: ExecutionMode) -> types.ModuleType: return pd +def get_array_pkg(execution_mode: ExecutionMode) -> types.ModuleType: + """ + Return the appropriate array package (CuPy for GPU, NumPy for CPU) based on the execution mode. + """ + + if execution_mode == ExecutionMode.GPU: + import cupy + return cupy + + return np + + @typing.overload def get_df_class(df_type_str: DataFrameTypeStr) -> type[DataFrameType]: ... From 0ca0be59b4657bf4c28f2c5d0f92df2045e27d04 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 10:22:26 -0700 Subject: [PATCH 097/347] Mark InMemorySinkStage (and thus CompareDataFrameStage and ValidationStage) as supporting both GPU and CPU --- python/morpheus/morpheus/stages/output/in_memory_sink_stage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/stages/output/in_memory_sink_stage.py b/python/morpheus/morpheus/stages/output/in_memory_sink_stage.py index e3b7d0af16..0dd1b2e91d 100644 --- a/python/morpheus/morpheus/stages/output/in_memory_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/in_memory_sink_stage.py @@ -18,11 +18,12 @@ import mrc.core.operators as ops from morpheus.config import Config +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage -class InMemorySinkStage(PassThruTypeMixin, SinglePortStage): +class InMemorySinkStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Collects incoming messages into a list that can be accessed after the pipeline is complete. Useful for testing. From b89d978fc7d2e3bfa8bc735301acd3f669533ca0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 10:22:47 -0700 Subject: [PATCH 098/347] Misc cleanups --- python/morpheus/morpheus/io/serializers.py | 4 ++-- .../morpheus/stages/output/compare_dataframe_stage.py | 4 ++-- python/morpheus/morpheus/utils/column_info.py | 8 ++++---- python/morpheus/morpheus/utils/concat_df.py | 3 ++- python/morpheus/morpheus/utils/module_utils.py | 8 ++------ python/morpheus/morpheus/utils/schema_transforms.py | 8 ++++---- 6 files changed, 16 insertions(+), 19 deletions(-) diff --git a/python/morpheus/morpheus/io/serializers.py b/python/morpheus/morpheus/io/serializers.py index f6c9e4d4f3..b82b82e99c 100644 --- a/python/morpheus/morpheus/io/serializers.py +++ b/python/morpheus/morpheus/io/serializers.py @@ -24,6 +24,7 @@ from morpheus.common import write_df_to_file as write_df_to_file_cpp from morpheus.config import CppConfig from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import is_cudf_type def df_to_stream_csv(df: DataFrameType, stream: IOBase, include_header=False, include_index_col=True): @@ -202,8 +203,7 @@ def write_df_to_file(df: DataFrameType, file_name: str, file_type: FileTypes = F function is one of `write_df_to_file_cpp`, `df_to_stream_csv`, or `df_to_stream_json`. """ if (CppConfig.get_should_use_cpp()): - import cudf - if (isinstance(df, cudf.DataFrame)): + if (is_cudf_type(df)): # Use the C++ implementation write_df_to_file_cpp(df=df, filename=file_name, file_type=file_type, **kwargs) return diff --git a/python/morpheus/morpheus/stages/output/compare_dataframe_stage.py b/python/morpheus/morpheus/stages/output/compare_dataframe_stage.py index ecd94563c5..ab7ec49f40 100644 --- a/python/morpheus/morpheus/stages/output/compare_dataframe_stage.py +++ b/python/morpheus/morpheus/stages/output/compare_dataframe_stage.py @@ -27,6 +27,7 @@ from morpheus.utils import compare_df as compare_df_module from morpheus.utils import concat_df from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import is_cudf_type class CompareDataFrameStage(InMemorySinkStage): @@ -79,8 +80,7 @@ def __init__(self, tmp_dfs.append(tmp_df) compare_df = pd.concat(tmp_dfs) compare_df.reset_index(inplace=True, drop=True) - elif not isinstance(compare_df, pd.DataFrame): - # assume it is a cudf DataFrame + elif is_cudf_type(compare_df): compare_df = compare_df.to_pandas() self._compare_df = compare_df diff --git a/python/morpheus/morpheus/utils/column_info.py b/python/morpheus/morpheus/utils/column_info.py index d751dd6bb0..32eda6805a 100644 --- a/python/morpheus/morpheus/utils/column_info.py +++ b/python/morpheus/morpheus/utils/column_info.py @@ -23,6 +23,7 @@ import pandas as pd from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import is_cudf_type logger = logging.getLogger(f"morpheus.{__name__}") @@ -640,10 +641,9 @@ def _json_flatten(df_input: DataFrameType, # Check if we even have any JSON columns to flatten if (not df_input.columns.intersection(json_cols).empty): - convert_to_cudf = False + is_cudf = is_cudf_type(df_input) - if (not isinstance(df_input, pd.DataFrame)): - convert_to_cudf = True + if (is_cudf): df_input = df_input.to_pandas() json_normalized = [] @@ -672,7 +672,7 @@ def _json_flatten(df_input: DataFrameType, # Combine the original DataFrame with the normalized JSON columns df_input = pd.concat([df_input[columns_to_keep]] + json_normalized, axis=1) - if (convert_to_cudf): + if (is_cudf): import cudf df_input = cudf.from_pandas(df_input).reset_index(drop=True) diff --git a/python/morpheus/morpheus/utils/concat_df.py b/python/morpheus/morpheus/utils/concat_df.py index b3ec2fbc7b..90b351b076 100644 --- a/python/morpheus/morpheus/utils/concat_df.py +++ b/python/morpheus/morpheus/utils/concat_df.py @@ -19,6 +19,7 @@ from morpheus.messages import ControlMessage from morpheus.messages import MessageBase from morpheus.messages import MultiMessage +from morpheus.utils.type_utils import is_cudf_type def concat_dataframes(messages: typing.List[MessageBase]) -> pd.DataFrame: @@ -44,7 +45,7 @@ def concat_dataframes(messages: typing.List[MessageBase]) -> pd.DataFrame: else: df = x.df - if not isinstance(df, pd.DataFrame): + if is_cudf_type(df): df = df.to_pandas() all_meta.append(df) diff --git a/python/morpheus/morpheus/utils/module_utils.py b/python/morpheus/morpheus/utils/module_utils.py index f756a65cf4..18adb0e417 100644 --- a/python/morpheus/morpheus/utils/module_utils.py +++ b/python/morpheus/morpheus/utils/module_utils.py @@ -25,6 +25,7 @@ from pydantic import BaseModel from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import get_df_pkg_from_obj logger = logging.getLogger(__name__) @@ -214,12 +215,7 @@ def to_period_approximation(data_df: DataFrameType, period: str) -> DataFrameTyp strptime_format = period_to_strptime[period] - if isinstance(data_df, pd.DataFrame): - df_pkg = pd - else: - import cudf - df_pkg = cudf - + df_pkg = get_df_pkg_from_obj(data_df) data_df["period"] = df_pkg.to_datetime(data_df["ts"].dt.strftime(strptime_format) + '-1', format=f"{strptime_format}-%w") diff --git a/python/morpheus/morpheus/utils/schema_transforms.py b/python/morpheus/morpheus/utils/schema_transforms.py index 787fd2c441..162a2064db 100644 --- a/python/morpheus/morpheus/utils/schema_transforms.py +++ b/python/morpheus/morpheus/utils/schema_transforms.py @@ -19,6 +19,7 @@ from morpheus.utils.column_info import DataFrameInputSchema from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import is_cudf_type if typing.TYPE_CHECKING: import cudf @@ -74,10 +75,9 @@ def process_dataframe( output_df = pd.DataFrame() - convert_to_cudf = False - if (not isinstance(df_in, pd.DataFrame)): + is_cudf = is_cudf_type(df_in) + if (is_cudf): df_in = df_in.to_pandas() - convert_to_cudf = True # Iterate over the column info for ci in input_schema.column_info: @@ -96,7 +96,7 @@ def process_dataframe( output_df[match_columns] = df_in[match_columns] - if (convert_to_cudf): + if (is_cudf): import cudf return cudf.from_pandas(output_df) From 396ac6e1c647671c6a1282bea2c76e5afee5d548 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 10:37:57 -0700 Subject: [PATCH 099/347] Misc Cleanups --- .../messages/memory/inference_memory.py | 22 +++++++++---------- .../messages/memory/response_memory.py | 16 ++++++++------ .../morpheus/messages/memory/tensor_memory.py | 14 ++++++------ 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/python/morpheus/morpheus/messages/memory/inference_memory.py b/python/morpheus/morpheus/messages/memory/inference_memory.py index 9a11e58788..6913515fe8 100644 --- a/python/morpheus/morpheus/messages/memory/inference_memory.py +++ b/python/morpheus/morpheus/messages/memory/inference_memory.py @@ -25,7 +25,7 @@ class InferenceMemory(TensorMemory, cpp_class=_messages.InferenceMemory): """ This is a base container class for data that will be used for inference stages. This class is designed to - hold generic tensor data in cupy arrays. + hold generic tensor data in either CuPy or NumPy arrays. """ def get_input(self, name: str): @@ -39,7 +39,7 @@ def get_input(self, name: str): Returns ------- - cupy.ndarray + NDArrayType Inputs corresponding to name. Raises @@ -57,8 +57,8 @@ def set_input(self, name: str, tensor: NDArrayType): ---------- name : str Key used to do lookup in inputs dict of the container. - tensor : cupy.ndarray - Tensor as a CuPy array. + tensor : NDArrayType + Tensor as either CuPy or NumPy array. """ self.set_tensor(name, tensor) @@ -71,11 +71,11 @@ class InferenceMemoryNLP(InferenceMemory, cpp_class=_messages.InferenceMemoryNLP Parameters ---------- - input_ids : cupy.ndarray + input_ids : NDArrayType The token-ids for each string padded with 0s to max_length. - input_mask : cupy.ndarray + input_mask : NDArrayType The mask for token-ids result where corresponding positions identify valid token-id values. - seq_ids : cupy.ndarray + seq_ids : NDArrayType Ids used to index from an inference input to a message. Necessary since there can be more inference inputs than messages (i.e., if some messages get broken into multiple inference requests). @@ -99,9 +99,9 @@ class InferenceMemoryFIL(InferenceMemory, cpp_class=_messages.InferenceMemoryFIL Parameters ---------- - input__0 : cupy.ndarray + input__0 : NDArrayType Inference input. - seq_ids : cupy.ndarray + seq_ids : NDArrayType Ids used to index from an inference input to a message. Necessary since there can be more inference inputs than messages (i.e., if some messages get broken into multiple inference requests). @@ -122,9 +122,9 @@ class InferenceMemoryAE(InferenceMemory, cpp_class=None): Parameters ---------- - inputs : cupy.ndarray + inputs : NDArrayType Inference input. - seq_ids : cupy.ndarray + seq_ids : NDArrayType Ids used to index from an inference input to a message. Necessary since there can be more inference inputs than messages (i.e., if some messages get broken into multiple inference requests). """ diff --git a/python/morpheus/morpheus/messages/memory/response_memory.py b/python/morpheus/morpheus/messages/memory/response_memory.py index ffce421c7f..bcf6a4c61b 100644 --- a/python/morpheus/morpheus/messages/memory/response_memory.py +++ b/python/morpheus/morpheus/messages/memory/response_memory.py @@ -16,6 +16,8 @@ import dataclasses import logging +import pandas as pd + import morpheus._lib.messages as _messages from morpheus.messages.data_class_prop import DataClassProp from morpheus.messages.memory.tensor_memory import TensorMemory @@ -44,7 +46,7 @@ def get_output(self, name: str): Returns ------- - cupy.ndarray + NDArrayType Tensors corresponding to name. Raises @@ -63,8 +65,8 @@ def set_output(self, name: str, tensor: NDArrayType): ---------- name : str Key used to do lookup in tensors dict of the container. - tensor : cupy.ndarray - Tensor as a CuPy array. + tensor : NDArrayType + Tensor as either a CuPy or NumPy array. Raises ------ @@ -81,7 +83,7 @@ class ResponseMemoryProbs(ResponseMemory, cpp_class=_messages.ResponseMemoryProb Parameters ---------- - probs : cupy.ndarray + probs : NDArrayType Probabilities tensor """ probs: dataclasses.InitVar[NDArrayType] = DataClassProp(ResponseMemory._get_tensor_prop, ResponseMemory.set_output) @@ -97,7 +99,7 @@ class ResponseMemoryAE(ResponseMemory, cpp_class=None): Parameters ---------- - probs : cupy.ndarray + probs : NDArrayType Probabilities tensor user_id : str @@ -108,8 +110,8 @@ class ResponseMemoryAE(ResponseMemory, cpp_class=None): containing the loss z-score along with `max_abs_z` and `mean_abs_z` columns """ probs: dataclasses.InitVar[NDArrayType] = DataClassProp(ResponseMemory._get_tensor_prop, ResponseMemory.set_output) - user_id = "" - explain_df = None + user_id: str = "" + explain_df: pd.DataFrame = None def __init__(self, *, count: int, probs: NDArrayType): super().__init__(count=count, tensors={'probs': probs}) diff --git a/python/morpheus/morpheus/messages/memory/tensor_memory.py b/python/morpheus/morpheus/messages/memory/tensor_memory.py index fc34ded920..ce193cd5da 100644 --- a/python/morpheus/morpheus/messages/memory/tensor_memory.py +++ b/python/morpheus/morpheus/messages/memory/tensor_memory.py @@ -28,13 +28,13 @@ class TensorMemory(MessageData, cpp_class=_messages.TensorMemory): """ This is a base container class for data that will be used for inference stages. This class is designed to - hold generic tensor data in cupy arrays. + hold generic tensor data in either CuPy or NumPy arrays. Parameters ---------- count : int Length of each tensor contained in `tensors`. - tensors : typing.Dict[str, cupy.ndarray] + tensors : TensorMapType Collection of tensors uniquely identified by a name. """ @@ -109,7 +109,7 @@ def set_tensors(self, tensors: TensorMapType): Parameters ---------- - tensors : typing.Dict[str, cupy.ndarray] + tensors : TensorMapType Collection of tensors uniquely identified by a name. """ self._check_tensors(tensors) @@ -126,7 +126,7 @@ def get_tensor(self, name: str): Returns ------- - cupy.ndarray + NDArrayType Tensor. Raises @@ -147,7 +147,7 @@ def _get_tensor_prop(self, name: str): Returns ------- - cupy.ndarray + NDArrayType Tensor. Raises @@ -168,8 +168,8 @@ def set_tensor(self, name: str, tensor: NDArrayType): ---------- name : str Tensor key name. - tensor : cupy.ndarray - Tensor as a CuPy array. + tensor : NDArrayType + Tensor as either a CuPy or NumPy array. Raises ------ From cbc59c2828a68f58985e9e49432783ab8ea2957c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 14:44:09 -0700 Subject: [PATCH 100/347] Optionally run the container without access to GPUs, useful for testing --- docker/run_container_dev.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/run_container_dev.sh b/docker/run_container_dev.sh index 9a2db756af..fd05aa7031 100755 --- a/docker/run_container_dev.sh +++ b/docker/run_container_dev.sh @@ -28,7 +28,15 @@ DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"morpheus"} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"dev-$(date +'%y%m%d')"} DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} -DOCKER_ARGS="--runtime=nvidia --env WORKSPACE_VOLUME=${PWD} -v $PWD:/workspace --net=host --gpus=all --cap-add=sys_nice" +DOCKER_ARGS=" --env WORKSPACE_VOLUME=${PWD} -v $PWD:/workspace --net=host --cap-add=sys_nice" + +if [[ -n "${CPU_ONLY}" ]]; then + echo -e "${b}Executing in CPU only mode${x}" + DOCKER_ARGS="${DOCKER_ARGS} --runtime=runc" +else + echo -e "${b}Executing in GPU mode${x}" + DOCKER_ARGS="${DOCKER_ARGS} --runtime=nvidia --gpus=all" +fi if [[ -n "${SSH_AUTH_SOCK}" ]]; then echo -e "${b}Setting up ssh-agent auth socket${x}" From 1457d5569f8444d42ebf43944cffca880ace9989 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 14:46:10 -0700 Subject: [PATCH 101/347] Disable default iteration on the use_cpp fixture, the primary reason is that now that Python mode is synonymous with CPU mode, execution in CPU mode will now be considered a special case and not the general case --- tests/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a7278db60a..560be53ef2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -312,7 +312,7 @@ def df_type(request: pytest.FixtureRequest): @pytest.fixture(scope="function") -def config(use_cpp: bool): +def config(): """ For new pytest style tests, get the config by using this fixture. It will setup the config based on the marks set on the object. If no marks are added to the test, it will be parameterized for both C++ and python. For example, @@ -325,10 +325,7 @@ def my_python_test(config: Config): """ from morpheus.config import Config - from morpheus.config import ExecutionMode config = Config() - if not use_cpp: - config.execution_mode = ExecutionMode.CPU yield config From 88057d7a2b772efff85f74f2655a714e1bcdc652 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 14:50:35 -0700 Subject: [PATCH 102/347] Remove the execution_modes argument from the register_stage decorator, this is a hold-over from the POC --- python/morpheus/morpheus/cli/register_stage.py | 2 -- python/morpheus/morpheus/stages/input/arxiv_source.py | 2 +- .../stages/input/databricks_deltalake_source_stage.py | 2 +- python/morpheus/morpheus/stages/input/file_source_stage.py | 4 +--- .../morpheus/stages/input/http_client_source_stage.py | 4 +--- .../morpheus/stages/input/http_server_source_stage.py | 2 +- .../morpheus/morpheus/stages/output/http_server_sink_stage.py | 4 +--- python/morpheus/morpheus/stages/output/write_to_file_stage.py | 4 +--- .../morpheus/morpheus/stages/postprocess/serialize_stage.py | 2 +- .../morpheus/morpheus/stages/preprocess/deserialize_stage.py | 1 - 10 files changed, 8 insertions(+), 19 deletions(-) diff --git a/python/morpheus/morpheus/cli/register_stage.py b/python/morpheus/morpheus/cli/register_stage.py index 48000a273f..f1b47e67a9 100644 --- a/python/morpheus/morpheus/cli/register_stage.py +++ b/python/morpheus/morpheus/cli/register_stage.py @@ -34,7 +34,6 @@ from morpheus.cli.utils import parse_enum from morpheus.cli.utils import prepare_command from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.utils.type_utils import _DecoratorType from morpheus.utils.type_utils import get_full_qualname @@ -249,7 +248,6 @@ def compute_option_name(stage_arg_name: str, rename_options: typing.Dict[str, st def register_stage(command_name: str = None, modes: typing.Sequence[PipelineModes] = None, - execution_modes: tuple[ExecutionMode] = (ExecutionMode.GPU, ), ignore_args: typing.List[str] = None, command_args: dict = None, option_args: typing.Dict[str, dict] = None, diff --git a/python/morpheus/morpheus/stages/input/arxiv_source.py b/python/morpheus/morpheus/stages/input/arxiv_source.py index 385e8be2a5..301fb54852 100644 --- a/python/morpheus/morpheus/stages/input/arxiv_source.py +++ b/python/morpheus/morpheus/stages/input/arxiv_source.py @@ -40,7 +40,7 @@ "--file conda/environments/all_cuda-121_arch-x86_64.yaml --prune`") -@register_stage("from-arxiv", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("from-arxiv") class ArxivSource(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage that downloads PDFs from arxiv and converts them to dataframes. diff --git a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py index 77a63a2165..a50488123b 100644 --- a/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py +++ b/python/morpheus/morpheus/stages/input/databricks_deltalake_source_stage.py @@ -38,7 +38,7 @@ IMPORT_EXCEPTION = import_exc -@register_stage("from-databricks-deltalake", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("from-databricks-deltalake") class DataBricksDeltaLakeSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage used to load messages from a DeltaLake table. diff --git a/python/morpheus/morpheus/stages/input/file_source_stage.py b/python/morpheus/morpheus/stages/input/file_source_stage.py index ad3c5d75c0..f075b41bcd 100644 --- a/python/morpheus/morpheus/stages/input/file_source_stage.py +++ b/python/morpheus/morpheus/stages/input/file_source_stage.py @@ -34,9 +34,7 @@ logger = logging.getLogger(__name__) -@register_stage("from-file", - modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], - execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("from-file", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER]) class FileSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Load messages from a file. diff --git a/python/morpheus/morpheus/stages/input/http_client_source_stage.py b/python/morpheus/morpheus/stages/input/http_client_source_stage.py index 8309ba5921..6936125659 100644 --- a/python/morpheus/morpheus/stages/input/http_client_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_client_source_stage.py @@ -36,9 +36,7 @@ logger = logging.getLogger(__name__) -@register_stage("from-http-client", - execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), - ignore_args=["query_params", "headers", "**request_kwargs"]) +@register_stage("from-http-client", ignore_args=["query_params", "headers", "**request_kwargs"]) class HttpClientSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage that polls a remote HTTP server for incoming data. diff --git a/python/morpheus/morpheus/stages/input/http_server_source_stage.py b/python/morpheus/morpheus/stages/input/http_server_source_stage.py index 2d2afe53e4..73652f3a18 100644 --- a/python/morpheus/morpheus/stages/input/http_server_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_server_source_stage.py @@ -43,7 +43,7 @@ HEALTH_SUPPORTED_METHODS = (HTTPMethod.GET, HTTPMethod.POST) -@register_stage("from-http", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("from-http") class HttpServerSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Source stage that starts an HTTP server and listens for incoming requests on a specified endpoint. diff --git a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py index b02de333f8..2788702291 100644 --- a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py @@ -41,9 +41,7 @@ logger = logging.getLogger(__name__) -@register_stage("to-http-server", - execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), - ignore_args=["df_serializer_fn"]) +@register_stage("to-http-server", ignore_args=["df_serializer_fn"]) class HttpServerSinkStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Sink stage that starts an HTTP server and listens for incoming requests on a specified endpoint. diff --git a/python/morpheus/morpheus/stages/output/write_to_file_stage.py b/python/morpheus/morpheus/stages/output/write_to_file_stage.py index e3f9b06cbf..f1316f7f9a 100644 --- a/python/morpheus/morpheus/stages/output/write_to_file_stage.py +++ b/python/morpheus/morpheus/stages/output/write_to_file_stage.py @@ -29,9 +29,7 @@ from morpheus.pipeline.single_port_stage import SinglePortStage -@register_stage("to-file", - rename_options={"include_index_col": "--include-index-col"}, - execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("to-file", rename_options={"include_index_col": "--include-index-col"}) class WriteToFileStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Write all messages to a file. diff --git a/python/morpheus/morpheus/stages/postprocess/serialize_stage.py b/python/morpheus/morpheus/stages/postprocess/serialize_stage.py index 0372cd83ca..36f23bdd3a 100644 --- a/python/morpheus/morpheus/stages/postprocess/serialize_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/serialize_stage.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) -@register_stage("serialize", execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) +@register_stage("serialize") class SerializeStage(GpuAndCpuMixin, SinglePortStage): """ Includes & excludes columns from messages. diff --git a/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py b/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py index f97d9bc2ec..42e2a05500 100644 --- a/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py @@ -35,7 +35,6 @@ @register_stage("deserialize", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER], - execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU), ignore_args=["message_type", "task_type", "task_payload"]) class DeserializeStage(GpuAndCpuMixin, MultiMessageStage): """ From 27384f05350a9dd0bc372bf18765c65419f2a47d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 15:55:02 -0700 Subject: [PATCH 103/347] WIP --- python/morpheus/morpheus/io/deserializers.py | 1 - python/morpheus/morpheus/modules/payload_batcher.py | 1 - python/morpheus/morpheus/parsers/ip.py | 10 +++++----- .../morpheus/stages/input/file_source_stage.py | 9 ++------- .../morpheus/stages/input/http_client_source_stage.py | 1 - .../morpheus/stages/input/http_server_source_stage.py | 1 - .../morpheus/stages/output/http_server_sink_stage.py | 1 - .../morpheus/stages/output/write_to_file_stage.py | 1 - .../morpheus/stages/postprocess/serialize_stage.py | 1 - .../morpheus/stages/preprocess/deserialize_stage.py | 1 - .../morpheus/stages/preprocess/preprocess_ae_stage.py | 1 - .../stages/preprocess/preprocess_base_stage.py | 3 --- .../morpheus/stages/preprocess/preprocess_fil_stage.py | 9 --------- python/morpheus/morpheus/utils/module_utils.py | 1 - python/morpheus/morpheus/utils/type_utils.py | 4 ++-- .../morpheus_llm/stages/llm/llm_engine_stage.py | 2 +- tests/pipeline/test_execution_modes.py | 8 ++++---- 17 files changed, 14 insertions(+), 41 deletions(-) diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index 7b0709c2ce..0e93f11ae5 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -18,7 +18,6 @@ import typing import numpy as np -import pandas as pd from morpheus.common import FileTypes from morpheus.common import determine_file_type diff --git a/python/morpheus/morpheus/modules/payload_batcher.py b/python/morpheus/morpheus/modules/payload_batcher.py index 3ba8ad5c14..d3372e40e3 100644 --- a/python/morpheus/morpheus/modules/payload_batcher.py +++ b/python/morpheus/morpheus/modules/payload_batcher.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -import typing import warnings import mrc diff --git a/python/morpheus/morpheus/parsers/ip.py b/python/morpheus/morpheus/parsers/ip.py index 8c80f09bec..5839f269a5 100644 --- a/python/morpheus/morpheus/parsers/ip.py +++ b/python/morpheus/morpheus/parsers/ip.py @@ -348,12 +348,12 @@ def _mask_kernel(idx, out1, out2, out3, out4, kwarg1): out4[i] = int(kwarg1) % 256 -def _mask_pandas(df_cols: tuple[int], mask: int, series_name: str) -> pd.Series: - outputs = [int(mask / 16777216) % 256, int(mask / 65536) % 256, int(mask / 256) % 256, int(mask) % 256] +def _mask_pandas(df_cols: tuple[int], mask_: int, series_name: str) -> pd.Series: + outputs = [int(mask_ / 16777216) % 256, int(mask_ / 65536) % 256, int(mask_ / 256) % 256, int(mask_) % 256] return pd.Series([df_cols.idx, ".".join(map(str, outputs))], index=["idx", series_name]) -def _compute_mask_impl(ips: SeriesType, mask: int, series_name: str) -> SeriesType: +def _compute_mask_impl(ips: SeriesType, mask_: int, series_name: str) -> SeriesType: df_pkg = get_df_pkg_from_obj(ips) if is_cudf_type(ips): df = df_pkg.DataFrame() @@ -364,7 +364,7 @@ def _compute_mask_impl(ips: SeriesType, mask: int, series_name: str) -> SeriesTy outcols={ "out1": np.int64, "out2": np.int64, "out3": np.int64, "out4": np.int64 }, - kwargs={"kwarg1": mask}, + kwargs={"kwarg1": mask_}, ) out1 = x["out1"].astype(str) @@ -374,7 +374,7 @@ def _compute_mask_impl(ips: SeriesType, mask: int, series_name: str) -> SeriesTy df[series_name] = out1.str.cat(out2, sep=".").str.cat(out3, sep=".").str.cat(out4, sep=".") else: df = df_pkg.DataFrame({"idx": ips.index}) - df = df.apply(_mask_pandas, axis=1, args=(mask, series_name)) + df = df.apply(_mask_pandas, axis=1, args=(mask_, series_name)) return df[series_name] diff --git a/python/morpheus/morpheus/stages/input/file_source_stage.py b/python/morpheus/morpheus/stages/input/file_source_stage.py index f075b41bcd..1f67d91624 100644 --- a/python/morpheus/morpheus/stages/input/file_source_stage.py +++ b/python/morpheus/morpheus/stages/input/file_source_stage.py @@ -22,7 +22,6 @@ from morpheus.cli import register_stage from morpheus.common import FileTypes from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta @@ -30,6 +29,7 @@ from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_utils import exec_mode_to_df_type_str logger = logging.getLogger(__name__) @@ -99,12 +99,7 @@ def __init__(self, self._iterative = iterative self._repeat_count = repeat - if c.execution_mode == ExecutionMode.GPU: - print("GPU MODE Using cudf") - self._df_type = "cudf" - else: - print("CPU MODE Using pandas") - self._df_type = "pandas" + self._df_type = exec_mode_to_df_type_str(c.execution_mode) @property def name(self) -> str: diff --git a/python/morpheus/morpheus/stages/input/http_client_source_stage.py b/python/morpheus/morpheus/stages/input/http_client_source_stage.py index 6936125659..2fc364bc17 100644 --- a/python/morpheus/morpheus/stages/input/http_client_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_client_source_stage.py @@ -23,7 +23,6 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin diff --git a/python/morpheus/morpheus/stages/input/http_server_source_stage.py b/python/morpheus/morpheus/stages/input/http_server_source_stage.py index 73652f3a18..c848147d58 100644 --- a/python/morpheus/morpheus/stages/input/http_server_source_stage.py +++ b/python/morpheus/morpheus/stages/input/http_server_source_stage.py @@ -24,7 +24,6 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin diff --git a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py index 2788702291..00e7c4530b 100644 --- a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py @@ -26,7 +26,6 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.io import serializers from morpheus.messages import MessageMeta from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin diff --git a/python/morpheus/morpheus/stages/output/write_to_file_stage.py b/python/morpheus/morpheus/stages/output/write_to_file_stage.py index f1316f7f9a..9f3298bc61 100644 --- a/python/morpheus/morpheus/stages/output/write_to_file_stage.py +++ b/python/morpheus/morpheus/stages/output/write_to_file_stage.py @@ -21,7 +21,6 @@ from morpheus.cli.register_stage import register_stage from morpheus.common import FileTypes from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.controllers.write_to_file_controller import WriteToFileController from morpheus.messages import MessageMeta from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin diff --git a/python/morpheus/morpheus/stages/postprocess/serialize_stage.py b/python/morpheus/morpheus/stages/postprocess/serialize_stage.py index 36f23bdd3a..ee0b647d02 100644 --- a/python/morpheus/morpheus/stages/postprocess/serialize_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/serialize_stage.py @@ -21,7 +21,6 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.controllers.serialize_controller import SerializeController from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta diff --git a/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py b/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py index 42e2a05500..1ab5b6e7a7 100644 --- a/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/deserialize_stage.py @@ -20,7 +20,6 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py index 3892d99bbc..d62eb3162f 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py @@ -17,7 +17,6 @@ from functools import partial import cupy as cp -import mrc from morpheus.cli.register_stage import register_stage from morpheus.config import Config diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py index cf35449773..71a2c4e595 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py @@ -12,12 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect import typing -from abc import abstractmethod import mrc -import typing_utils from mrc.core import operators as ops from morpheus.config import Config diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py index bfe9535c55..069d3bee6d 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py @@ -13,22 +13,13 @@ # limitations under the License. import logging -import typing -from functools import partial import mrc -import numpy as np -import pandas as pd from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes -from morpheus.messages import ControlMessage -from morpheus.messages import InferenceMemoryFIL from morpheus.messages import MultiInferenceFILMessage -from morpheus.messages import MultiInferenceMessage -from morpheus.messages import MultiMessage -from morpheus.messages import TensorMemory from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage logger = logging.getLogger(__name__) diff --git a/python/morpheus/morpheus/utils/module_utils.py b/python/morpheus/morpheus/utils/module_utils.py index 18adb0e417..a250f1a650 100644 --- a/python/morpheus/morpheus/utils/module_utils.py +++ b/python/morpheus/morpheus/utils/module_utils.py @@ -21,7 +21,6 @@ from typing import Type import mrc -import pandas as pd from pydantic import BaseModel from morpheus.utils.type_aliases import DataFrameType diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 17bb66b35e..baa8021d60 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -177,7 +177,7 @@ def df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr) -> ExecutionMode: """ if df_type_str == "cudf": return ExecutionMode.GPU - elif df_type_str == "pandas": + if df_type_str == "pandas": return ExecutionMode.CPU valid_values = ", ".join(typing.get_args(DataFrameTypeStr)) @@ -198,7 +198,7 @@ def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: if df_type_str == "cudf": import cudf return cudf - elif df_type_str == "pandas": + if df_type_str == "pandas": return pd valid_values = ", ".join(typing.get_args(DataFrameTypeStr)) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index 4188b33b55..f16442ac59 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -78,7 +78,7 @@ def _cast_control_message(self, message: ControlMessage, *, cpp_messages_lib: ty return cpp_messages_lib.ControlMessage(message) def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: - import morpheus._lib.llm as _llm + import morpheus_llm._lib.llm as _llm node = _llm.LLMEngineStage(builder, self.unique_name, self._engine) node.launch_options.pe_count = 1 diff --git a/tests/pipeline/test_execution_modes.py b/tests/pipeline/test_execution_modes.py index 7d1ef2e81b..28499d24ab 100755 --- a/tests/pipeline/test_execution_modes.py +++ b/tests/pipeline/test_execution_modes.py @@ -90,8 +90,8 @@ def test_execution_mode_mixins(stage_cls: type[ConvMsg], expected_modes: set): else: config.execution_mode = ExecutionMode.GPU - stage = stage_cls(config) - assert set(stage.supported_execution_modes()) == expected_modes + stage_ = stage_cls(config) + assert set(stage_.supported_execution_modes()) == expected_modes @pytest.mark.parametrize("stage_cls, execution_mode", @@ -110,5 +110,5 @@ def test_unsupported_mode_error(stage_cls: type[ConvMsg], execution_mode: Execut config.execution_mode = execution_mode with pytest.raises(RuntimeError, match="Unsupported execution mode"): - stage = stage_cls(config) - stage._pre_build(do_propagate=False) + stage_ = stage_cls(config) + stage_._pre_build(do_propagate=False) From a7fbaf599f4f1a0a6d722bc639a428fcb5fb446d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 16:13:22 -0700 Subject: [PATCH 104/347] When in doubt default to C++ mode --- tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 560be53ef2..dcee03d68d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -102,7 +102,6 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): This function will add parameterizations for the `config` fixture depending on what types of config the test supports """ - # === use_cpp Parameterize === use_cpp = metafunc.definition.get_closest_marker("use_cpp") is not None use_python = metafunc.definition.get_closest_marker("use_python") is not None @@ -126,6 +125,9 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): elif (use_cpp and use_python): # Need to parameterize since we have multiple _set_use_cpp_params.extend([use_cpp_param, use_python_param]) + elif (not use_cpp and not use_python): + # If neither are set, default to cpp + _set_use_cpp_params.append(use_cpp_param) if (len(_set_use_cpp_params) > 0): metafunc.parametrize("_set_use_cpp", _set_use_cpp_params, indirect=True) From 280329509583fd46336c6fe2ab36b40fe3fd8884 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 16:54:49 -0700 Subject: [PATCH 105/347] Mark supporting gpu and cpu --- python/morpheus/morpheus/stages/input/kafka_source_stage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/stages/input/kafka_source_stage.py b/python/morpheus/morpheus/stages/input/kafka_source_stage.py index 1d85cd8480..6f5a7b8b4c 100644 --- a/python/morpheus/morpheus/stages/input/kafka_source_stage.py +++ b/python/morpheus/morpheus/stages/input/kafka_source_stage.py @@ -28,6 +28,7 @@ from morpheus.config import auto_determine_bootstrap from morpheus.io.utils import get_json_reader from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema @@ -44,7 +45,7 @@ class AutoOffsetReset(Enum): @register_stage("from-kafka", modes=[PipelineModes.FIL, PipelineModes.NLP, PipelineModes.OTHER]) -class KafkaSourceStage(PreallocatorMixin, SingleOutputSource): +class KafkaSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): """ Load messages from a Kafka cluster. From cd3c24a82d751810719d59a2de8eb70f8da93880 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 16:55:14 -0700 Subject: [PATCH 106/347] Fix python only test --- tests/test_add_classifications_stage.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/test_add_classifications_stage.py b/tests/test_add_classifications_stage.py index e3bbf70c1a..608c132de0 100755 --- a/tests/test_add_classifications_stage.py +++ b/tests/test_add_classifications_stage.py @@ -16,17 +16,15 @@ import typing -import cupy as cp +import numpy as np +import pandas as pd import pytest import typing_utils -import cudf - from _utils.dataset_manager import DatasetManager -# pylint: disable=morpheus-incorrect-lib-from-import -from morpheus._lib.messages import TensorMemory as CppTensorMemory from morpheus.config import Config from morpheus.messages import ControlMessage +from morpheus.messages import TensorMemory from morpheus.messages.memory.tensor_memory import TensorMemory from morpheus.messages.message_meta import MessageMeta from morpheus.messages.multi_response_message import MultiResponseMessage @@ -70,8 +68,8 @@ def test_add_labels_with_multi_response_message_and_contgrol_message(): threshold = 0.6 - df = cudf.DataFrame([0, 1], columns=["dummy"]) - probs_array = cp.array([[0.1, 0.6, 0.8], [0.3, 0.61, 0.9]]) + df = pd.DataFrame([0, 1], columns=["dummy"]) + probs_array = np.array([[0.1, 0.6, 0.8], [0.3, 0.61, 0.9]]) probs_array_bool = probs_array > threshold mrm = MultiResponseMessage(meta=MessageMeta(df), memory=TensorMemory(count=2, tensors={"probs": probs_array})) @@ -84,7 +82,7 @@ def test_add_labels_with_multi_response_message_and_contgrol_message(): cm = ControlMessage() cm.payload(MessageMeta(df)) - cm.tensors(CppTensorMemory(count=2, tensors={"probs": probs_array})) + cm.tensors(TensorMemory(count=2, tensors={"probs": probs_array})) labeled_cm = AddClassificationsStage._add_labels(cm, idx2label=class_labels, threshold=threshold) @@ -122,7 +120,7 @@ def test_add_labels_with_multi_response_message_and_contgrol_message(): cm = ControlMessage() cm.payload(MessageMeta(df)) - cm.tensors(CppTensorMemory(count=2, tensors={"probs": probs_array[:, 0:-1]})) + cm.tensors(TensorMemory(count=2, tensors={"probs": probs_array[:, 0:-1]})) with pytest.raises(RuntimeError): AddClassificationsStage._add_labels(cm, idx2label=class_labels, threshold=threshold) From 7fbc0581ffee58cdce68a807c8cbf97f523f7adb Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 16:58:08 -0700 Subject: [PATCH 107/347] WIP --- tests/test_add_scores_stage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_add_scores_stage.py b/tests/test_add_scores_stage.py index 0e347c7d78..0722df95f4 100755 --- a/tests/test_add_scores_stage.py +++ b/tests/test_add_scores_stage.py @@ -17,6 +17,8 @@ import typing import cupy as cp +import numpy as np +import pandas as pd import pytest import typing_utils @@ -68,8 +70,8 @@ def test_constructor_errors(config: Config): def test_add_labels_with_multi_response_message_and_control_message(): class_labels = {0: "frogs", 1: "lizards", 2: "toads"} - df = cudf.DataFrame([0, 1], columns=["dummy"]) - probs_array = cp.array([[0.1, 0.5, 0.8], [0.2, 0.6, 0.9]]) + df = pd.DataFrame([0, 1], columns=["dummy"]) + probs_array = np.array([[0.1, 0.5, 0.8], [0.2, 0.6, 0.9]]) mrm = MultiResponseMessage(meta=MessageMeta(df), memory=TensorMemory(count=2, tensors={"probs": probs_array})) @@ -81,7 +83,7 @@ def test_add_labels_with_multi_response_message_and_control_message(): cm = ControlMessage() cm.payload(MessageMeta(df)) - cm.tensors(_messages.TensorMemory(count=2, tensors={"probs": probs_array})) + cm.tensors(TensorMemory(count=2, tensors={"probs": probs_array})) labeled_cm = AddClassificationsStage._add_labels(cm, idx2label=class_labels, threshold=None) @@ -119,7 +121,7 @@ def test_add_labels_with_multi_response_message_and_control_message(): cm = ControlMessage() cm.payload(MessageMeta(df)) - cm.tensors(_messages.TensorMemory(count=2, tensors={"probs": probs_array[:, 0:-1]})) + cm.tensors(TensorMemory(count=2, tensors={"probs": probs_array[:, 0:-1]})) with pytest.raises(RuntimeError): AddClassificationsStage._add_labels(cm, idx2label=class_labels, threshold=None) From f6b6175ae026006153a1dfa6effe36b18ae6cf9a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 16:59:01 -0700 Subject: [PATCH 108/347] WIP --- tests/stages/test_filter_detections_stage.py | 35 +++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tests/stages/test_filter_detections_stage.py b/tests/stages/test_filter_detections_stage.py index 73633f2f99..c58b312eb2 100644 --- a/tests/stages/test_filter_detections_stage.py +++ b/tests/stages/test_filter_detections_stage.py @@ -15,6 +15,7 @@ # limitations under the License. import cupy as cp +import numpy as np import pytest from morpheus.common import FilterSource @@ -52,11 +53,11 @@ def test_constructor(config): assert len(accepted_types) > 0 -@pytest.mark.use_cudf +@pytest.mark.use_pandas def test_filter_copy(config, filter_probs_df): fds = FilterDetectionsStage(config, threshold=0.5, filter_source=FilterSource.TENSOR) - probs = cp.array([[0.1, 0.5, 0.3], [0.2, 0.3, 0.4]]) + probs = np.array([[0.1, 0.5, 0.3], [0.2, 0.3, 0.4]]) mock_multi_response_message = _make_multi_response_message(filter_probs_df, probs) mock_control_message = _make_control_message(filter_probs_df, probs) @@ -67,7 +68,7 @@ def test_filter_copy(config, filter_probs_df): assert output_control_message is None # Only one row has a value above the threshold - probs = cp.array([ + probs = np.array([ [0.2, 0.4, 0.3], [0.1, 0.5, 0.8], [0.2, 0.4, 0.3], @@ -75,14 +76,15 @@ def test_filter_copy(config, filter_probs_df): mock_multi_response_message = _make_multi_response_message(filter_probs_df, probs) output_multi_response_message = fds._controller.filter_copy(mock_multi_response_message) - assert output_multi_response_message.get_meta().to_cupy().tolist() == filter_probs_df.loc[1:1, :].to_cupy().tolist() + assert output_multi_response_message.get_meta().to_numpy().tolist() == filter_probs_df.loc[ + 1:1, :].to_numpy().tolist() mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_copy(mock_control_message) - assert output_control_message.payload().get_data().to_cupy().tolist() == output_multi_response_message.get_meta( - ).to_cupy().tolist() + assert output_control_message.payload().get_data().to_numpy().tolist() == output_multi_response_message.get_meta( + ).to_numpy().tolist() # Two adjacent rows have a value above the threashold - probs = cp.array([ + probs = np.array([ [0.2, 0.4, 0.3], [0.1, 0.2, 0.3], [0.1, 0.5, 0.8], @@ -92,14 +94,15 @@ def test_filter_copy(config, filter_probs_df): mock_multi_response_message = _make_multi_response_message(filter_probs_df, probs) output_multi_response_message = fds._controller.filter_copy(mock_multi_response_message) - assert output_multi_response_message.get_meta().to_cupy().tolist() == filter_probs_df.loc[2:3, :].to_cupy().tolist() + assert output_multi_response_message.get_meta().to_numpy().tolist() == filter_probs_df.loc[ + 2:3, :].to_numpy().tolist() mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_copy(mock_control_message) - assert output_control_message.payload().get_data().to_cupy().tolist() == output_multi_response_message.get_meta( - ).to_cupy().tolist() + assert output_control_message.payload().get_data().to_numpy().tolist() == output_multi_response_message.get_meta( + ).to_numpy().tolist() # Two non-adjacent rows have a value above the threashold - probs = cp.array([ + probs = np.array([ [0.2, 0.4, 0.3], [0.1, 0.2, 0.3], [0.1, 0.5, 0.8], @@ -108,18 +111,18 @@ def test_filter_copy(config, filter_probs_df): [0.2, 0.4, 0.3], ]) - mask = cp.zeros(len(filter_probs_df), dtype=cp.bool_) + mask = np.zeros(len(filter_probs_df), dtype=np.bool_) mask[2] = True mask[4] = True mock_multi_response_message = _make_multi_response_message(filter_probs_df, probs) output_multi_response_message = fds._controller.filter_copy(mock_multi_response_message) - assert output_multi_response_message.get_meta().to_cupy().tolist() == filter_probs_df.loc[ - mask, :].to_cupy().tolist() + assert output_multi_response_message.get_meta().to_numpy().tolist() == filter_probs_df.loc[ + mask, :].to_numpy().tolist() mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_copy(mock_control_message) - assert output_control_message.payload().get_data().to_cupy().tolist() == output_multi_response_message.get_meta( - ).to_cupy().tolist() + assert output_control_message.payload().get_data().to_numpy().tolist() == output_multi_response_message.get_meta( + ).to_numpy().tolist() @pytest.mark.use_cudf From f6f814d0cdd25795fbf6ffec9eb668216acd0edf Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 Aug 2024 17:08:26 -0700 Subject: [PATCH 109/347] WIP --- tests/test_cli.py | 1 + tests/test_column_info.py | 9 --------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 718562154a..67f82b375e 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -177,6 +177,7 @@ def test_manual_seed(self, mock_manual_seed: mock.MagicMock, value: int, use_env assert result.exit_code == 0, result.output mock_manual_seed.assert_called_once_with(value) + @pytest.mark.xfail(reason="TODO: Fix this") @pytest.mark.replace_callback('pipeline_ae') def test_pipeline_ae(self, config, callback_values): """ diff --git a/tests/test_column_info.py b/tests/test_column_info.py index c40e7854ac..52c80f4e64 100644 --- a/tests/test_column_info.py +++ b/tests/test_column_info.py @@ -24,8 +24,6 @@ import pandas as pd import pytest -import cudf - from _utils import TEST_DIRS from morpheus.io.deserializers import read_file_to_df from morpheus.utils.column_info import ColumnInfo @@ -52,13 +50,6 @@ def azure_ad_logs_pdf_fixture(_azure_ad_logs_pdf: pd.DataFrame): yield _azure_ad_logs_pdf.copy(deep=True) -@pytest.fixture(name="azure_ad_logs_cdf", scope="function") -def azure_ad_logs_cdf_fixture(_azure_ad_logs_pdf: pd.DataFrame): - # cudf.from_pandas essentially does a deep copy, so we can use this to ensure that the source pandas df is not - # modified - yield cudf.from_pandas(_azure_ad_logs_pdf) - - @pytest.mark.use_python def test_dataframe_input_schema_without_json_cols(azure_ad_logs_pdf: pd.DataFrame): assert len(azure_ad_logs_pdf.columns) == 16 From c2a87467931a1b6e8a7091bdfb5d100d91459ffa Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 13:01:23 -0700 Subject: [PATCH 110/347] Add return type-hint --- python/morpheus/morpheus/stages/input/rss_source_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/stages/input/rss_source_stage.py b/python/morpheus/morpheus/stages/input/rss_source_stage.py index f548f54716..0e76d7dd5f 100644 --- a/python/morpheus/morpheus/stages/input/rss_source_stage.py +++ b/python/morpheus/morpheus/stages/input/rss_source_stage.py @@ -90,7 +90,7 @@ def __init__(self, def name(self) -> str: return "from-rss" - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return False def compute_schema(self, schema: StageSchema): From 587d567b0db9f89c86c5c956ff6e04f160f2dcbd Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 13:01:49 -0700 Subject: [PATCH 111/347] wip --- tests/test_rss_source_stage_pipe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_rss_source_stage_pipe.py b/tests/test_rss_source_stage_pipe.py index ab5a3f0951..973f00bb3f 100644 --- a/tests/test_rss_source_stage_pipe.py +++ b/tests/test_rss_source_stage_pipe.py @@ -28,10 +28,9 @@ invalid_feed_input = os.path.join(TEST_DIRS.tests_data_dir, "rss_feed_atom.xm") -@pytest.mark.use_python def test_support_cpp_node(config): url_feed_input = "https://fake.nvidia.com/rss/HomePage.xml" - rss_source_stage = RSSSourceStage(config, feed_input=url_feed_input) + rss_source_stage = RSSSourceStage(config, feed_input=[url_feed_input]) assert rss_source_stage.supports_cpp_node() is False From 86a571d1eeb51ec509eba2929595e17aa6f5e877 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 14:15:06 -0700 Subject: [PATCH 112/347] Remove support for inferring execution mode based upon the test name, this doesn't appear to be used anymore and we're changing the definition of C++ & Python mode --- tests/conftest.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index dcee03d68d..56583d0483 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -169,18 +169,12 @@ def pytest_runtest_setup(item): def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: typing.List[pytest.Item]): """ - To support old unittest style tests, try to determine the mark from the name + Remove tests that are incompatible with the current configuration. """ if config.getoption("--run_kafka") and not PYTEST_KAFKA_AVAIL: raise RuntimeError(f"--run_kafka requested but pytest_kafka not available due to: {PYTEST_KAFKA_ERROR}") - for item in items: - if "no_cpp" in item.nodeid and item.get_closest_marker("use_python") is None: - item.add_marker(pytest.mark.use_python(added_in="collection_modifyitems")) - elif "cpp" in item.nodeid and item.get_closest_marker("use_cpp") is None: - item.add_marker(pytest.mark.use_cpp(added_in="collection_modifyitems")) - def should_filter_test(item: pytest.Item): use_cpp = item.get_closest_marker("use_cpp") From 525ef2a25987202f06c92a2a2a0cac5964844596 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 14:17:07 -0700 Subject: [PATCH 113/347] Revert change --- tests/test_rss_source_stage_pipe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_rss_source_stage_pipe.py b/tests/test_rss_source_stage_pipe.py index 973f00bb3f..2f24c8b3ae 100644 --- a/tests/test_rss_source_stage_pipe.py +++ b/tests/test_rss_source_stage_pipe.py @@ -28,6 +28,7 @@ invalid_feed_input = os.path.join(TEST_DIRS.tests_data_dir, "rss_feed_atom.xm") +@pytest.mark.use_python def test_support_cpp_node(config): url_feed_input = "https://fake.nvidia.com/rss/HomePage.xml" rss_source_stage = RSSSourceStage(config, feed_input=[url_feed_input]) From 6c75451ff2cc521d6081d05e2c492faa40c17b4c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 14:57:34 -0700 Subject: [PATCH 114/347] Remove mock of open, this broke cudf in a weird way --- tests/test_write_to_file_stage.py | 35 +++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tests/test_write_to_file_stage.py b/tests/test_write_to_file_stage.py index 002d3d4808..1efc390a37 100755 --- a/tests/test_write_to_file_stage.py +++ b/tests/test_write_to_file_stage.py @@ -15,34 +15,47 @@ # limitations under the License. import os -from unittest import mock import pytest from _utils import TEST_DIRS +from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.pipeline import LinearPipeline from morpheus.stages.input.file_source_stage import FileSourceStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage +from morpheus.stages.postprocess.serialize_stage import SerializeStage +from morpheus.stages.preprocess.deserialize_stage import DeserializeStage @pytest.mark.use_python +@pytest.mark.parametrize("use_deserialize", [False, True]) @pytest.mark.parametrize("flush", [False, True]) @pytest.mark.parametrize("output_type", ["csv", "json", "jsonlines"]) -def test_file_rw_pipe(tmp_path: str, config: Config, output_type: str, flush: bool): +def test_file_rw_pipe(tmp_path: str, + config: Config, + dataset: DatasetManager, + output_type: str, + flush: bool, + use_deserialize: bool): """ Test the flush functionality of the WriteToFileStage. """ input_file = os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv") out_file = os.path.join(tmp_path, f'results.{output_type}') - # This currently works because the FileSourceStage doesn't use the builtin open function, but WriteToFileStage does - mock_open = mock.mock_open() - with mock.patch('builtins.open', mock_open): - pipe = LinearPipeline(config) - pipe.set_source(FileSourceStage(config, filename=input_file)) - pipe.add_stage(WriteToFileStage(config, filename=out_file, overwrite=False, flush=flush)) - pipe.run() + pipe = LinearPipeline(config) + pipe.set_source(FileSourceStage(config, filename=input_file)) - assert not os.path.exists(out_file) - assert mock_open().flush.called == flush + if use_deserialize: + pipe.add_stage(DeserializeStage(config)) + pipe.add_stage(SerializeStage(config)) + + pipe.add_stage(WriteToFileStage(config, filename=out_file, overwrite=False, flush=flush)) + pipe.run() + + assert os.path.exists(out_file) + + expected_df = dataset['filter_probs.csv'] + actual_df = dataset.get_df(out_file, no_cache=True) + dataset.assert_compare_df(expected_df, actual_df) From d412c89653b54a782c1e67cea9c024c6ccc0fff5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 15:03:27 -0700 Subject: [PATCH 115/347] Mark GNN Fraud tests as xfail, need to determine what to do about GPU pipelines which require Python messages --- .../gnn_fraud_detection_pipeline/test_classification_stage.py | 1 + .../gnn_fraud_detection_pipeline/test_graph_sage_stage.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py index e4935ae592..e0924d4448 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py @@ -24,6 +24,7 @@ # pylint: disable=no-name-in-module +@pytest.mark.xfail(reason="TODO: Need to determine what to do with GPU pipelines which only run in Python mode") @pytest.mark.use_python class TestClassificationStage: diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py index 886c339962..89fd0c6934 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py @@ -24,6 +24,7 @@ # pylint: disable=no-name-in-module +@pytest.mark.xfail(reason="TODO: Need to determine what to do with GPU pipelines which only run in Python mode") @pytest.mark.usefixtures("manual_seed") @pytest.mark.use_python class TestGraphSageStage: From f5c70fd14335c5c1c1d63a4aaef3df6cc3f91be1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:02:45 -0700 Subject: [PATCH 116/347] Update comment and add a dvlog --- python/morpheus/morpheus/_lib/src/messages/meta.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/_lib/src/messages/meta.cpp b/python/morpheus/morpheus/_lib/src/messages/meta.cpp index c9de4eda7f..ad500908e1 100644 --- a/python/morpheus/morpheus/_lib/src/messages/meta.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/meta.cpp @@ -252,7 +252,7 @@ std::shared_ptr MessageMetaInterfaceProxy::init_python(py::object&& auto cudf_df_cls = py::module_::import("cudf").attr("DataFrame"); if (!py::isinstance(data_frame, cudf_df_cls)) { - // Convert to cudf if it's a Pandas DF, thrown an error otherwise + // Check if we received a Pandas DF or the Python impl of MessageMeta, throw an error otherwise auto pd_df_cls = py::module_::import("pandas").attr("DataFrame"); if (py::isinstance(data_frame, pd_df_cls)) { @@ -265,6 +265,7 @@ std::shared_ptr MessageMetaInterfaceProxy::init_python(py::object&& auto msg_meta_cls = py::module_::import("morpheus.messages").attr("MessageMeta"); if (py::isinstance(data_frame, msg_meta_cls)) { + DVLOG(10) << "Converting from a Python impl of MessageMeta to C++ impl"; return init_python_meta(data_frame); } else From 4b9ba967b28a23e9047007ecc3e3bc85c9c2f048 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:03:39 -0700 Subject: [PATCH 117/347] Support comparing series --- tests/_utils/dataset_manager.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/_utils/dataset_manager.py b/tests/_utils/dataset_manager.py index c6aeb09892..da389d49c6 100644 --- a/tests/_utils/dataset_manager.py +++ b/tests/_utils/dataset_manager.py @@ -192,7 +192,7 @@ def dup_index(cls, df: DataFrameType, count: int = 1) -> DataFrameType: return cls.replace_index(df, replace_dict) @staticmethod - def _value_as_pandas(val: typing.Union[pd.DataFrame, cdf.DataFrame, cdf.Series], assert_is_pandas=True): + def _value_as_pandas(val: typing.Union[pd.DataFrame, pd.Series, cdf.DataFrame, cdf.Series], assert_is_pandas=True): if (isinstance(val, (cdf.DataFrame, cdf.Series))): return val.to_pandas() @@ -201,6 +201,16 @@ def _value_as_pandas(val: typing.Union[pd.DataFrame, cdf.DataFrame, cdf.Series], return val + @classmethod + def _value_as_pandas_df(cls, + val: typing.Union[pd.DataFrame, pd.Series, cdf.DataFrame, cdf.Series], + assert_is_pandas=True): + pval = cls._value_as_pandas(val) + if isinstance(pval, pd.Series): + pval = pval.to_frame() + + return pval + @classmethod def df_equal(cls, df_to_check: typing.Union[pd.DataFrame, cdf.DataFrame], val_to_check: typing.Any): """ @@ -242,7 +252,7 @@ def compare_df(cls, with warnings.catch_warnings(): # Ignore performance warnings from pandas triggered by the comparison warnings.filterwarnings("ignore", category=pd.errors.PerformanceWarning) - return compare_df.compare_df(cls._value_as_pandas(dfa), cls._value_as_pandas(dfb), **compare_args) + return compare_df.compare_df(cls._value_as_pandas_df(dfa), cls._value_as_pandas_df(dfb), **compare_args) @classmethod def assert_compare_df(cls, From 39b18f80fedd5cfafa1526afa1c6d86921264a48 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:04:58 -0700 Subject: [PATCH 118/347] Set log level to debug if GLOG_v is defined, add a fixture to ensure cudf helpers are loaded --- tests/conftest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 56583d0483..7128a9c0c2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -788,6 +788,9 @@ def configure_tests_logging(pytestconfig: pytest.Config): if (trace_module is not None and trace_module.find("pydevd") != -1): log_level = logging.DEBUG + if os.environ.get("GLOG_v") is not None: + log_level = logging.DEBUG + config_log_level = pytestconfig.getoption("log_level") # Overwrite the logging level if specified @@ -1125,3 +1128,10 @@ def mock_nemollm_fixture(): mock_nemollm.post_process_generate_response.return_value = {"text": "test_output"} yield mock_nemollm + + +@pytest.fixture(name="load_cudf_helper", scope="session", autouse=True) +def load_cudf_helper_fixture(): + if os.environ.get("MORPHEUS_CPU_ONLY") is None: + from morpheus.common import load_cudf_helper + load_cudf_helper() From ab8045b2fa21d92363753ddd22d6a50b78fd6c5e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:06:08 -0700 Subject: [PATCH 119/347] Replace the source file to test with, works-around an issue where pandas & cudf object columns are handled --- tests/messages/test_message_meta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/messages/test_message_meta.py b/tests/messages/test_message_meta.py index b5e2606976..740b48a0e9 100644 --- a/tests/messages/test_message_meta.py +++ b/tests/messages/test_message_meta.py @@ -329,7 +329,7 @@ def test_cast_python_to_cpp(dataset: DatasetManager): """ Test that we can cast a python MessageMeta to a C++ MessageMeta """ - df = dataset["test_dataframe.jsonlines"] + df = dataset["filter_probs.csv"] py_meta = MessageMeta(df) assert isinstance(py_meta, MessageMeta) @@ -348,7 +348,7 @@ def test_cast_cpp_to_python(dataset: DatasetManager): """ Test that we can cast a a C++ MessageMeta to a python MessageMeta """ - df = dataset["test_dataframe.jsonlines"] + df = dataset["filter_probs.csv"] cpp_meta = MessageMetaCpp(df) py_meta = MessageMeta(cpp_meta) From c650528bbe1ea5b1c5d0bb75945700964a6da164 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:16:07 -0700 Subject: [PATCH 120/347] Update python tests to use pandas/numpy --- tests/stages/test_filter_detections_stage.py | 48 ++++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/tests/stages/test_filter_detections_stage.py b/tests/stages/test_filter_detections_stage.py index c58b312eb2..5bf5456c99 100644 --- a/tests/stages/test_filter_detections_stage.py +++ b/tests/stages/test_filter_detections_stage.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import cupy as cp import numpy as np import pytest @@ -125,7 +124,7 @@ def test_filter_copy(config, filter_probs_df): ).to_numpy().tolist() -@pytest.mark.use_cudf +@pytest.mark.use_pandas @pytest.mark.parametrize('do_copy', [True, False]) @pytest.mark.parametrize('threshold', [0.1, 0.5, 0.8]) @pytest.mark.parametrize('field_name', ['v1', 'v2', 'v3', 'v4']) @@ -135,25 +134,24 @@ def test_filter_column(config, filter_probs_df, do_copy, threshold, field_name): copy=do_copy, filter_source=FilterSource.DATAFRAME, field_name=field_name) - expected_df = filter_probs_df.to_pandas() - expected_df = expected_df[expected_df[field_name] > threshold] + expected_df = filter_probs_df[filter_probs_df[field_name] > threshold] - probs = cp.zeros([len(filter_probs_df), 3], 'float') + probs = np.zeros([len(filter_probs_df), 3], 'float') mock_multi_response_message = _make_multi_response_message(filter_probs_df, probs) # All values are at or below the threshold output_multi_response_message = fds._controller.filter_copy(mock_multi_response_message) - assert output_multi_response_message.get_meta().to_cupy().tolist() == expected_df.to_numpy().tolist() + assert output_multi_response_message.get_meta().to_numpy().tolist() == expected_df.to_numpy().tolist() mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_copy(mock_control_message) - assert output_control_message.payload().get_data().to_cupy().tolist() == output_multi_response_message.get_meta( - ).to_cupy().tolist() + assert output_control_message.payload().get_data().to_numpy().tolist() == output_multi_response_message.get_meta( + ).to_numpy().tolist() -@pytest.mark.use_cudf +@pytest.mark.use_pandas def test_filter_slice(config, filter_probs_df): fds = FilterDetectionsStage(config, threshold=0.5, filter_source=FilterSource.TENSOR) - probs = cp.array([[0.1, 0.5, 0.3], [0.2, 0.3, 0.4]]) + probs = np.array([[0.1, 0.5, 0.3], [0.2, 0.3, 0.4]]) mock_multi_response_message = _make_multi_response_message(filter_probs_df, probs) # All values are at or below the threshold @@ -164,7 +162,7 @@ def test_filter_slice(config, filter_probs_df): assert len(output_control_message) == len(output_multi_response_messages) # Only one row has a value above the threshold - probs = cp.array([ + probs = np.array([ [0.2, 0.4, 0.3], [0.1, 0.5, 0.8], [0.2, 0.4, 0.3], @@ -174,17 +172,17 @@ def test_filter_slice(config, filter_probs_df): output_multi_response_messages = fds._controller.filter_slice(mock_multi_response_message) assert len(output_multi_response_messages) == 1 - assert output_multi_response_messages[0].get_meta().to_cupy().tolist() == filter_probs_df.loc[ - 1:1, :].to_cupy().tolist() + assert output_multi_response_messages[0].get_meta().to_numpy().tolist() == filter_probs_df.loc[ + 1:1, :].to_numpy().tolist() mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_slice(mock_control_message) assert len(output_control_message) == len(output_multi_response_messages) - assert output_control_message[0].payload().get_data().to_cupy().tolist( - ) == output_multi_response_messages[0].get_meta().to_cupy().tolist() + assert output_control_message[0].payload().get_data().to_numpy().tolist( + ) == output_multi_response_messages[0].get_meta().to_numpy().tolist() # Two adjacent rows have a value above the threashold - probs = cp.array([ + probs = np.array([ [0.2, 0.4, 0.3], [0.1, 0.2, 0.3], [0.1, 0.5, 0.8], @@ -198,17 +196,17 @@ def test_filter_slice(config, filter_probs_df): assert len(output_multi_response_messages) == 1 assert output_multi_response_messages[0].offset == 2 assert output_multi_response_messages[0].count == 2 - assert output_multi_response_messages[0].get_meta().to_cupy().tolist() == filter_probs_df.loc[ - 2:3, :].to_cupy().tolist() + assert output_multi_response_messages[0].get_meta().to_numpy().tolist() == filter_probs_df.loc[ + 2:3, :].to_numpy().tolist() mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_slice(mock_control_message) assert len(output_control_message) == len(output_multi_response_messages) - assert output_control_message[0].payload().get_data().to_cupy().tolist( - ) == output_multi_response_messages[0].get_meta().to_cupy().tolist() + assert output_control_message[0].payload().get_data().to_numpy().tolist( + ) == output_multi_response_messages[0].get_meta().to_numpy().tolist() # Two non-adjacent rows have a value above the threashold - probs = cp.array([ + probs = np.array([ [0.2, 0.4, 0.3], [0.1, 0.2, 0.3], [0.1, 0.5, 0.8], @@ -230,8 +228,8 @@ def test_filter_slice(config, filter_probs_df): assert multi_response_msg2.offset == 4 assert multi_response_msg2.count == 1 - assert multi_response_msg1.get_meta().to_cupy().tolist() == filter_probs_df.loc[2:2, :].to_cupy().tolist() - assert multi_response_msg2.get_meta().to_cupy().tolist() == filter_probs_df.loc[4:4, :].to_cupy().tolist() + assert multi_response_msg1.get_meta().to_numpy().tolist() == filter_probs_df.loc[2:2, :].to_numpy().tolist() + assert multi_response_msg2.get_meta().to_numpy().tolist() == filter_probs_df.loc[4:4, :].to_numpy().tolist() mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_slice(mock_control_message) @@ -240,5 +238,5 @@ def test_filter_slice(config, filter_probs_df): assert control_msg1.payload().count == multi_response_msg1.count assert control_msg2.payload().count == multi_response_msg2.count - assert control_msg1.payload().get_data().to_cupy().tolist() == multi_response_msg1.get_meta().to_cupy().tolist() - assert control_msg2.payload().get_data().to_cupy().tolist() == multi_response_msg2.get_meta().to_cupy().tolist() + assert control_msg1.payload().get_data().to_numpy().tolist() == multi_response_msg1.get_meta().to_numpy().tolist() + assert control_msg2.payload().get_data().to_numpy().tolist() == multi_response_msg2.get_meta().to_numpy().tolist() From 92b97bb5d75584081419211e567c45109b298b8f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:24:51 -0700 Subject: [PATCH 121/347] Remove tests for removed Python impls --- tests/stages/test_preprocess_fil_stage.py | 50 ------------ tests/stages/test_preprocess_nlp_stage.py | 97 ----------------------- 2 files changed, 147 deletions(-) diff --git a/tests/stages/test_preprocess_fil_stage.py b/tests/stages/test_preprocess_fil_stage.py index cdbe66dafe..5c3bfb518b 100644 --- a/tests/stages/test_preprocess_fil_stage.py +++ b/tests/stages/test_preprocess_fil_stage.py @@ -46,53 +46,3 @@ def test_constructor(config: Config): accepted_union = typing.Union[stage.accepted_types()] assert typing_utils.issubtype(MultiMessage, accepted_union) assert typing_utils.issubtype(ControlMessage, accepted_union) - - -def test_process_control_message(config: Config): - stage = PreprocessFILStage(config) - input_cm = ControlMessage() - df = cudf.DataFrame({"data": [1, 2, 3]}) - meta = MessageMeta(df) - input_cm.payload(meta) - - output_cm = stage.pre_process_batch(input_cm, stage._fea_length, stage.features) - assert cp.array_equal(output_cm.tensors().get_tensor("input__0"), cp.asarray(df.to_cupy())) - expect_seg_ids = cp.zeros((df.shape[0], 3), dtype=cp.uint32) - expect_seg_ids[:, 0] = cp.arange(0, df.shape[0], dtype=cp.uint32) - expect_seg_ids[:, 2] = stage._fea_length - 1 - assert cp.array_equal(output_cm.tensors().get_tensor("seq_ids"), expect_seg_ids) - - -def test_process_multi_message(config: Config): - stage = PreprocessFILStage(config) - df = cudf.DataFrame({"data": [1, 2, 3]}) - meta = MessageMeta(df) - mess_offset = 0 - input_multi_message = MultiMessage(meta=meta, mess_offset=mess_offset, mess_count=3) - - output_infer_message = stage.pre_process_batch(input_multi_message, stage._fea_length, stage.features) - assert cp.array_equal(output_infer_message.input__0, cp.asarray(df.to_cupy())) - expect_seg_ids = cp.zeros((df.shape[0], 3), dtype=cp.uint32) - expect_seg_ids[:, 0] = cp.arange(0, df.shape[0], dtype=cp.uint32) - expect_seg_ids[:, 2] = stage._fea_length - 1 - assert cp.array_equal(output_infer_message.seq_ids, expect_seg_ids) - - -def test_process_control_message_and_multi_message(config: Config): - stage = PreprocessFILStage(config) - df = cudf.DataFrame({"data": [1, 2, 3]}) - meta = MessageMeta(df) - input_control_message = ControlMessage() - input_control_message.payload(meta) - - mess_offset = 0 - input_multi_message = MultiMessage(meta=meta, mess_offset=mess_offset, mess_count=3) - - output_control_message = stage.pre_process_batch(input_control_message, stage._fea_length, stage.features) - - output_infer_message = stage.pre_process_batch(input_multi_message, stage._fea_length, stage.features) - - # Check if each tensor in the control message is equal to the corresponding tensor in the inference message - for tensor_key in output_control_message.tensors().tensor_names: - assert cp.array_equal(output_control_message.tensors().get_tensor(tensor_key), - getattr(output_infer_message, tensor_key)) diff --git a/tests/stages/test_preprocess_nlp_stage.py b/tests/stages/test_preprocess_nlp_stage.py index 9c202a168d..a8328ee018 100644 --- a/tests/stages/test_preprocess_nlp_stage.py +++ b/tests/stages/test_preprocess_nlp_stage.py @@ -66,100 +66,3 @@ def test_constructor(config: Config): accepted_union = typing.Union[stage.accepted_types()] assert typing_utils.issubtype(MultiMessage, accepted_union) assert typing_utils.issubtype(ControlMessage, accepted_union) - - -@patch("morpheus.stages.preprocess.preprocess_nlp_stage.tokenize_text_series") -def test_process_control_message(mock_tokenize_text_series, config: Config): - mock_tokenized = Mock() - mock_tokenized.input_ids = cp.array([[1, 2], [1, 2]]) - mock_tokenized.input_mask = cp.array([[3, 4], [3, 4]]) - mock_tokenized.segment_ids = cp.array([[0, 0], [1, 1]]) - mock_tokenize_text_series.return_value = mock_tokenized - - stage = PreprocessNLPStage(config) - input_cm = ControlMessage() - df = cudf.DataFrame({"data": ["a", "b", "c"]}) - meta = MessageMeta(df) - input_cm.payload(meta) - - output_cm = stage.pre_process_batch(input_cm, - stage._vocab_hash_file, - stage._do_lower_case, - stage._seq_length, - stage._stride, - stage._truncation, - stage._add_special_tokens, - stage._column) - assert output_cm.get_metadata("inference_memory_params") == {"inference_type": "nlp"} - assert cp.array_equal(output_cm.tensors().get_tensor("input_ids"), mock_tokenized.input_ids) - assert cp.array_equal(output_cm.tensors().get_tensor("input_mask"), mock_tokenized.input_mask) - assert cp.array_equal(output_cm.tensors().get_tensor("seq_ids"), mock_tokenized.segment_ids) - - -@patch("morpheus.stages.preprocess.preprocess_nlp_stage.tokenize_text_series") -def test_process_multi_message(mock_tokenize_text_series, config: Config): - mock_tokenized = Mock() - mock_tokenized.input_ids = cp.array([[1, 2], [1, 2]]) - mock_tokenized.input_mask = cp.array([[3, 4], [3, 4]]) - mock_tokenized.segment_ids = cp.array([[0, 0], [1, 1]]) - mock_tokenize_text_series.return_value = mock_tokenized - - stage = PreprocessNLPStage(config) - df = cudf.DataFrame({"data": ["a", "b", "c"]}) - meta = MessageMeta(df) - mess_offset = 0 - input_multi_message = MultiMessage(meta=meta, mess_offset=mess_offset, mess_count=2) - - output_infer_message = stage.pre_process_batch(input_multi_message, - stage._vocab_hash_file, - stage._do_lower_case, - stage._seq_length, - stage._stride, - stage._truncation, - stage._add_special_tokens, - stage._column) - assert cp.array_equal(output_infer_message.input_ids, mock_tokenized.input_ids) - assert cp.array_equal(output_infer_message.input_mask, mock_tokenized.input_mask) - mock_tokenized.segment_ids[:, 0] = mock_tokenized.segment_ids[:, 0] + mess_offset - assert cp.array_equal(output_infer_message.seq_ids, mock_tokenized.segment_ids) - - -@patch("morpheus.stages.preprocess.preprocess_nlp_stage.tokenize_text_series") -def test_process_control_message_and_multi_message(mock_tokenize_text_series, config: Config): - mock_tokenized = Mock() - mock_tokenized.input_ids = cp.array([[1, 2], [1, 2]]) - mock_tokenized.input_mask = cp.array([[3, 4], [3, 4]]) - mock_tokenized.segment_ids = cp.array([[0, 0], [1, 1]]) - mock_tokenize_text_series.return_value = mock_tokenized - - stage = PreprocessNLPStage(config) - df = cudf.DataFrame({"data": ["a", "b", "c"]}) - meta = MessageMeta(df) - input_control_message = ControlMessage() - input_control_message.payload(meta) - - mess_offset = 0 - input_multi_message = MultiMessage(meta=meta, mess_offset=mess_offset, mess_count=2) - - output_control_message = stage.pre_process_batch(input_control_message, - stage._vocab_hash_file, - stage._do_lower_case, - stage._seq_length, - stage._stride, - stage._truncation, - stage._add_special_tokens, - stage._column) - - output_infer_message = stage.pre_process_batch(input_multi_message, - stage._vocab_hash_file, - stage._do_lower_case, - stage._seq_length, - stage._stride, - stage._truncation, - stage._add_special_tokens, - stage._column) - - # Check if each tensor in the control message is equal to the corresponding tensor in the inference message - for tensor_key in output_control_message.tensors().tensor_names: - assert cp.array_equal(output_control_message.tensors().get_tensor(tensor_key), - getattr(output_infer_message, tensor_key)) From c43b95c60dc7decf47ce660476853d8a8baa9dfd Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:55:27 -0700 Subject: [PATCH 122/347] Misc cleanups --- python/morpheus/morpheus/utils/type_utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index baa8021d60..daacfa9b09 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -206,16 +206,23 @@ def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: @typing.overload -def get_df_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: +def get_df_pkg(selector: DataFrameTypeStr) -> types.ModuleType: ... -def get_df_pkg(execution_mode: ExecutionMode) -> types.ModuleType: +@typing.overload +def get_df_pkg(selector: ExecutionMode) -> types.ModuleType: + ... + + +def get_df_pkg(selector: ExecutionMode | DataFrameTypeStr) -> types.ModuleType: """ Return the appropriate DataFrame package based on the execution mode. """ - if not isinstance(execution_mode, ExecutionMode): - execution_mode = df_type_str_to_exec_mode(execution_mode) + if not isinstance(selector, ExecutionMode): + execution_mode = df_type_str_to_exec_mode(selector) + else: + execution_mode = selector if execution_mode == ExecutionMode.GPU: import cudf From 7e7a9f520ad1f20be49b33e570e87543c4627805 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Sep 2024 16:56:11 -0700 Subject: [PATCH 123/347] Reload type_utils along with config, allowing the enum in config to match the one imported in type_utils --- tests/test_messages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_messages.py b/tests/test_messages.py index 22c7947188..4b365f17fd 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -23,6 +23,7 @@ import morpheus._lib.messages as _messages import morpheus.config +import morpheus.utils.type_utils from morpheus import messages from morpheus.messages.memory import inference_memory from morpheus.messages.memory import response_memory @@ -172,7 +173,7 @@ def test_constructor_cpp(): check_all_messages(morpheus.config.CppConfig.get_should_use_cpp(), False) -@pytest.mark.reload_modules(morpheus.config) +@pytest.mark.reload_modules([morpheus.config, morpheus.utils.type_utils]) @pytest.mark.usefixtures("reload_modules", "restore_environ") def test_constructor_env(): # Set the NO_CPP flag which should disable C++ regardless From 688279ae34a9a5c430f1f4523f238530b3f082ef Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 08:41:55 -0700 Subject: [PATCH 124/347] Remove test that requires reloading the morpheus.config module, this breaks too many other tests that rely on calling isinstance(mode, ExecutionMode) --- tests/test_messages.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/test_messages.py b/tests/test_messages.py index 4b365f17fd..d0dfdb3101 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -171,16 +171,3 @@ def check_all_messages(should_be_cpp: bool, no_cpp_class: bool): @pytest.mark.usefixtures("use_cpp") def test_constructor_cpp(): check_all_messages(morpheus.config.CppConfig.get_should_use_cpp(), False) - - -@pytest.mark.reload_modules([morpheus.config, morpheus.utils.type_utils]) -@pytest.mark.usefixtures("reload_modules", "restore_environ") -def test_constructor_env(): - # Set the NO_CPP flag which should disable C++ regardless - os.environ['MORPHEUS_NO_CPP'] = '1' - - # Reload the CppConfig class just in case - importlib.reload(morpheus.config) - - # Check all messages. Should be False regardless due to the environment variable - check_all_messages(False, False) From 2a0c5ead153a0db4e7382056dbed2ca2ee367d5f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 08:57:24 -0700 Subject: [PATCH 125/347] Remove python-mode tests for GPU based tests --- tests/test_abp.py | 147 --------------------------- tests/test_dfp.py | 3 + tests/test_dfp_kafka.py | 2 + tests/test_phishing.py | 68 ------------- tests/test_phishing_kafka.py | 96 ----------------- tests/test_triton_inference_stage.py | 56 ---------- 6 files changed, 5 insertions(+), 367 deletions(-) diff --git a/tests/test_abp.py b/tests/test_abp.py index 334f87afeb..0cec1331a9 100755 --- a/tests/test_abp.py +++ b/tests/test_abp.py @@ -50,70 +50,6 @@ MODEL_MAX_BATCH_SIZE = 1024 -@pytest.mark.slow -@pytest.mark.use_python -@mock.patch('tritonclient.grpc.InferenceServerClient') -def test_abp_no_cpp(mock_triton_client: mock.MagicMock, config: Config, tmp_path: str, morpheus_log_level: int): - mock_metadata = { - "inputs": [{ - 'name': 'input__0', 'datatype': 'FP32', "shape": [-1, FEATURE_LENGTH] - }], - "outputs": [{ - 'name': 'output__0', 'datatype': 'FP32', 'shape': ['-1', '1'] - }] - } - mock_model_config = {"config": {"max_batch_size": MODEL_MAX_BATCH_SIZE}} - - mock_triton_client.return_value = mock_triton_client - mock_triton_client.is_server_live.return_value = True - mock_triton_client.is_server_ready.return_value = True - mock_triton_client.is_model_ready.return_value = True - mock_triton_client.get_model_metadata.return_value = mock_metadata - mock_triton_client.get_model_config.return_value = mock_model_config - - data = np.loadtxt(os.path.join(TEST_DIRS.tests_data_dir, 'triton_abp_inf_results.csv'), delimiter=',') - inf_results = np.split(data, range(MODEL_MAX_BATCH_SIZE, len(data), MODEL_MAX_BATCH_SIZE)) - - async_infer = mk_async_infer(inf_results) - - mock_triton_client.async_infer.side_effect = async_infer - - config.mode = PipelineModes.FIL - config.class_labels = ["mining"] - config.model_max_batch_size = MODEL_MAX_BATCH_SIZE - config.pipeline_batch_size = 1024 - config.feature_length = FEATURE_LENGTH - config.edge_buffer_size = 128 - config.num_threads = 1 - - config.fil = ConfigFIL() - config.fil.feature_columns = load_labels_file(os.path.join(TEST_DIRS.data_dir, 'columns_fil.txt')) - - val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'abp-validation-data.jsonlines') - out_file = os.path.join(tmp_path, 'results.csv') - results_file_name = os.path.join(tmp_path, 'results.json') - - pipe = LinearPipeline(config) - pipe.set_source(FileSourceStage(config, filename=val_file_name, iterative=False)) - pipe.add_stage(DeserializeStage(config)) - pipe.add_stage(PreprocessFILStage(config)) - pipe.add_stage( - TritonInferenceStage(config, model_name='abp-nvsmi-xgb', server_url='test:0000', force_convert_inputs=True)) - pipe.add_stage( - MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf", log_level=morpheus_log_level)) - pipe.add_stage(AddClassificationsStage(config)) - pipe.add_stage(AddScoresStage(config, prefix="score_")) - pipe.add_stage( - ValidationStage(config, val_file_name=val_file_name, results_file_name=results_file_name, rel_tol=0.05)) - pipe.add_stage(SerializeStage(config)) - pipe.add_stage(WriteToFileStage(config, filename=out_file, overwrite=False)) - - pipe.run() - compare_class_to_scores(out_file, config.class_labels, '', 'score_', threshold=0.5) - results = calc_error_val(results_file_name) - assert results.diff_rows == 0 - - @pytest.mark.slow @pytest.mark.use_cpp @pytest.mark.usefixtures("launch_mock_triton") @@ -164,89 +100,6 @@ def test_abp_cpp(config: Config, tmp_path: str, message_type: type, morpheus_log assert results.diff_rows == 0 -@pytest.mark.slow -@pytest.mark.use_python -@mock.patch('tritonclient.grpc.InferenceServerClient') -def test_abp_multi_segment_no_cpp(mock_triton_client: mock.MagicMock, - config: Config, - tmp_path: str, - morpheus_log_level: int): - mock_metadata = { - "inputs": [{ - 'name': 'input__0', 'datatype': 'FP32', "shape": [-1, FEATURE_LENGTH] - }], - "outputs": [{ - 'name': 'output__0', 'datatype': 'FP32', 'shape': ['-1', '1'] - }] - } - mock_model_config = {"config": {"max_batch_size": MODEL_MAX_BATCH_SIZE}} - - mock_triton_client.return_value = mock_triton_client - mock_triton_client.is_server_live.return_value = True - mock_triton_client.is_server_ready.return_value = True - mock_triton_client.is_model_ready.return_value = True - mock_triton_client.get_model_metadata.return_value = mock_metadata - mock_triton_client.get_model_config.return_value = mock_model_config - - data = np.loadtxt(os.path.join(TEST_DIRS.tests_data_dir, 'triton_abp_inf_results.csv'), delimiter=',') - inf_results = np.split(data, range(MODEL_MAX_BATCH_SIZE, len(data), MODEL_MAX_BATCH_SIZE)) - - async_infer = mk_async_infer(inf_results) - - mock_triton_client.async_infer.side_effect = async_infer - - config.mode = PipelineModes.FIL - config.class_labels = ["mining"] - config.model_max_batch_size = MODEL_MAX_BATCH_SIZE - config.pipeline_batch_size = 1024 - config.feature_length = FEATURE_LENGTH - config.edge_buffer_size = 128 - config.num_threads = 1 - - config.fil = ConfigFIL() - config.fil.feature_columns = load_labels_file(os.path.join(TEST_DIRS.data_dir, 'columns_fil.txt')) - - val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'abp-validation-data.jsonlines') - out_file = os.path.join(tmp_path, 'results.csv') - results_file_name = os.path.join(tmp_path, 'results.json') - - pipe = LinearPipeline(config) - pipe.set_source(FileSourceStage(config, filename=val_file_name, iterative=False)) - pipe.add_stage(DeserializeStage(config)) - - pipe.add_segment_boundary(MultiMessage) # Boundary 1 - - pipe.add_stage(PreprocessFILStage(config)) - - pipe.add_segment_boundary(MultiInferenceMessage) # Boundary 2 - - pipe.add_stage( - TritonInferenceStage(config, model_name='abp-nvsmi-xgb', server_url='test:0000', force_convert_inputs=True)) - - pipe.add_segment_boundary(MultiResponseMessage) # Boundary 3 - - pipe.add_stage( - MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf", log_level=morpheus_log_level)) - pipe.add_stage(AddClassificationsStage(config)) - - pipe.add_segment_boundary(MultiResponseMessage) # Boundary 4 - - pipe.add_stage( - ValidationStage(config, val_file_name=val_file_name, results_file_name=results_file_name, rel_tol=0.05)) - - pipe.add_segment_boundary(MultiResponseMessage) # Boundary 5 - - pipe.add_stage(SerializeStage(config)) - - pipe.add_segment_boundary(MessageMeta) # Boundary 6 - - pipe.add_stage(WriteToFileStage(config, filename=out_file, overwrite=False)) - - pipe.run() - results = calc_error_val(results_file_name) - assert results.diff_rows == 0 - - @pytest.mark.slow @pytest.mark.use_cpp @pytest.mark.usefixtures("launch_mock_triton") diff --git a/tests/test_dfp.py b/tests/test_dfp.py index 2f3bacbdae..2a8a9ac5df 100755 --- a/tests/test_dfp.py +++ b/tests/test_dfp.py @@ -46,6 +46,7 @@ # End-to-end test intended to imitate the DFP validation test +@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow @pytest.mark.use_python @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @@ -132,6 +133,7 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, config: Config, tmp_path: str, morph assert results.diff_rows == 0 +@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow @pytest.mark.use_python @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @@ -215,6 +217,7 @@ def test_dfp_user123(mock_ae: mock.MagicMock, config: Config, tmp_path: str, mor assert results.diff_rows == 0 +@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow @pytest.mark.use_python @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) diff --git a/tests/test_dfp_kafka.py b/tests/test_dfp_kafka.py index d952b00ae7..9046d39a80 100755 --- a/tests/test_dfp_kafka.py +++ b/tests/test_dfp_kafka.py @@ -48,6 +48,7 @@ from kafka import KafkaConsumer +@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.kafka @pytest.mark.slow @pytest.mark.use_python @@ -152,6 +153,7 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, assert results['diff_rows'] == 0 +@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.kafka @pytest.mark.slow @pytest.mark.use_python diff --git a/tests/test_phishing.py b/tests/test_phishing.py index 77e752ef3f..73e453fd49 100755 --- a/tests/test_phishing.py +++ b/tests/test_phishing.py @@ -42,74 +42,6 @@ MODEL_MAX_BATCH_SIZE = 32 -@pytest.mark.slow -@pytest.mark.use_python -@mock.patch('tritonclient.grpc.InferenceServerClient') -def test_email_no_cpp(mock_triton_client: mock.MagicMock, config: Config, tmp_path: str, morpheus_log_level: int): - mock_metadata = { - "inputs": [{ - "name": "input_ids", "datatype": "INT64", "shape": [-1, FEATURE_LENGTH] - }, { - "name": "attention_mask", "datatype": "INT64", "shape": [-1, FEATURE_LENGTH] - }], - "outputs": [{ - "name": "output", "datatype": "FP32", "shape": [-1, 2] - }] - } - mock_model_config = {"config": {"max_batch_size": MODEL_MAX_BATCH_SIZE}} - - mock_triton_client.return_value = mock_triton_client - mock_triton_client.is_server_live.return_value = True - mock_triton_client.is_server_ready.return_value = True - mock_triton_client.is_model_ready.return_value = True - mock_triton_client.get_model_metadata.return_value = mock_metadata - mock_triton_client.get_model_config.return_value = mock_model_config - - data = np.loadtxt(os.path.join(TEST_DIRS.tests_data_dir, 'triton_phishing_inf_results.csv'), delimiter=',') - inf_results = np.split(data, range(MODEL_MAX_BATCH_SIZE, len(data), MODEL_MAX_BATCH_SIZE)) - - async_infer = mk_async_infer(inf_results) - - mock_triton_client.async_infer.side_effect = async_infer - - config.mode = PipelineModes.NLP - config.class_labels = load_labels_file(os.path.join(TEST_DIRS.data_dir, "labels_phishing.txt")) - config.model_max_batch_size = MODEL_MAX_BATCH_SIZE - config.pipeline_batch_size = 1024 - config.feature_length = FEATURE_LENGTH - config.edge_buffer_size = 128 - config.num_threads = 1 - - val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'phishing-email-validation-data.jsonlines') - vocab_file_name = os.path.join(TEST_DIRS.data_dir, 'bert-base-uncased-hash.txt') - out_file = os.path.join(tmp_path, 'results.csv') - results_file_name = os.path.join(tmp_path, 'results.json') - - pipe = LinearPipeline(config) - pipe.set_source(FileSourceStage(config, filename=val_file_name, iterative=False)) - pipe.add_stage(DeserializeStage(config)) - pipe.add_stage( - PreprocessNLPStage(config, - vocab_hash_file=vocab_file_name, - truncation=True, - do_lower_case=True, - add_special_tokens=False)) - pipe.add_stage( - TritonInferenceStage(config, model_name='phishing-bert-onnx', server_url='test:0000', - force_convert_inputs=True)) - pipe.add_stage( - MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf", log_level=morpheus_log_level)) - pipe.add_stage(AddClassificationsStage(config, labels=["is_phishing"], threshold=0.7)) - pipe.add_stage( - ValidationStage(config, val_file_name=val_file_name, results_file_name=results_file_name, rel_tol=0.05)) - pipe.add_stage(SerializeStage(config)) - pipe.add_stage(WriteToFileStage(config, filename=out_file, overwrite=False)) - - pipe.run() - results = calc_error_val(results_file_name) - assert results.diff_rows == 153 - - @pytest.mark.slow @pytest.mark.use_cpp @pytest.mark.usefixtures("launch_mock_triton") diff --git a/tests/test_phishing_kafka.py b/tests/test_phishing_kafka.py index 1a04061cc9..d2331601c9 100755 --- a/tests/test_phishing_kafka.py +++ b/tests/test_phishing_kafka.py @@ -51,102 +51,6 @@ MODEL_MAX_BATCH_SIZE = 32 -@pytest.mark.kafka -@pytest.mark.slow -@pytest.mark.use_python -@mock.patch('tritonclient.grpc.InferenceServerClient') -def test_email_no_cpp(mock_triton_client: mock.MagicMock, - dataset_pandas: DatasetManager, - config: Config, - kafka_bootstrap_servers: str, - kafka_topics: KafkaTopics, - kafka_consumer: "KafkaConsumer", - morpheus_log_level: int): - mock_metadata = { - "inputs": [{ - "name": "input_ids", "datatype": "INT64", "shape": [-1, FEATURE_LENGTH] - }, { - "name": "attention_mask", "datatype": "INT64", "shape": [-1, FEATURE_LENGTH] - }], - "outputs": [{ - "name": "output", "datatype": "FP32", "shape": [-1, 2] - }] - } - mock_model_config = {"config": {"max_batch_size": MODEL_MAX_BATCH_SIZE}} - - mock_triton_client.return_value = mock_triton_client - mock_triton_client.is_server_live.return_value = True - mock_triton_client.is_server_ready.return_value = True - mock_triton_client.is_model_ready.return_value = True - mock_triton_client.get_model_metadata.return_value = mock_metadata - mock_triton_client.get_model_config.return_value = mock_model_config - - data = np.loadtxt(os.path.join(TEST_DIRS.tests_data_dir, 'triton_phishing_inf_results.csv'), delimiter=',') - inf_results = np.split(data, range(MODEL_MAX_BATCH_SIZE, len(data), MODEL_MAX_BATCH_SIZE)) - - async_infer = mk_async_infer(inf_results) - - mock_triton_client.async_infer.side_effect = async_infer - - config.mode = PipelineModes.NLP - config.class_labels = load_labels_file(os.path.join(TEST_DIRS.data_dir, "labels_phishing.txt")) - config.model_max_batch_size = MODEL_MAX_BATCH_SIZE - config.pipeline_batch_size = 1024 - config.feature_length = FEATURE_LENGTH - config.edge_buffer_size = 128 - config.num_threads = 1 - - val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'phishing-email-validation-data.jsonlines') - vocab_file_name = os.path.join(TEST_DIRS.data_dir, 'bert-base-uncased-hash.txt') - - num_records = write_file_to_kafka(kafka_bootstrap_servers, kafka_topics.input_topic, val_file_name) - - # Disabling commits due to known issue in Python impl: https://github.com/nv-morpheus/Morpheus/issues/294 - pipe = LinearPipeline(config) - pipe.set_source( - KafkaSourceStage(config, - bootstrap_servers=kafka_bootstrap_servers, - input_topic=kafka_topics.input_topic, - auto_offset_reset="earliest", - poll_interval="1seconds", - disable_commit=True, - stop_after=num_records)) - pipe.add_stage(DeserializeStage(config)) - pipe.add_stage( - PreprocessNLPStage(config, - vocab_hash_file=vocab_file_name, - truncation=True, - do_lower_case=True, - add_special_tokens=False)) - pipe.add_stage( - TritonInferenceStage(config, model_name='phishing-bert-onnx', server_url='test:0000', - force_convert_inputs=True)) - pipe.add_stage( - MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf", log_level=morpheus_log_level)) - pipe.add_stage(AddClassificationsStage(config, labels=["is_phishing"], threshold=0.7)) - pipe.add_stage(SerializeStage(config)) - pipe.add_stage( - WriteToKafkaStage(config, bootstrap_servers=kafka_bootstrap_servers, output_topic=kafka_topics.output_topic)) - - pipe.run() - - val_df = dataset_pandas[val_file_name] - - output_buf = StringIO() - for rec in kafka_consumer: - output_buf.write(f"{rec.value.decode('utf-8')}\n") - - output_buf.seek(0) - output_df = pandas.read_json(output_buf, lines=True) - output_df = filter_null_data(output_df) - - assert len(output_df) == num_records - - results = compare_df(val_df, output_df, exclude_columns=[r'^ID$', r'^_ts_'], rel_tol=0.05) - - assert results['diff_rows'] == 153 - - @pytest.mark.kafka @pytest.mark.slow @pytest.mark.use_cpp diff --git a/tests/test_triton_inference_stage.py b/tests/test_triton_inference_stage.py index a361c712a1..fee1495cef 100644 --- a/tests/test_triton_inference_stage.py +++ b/tests/test_triton_inference_stage.py @@ -147,59 +147,3 @@ def test_stage_get_inference_worker(config: Config, pipeline_mode: PipelineModes worker = stage._get_inference_worker(ProducerConsumerQueue()) assert isinstance(worker, TritonInferenceWorker) assert worker.needs_logits == expexted_needs_logits - - -@pytest.mark.slow -@pytest.mark.use_python -@pytest.mark.parametrize('num_records', [1000, 2000, 4000]) -@mock.patch('tritonclient.grpc.InferenceServerClient') -def test_triton_stage_pipe(mock_triton_client, config, num_records): - mock_metadata = { - "inputs": [{ - 'name': 'input__0', 'datatype': 'FP32', "shape": [-1, 1] - }], - "outputs": [{ - 'name': 'output__0', 'datatype': 'FP32', 'shape': ['-1', '1'] - }] - } - mock_model_config = {"config": {"max_batch_size": MODEL_MAX_BATCH_SIZE}} - - input_df = pd.DataFrame(data={'v': (i * 2 for i in range(num_records))}) - expected_df = pd.DataFrame(data={'v': input_df['v'], 'score_test': input_df['v']}) - - mock_triton_client.return_value = mock_triton_client - mock_triton_client.is_server_live.return_value = True - mock_triton_client.is_server_ready.return_value = True - mock_triton_client.is_model_ready.return_value = True - mock_triton_client.get_model_metadata.return_value = mock_metadata - mock_triton_client.get_model_config.return_value = mock_model_config - - inf_results = np.split(input_df.values, range(MODEL_MAX_BATCH_SIZE, len(input_df), MODEL_MAX_BATCH_SIZE)) - - async_infer = mk_async_infer(inf_results) - mock_triton_client.async_infer.side_effect = async_infer - - config.mode = PipelineModes.FIL - config.class_labels = ["test"] - config.model_max_batch_size = MODEL_MAX_BATCH_SIZE - config.pipeline_batch_size = 1024 - config.feature_length = 1 - config.edge_buffer_size = 128 - config.num_threads = 1 - - config.fil = ConfigFIL() - config.fil.feature_columns = ['v'] - - pipe = LinearPipeline(config) - pipe.set_source(InMemorySourceStage(config, [cudf.DataFrame(input_df)])) - pipe.add_stage(DeserializeStage(config)) - pipe.add_stage(PreprocessFILStage(config)) - pipe.add_stage( - TritonInferenceStage(config, model_name='abp-nvsmi-xgb', server_url='test:0000', force_convert_inputs=True)) - pipe.add_stage(AddScoresStage(config, prefix="score_")) - pipe.add_stage(SerializeStage(config)) - comp_stage = pipe.add_stage(CompareDataFrameStage(config, expected_df)) - - pipe.run() - - assert_results(comp_stage.get_results()) From de78d228f4fbbf9e71adb27aeb912b8c08f3ebc8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 09:20:26 -0700 Subject: [PATCH 126/347] Remove python-mode tests for GPU based tests --- tests/test_abp_kafka.py | 93 ---------------------------------------- tests/test_sid_kafka.py | 95 ----------------------------------------- 2 files changed, 188 deletions(-) diff --git a/tests/test_abp_kafka.py b/tests/test_abp_kafka.py index 46306ff29c..02a6e795b5 100755 --- a/tests/test_abp_kafka.py +++ b/tests/test_abp_kafka.py @@ -52,99 +52,6 @@ MODEL_MAX_BATCH_SIZE = 1024 -@pytest.mark.kafka -@pytest.mark.slow -@pytest.mark.use_python -@mock.patch('tritonclient.grpc.InferenceServerClient') -def test_abp_no_cpp(mock_triton_client: mock.MagicMock, - dataset_pandas: DatasetManager, - config: Config, - kafka_bootstrap_servers: str, - kafka_topics: KafkaTopics, - kafka_consumer: "KafkaConsumer", - morpheus_log_level: int): - mock_metadata = { - "inputs": [{ - 'name': 'input__0', 'datatype': 'FP32', "shape": [-1, FEATURE_LENGTH] - }], - "outputs": [{ - 'name': 'output__0', 'datatype': 'FP32', 'shape': ['-1', '1'] - }] - } - mock_model_config = {"config": {"max_batch_size": MODEL_MAX_BATCH_SIZE}} - - mock_triton_client.return_value = mock_triton_client - mock_triton_client.is_server_live.return_value = True - mock_triton_client.is_server_ready.return_value = True - mock_triton_client.is_model_ready.return_value = True - mock_triton_client.get_model_metadata.return_value = mock_metadata - mock_triton_client.get_model_config.return_value = mock_model_config - - data = np.loadtxt(os.path.join(TEST_DIRS.tests_data_dir, 'triton_abp_inf_results.csv'), delimiter=',') - inf_results = np.split(data, range(MODEL_MAX_BATCH_SIZE, len(data), MODEL_MAX_BATCH_SIZE)) - - async_infer = mk_async_infer(inf_results) - - mock_triton_client.async_infer.side_effect = async_infer - - config.mode = PipelineModes.FIL - config.class_labels = ["mining"] - config.model_max_batch_size = MODEL_MAX_BATCH_SIZE - config.pipeline_batch_size = 1024 - config.feature_length = FEATURE_LENGTH - config.edge_buffer_size = 128 - config.num_threads = 1 - - config.fil = ConfigFIL() - config.fil.feature_columns = load_labels_file(os.path.join(TEST_DIRS.data_dir, 'columns_fil.txt')) - - val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'abp-validation-data.jsonlines') - - # Fill our topic with the input data - num_records = write_file_to_kafka(kafka_bootstrap_servers, kafka_topics.input_topic, val_file_name) - - pipe = LinearPipeline(config) - pipe.set_source( - KafkaSourceStage(config, - bootstrap_servers=kafka_bootstrap_servers, - input_topic=kafka_topics.input_topic, - auto_offset_reset="earliest", - poll_interval="1seconds", - stop_after=num_records, - client_id="test_abp_no_cpp_reader")) - pipe.add_stage(DeserializeStage(config)) - pipe.add_stage(PreprocessFILStage(config)) - pipe.add_stage( - TritonInferenceStage(config, model_name='abp-nvsmi-xgb', server_url='test:0000', force_convert_inputs=True)) - pipe.add_stage( - MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf", log_level=morpheus_log_level)) - pipe.add_stage(AddClassificationsStage(config)) - pipe.add_stage(SerializeStage(config)) - pipe.add_stage( - WriteToKafkaStage(config, - bootstrap_servers=kafka_bootstrap_servers, - output_topic=kafka_topics.output_topic, - client_id="test_abp_no_cpp_writer")) - - pipe.run() - - val_df = dataset_pandas[val_file_name] - - output_buf = StringIO() - for rec in kafka_consumer: - output_buf.write(f'{rec.value.decode("utf-8")}\n') - - output_buf.seek(0) - output_df = pandas.read_json(output_buf, lines=True) - output_df = filter_null_data(output_df) - - assert len(output_df) == num_records - - results = compare_df(val_df, output_df, exclude_columns=[r'^ID$', r'^_ts_'], rel_tol=0.05) - - assert results['diff_rows'] == 0 - - @pytest.mark.kafka @pytest.mark.slow @pytest.mark.use_cpp diff --git a/tests/test_sid_kafka.py b/tests/test_sid_kafka.py index a50544c9c9..47e86b433e 100755 --- a/tests/test_sid_kafka.py +++ b/tests/test_sid_kafka.py @@ -49,101 +49,6 @@ MODEL_MAX_BATCH_SIZE = 32 -@pytest.mark.kafka -@pytest.mark.slow -@pytest.mark.use_python -@mock.patch('tritonclient.grpc.InferenceServerClient') -def test_minibert_no_cpp(mock_triton_client: mock.MagicMock, - dataset_pandas: DatasetManager, - config: Config, - kafka_bootstrap_servers: str, - kafka_topics: KafkaTopics, - kafka_consumer: "KafkaConsumer", - morpheus_log_level: int): - mock_metadata = { - "inputs": [{ - "name": "input_ids", "datatype": "INT32", "shape": [-1, FEATURE_LENGTH] - }, { - "name": "attention_mask", "datatype": "INT32", "shape": [-1, FEATURE_LENGTH] - }], - "outputs": [{ - "name": "output", "datatype": "FP32", "shape": [-1, 10] - }] - } - mock_model_config = {"config": {"max_batch_size": MODEL_MAX_BATCH_SIZE}} - - mock_triton_client.return_value = mock_triton_client - mock_triton_client.is_server_live.return_value = True - mock_triton_client.is_server_ready.return_value = True - mock_triton_client.is_model_ready.return_value = True - mock_triton_client.get_model_metadata.return_value = mock_metadata - mock_triton_client.get_model_config.return_value = mock_model_config - - data = np.loadtxt(os.path.join(TEST_DIRS.tests_data_dir, 'triton_sid_inf_results.csv'), delimiter=',') - inf_results = np.split(data, range(MODEL_MAX_BATCH_SIZE, len(data), MODEL_MAX_BATCH_SIZE)) - - async_infer = mk_async_infer(inf_results) - mock_triton_client.async_infer.side_effect = async_infer - - config.mode = PipelineModes.NLP - config.class_labels = [ - "address", - "bank_acct", - "credit_card", - "email", - "govt_id", - "name", - "password", - "phone_num", - "secret_keys", - "user" - ] - config.model_max_batch_size = MODEL_MAX_BATCH_SIZE - config.pipeline_batch_size = 1024 - config.feature_length = FEATURE_LENGTH - config.edge_buffer_size = 128 - config.num_threads = 1 - - val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'sid-validation-data.csv') - vocab_file_name = os.path.join(TEST_DIRS.data_dir, 'bert-base-uncased-hash.txt') - - pipe = LinearPipeline(config) - pipe.set_source(FileSourceStage(config, filename=val_file_name, iterative=False)) - pipe.add_stage(DeserializeStage(config)) - pipe.add_stage( - PreprocessNLPStage(config, - vocab_hash_file=vocab_file_name, - truncation=True, - do_lower_case=True, - add_special_tokens=False)) - pipe.add_stage( - TritonInferenceStage(config, model_name='sid-minibert-onnx', server_url='fake:001', force_convert_inputs=True)) - pipe.add_stage( - MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf", log_level=morpheus_log_level)) - pipe.add_stage(AddClassificationsStage(config, threshold=0.5, prefix="si_")) - pipe.add_stage(SerializeStage(config)) - pipe.add_stage( - WriteToKafkaStage(config, bootstrap_servers=kafka_bootstrap_servers, output_topic=kafka_topics.output_topic)) - - pipe.run() - - val_df = dataset_pandas[val_file_name] - - output_buf = StringIO() - for rec in kafka_consumer: - output_buf.write(f"{rec.value.decode('utf-8')}\n") - - output_buf.seek(0) - output_df = pandas.read_json(output_buf, lines=True) - output_df = filter_null_data(output_df) - - assert len(output_df) == len(val_df) - - results = compare_df(val_df, output_df, exclude_columns=[r'^ID$', r'^_ts_'], rel_tol=0.05) - - assert results['diff_rows'] == 1333 - - @pytest.mark.kafka @pytest.mark.slow @pytest.mark.use_cpp From b44ba66ca8d58813eb237dfd367005856dba502f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 10:39:07 -0700 Subject: [PATCH 127/347] Remove the use_python marker from tests where it doesn't make sense --- tests/controllers/test_elasticsearch_controller.py | 6 ------ tests/test_cli.py | 1 - tests/test_column_info.py | 8 -------- tests/test_directory_watcher.py | 1 - tests/test_error_pipe.py | 1 + tests/test_inference_stage.py | 1 + tests/test_inference_worker.py | 1 + tests/test_milvus_write_to_vector_db_stage_pipe.py | 2 +- tests/test_module_utils.py | 1 - tests/test_rss_source_stage_pipe.py | 12 +++++------- tests/test_tensor_memory.py | 2 ++ tests/test_triton_inference_stage.py | 2 ++ tests/test_write_to_elasticsearch_stage_pipe.py | 3 --- 13 files changed, 13 insertions(+), 28 deletions(-) diff --git a/tests/controllers/test_elasticsearch_controller.py b/tests/controllers/test_elasticsearch_controller.py index 903e4bf14f..3a136b0cd8 100644 --- a/tests/controllers/test_elasticsearch_controller.py +++ b/tests/controllers/test_elasticsearch_controller.py @@ -48,14 +48,12 @@ def inner_create_controller(*, connection_kwargs=connection_kwargs, refresh_peri yield inner_create_controller -@pytest.mark.use_python def test_constructor(create_controller: typing.Callable[..., ElasticsearchController], connection_kwargs: dict): assert create_controller(raise_on_exception=True)._raise_on_exception is True assert create_controller(refresh_period_secs=1.5)._refresh_period_secs == 1.5 assert create_controller()._connection_kwargs == connection_kwargs -@pytest.mark.use_python def test_refresh_client_force(create_controller: typing.Callable[..., ElasticsearchController]): controller = create_controller(refresh_period_secs=1) @@ -68,7 +66,6 @@ def test_refresh_client_force(create_controller: typing.Callable[..., Elasticsea assert controller._last_refresh_time > 0 -@pytest.mark.use_python def test_refresh_client_not_needed(create_controller: typing.Callable[..., ElasticsearchController]): controller = create_controller() client = controller._client @@ -81,7 +78,6 @@ def test_refresh_client_not_needed(create_controller: typing.Callable[..., Elast assert is_refreshed is False -@pytest.mark.use_python def test_refresh_client_needed(create_controller: typing.Callable[..., ElasticsearchController]): # Set a 1 second refresh period @@ -98,7 +94,6 @@ def test_refresh_client_needed(create_controller: typing.Callable[..., Elasticse assert is_refreshed is True -@pytest.mark.use_python @patch("morpheus.controllers.elasticsearch_controller.parallel_bulk", return_value=[(True, None)]) def test_parallel_bulk_write(mock_parallel_bulk, create_controller: typing.Callable[..., ElasticsearchController]): # Define your mock actions @@ -108,7 +103,6 @@ def test_parallel_bulk_write(mock_parallel_bulk, create_controller: typing.Calla mock_parallel_bulk.assert_called_once() -@pytest.mark.use_python @patch("morpheus.controllers.elasticsearch_controller.parallel_bulk", return_value=[(True, None)]) def test_df_to_parallel_bulk_write(mock_parallel_bulk: typing.Callable, create_controller: typing.Callable[..., ElasticsearchController]): diff --git a/tests/test_cli.py b/tests/test_cli.py index 67f82b375e..0e35b4a364 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -141,7 +141,6 @@ def config_warning_fixture(): @pytest.mark.reload_modules(commands) @pytest.mark.usefixtures("chdir_tmpdir", "reload_modules") -@pytest.mark.use_python class TestCLI: @pytest.mark.parametrize('cmd', diff --git a/tests/test_column_info.py b/tests/test_column_info.py index 52c80f4e64..18abe2ae2b 100644 --- a/tests/test_column_info.py +++ b/tests/test_column_info.py @@ -50,7 +50,6 @@ def azure_ad_logs_pdf_fixture(_azure_ad_logs_pdf: pd.DataFrame): yield _azure_ad_logs_pdf.copy(deep=True) -@pytest.mark.use_python def test_dataframe_input_schema_without_json_cols(azure_ad_logs_pdf: pd.DataFrame): assert len(azure_ad_logs_pdf.columns) == 16 @@ -97,7 +96,6 @@ def test_dataframe_input_schema_without_json_cols(azure_ad_logs_pdf: pd.DataFram process_dataframe(azure_ad_logs_pdf, schema2) -@pytest.mark.use_python def test_string_cat_column(): cities = pd.Series([ "New York", @@ -144,7 +142,6 @@ def test_string_cat_column(): string_cat_col_with_int._process_column(df) -@pytest.mark.use_python def test_string_join_column(): cities = pd.Series([ "Boston", @@ -163,7 +160,6 @@ def test_string_join_column(): assert actual.equals(expected) -@pytest.mark.use_python def test_column_info(): cities = pd.Series([ "Boston", @@ -181,7 +177,6 @@ def test_column_info(): assert string_join_col.name == "city" -@pytest.mark.use_python def test_date_column(): time_series = pd.Series([ "2022-08-29T21:21:41.645157Z", @@ -200,7 +195,6 @@ def test_date_column(): assert datetime_series.dtype == np.dtype("datetime64[ns]") -@pytest.mark.use_python def test_rename_column(): time_series = pd.Series([ "2022-08-29T21:21:41.645157Z", @@ -223,7 +217,6 @@ def convert_to_upper(df, column_name: str): return df[column_name].str.upper() -@pytest.mark.use_python def test_custom_column(): cities = pd.Series([ "New York", @@ -244,7 +237,6 @@ def test_custom_column(): assert actutal.equals(expected) -@pytest.mark.use_python def test_type_cast(): """ Test reproduces issue reported in #922 diff --git a/tests/test_directory_watcher.py b/tests/test_directory_watcher.py index d7943bfb29..cc7f3dcccd 100644 --- a/tests/test_directory_watcher.py +++ b/tests/test_directory_watcher.py @@ -22,7 +22,6 @@ from morpheus.utils.directory_watcher import DirectoryWatcher -@pytest.mark.use_python @pytest.mark.parametrize('watch_directory', [True]) @pytest.mark.parametrize('max_files', [-1]) @pytest.mark.parametrize('sort_glob', [True]) diff --git a/tests/test_error_pipe.py b/tests/test_error_pipe.py index 7f1e044286..ef0cf54cbf 100755 --- a/tests/test_error_pipe.py +++ b/tests/test_error_pipe.py @@ -43,6 +43,7 @@ def test_stage_raises_exception(config: Config, filter_probs_df: pd.DataFrame, e assert len(sink_stage.get_messages()) == 0 +# TODO should work in both GPU and CPU @pytest.mark.use_python @pytest.mark.parametrize("delayed_start", [False, True]) def test_monitor_not_impl(config: Config, delayed_start: bool): diff --git a/tests/test_inference_stage.py b/tests/test_inference_stage.py index e34f5a5bd4..7df1ade557 100755 --- a/tests/test_inference_stage.py +++ b/tests/test_inference_stage.py @@ -120,6 +120,7 @@ def test_split_batches(): mock_message.get_slice.assert_has_calls([mock.call(0, 3), mock.call(3, 7), mock.call(7, 10)]) +@pytest.mark.skip(reason="Test is passing, but python only impls for inference remains TBD") @pytest.mark.use_python def test_convert_one_response(): # Pylint currently fails to work with classmethod: https://github.com/pylint-dev/pylint/issues/981 diff --git a/tests/test_inference_worker.py b/tests/test_inference_worker.py index d3877c1336..5953796169 100755 --- a/tests/test_inference_worker.py +++ b/tests/test_inference_worker.py @@ -33,6 +33,7 @@ def test_constructor(): worker.stop() +@pytest.mark.skip(reason="Test is passing, but python only impls for inference remains TBD") @pytest.mark.use_python @pytest.mark.usefixtures("config") def test_build_output_message(): diff --git a/tests/test_milvus_write_to_vector_db_stage_pipe.py b/tests/test_milvus_write_to_vector_db_stage_pipe.py index d5e8efee99..3f91ec3375 100755 --- a/tests/test_milvus_write_to_vector_db_stage_pipe.py +++ b/tests/test_milvus_write_to_vector_db_stage_pipe.py @@ -134,7 +134,7 @@ def test_write_to_vector_db_stage_from_cm_pipe(milvus_server_uri: str, @pytest.mark.milvus -@pytest.mark.use_python +@pytest.mark.use_cpp @pytest.mark.parametrize("is_multiresponse_message", [True, False]) def test_write_to_vector_db_stage_from_mm_pipe(milvus_server_uri: str, idx_part_collection_config: dict, diff --git a/tests/test_module_utils.py b/tests/test_module_utils.py index adcdc3e660..baf8027a9e 100644 --- a/tests/test_module_utils.py +++ b/tests/test_module_utils.py @@ -28,7 +28,6 @@ # pylint: disable=unused-argument,too-many-function-args -@pytest.mark.use_python def test_mrc_version(): assert len(mrc_version) == 3 assert isinstance(mrc_version, list) diff --git a/tests/test_rss_source_stage_pipe.py b/tests/test_rss_source_stage_pipe.py index 2f24c8b3ae..4d73f01524 100644 --- a/tests/test_rss_source_stage_pipe.py +++ b/tests/test_rss_source_stage_pipe.py @@ -28,7 +28,6 @@ invalid_feed_input = os.path.join(TEST_DIRS.tests_data_dir, "rss_feed_atom.xm") -@pytest.mark.use_python def test_support_cpp_node(config): url_feed_input = "https://fake.nvidia.com/rss/HomePage.xml" rss_source_stage = RSSSourceStage(config, feed_input=[url_feed_input]) @@ -61,9 +60,8 @@ def test_rss_source_stage_pipe(config: Config, assert len(sink_stage.get_messages()) == expected_count -# TODO(Devin): Remove before merge, this isn't a stage test, this is a test of RSSController -# @pytest.mark.use_python -# def test_invalid_input_rss_source_stage(config: Config): -# -# with pytest.raises(ValueError, match=f"Invalid URL or file path: {invalid_feed_input}"): -# RSSSourceStage(config, feed_input=[invalid_feed_input], interval_secs=1, cooldown_interval=500) +@pytest.mark.use_python +def test_invalid_input_rss_source_stage(config: Config): + + with pytest.raises(ValueError, match=f"Invalid URL or file path: {invalid_feed_input}"): + RSSSourceStage(config, feed_input=[invalid_feed_input], interval_secs=1, cooldown_interval=500) diff --git a/tests/test_tensor_memory.py b/tests/test_tensor_memory.py index e3f072277c..0d71fb6817 100644 --- a/tests/test_tensor_memory.py +++ b/tests/test_tensor_memory.py @@ -87,6 +87,7 @@ def test_tensor_memory(config: Config): check_tensor_memory(cls, count, tensors) +@pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") @pytest.mark.use_python def test_inference_memory_ae(config: Config): test_data = cp.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) @@ -156,6 +157,7 @@ def check_response_memory_probs_and_ae(cls: type): return mem +@pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") @pytest.mark.use_python def test_response_memory_ae(config: Config, filter_probs_df: DataFrameType): mem = check_response_memory_probs_and_ae(ResponseMemoryAE) diff --git a/tests/test_triton_inference_stage.py b/tests/test_triton_inference_stage.py index fee1495cef..34a411e734 100644 --- a/tests/test_triton_inference_stage.py +++ b/tests/test_triton_inference_stage.py @@ -122,6 +122,7 @@ def test_resource_pool_create_raises_error(): assert pool.borrow_obj() == 20 +@pytest.mark.skip(reason="TODO: determine what to do about python impls") @pytest.mark.use_python @pytest.mark.parametrize("pipeline_mode", list(PipelineModes)) def test_stage_constructor_worker_class(config: Config, pipeline_mode: PipelineModes): @@ -131,6 +132,7 @@ def test_stage_constructor_worker_class(config: Config, pipeline_mode: PipelineM assert isinstance(worker, TritonInferenceWorker) +@pytest.mark.skip(reason="TODO: determine what to do about python impls") @pytest.mark.use_python @pytest.mark.parametrize("pipeline_mode", list(PipelineModes)) @pytest.mark.parametrize("needs_logits", [True, False, None]) diff --git a/tests/test_write_to_elasticsearch_stage_pipe.py b/tests/test_write_to_elasticsearch_stage_pipe.py index 199ff8319d..7660c5a4e5 100644 --- a/tests/test_write_to_elasticsearch_stage_pipe.py +++ b/tests/test_write_to_elasticsearch_stage_pipe.py @@ -47,7 +47,6 @@ def connection_conf_file_fixture(tmp_path): yield connection_conf_file -@pytest.mark.use_python @pytest.mark.parametrize("conf_file, exception", [("connection_conf.yaml", FileNotFoundError), (None, Exception)]) def test_constructor_invalid_conf_file(config: Config, conf_file: str, @@ -56,7 +55,6 @@ def test_constructor_invalid_conf_file(config: Config, WriteToElasticsearchStage(config, index="t_index", connection_conf_file=conf_file) -@pytest.mark.use_python @patch("morpheus.controllers.elasticsearch_controller.Elasticsearch") def test_constructor_with_custom_func(config: Config, connection_conf_file: str): expected_connection_kwargs = { @@ -73,7 +71,6 @@ def test_constructor_with_custom_func(config: Config, connection_conf_file: str) assert stage._controller._connection_kwargs == expected_connection_kwargs -@pytest.mark.use_python @patch("morpheus.stages.output.write_to_elasticsearch_stage.ElasticsearchController") def test_write_to_elasticsearch_stage_pipe(mock_controller: typing.Any, connection_conf_file: str, From a3107e19e9bff418d6c8afa30bee8fbabcdef7ed Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 11:27:45 -0700 Subject: [PATCH 128/347] Rename use_python to cpu_mode and rename use_cpp to gpu_mode --- .../test_bench_agents_simple_pipeline.py | 2 +- .../test_bench_completion_pipeline.py | 2 +- .../test_bench_rag_standalone_pipeline.py | 2 +- .../test_bench_vdb_upload_pipeline.py | 2 +- tests/conftest.py | 106 +++++++++--------- tests/dfencoder/test_autoencoder.py | 2 +- .../gnn_fraud_detection_pipeline/conftest.py | 2 +- .../test_classification_stage.py | 2 +- .../test_graph_construction_stage.py | 2 +- .../test_graph_sage_stage.py | 2 +- .../common/test_content_extractor_module.py | 2 +- .../llm/common/test_web_scraper_module.py | 2 +- .../llm/common/test_web_scraper_stage.py | 2 +- .../test_schema_transform_module.py | 2 +- tests/examples/log_parsing/conftest.py | 2 +- .../examples/ransomware_detection/conftest.py | 2 +- .../test_create_features.py | 2 +- .../test_preprocessing.py | 2 +- tests/llm/test_vdb_upload_pipe.py | 2 +- tests/messages/test_message_meta.py | 8 +- tests/modules/test_from_control_message.py | 4 +- tests/modules/test_payload_batcher.py | 4 +- tests/modules/test_to_control_message.py | 2 +- tests/pipeline/test_preallocation_pipe.py | 2 +- tests/pipeline/test_stage_decorator.py | 38 +++---- tests/stages/test_deserialize_stage_pipe.py | 2 +- tests/stages/test_http_server_source_stage.py | 4 +- tests/stages/test_llm_engine_stage_pipe.py | 2 +- tests/stages/test_ml_flow_drift_stage.py | 2 +- tests/stages/test_preprocess_fil_stage.py | 2 +- tests/stages/test_preprocess_nlp_stage.py | 2 +- tests/stages/test_timeseries_stage.py | 2 +- tests/test_abp.py | 4 +- tests/test_abp_kafka.py | 2 +- tests/test_add_classifications_stage.py | 4 +- tests/test_add_scores_stage.py | 4 +- tests/test_conftest.py | 88 +++++++-------- tests/test_dfp.py | 6 +- tests/test_dfp_kafka.py | 4 +- tests/test_error_pipe.py | 2 +- tests/test_file_in_out.py | 4 +- tests/test_http_server_sink_stage.py | 2 +- tests/test_inference_stage.py | 2 +- tests/test_inference_worker.py | 2 +- tests/test_linear_modules_stage.py | 6 +- tests/test_messages.py | 2 +- ...st_milvus_write_to_vector_db_stage_pipe.py | 4 +- tests/test_monitor_stage.py | 2 +- tests/test_multi_message.py | 22 ++-- tests/test_multi_port_modules_stage.py | 2 +- tests/test_phishing.py | 2 +- tests/test_phishing_kafka.py | 2 +- tests/test_rss_source_stage_pipe.py | 4 +- tests/test_serialize_stage.py | 2 +- tests/test_sid.py | 2 +- tests/test_sid_kafka.py | 2 +- tests/test_tensor_memory.py | 4 +- tests/test_triton_inference_stage.py | 4 +- tests/test_write_to_file_stage.py | 2 +- 59 files changed, 201 insertions(+), 201 deletions(-) diff --git a/tests/benchmarks/test_bench_agents_simple_pipeline.py b/tests/benchmarks/test_bench_agents_simple_pipeline.py index 5631d6e6ac..5c46fa3367 100644 --- a/tests/benchmarks/test_bench_agents_simple_pipeline.py +++ b/tests/benchmarks/test_bench_agents_simple_pipeline.py @@ -95,7 +95,7 @@ def _run_pipeline(config: Config, source_dfs: list[cudf.DataFrame], model_name: @pytest.mark.usefixtures("openai", "restore_environ") -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.benchmark @mock.patch("langchain.utilities.serpapi.SerpAPIWrapper.aresults") @mock.patch("langchain.OpenAI._agenerate", autospec=True) # autospec is needed as langchain will inspect the function diff --git a/tests/benchmarks/test_bench_completion_pipeline.py b/tests/benchmarks/test_bench_completion_pipeline.py index 99d7157a5e..fde17efebf 100644 --- a/tests/benchmarks/test_bench_completion_pipeline.py +++ b/tests/benchmarks/test_bench_completion_pipeline.py @@ -76,7 +76,7 @@ def _run_pipeline(config: Config, @pytest.mark.use_cudf -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.benchmark @pytest.mark.usefixtures("mock_nemollm", "mock_chat_completion") @pytest.mark.parametrize("llm_service_cls", [NeMoLLMService, OpenAIChatService]) diff --git a/tests/benchmarks/test_bench_rag_standalone_pipeline.py b/tests/benchmarks/test_bench_rag_standalone_pipeline.py index ee4cd1e974..5a42fd5bdc 100644 --- a/tests/benchmarks/test_bench_rag_standalone_pipeline.py +++ b/tests/benchmarks/test_bench_rag_standalone_pipeline.py @@ -123,7 +123,7 @@ def _run_pipeline(config: Config, @pytest.mark.milvus -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.use_cudf @pytest.mark.benchmark @pytest.mark.import_mod(os.path.join(TEST_DIRS.examples_dir, 'llm/common/utils.py')) diff --git a/tests/benchmarks/test_bench_vdb_upload_pipeline.py b/tests/benchmarks/test_bench_vdb_upload_pipeline.py index d54c536780..4cbae382e9 100644 --- a/tests/benchmarks/test_bench_vdb_upload_pipeline.py +++ b/tests/benchmarks/test_bench_vdb_upload_pipeline.py @@ -87,7 +87,7 @@ def _run_pipeline(config: Config, @pytest.mark.milvus -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.use_pandas @pytest.mark.benchmark @pytest.mark.import_mod([ diff --git a/tests/conftest.py b/tests/conftest.py index 7128a9c0c2..6a6aeebf81 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -102,35 +102,35 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): This function will add parameterizations for the `config` fixture depending on what types of config the test supports """ - # === use_cpp Parameterize === - use_cpp = metafunc.definition.get_closest_marker("use_cpp") is not None - use_python = metafunc.definition.get_closest_marker("use_python") is not None + # === gpu_mode Parameterize === + gpu_mode = metafunc.definition.get_closest_marker("gpu_mode") is not None + cpu_mode = metafunc.definition.get_closest_marker("cpu_mode") is not None - use_cpp_param = pytest.param(True, marks=pytest.mark.use_cpp(added_by="generate_tests"), id="use_cpp") - use_python_param = pytest.param(False, marks=pytest.mark.use_python(added_by="generate_tests"), id="use_python") + gpu_mode_param = pytest.param(True, marks=pytest.mark.gpu_mode(added_by="generate_tests"), id="gpu_mode") + cpu_mode_param = pytest.param(False, marks=pytest.mark.cpu_mode(added_by="generate_tests"), id="cpu_mode") - _set_use_cpp_params = [] + _set_gpu_mode_params = [] - if ("use_cpp" in metafunc.fixturenames): + if ("gpu_mode" in metafunc.fixturenames): # Need to add some params since the fixture was requested - # Add cpp unless use_cpp == True and use_python == False - if not (use_python and not use_cpp): - _set_use_cpp_params.append(use_cpp_param) + # Add cpp unless gpu_mode == True and cpu_mode == False + if not (cpu_mode and not gpu_mode): + _set_gpu_mode_params.append(gpu_mode_param) - # Add python unless use_cpp == False and use_python == True - if not (not use_python and use_cpp): - _set_use_cpp_params.append(use_python_param) + # Add python unless gpu_mode == False and cpu_mode == True + if not (not cpu_mode and gpu_mode): + _set_gpu_mode_params.append(cpu_mode_param) - elif (use_cpp and use_python): + elif (gpu_mode and cpu_mode): # Need to parameterize since we have multiple - _set_use_cpp_params.extend([use_cpp_param, use_python_param]) - elif (not use_cpp and not use_python): + _set_gpu_mode_params.extend([gpu_mode_param, cpu_mode_param]) + elif (not gpu_mode and not cpu_mode): # If neither are set, default to cpp - _set_use_cpp_params.append(use_cpp_param) + _set_gpu_mode_params.append(gpu_mode_param) - if (len(_set_use_cpp_params) > 0): - metafunc.parametrize("_set_use_cpp", _set_use_cpp_params, indirect=True) + if (len(_set_gpu_mode_params) > 0): + metafunc.parametrize("_set_gpu_mode", _set_gpu_mode_params, indirect=True) # === df_type Parameterize === if ("df_type" in metafunc.fixturenames): @@ -177,15 +177,15 @@ def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config def should_filter_test(item: pytest.Item): - use_cpp = item.get_closest_marker("use_cpp") + gpu_mode = item.get_closest_marker("gpu_mode") use_pandas = item.get_closest_marker("use_pandas") use_cudf = item.get_closest_marker("use_cudf") - use_python = item.get_closest_marker("use_python") + cpu_mode = item.get_closest_marker("cpu_mode") - if (use_cpp and use_pandas): + if (gpu_mode and use_pandas): return False - if (use_cudf and use_python): + if (use_cudf and cpu_mode): return False return True @@ -208,45 +208,45 @@ def pytest_runtest_teardown(item, nextitem): reset_logging(logger_name=None) # Reset the root logger as well -def _get_use_cpp(request: pytest.FixtureRequest) -> bool: - do_use_cpp: bool = True +def _get_gpu_mode(request: pytest.FixtureRequest) -> bool: + do_gpu_mode: bool = True # Check for the param if this was indirectly set if (hasattr(request, "param") and isinstance(request.param, bool)): - do_use_cpp = request.param + do_gpu_mode = request.param else: # If not, check for the marker and use that - use_cpp = request.node.get_closest_marker("use_cpp") is not None - use_python = request.node.get_closest_marker("use_python") is not None + gpu_mode = request.node.get_closest_marker("gpu_mode") is not None + cpu_mode = request.node.get_closest_marker("cpu_mode") is not None - if (use_cpp and use_python): - raise RuntimeError(f"Both markers (use_cpp and use_python) were added to function {request.node.nodeid}. " + if (gpu_mode and cpu_mode): + raise RuntimeError(f"Both markers (gpu_mode and cpu_mode) were added to function {request.node.nodeid}. " "Remove markers to support both.") - # This will default to True or follow use_cpp - do_use_cpp = not use_python + # This will default to True or follow gpu_mode + do_gpu_mode = not cpu_mode - return do_use_cpp + return do_gpu_mode # This fixture will be used by all tests. @pytest.fixture(scope="function", autouse=True) -def _set_use_cpp(request: pytest.FixtureRequest): - do_use_cpp = _get_use_cpp(request) +def _set_gpu_mode(request: pytest.FixtureRequest): + do_gpu_mode = _get_gpu_mode(request) from morpheus.config import CppConfig - CppConfig.set_should_use_cpp(do_use_cpp) + CppConfig.set_should_use_cpp(do_gpu_mode) - yield do_use_cpp + yield do_gpu_mode # This fixture will be used by all tests. @pytest.fixture(scope="function") -def use_cpp(_set_use_cpp: bool): +def gpu_mode(_set_gpu_mode: bool): # Just return the set value - yield _set_use_cpp + yield _set_gpu_mode @pytest.fixture(scope="function") @@ -298,7 +298,7 @@ def df_type(request: pytest.FixtureRequest): use_cudf = request.node.get_closest_marker("use_cudf") is not None if (use_pandas and use_cudf): - raise RuntimeError(f"Both markers (use_cpp and use_python) were added to function {request.node.nodeid}. " + raise RuntimeError(f"Both markers (gpu_mode and cpu_mode) were added to function {request.node.nodeid}. " "Remove markers to support both.") # This will default to "cudf" or follow use_pandas @@ -314,7 +314,7 @@ def config(): the object. If no marks are added to the test, it will be parameterized for both C++ and python. For example, ``` - @pytest.mark.use_python + @pytest.mark.cpu_mode def my_python_test(config: Config): ... ``` @@ -907,33 +907,33 @@ def test_something(dataset: DatasetManager): ``` A test that requests this fixture will parameterize on the type of DataFrame returned by the DatasetManager. - If a test requests both this fixture and the `use_cpp` fixture, or indirectly via the `config` fixture, then - the test will parameterize over both df_type:[cudf, pandas] and use_cpp[True, False]. However it will remove the - df_type=pandas & use_cpp=True and df_type=cudf & use_cpp=False combinations as this will cause an unsupported usage + If a test requests both this fixture and the `gpu_mode` fixture, or indirectly via the `config` fixture, then + the test will parameterize over both df_type:[cudf, pandas] and gpu_mode[True, False]. However it will remove the + df_type=pandas & gpu_mode=True and df_type=cudf & gpu_mode=False combinations as this will cause an unsupported usage of Pandas dataframes with the C++ implementation of message classes, and cuDF with CPU-only implementations. - This behavior can also be overridden by using the `use_cudf`, `use_pandas`, `use_cpp` or `use_pandas` marks ex: + This behavior can also be overridden by using the `use_cudf`, `use_pandas`, `gpu_mode` or `use_pandas` marks ex: ``` # This test will only run once with C++ enabled and cudf dataframes - @pytest.mark.use_cpp + @pytest.mark.gpu_mode def test something(dataset: DatasetManager): ... # This test will run once for with pandas and C++ disabled - @pytest.mark.use_python + @pytest.mark.cpu_mode def test something(dataset: DatasetManager): ... # This test will run once with C++ mode enabled, using cudf dataframes @pytest.mark.use_cudf - def test something(use_cpp: bool, dataset: DatasetManager): + def test something(gpu_mode: bool, dataset: DatasetManager): ... # This test creates an incompatible combination and will raise a RuntimeError without being executed @pytest.mark.use_cudf - @pytest.mark.use_python + @pytest.mark.cpu_mode def test something(dataset: DatasetManager): ... # This test creates an incompatible combination and will raise a RuntimeError without being executed @pytest.mark.use_pandas - @pytest.mark.use_cpp + @pytest.mark.gpu_mode def test something(dataset: DatasetManager): ``` @@ -955,7 +955,7 @@ def dataset_pandas(): In addition to this, users can use this fixture to explicitly request a cudf Dataframe as well, allowing for a test that looks like: ``` - @pytest.mark.use_cpp + @pytest.mark.gpu_mode def test_something(dataset_pandas: DatasetManager): input_df = dataset_pandas.cudf["filter_probs.csv"] # Feed our source stage a cudf DF @@ -983,12 +983,12 @@ def test_something(dataset_cudf: DatasetManager): @pytest.fixture(scope="function") -def filter_probs_df(dataset, use_cpp: bool): +def filter_probs_df(dataset, gpu_mode: bool): """ Shortcut fixture for loading the filter_probs.csv dataset. Unless your test uses the `use_pandas` or `use_cudf` marks this fixture will parametarize over the two dataframe - types. Similarly unless your test uses the `use_cpp` or `use_python` marks this fixture will also parametarize over + types. Similarly unless your test uses the `gpu_mode` or `cpu_mode` marks this fixture will also parametarize over that as well, while excluding the combination of C++ execution and Pandas dataframes. """ yield dataset["filter_probs.csv"] diff --git a/tests/dfencoder/test_autoencoder.py b/tests/dfencoder/test_autoencoder.py index 43a1b7574b..421194a971 100755 --- a/tests/dfencoder/test_autoencoder.py +++ b/tests/dfencoder/test_autoencoder.py @@ -37,7 +37,7 @@ from morpheus.models.dfencoder.dataloader import FileSystemDataset # Only pandas and Python is supported -pytestmark = [pytest.mark.use_pandas, pytest.mark.use_python] +pytestmark = [pytest.mark.use_pandas, pytest.mark.cpu_mode] BIN_COLS = ['ts_anomaly'] diff --git a/tests/examples/gnn_fraud_detection_pipeline/conftest.py b/tests/examples/gnn_fraud_detection_pipeline/conftest.py index 30176f71e4..2354949be7 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/conftest.py +++ b/tests/examples/gnn_fraud_detection_pipeline/conftest.py @@ -44,7 +44,7 @@ def cuml_fixture(fail_missing: bool): @pytest.fixture(name="config") -def config_fixture(config, use_cpp: bool): # pylint: disable=unused-argument +def config_fixture(config, gpu_mode: bool): # pylint: disable=unused-argument """ The GNN fraud detection pipeline utilizes the "other" pipeline mode. """ diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py index e0924d4448..0271782e65 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py @@ -25,7 +25,7 @@ @pytest.mark.xfail(reason="TODO: Need to determine what to do with GPU pipelines which only run in Python mode") -@pytest.mark.use_python +@pytest.mark.cpu_mode class TestClassificationStage: def test_constructor(self, config: Config, xgb_model: str, cuml: types.ModuleType): diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py index e1785076eb..3e79c2ee7e 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py @@ -28,7 +28,7 @@ # pylint: disable=no-name-in-module -@pytest.mark.use_python +@pytest.mark.cpu_mode class TestGraphConstructionStage: def test_constructor(self, config: Config, training_file: str): diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py index 89fd0c6934..2e717efe67 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py @@ -26,7 +26,7 @@ # pylint: disable=no-name-in-module @pytest.mark.xfail(reason="TODO: Need to determine what to do with GPU pipelines which only run in Python mode") @pytest.mark.usefixtures("manual_seed") -@pytest.mark.use_python +@pytest.mark.cpu_mode class TestGraphSageStage: def test_constructor(self, config: Config, model_dir: str): diff --git a/tests/examples/llm/common/test_content_extractor_module.py b/tests/examples/llm/common/test_content_extractor_module.py index 2c77737681..0cd4e78ed9 100644 --- a/tests/examples/llm/common/test_content_extractor_module.py +++ b/tests/examples/llm/common/test_content_extractor_module.py @@ -88,7 +88,7 @@ def generate_random_string(length: int) -> str: return ''.join(random.choices(string.ascii_letters + string.digits, k=length)) -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.use_cudf @pytest.mark.parametrize("data_len, num_rows_per_file, batch_size", [(40, 5, 2), (51, 3, 1), (150, 10, 5), (500, 3, 2), (1000, 5, 3), (50, 10, 2), (100, 20, 3), diff --git a/tests/examples/llm/common/test_web_scraper_module.py b/tests/examples/llm/common/test_web_scraper_module.py index 592f5d38fb..c79026e7e1 100644 --- a/tests/examples/llm/common/test_web_scraper_module.py +++ b/tests/examples/llm/common/test_web_scraper_module.py @@ -30,7 +30,7 @@ @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.use_cudf @pytest.mark.import_mod(os.path.join(TEST_DIRS.examples_dir, 'llm/vdb_upload/module/web_scraper_module.py')) def test_web_scraper_module(config: Config, mock_rest_server: str, import_mod: types.ModuleType): diff --git a/tests/examples/llm/common/test_web_scraper_stage.py b/tests/examples/llm/common/test_web_scraper_stage.py index 418d245043..eac1604ff2 100644 --- a/tests/examples/llm/common/test_web_scraper_stage.py +++ b/tests/examples/llm/common/test_web_scraper_stage.py @@ -28,7 +28,7 @@ @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.use_cudf @pytest.mark.import_mod(os.path.join(TEST_DIRS.examples_dir, 'llm/vdb_upload/module/web_scraper_stage.py')) def test_http_client_source_stage_pipe(config: Config, mock_rest_server: str, import_mod: types.ModuleType): diff --git a/tests/examples/llm/vdb_upload/test_schema_transform_module.py b/tests/examples/llm/vdb_upload/test_schema_transform_module.py index 8a4ed6e870..7aed8642f8 100644 --- a/tests/examples/llm/vdb_upload/test_schema_transform_module.py +++ b/tests/examples/llm/vdb_upload/test_schema_transform_module.py @@ -27,7 +27,7 @@ from morpheus.stages.output.compare_dataframe_stage import CompareDataFrameStage -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.use_cudf @pytest.mark.parametrize("num_select, num_renames", [(1, 0), (0, 1), (1, 1), (6, 6), (13, 10), (10, 13)]) def test_schema_transform_module(num_select, diff --git a/tests/examples/log_parsing/conftest.py b/tests/examples/log_parsing/conftest.py index f927c3fcc1..e63ceac2f6 100644 --- a/tests/examples/log_parsing/conftest.py +++ b/tests/examples/log_parsing/conftest.py @@ -17,7 +17,7 @@ @pytest.fixture(name="config") -def config_fixture(config, use_cpp: bool): # pylint: disable=unused-argument +def config_fixture(config, gpu_mode: bool): # pylint: disable=unused-argument """ The log_parsing pipelie requires NLP mode. Set this here so all the tests don't need to set it themselves. """ diff --git a/tests/examples/ransomware_detection/conftest.py b/tests/examples/ransomware_detection/conftest.py index a92786555a..b24f321a29 100644 --- a/tests/examples/ransomware_detection/conftest.py +++ b/tests/examples/ransomware_detection/conftest.py @@ -39,7 +39,7 @@ def dask_distributed(fail_missing: bool): @pytest.fixture(name="config") -def config_fixture(config, use_cpp: bool): # pylint: disable=unused-argument +def config_fixture(config, gpu_mode: bool): # pylint: disable=unused-argument """ The ransomware detection pipeline utilizes the FIL pipeline mode. """ diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index 4d095ccf2f..f27ea725bb 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -30,7 +30,7 @@ from morpheus.stages.input.appshield_source_stage import AppShieldSourceStage -@pytest.mark.use_python +@pytest.mark.cpu_mode class TestCreateFeaturesRWStage: # pylint: disable=no-name-in-module diff --git a/tests/examples/ransomware_detection/test_preprocessing.py b/tests/examples/ransomware_detection/test_preprocessing.py index a72225edbf..35f5eaf425 100644 --- a/tests/examples/ransomware_detection/test_preprocessing.py +++ b/tests/examples/ransomware_detection/test_preprocessing.py @@ -25,7 +25,7 @@ from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage -@pytest.mark.use_python +@pytest.mark.cpu_mode class TestPreprocessingRWStage: # pylint: disable=no-name-in-module diff --git a/tests/llm/test_vdb_upload_pipe.py b/tests/llm/test_vdb_upload_pipe.py index f6051ebb55..9487d5322d 100755 --- a/tests/llm/test_vdb_upload_pipe.py +++ b/tests/llm/test_vdb_upload_pipe.py @@ -31,7 +31,7 @@ @pytest.mark.milvus -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.use_pandas @pytest.mark.import_mod([ os.path.join(TEST_DIRS.examples_dir, 'llm/common'), diff --git a/tests/messages/test_message_meta.py b/tests/messages/test_message_meta.py index 740b48a0e9..1e738b0118 100644 --- a/tests/messages/test_message_meta.py +++ b/tests/messages/test_message_meta.py @@ -37,7 +37,7 @@ def fixture_index_type(request: pytest.FixtureRequest) -> typing.Literal["normal @pytest.fixture(name="df", scope="function") def fixture_df( - use_cpp: bool, # pylint: disable=unused-argument + gpu_mode: bool, # pylint: disable=unused-argument dataset: DatasetManager, index_type: typing.Literal['normal', 'skip', 'dup', 'down', 'updown']) -> typing.Union[cudf.DataFrame, pd.DataFrame]: @@ -296,7 +296,7 @@ def test_update_dataframe(df: DataFrameType): assert meta.get_data()[col_new_int_name].isin(col_new_int).all() # pylint: disable=unsubscriptable-object -@pytest.mark.use_cpp +@pytest.mark.gpu_mode def test_pandas_df_cpp(dataset_pandas: DatasetManager): """ Test for issue #821, calling the `df` property returns an empty cudf dataframe. @@ -324,7 +324,7 @@ def test_cast(config: Config, dataset: DatasetManager): # pylint: disable=unuse @pytest.mark.use_pandas -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_cast_python_to_cpp(dataset: DatasetManager): """ Test that we can cast a python MessageMeta to a C++ MessageMeta @@ -343,7 +343,7 @@ def test_cast_python_to_cpp(dataset: DatasetManager): @pytest.mark.use_pandas -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_cast_cpp_to_python(dataset: DatasetManager): """ Test that we can cast a a C++ MessageMeta to a python MessageMeta diff --git a/tests/modules/test_from_control_message.py b/tests/modules/test_from_control_message.py index b129bbbcc8..514dc68234 100644 --- a/tests/modules/test_from_control_message.py +++ b/tests/modules/test_from_control_message.py @@ -71,7 +71,7 @@ def test_get_module(): fn_constructor("FromControlMessageTest", config) # pylint: disable=not-callable -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.parametrize("filename, expected_count", [("train_infer.json", 0), ("train.json", 0)], indirect=["filename"]) def test_cm_with_no_payload(config, filename, expected_count): @@ -97,7 +97,7 @@ def test_cm_with_no_payload(config, filename, expected_count): assert len(sink_stage.get_messages()) == expected_count -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.parametrize("filename, expected_count", [("train_infer.json", 2), ("train.json", 1)], indirect=["filename"]) def test_cm_with_with_payload(config, filename, expected_count): diff --git a/tests/modules/test_payload_batcher.py b/tests/modules/test_payload_batcher.py index 02acd6b8ee..8fa39b18a5 100644 --- a/tests/modules/test_payload_batcher.py +++ b/tests/modules/test_payload_batcher.py @@ -83,7 +83,7 @@ def test_get_module(): assert isinstance(module_instance, mrc.core.segment.SegmentModule) -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.parametrize( "max_batch_size, raise_on_failure, group_by_columns, disable_max_batch_size, timestamp_column_name, " "timestamp_pattern, period, expected_count, expected_exception", @@ -193,7 +193,7 @@ def test_custom_params(config, assert len(sink_stage.get_messages()) == expected_count -@pytest.mark.use_cpp +@pytest.mark.gpu_mode def test_default_params(config, filter_probs_df): pipe = Pipeline(config) diff --git a/tests/modules/test_to_control_message.py b/tests/modules/test_to_control_message.py index 96f91a2fee..ce2218b8aa 100644 --- a/tests/modules/test_to_control_message.py +++ b/tests/modules/test_to_control_message.py @@ -61,7 +61,7 @@ def test_get_module(): assert isinstance(module_instance, mrc.core.segment.SegmentModule) -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.parametrize("expected_count", [1, 2]) def test_to_control_message_module(config, filter_probs_df, expected_count): dataframes = [filter_probs_df for _ in range(expected_count)] diff --git a/tests/pipeline/test_preallocation_pipe.py b/tests/pipeline/test_preallocation_pipe.py index a1bd8b7c4a..afd1239be3 100755 --- a/tests/pipeline/test_preallocation_pipe.py +++ b/tests/pipeline/test_preallocation_pipe.py @@ -100,7 +100,7 @@ def test_preallocation_multi_segment_pipe(config, filter_probs_df, probs_type): assert_results(comp_stage.get_results()) -@pytest.mark.use_cpp +@pytest.mark.gpu_mode def test_preallocation_error(config, filter_probs_df): """ Verify that we get a raised exception when add_scores attempts to use columns that don't exist diff --git a/tests/pipeline/test_stage_decorator.py b/tests/pipeline/test_stage_decorator.py index 936ed5b215..91812b824f 100644 --- a/tests/pipeline/test_stage_decorator.py +++ b/tests/pipeline/test_stage_decorator.py @@ -56,7 +56,7 @@ def _mk_compute_schema_fn(return_type: type) -> ComputeSchemaType: return lambda schema: schema.output_schema.set_type(return_type) -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("generator_type", [None, typing.Iterator, typing.Generator, collections.abc.Iterator, collections.abc.Generator]) @pytest.mark.parametrize("return_type, is_prealloc", @@ -94,7 +94,7 @@ def test_source_gen() -> return_annotation: mock_compute_schema_fn.assert_called_once_with(schema) -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("src_cls", [WrappedFunctionSourceStage, PreAllocatedWrappedFunctionStage]) def test_wrapped_function_source_stage_not_generator_error(config: Config, src_cls: type): @@ -108,7 +108,7 @@ def test_source_gen() -> MessageMeta: compute_schema_fn=_mk_compute_schema_fn(MessageMeta)) -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("generator_type", [None, typing.Iterator, typing.Generator, collections.abc.Iterator, collections.abc.Generator]) @pytest.mark.parametrize("return_type, is_prealloc", @@ -132,7 +132,7 @@ def test_source_gen() -> return_annotation: assert schema.output_schema.get_type() is return_type -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_source_decorator_name(config: Config): @source @@ -143,7 +143,7 @@ def test_source_gen(value: int) -> int: assert source_stage.name == 'test_source_gen' # pylint: disable=no-member -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_source_decorator_explicit_name(config: Config): @source(name="source_gen") @@ -154,7 +154,7 @@ def test_source_gen(value: int) -> int: assert source_stage.name == 'source_gen' # pylint: disable=no-member -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_source_decorator_explicit_compute_schema(config: Config): mock_compute_schema_fn = mock.MagicMock() mock_compute_schema_fn.side_effect = _mk_compute_schema_fn(int) @@ -170,7 +170,7 @@ def test_source_gen(value: int) -> int: mock_compute_schema_fn.assert_called_once_with(schema) -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_source_decorator_no_annoation_error(config: Config): @source @@ -181,7 +181,7 @@ def test_source_gen(): test_source_gen(config) # pylint: disable=too-many-function-args -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_not_generator_error(config: Config): @source @@ -192,7 +192,7 @@ def test_fn() -> int: test_fn(config) # pylint: disable=too-many-function-args -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_source_stage_arg_no_value_error(config: Config): @source @@ -203,7 +203,7 @@ def test_source_gen(value: int) -> int: test_source_gen(config) -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), (typing.Union[float, int], float), (float, typing.Any), (typing.Any, float), @@ -219,7 +219,7 @@ def test_wrapped_function_stage_constructor(config: Config, accept_type: type, r assert wrapped_stage.accepted_types() == (accept_type, ) -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), (typing.Union[float, int], float), (float, float), (typing.Any, float), @@ -255,7 +255,7 @@ def source_fn(): assert schema.output_schema.get_type() is return_type -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_wrapped_function_stage_name(config: Config): def multiplier(message: MessageMeta, column: str, value: int | float) -> MessageMeta: @@ -272,7 +272,7 @@ def multiplier(message: MessageMeta, column: str, value: int | float) -> Message assert wrapped_stage.name == 'multiplier' -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("needed_columns", [None, { 'result': TypeId.INT64 @@ -294,7 +294,7 @@ def test_fn(message: MessageMeta) -> MessageMeta: assert wrapped_stage._needed_columns == expected_needed_columns -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("use_accept_type_annotation", [True, False]) @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), @@ -319,7 +319,7 @@ def test_fn(message) -> return_type: assert wrapped_stage.accepted_types() == (accept_type, ) -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("name", [None, "unittest-stage"]) def test_stage_decorator_name(config: Config, name: str): if name is None: @@ -335,7 +335,7 @@ def test_fn(message: float, value: float) -> float: assert wrapped_stage.name == expected_name -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("explicit_compute_schema_fn", [True, False]) @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), @@ -376,7 +376,7 @@ def test_stage(message: accept_type) -> return_type: assert schema.output_schema.get_type() is return_type -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_stage_decorator_no_annotation_error(config: Config): @stage @@ -387,7 +387,7 @@ def test_fn(message): test_fn(config) -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_stage_arg_no_value_error(config: Config): @stage @@ -398,7 +398,7 @@ def test_fn(message: float, value: float) -> float: test_fn(config) # pylint: disable=no-value-for-parameter -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("needed_columns", [None, { 'result': TypeId.INT64 diff --git a/tests/stages/test_deserialize_stage_pipe.py b/tests/stages/test_deserialize_stage_pipe.py index 5231002a20..4044789de5 100755 --- a/tests/stages/test_deserialize_stage_pipe.py +++ b/tests/stages/test_deserialize_stage_pipe.py @@ -32,7 +32,7 @@ @pytest.mark.use_cudf -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_fixing_non_unique_indexes(dataset: DatasetManager): # Set 2 ids equal to others df = dataset.dup_index(dataset["filter_probs.csv"], count=2) diff --git a/tests/stages/test_http_server_source_stage.py b/tests/stages/test_http_server_source_stage.py index 15c8e585da..4a67a2e65d 100644 --- a/tests/stages/test_http_server_source_stage.py +++ b/tests/stages/test_http_server_source_stage.py @@ -58,7 +58,7 @@ def join(self, timeout=None): @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("lines", [False, True], ids=["json", "lines"]) @pytest.mark.parametrize("use_payload_to_df_fn", [False, True], ids=["no_payload_to_df_fn", "payload_to_df_fn"]) def test_generate_frames(config: Config, dataset_pandas: DatasetManager, lines: bool, use_payload_to_df_fn: bool): @@ -154,7 +154,7 @@ def test_constructor_invalid_accept_status(config: Config, invalid_accept_status @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize( "lines", [False, pytest.param(True, marks=pytest.mark.skip(reason="https://github.com/rapidsai/cudf/issues/15820"))], diff --git a/tests/stages/test_llm_engine_stage_pipe.py b/tests/stages/test_llm_engine_stage_pipe.py index 16151bca26..5f445d853b 100644 --- a/tests/stages/test_llm_engine_stage_pipe.py +++ b/tests/stages/test_llm_engine_stage_pipe.py @@ -41,7 +41,7 @@ def _build_engine() -> LLMEngine: @pytest.mark.use_cudf -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_pipeline(config: Config, dataset_cudf: DatasetManager): test_data = os.path.join(TEST_DIRS.validation_data_dir, 'root-cause-validation-data-input.jsonlines') input_df = dataset_cudf[test_data] diff --git a/tests/stages/test_ml_flow_drift_stage.py b/tests/stages/test_ml_flow_drift_stage.py index 3f41683315..501bb3eb1a 100644 --- a/tests/stages/test_ml_flow_drift_stage.py +++ b/tests/stages/test_ml_flow_drift_stage.py @@ -56,7 +56,7 @@ def test_constructor(config): @pytest.mark.use_cudf -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_calc_drift(config, filter_probs_df): with patch("morpheus.stages.postprocess.ml_flow_drift_stage.mlflow.start_run"): labels = ["a", "b", "c"] diff --git a/tests/stages/test_preprocess_fil_stage.py b/tests/stages/test_preprocess_fil_stage.py index 5c3bfb518b..85cb84347d 100644 --- a/tests/stages/test_preprocess_fil_stage.py +++ b/tests/stages/test_preprocess_fil_stage.py @@ -30,7 +30,7 @@ @pytest.fixture(name='config') -def fixture_config(config: Config, use_cpp: bool): # pylint: disable=unused-argument +def fixture_config(config: Config, gpu_mode: bool): # pylint: disable=unused-argument config.feature_length = 1 config.fil = ConfigFIL() config.fil.feature_columns = ["data"] diff --git a/tests/stages/test_preprocess_nlp_stage.py b/tests/stages/test_preprocess_nlp_stage.py index a8328ee018..bdf667fa7a 100644 --- a/tests/stages/test_preprocess_nlp_stage.py +++ b/tests/stages/test_preprocess_nlp_stage.py @@ -31,7 +31,7 @@ @pytest.fixture(name='config') -def fixture_config(config: Config, use_cpp: bool): # pylint: disable=unused-argument +def fixture_config(config: Config, gpu_mode: bool): # pylint: disable=unused-argument config.class_labels = [ "address", "bank_acct", diff --git a/tests/stages/test_timeseries_stage.py b/tests/stages/test_timeseries_stage.py index 8babd0e752..30265ed865 100644 --- a/tests/stages/test_timeseries_stage.py +++ b/tests/stages/test_timeseries_stage.py @@ -68,7 +68,7 @@ def test_constructor(config): @pytest.mark.use_cudf -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_call_timeseries_user(config): stage = TimeSeriesStage(config) diff --git a/tests/test_abp.py b/tests/test_abp.py index 0cec1331a9..ea55530094 100755 --- a/tests/test_abp.py +++ b/tests/test_abp.py @@ -51,7 +51,7 @@ @pytest.mark.slow -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("launch_mock_triton") @pytest.mark.parametrize("message_type", [MultiMessage, ControlMessage]) def test_abp_cpp(config: Config, tmp_path: str, message_type: type, morpheus_log_level: int): @@ -101,7 +101,7 @@ def test_abp_cpp(config: Config, tmp_path: str, message_type: type, morpheus_log @pytest.mark.slow -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("launch_mock_triton") @pytest.mark.parametrize("message_type", [MultiMessage, ControlMessage]) def test_abp_multi_segment_cpp(config, tmp_path, message_type): diff --git a/tests/test_abp_kafka.py b/tests/test_abp_kafka.py index 02a6e795b5..0a13444f74 100755 --- a/tests/test_abp_kafka.py +++ b/tests/test_abp_kafka.py @@ -54,7 +54,7 @@ @pytest.mark.kafka @pytest.mark.slow -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("launch_mock_triton") def test_abp_cpp(config: Config, dataset_pandas: DatasetManager, diff --git a/tests/test_add_classifications_stage.py b/tests/test_add_classifications_stage.py index 608c132de0..2d6a99b15f 100755 --- a/tests/test_add_classifications_stage.py +++ b/tests/test_add_classifications_stage.py @@ -32,7 +32,7 @@ @pytest.fixture(name="config") -def config_fixture(config: Config, use_cpp: bool): # pylint: disable=unused-argument +def config_fixture(config: Config, gpu_mode: bool): # pylint: disable=unused-argument config.class_labels = ['frogs', 'lizards', 'toads'] yield config @@ -61,7 +61,7 @@ def test_constructor_errors(config: Config): AddClassificationsStage(config, labels=['missing']) -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_add_labels_with_multi_response_message_and_contgrol_message(): class_labels = {0: "frogs", 1: "lizards", 2: "toads"} diff --git a/tests/test_add_scores_stage.py b/tests/test_add_scores_stage.py index 0722df95f4..1d026c8d4e 100755 --- a/tests/test_add_scores_stage.py +++ b/tests/test_add_scores_stage.py @@ -36,7 +36,7 @@ @pytest.fixture(name='config') -def fixture_config(config: Config, use_cpp: bool): # pylint: disable=unused-argument +def fixture_config(config: Config, gpu_mode: bool): # pylint: disable=unused-argument config.class_labels = ['frogs', 'lizards', 'toads'] config.feature_length = 12 yield config @@ -66,7 +66,7 @@ def test_constructor_errors(config: Config): AddScoresStage(config, labels=['missing']) -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_add_labels_with_multi_response_message_and_control_message(): class_labels = {0: "frogs", 1: "lizards", 2: "toads"} diff --git a/tests/test_conftest.py b/tests/test_conftest.py index cc37b918d3..76d18b59d5 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -27,12 +27,12 @@ @pytest.fixture(name="cpp_from_marker", scope="function") def cpp_from_marker_fixture(request: pytest.FixtureRequest) -> bool: - use_cpp = len([x for x in request.node.iter_markers("use_cpp") if "added_by" in x.kwargs]) > 0 - use_python = len([x for x in request.node.iter_markers("use_python") if "added_by" in x.kwargs]) > 0 + gpu_mode = len([x for x in request.node.iter_markers("gpu_mode") if "added_by" in x.kwargs]) > 0 + cpu_mode = len([x for x in request.node.iter_markers("cpu_mode") if "added_by" in x.kwargs]) > 0 - assert use_cpp != use_python + assert gpu_mode != cpu_mode - return use_cpp + return gpu_mode @pytest.fixture(name="df_type_from_marker", scope="function") @@ -117,37 +117,37 @@ def test_no_mark(): # === No Marks === -@pytest.mark.use_cpp -def test_mark_use_cpp(): +@pytest.mark.gpu_mode +def test_mark_gpu_mode(): assert CppConfig.get_should_use_cpp() -@pytest.mark.use_python -def test_mark_use_python(): +@pytest.mark.cpu_mode +def test_mark_cpu_mode(): assert not CppConfig.get_should_use_cpp() -@pytest.mark.use_cpp -@pytest.mark.use_python +@pytest.mark.gpu_mode +@pytest.mark.cpu_mode def test_mark_both(cpp_from_marker: bool): assert CppConfig.get_should_use_cpp() == cpp_from_marker # === Marks and Config === -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("config") -def test_mark_and_config_use_cpp(): +def test_mark_and_config_gpu_mode(): assert CppConfig.get_should_use_cpp() -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.usefixtures("config") -def test_mark_and_config_use_python(): +def test_mark_and_config_cpu_mode(): assert not CppConfig.get_should_use_cpp() -@pytest.mark.use_cpp -@pytest.mark.use_python +@pytest.mark.gpu_mode +@pytest.mark.cpu_mode @pytest.mark.usefixtures("config") def test_mark_and_config_both(cpp_from_marker: bool): assert CppConfig.get_should_use_cpp() == cpp_from_marker @@ -159,26 +159,26 @@ def test_mark_and_config_neither(cpp_from_marker: bool): # === Fixture === -@pytest.mark.use_cpp -def test_fixture_use_cpp(use_cpp: bool): - assert use_cpp +@pytest.mark.gpu_mode +def test_fixture_gpu_mode(gpu_mode: bool): + assert gpu_mode assert CppConfig.get_should_use_cpp() -@pytest.mark.use_python -def test_fixture_use_python(use_cpp: bool): - assert not use_cpp +@pytest.mark.cpu_mode +def test_fixture_cpu_mode(gpu_mode: bool): + assert not gpu_mode assert not CppConfig.get_should_use_cpp() -@pytest.mark.use_cpp -@pytest.mark.use_python -def test_fixture_both(use_cpp: bool): - assert CppConfig.get_should_use_cpp() == use_cpp +@pytest.mark.gpu_mode +@pytest.mark.cpu_mode +def test_fixture_both(gpu_mode: bool): + assert CppConfig.get_should_use_cpp() == gpu_mode -def test_fixture_neither(use_cpp: bool): - assert CppConfig.get_should_use_cpp() == use_cpp +def test_fixture_neither(gpu_mode: bool): + assert CppConfig.get_should_use_cpp() == gpu_mode # === Config Fixture === @@ -197,58 +197,58 @@ class TestNoMarkerClass: def test_no_marker(self): assert CppConfig.get_should_use_cpp() - @pytest.mark.use_python + @pytest.mark.cpu_mode def test_python_marker(self): assert not CppConfig.get_should_use_cpp() - @pytest.mark.use_cpp + @pytest.mark.gpu_mode def test_cpp_marker(self): assert CppConfig.get_should_use_cpp() - @pytest.mark.use_cpp - @pytest.mark.use_python + @pytest.mark.gpu_mode + @pytest.mark.cpu_mode def test_marker_both(self, cpp_from_marker: bool): assert CppConfig.get_should_use_cpp() == cpp_from_marker @pytest.mark.slow - def test_other_marker(self, use_cpp: bool): - assert CppConfig.get_should_use_cpp() == use_cpp + def test_other_marker(self, gpu_mode: bool): + assert CppConfig.get_should_use_cpp() == gpu_mode -@pytest.mark.use_python +@pytest.mark.cpu_mode class TestPythonMarkerClass: def test_no_marker(self): assert not CppConfig.get_should_use_cpp() - def test_with_fixture(self, use_cpp: bool): - assert not use_cpp + def test_with_fixture(self, gpu_mode: bool): + assert not gpu_mode assert not CppConfig.get_should_use_cpp() - @pytest.mark.use_python + @pytest.mark.cpu_mode def test_extra_marker(self): assert not CppConfig.get_should_use_cpp() - @pytest.mark.use_cpp + @pytest.mark.gpu_mode def test_add_marker(self, cpp_from_marker: bool): assert CppConfig.get_should_use_cpp() == cpp_from_marker -@pytest.mark.use_cpp +@pytest.mark.gpu_mode class TestCppMarkerClass: def test_no_marker(self): assert CppConfig.get_should_use_cpp() - def test_with_fixture(self, use_cpp: bool): - assert use_cpp + def test_with_fixture(self, gpu_mode: bool): + assert gpu_mode assert CppConfig.get_should_use_cpp() - @pytest.mark.use_cpp + @pytest.mark.gpu_mode def test_extra_marker(self): assert CppConfig.get_should_use_cpp() - @pytest.mark.use_python + @pytest.mark.cpu_mode def test_add_marker(self, cpp_from_marker: bool): assert CppConfig.get_should_use_cpp() == cpp_from_marker diff --git a/tests/test_dfp.py b/tests/test_dfp.py index 2a8a9ac5df..9fb636fc28 100755 --- a/tests/test_dfp.py +++ b/tests/test_dfp.py @@ -48,7 +48,7 @@ @pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -135,7 +135,7 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, config: Config, tmp_path: str, morph @pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -219,7 +219,7 @@ def test_dfp_user123(mock_ae: mock.MagicMock, config: Config, tmp_path: str, mor @pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') diff --git a/tests/test_dfp_kafka.py b/tests/test_dfp_kafka.py index 9046d39a80..63571ebb17 100755 --- a/tests/test_dfp_kafka.py +++ b/tests/test_dfp_kafka.py @@ -51,7 +51,7 @@ @pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.kafka @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.reload_modules([commands, preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules", "loglevel_debug") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -156,7 +156,7 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, @pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.kafka @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules", "loglevel_debug") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') diff --git a/tests/test_error_pipe.py b/tests/test_error_pipe.py index ef0cf54cbf..e130b72951 100755 --- a/tests/test_error_pipe.py +++ b/tests/test_error_pipe.py @@ -44,7 +44,7 @@ def test_stage_raises_exception(config: Config, filter_probs_df: pd.DataFrame, e # TODO should work in both GPU and CPU -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("delayed_start", [False, True]) def test_monitor_not_impl(config: Config, delayed_start: bool): diff --git a/tests/test_file_in_out.py b/tests/test_file_in_out.py index bd0d06d6c2..40909b7557 100755 --- a/tests/test_file_in_out.py +++ b/tests/test_file_in_out.py @@ -110,7 +110,7 @@ def test_file_read_json(config: Config): @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.usefixtures("chdir_tmpdir") def test_to_file_no_path(tmp_path: pathlib.Path, config: Config): """ @@ -196,7 +196,7 @@ def test_file_rw_index_pipe(tmp_path: pathlib.Path, config: Config, input_file: "include_header": True }), (os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.jsonlines"), {})], ids=["CSV", "CSV_ID", "JSON"]) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_file_roundtrip(tmp_path: pathlib.Path, input_file: str, extra_kwargs: dict[str, typing.Any]): # Output file should be same type as input diff --git a/tests/test_http_server_sink_stage.py b/tests/test_http_server_sink_stage.py index 9702ec1dd5..220ba43a92 100644 --- a/tests/test_http_server_sink_stage.py +++ b/tests/test_http_server_sink_stage.py @@ -89,7 +89,7 @@ def _custom_serializer(df: DataFrameType) -> str: @pytest.mark.slow -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("lines", [False, True]) @pytest.mark.parametrize("max_rows_per_response", [10000, 10]) @pytest.mark.parametrize("df_serializer_fn", [None, _custom_serializer]) diff --git a/tests/test_inference_stage.py b/tests/test_inference_stage.py index 7df1ade557..e71d8acb9b 100755 --- a/tests/test_inference_stage.py +++ b/tests/test_inference_stage.py @@ -121,7 +121,7 @@ def test_split_batches(): @pytest.mark.skip(reason="Test is passing, but python only impls for inference remains TBD") -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_convert_one_response(): # Pylint currently fails to work with classmethod: https://github.com/pylint-dev/pylint/issues/981 # pylint: disable=no-member diff --git a/tests/test_inference_worker.py b/tests/test_inference_worker.py index 5953796169..8c87dafbe1 100755 --- a/tests/test_inference_worker.py +++ b/tests/test_inference_worker.py @@ -34,7 +34,7 @@ def test_constructor(): @pytest.mark.skip(reason="Test is passing, but python only impls for inference remains TBD") -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.usefixtures("config") def test_build_output_message(): diff --git a/tests/test_linear_modules_stage.py b/tests/test_linear_modules_stage.py index 8209b96b6e..da3dc8979e 100755 --- a/tests/test_linear_modules_stage.py +++ b/tests/test_linear_modules_stage.py @@ -28,7 +28,7 @@ } -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_constructor(config): mod_stage = LinearModulesStage(config, module_config, input_port_name="test_in", output_port_name="test_out") @@ -44,7 +44,7 @@ def test_constructor(config): pytest.raises(NotImplementedError, mod_stage._get_cpp_module_node, None) -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_build_single_before_module_registration(config): mock_node = mock.MagicMock() @@ -70,7 +70,7 @@ def module_init_fn(_: mrc.Builder): registry.register_module("TestSimpleModule", "test_morpheus_modules", mrc_version, module_init_fn) -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_build_single_after_module_registration(config): register_test_module() diff --git a/tests/test_messages.py b/tests/test_messages.py index d0dfdb3101..48080aa75e 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -168,6 +168,6 @@ def check_all_messages(should_be_cpp: bool, no_cpp_class: bool): ) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_constructor_cpp(): check_all_messages(morpheus.config.CppConfig.get_should_use_cpp(), False) diff --git a/tests/test_milvus_write_to_vector_db_stage_pipe.py b/tests/test_milvus_write_to_vector_db_stage_pipe.py index 3f91ec3375..3dd72ac972 100755 --- a/tests/test_milvus_write_to_vector_db_stage_pipe.py +++ b/tests/test_milvus_write_to_vector_db_stage_pipe.py @@ -48,7 +48,7 @@ def get_test_df(num_input_rows): @pytest.mark.milvus -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.parametrize("use_instance, num_input_rows, expected_num_output_rows, resource_kwargs, recreate", [(True, 5, 5, { "partition_name": "age_partition" @@ -134,7 +134,7 @@ def test_write_to_vector_db_stage_from_cm_pipe(milvus_server_uri: str, @pytest.mark.milvus -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.parametrize("is_multiresponse_message", [True, False]) def test_write_to_vector_db_stage_from_mm_pipe(milvus_server_uri: str, idx_part_collection_config: dict, diff --git a/tests/test_monitor_stage.py b/tests/test_monitor_stage.py index 68b1b35ca7..148a3f53cc 100755 --- a/tests/test_monitor_stage.py +++ b/tests/test_monitor_stage.py @@ -178,7 +178,7 @@ def test_log_level(mock_progress_sink: mock.MagicMock, assert mock_sink_on_completed.call_count == expected_call_count -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_thread(config: Config, morpheus_log_level: int): """ Test ensures the monitor stage executes on the same thread as the parent stage diff --git a/tests/test_multi_message.py b/tests/test_multi_message.py index 26e349e10d..da823e4e6b 100644 --- a/tests/test_multi_message.py +++ b/tests/test_multi_message.py @@ -47,7 +47,7 @@ from morpheus.utils import logger as morpheus_logger -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_missing_explicit_init(): with pytest.raises(ValueError, match="improperly configured"): @@ -149,11 +149,11 @@ def test_get_meta(filter_probs_df: typing.Union[cudf.DataFrame, pd.DataFrame]): _test_get_meta(filter_probs_df) -# Ignore unused arguments warnigns due to using the `use_cpp` fixture +# Ignore unused arguments warnigns due to using the `gpu_mode` fixture # pylint:disable=unused-argument -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_get_meta_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta @@ -163,7 +163,7 @@ def test_get_meta_dup_index(dataset: DatasetManager): _test_get_meta(df) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_set_meta(dataset: DatasetManager): df_saved = dataset.pandas["filter_probs.csv"] @@ -238,12 +238,12 @@ def _test_set_meta_new_column(df: typing.Union[cudf.DataFrame, pd.DataFrame], df DatasetManager.assert_df_equal(multi.get_meta(["v2", "new_column2"]), val_to_set) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_set_meta_new_column(dataset: DatasetManager): _test_set_meta_new_column(dataset["filter_probs.csv"], dataset.default_df_type) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_set_meta_new_column_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta df = dataset.replace_index(dataset["filter_probs.csv"], replace_ids={3: 4, 5: 4}) @@ -311,7 +311,7 @@ def test_copy_ranges(filter_probs_df: typing.Union[cudf.DataFrame, pd.DataFrame] _test_copy_ranges(filter_probs_df) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_copy_ranges_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta @@ -435,7 +435,7 @@ def test_get_slice_values(filter_probs_df: cudf.DataFrame): _test_get_slice_values(filter_probs_df) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_get_slice_values_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta @@ -716,7 +716,7 @@ def test_tensor_constructor(filter_probs_df: cudf.DataFrame): memory=TensorMemory(count=mess_len, tensors={"id_tensor": invalid_id_tensor})) -@pytest.mark.usefixtures("use_cpp") +@pytest.mark.usefixtures("gpu_mode") def test_tensor_slicing(dataset: DatasetManager): # Pylint currently fails to work with classmethod: https://github.com/pylint-dev/pylint/issues/981 @@ -805,8 +805,8 @@ def test_tensor_slicing(dataset: DatasetManager): dataset.assert_df_equal(double_slice.get_meta(), single_slice.get_meta()) -@pytest.mark.usefixtures("use_cpp") -@pytest.mark.use_python +@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.cpu_mode def test_deprecation_message(filter_probs_df: cudf.DataFrame, caplog): meta = MessageMeta(filter_probs_df) diff --git a/tests/test_multi_port_modules_stage.py b/tests/test_multi_port_modules_stage.py index ce8388ae1c..fdeef14a44 100755 --- a/tests/test_multi_port_modules_stage.py +++ b/tests/test_multi_port_modules_stage.py @@ -50,7 +50,7 @@ def registered_module_conf(): yield registered_module_conf -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_constructor(config, unregistered_module_conf): mod_stage = MultiPortModulesStage(config, diff --git a/tests/test_phishing.py b/tests/test_phishing.py index 73e453fd49..f81a65095a 100755 --- a/tests/test_phishing.py +++ b/tests/test_phishing.py @@ -43,7 +43,7 @@ @pytest.mark.slow -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("launch_mock_triton") def test_email_cpp(config: Config, tmp_path: str, morpheus_log_level: int): config.mode = PipelineModes.NLP diff --git a/tests/test_phishing_kafka.py b/tests/test_phishing_kafka.py index d2331601c9..dd12af6dd9 100755 --- a/tests/test_phishing_kafka.py +++ b/tests/test_phishing_kafka.py @@ -53,7 +53,7 @@ @pytest.mark.kafka @pytest.mark.slow -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("launch_mock_triton") def test_email_cpp(dataset_pandas: DatasetManager, config: Config, diff --git a/tests/test_rss_source_stage_pipe.py b/tests/test_rss_source_stage_pipe.py index 4d73f01524..80772dca9d 100644 --- a/tests/test_rss_source_stage_pipe.py +++ b/tests/test_rss_source_stage_pipe.py @@ -35,7 +35,7 @@ def test_support_cpp_node(config): assert rss_source_stage.supports_cpp_node() is False -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize( "feed_input, batch_size, expected_count, enable_cache", [([valid_feed_input], 30, 1, False), ([valid_feed_input], 12, 3, True), @@ -60,7 +60,7 @@ def test_rss_source_stage_pipe(config: Config, assert len(sink_stage.get_messages()) == expected_count -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_invalid_input_rss_source_stage(config: Config): with pytest.raises(ValueError, match=f"Invalid URL or file path: {invalid_feed_input}"): diff --git a/tests/test_serialize_stage.py b/tests/test_serialize_stage.py index a84c7d37e6..12fdd42710 100755 --- a/tests/test_serialize_stage.py +++ b/tests/test_serialize_stage.py @@ -25,7 +25,7 @@ from morpheus.stages.postprocess.serialize_stage import SerializeStage -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_fixed_columns(config): df1 = cudf.DataFrame() df1['apples'] = range(0, 4) diff --git a/tests/test_sid.py b/tests/test_sid.py index d7190bce7a..6b4899f9aa 100755 --- a/tests/test_sid.py +++ b/tests/test_sid.py @@ -174,7 +174,7 @@ def _run_minibert(*, @pytest.mark.slow -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("launch_mock_triton") @pytest.mark.parametrize("message_type", [MultiMessage, ControlMessage]) def test_minibert_no_trunc(config: Config, tmp_path: str, message_type: type, morpheus_log_level: int): diff --git a/tests/test_sid_kafka.py b/tests/test_sid_kafka.py index 47e86b433e..57c8f6edb2 100755 --- a/tests/test_sid_kafka.py +++ b/tests/test_sid_kafka.py @@ -51,7 +51,7 @@ @pytest.mark.kafka @pytest.mark.slow -@pytest.mark.use_cpp +@pytest.mark.gpu_mode @pytest.mark.usefixtures("launch_mock_triton") def test_minibert_cpp(dataset_pandas: DatasetManager, config: Config, diff --git a/tests/test_tensor_memory.py b/tests/test_tensor_memory.py index 0d71fb6817..631f6493b2 100644 --- a/tests/test_tensor_memory.py +++ b/tests/test_tensor_memory.py @@ -88,7 +88,7 @@ def test_tensor_memory(config: Config): @pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_inference_memory_ae(config: Config): test_data = cp.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) count = test_data.shape[0] @@ -158,7 +158,7 @@ def check_response_memory_probs_and_ae(cls: type): @pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") -@pytest.mark.use_python +@pytest.mark.cpu_mode def test_response_memory_ae(config: Config, filter_probs_df: DataFrameType): mem = check_response_memory_probs_and_ae(ResponseMemoryAE) diff --git a/tests/test_triton_inference_stage.py b/tests/test_triton_inference_stage.py index 34a411e734..c91811693c 100644 --- a/tests/test_triton_inference_stage.py +++ b/tests/test_triton_inference_stage.py @@ -123,7 +123,7 @@ def test_resource_pool_create_raises_error(): @pytest.mark.skip(reason="TODO: determine what to do about python impls") -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("pipeline_mode", list(PipelineModes)) def test_stage_constructor_worker_class(config: Config, pipeline_mode: PipelineModes): config.mode = pipeline_mode @@ -133,7 +133,7 @@ def test_stage_constructor_worker_class(config: Config, pipeline_mode: PipelineM @pytest.mark.skip(reason="TODO: determine what to do about python impls") -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("pipeline_mode", list(PipelineModes)) @pytest.mark.parametrize("needs_logits", [True, False, None]) def test_stage_get_inference_worker(config: Config, pipeline_mode: PipelineModes, needs_logits: bool | None): diff --git a/tests/test_write_to_file_stage.py b/tests/test_write_to_file_stage.py index 1efc390a37..12a07ef76b 100755 --- a/tests/test_write_to_file_stage.py +++ b/tests/test_write_to_file_stage.py @@ -28,7 +28,7 @@ from morpheus.stages.preprocess.deserialize_stage import DeserializeStage -@pytest.mark.use_python +@pytest.mark.cpu_mode @pytest.mark.parametrize("use_deserialize", [False, True]) @pytest.mark.parametrize("flush", [False, True]) @pytest.mark.parametrize("output_type", ["csv", "json", "jsonlines"]) From f128ae44f3320bb009e2ec2918695089dab372c5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 14:33:35 -0700 Subject: [PATCH 129/347] Add debug target for debugging unittests --- morpheus.code-workspace | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/morpheus.code-workspace b/morpheus.code-workspace index 5fb3a58375..be0d8af1aa 100644 --- a/morpheus.code-workspace +++ b/morpheus.code-workspace @@ -304,6 +304,18 @@ "request": "launch", "type": "debugpy" }, + { + "args": [ + "-x" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}", + "justMyCode": false, + "module": "pytest", + "name": "Python: tests", + "request": "launch", + "type": "debugpy" + }, { "MIMode": "gdb", "args": [ From 6c6dc01dee2795943a275663b063dcc6c1c08fa7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 15:37:01 -0700 Subject: [PATCH 130/347] Replace use_cpp and use_python markers with gpu_mode and cpu_mode respectively. Since supporting both GPU and CPU execution modes will not be common, require test authors to explicitly mark a test with gpu_and_cpu_mode --- pyproject.toml | 5 +- tests/conftest.py | 123 ++++++++++++++++++----------------------- tests/test_conftest.py | 102 +++++++++++++++++----------------- 3 files changed, 106 insertions(+), 124 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a2490e0456..52f237117b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,9 @@ markers = [ "slow: Slow tests", "kafka: Tests that require a running instance of kafka", "milvus: Tests that require a running instance of milvus", - "use_cpp: Test support C++ nodes and objects", - "use_python: Test only supports Python nodes and objects", + "gpu_mode: Test support GPU nodes and objects", + "cpu_mode: Test only supports CPU nodes and objects", + "gpu_and_cpu_mode: Test supports both GPU and CPU nodes and objects", "use_cudf: Test supports cuDF datasets", "use_pandas: Test supports Pandas datasets", "replace_callback: Replaces the results_callback in cli", diff --git a/tests/conftest.py b/tests/conftest.py index 6a6aeebf81..4561ba1e46 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,9 @@ from _utils.kafka import kafka_consumer_fixture # noqa: F401 pylint:disable=unused-import from _utils.kafka import kafka_topics_fixture # noqa: F401 pylint:disable=unused-import +if typing.TYPE_CHECKING: + from morpheus.config import ExecutionMode + # Don't let pylint complain about pytest fixtures # pylint: disable=redefined-outer-name,unused-argument @@ -102,35 +105,12 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): This function will add parameterizations for the `config` fixture depending on what types of config the test supports """ - # === gpu_mode Parameterize === - gpu_mode = metafunc.definition.get_closest_marker("gpu_mode") is not None - cpu_mode = metafunc.definition.get_closest_marker("cpu_mode") is not None - - gpu_mode_param = pytest.param(True, marks=pytest.mark.gpu_mode(added_by="generate_tests"), id="gpu_mode") - cpu_mode_param = pytest.param(False, marks=pytest.mark.cpu_mode(added_by="generate_tests"), id="cpu_mode") - - _set_gpu_mode_params = [] - - if ("gpu_mode" in metafunc.fixturenames): - # Need to add some params since the fixture was requested - # Add cpp unless gpu_mode == True and cpu_mode == False - if not (cpu_mode and not gpu_mode): - _set_gpu_mode_params.append(gpu_mode_param) - - # Add python unless gpu_mode == False and cpu_mode == True - if not (not cpu_mode and gpu_mode): - _set_gpu_mode_params.append(cpu_mode_param) - - elif (gpu_mode and cpu_mode): - # Need to parameterize since we have multiple - _set_gpu_mode_params.extend([gpu_mode_param, cpu_mode_param]) - elif (not gpu_mode and not cpu_mode): - # If neither are set, default to cpp - _set_gpu_mode_params.append(gpu_mode_param) - - if (len(_set_gpu_mode_params) > 0): - metafunc.parametrize("_set_gpu_mode", _set_gpu_mode_params, indirect=True) + # A test can request a fixture by placing it in the function arguments, or with a mark + if ("gpu_and_cpu_mode" in metafunc.fixturenames or metafunc.definition.get_closest_marker("gpu_and_cpu_mode")): + gpu_mode_param = pytest.param(True, marks=pytest.mark.gpu_mode(added_by="generate_tests"), id="gpu_mode") + cpu_mode_param = pytest.param(False, marks=pytest.mark.cpu_mode(added_by="generate_tests"), id="cpu_mode") + metafunc.parametrize("execution_mode", [gpu_mode_param, cpu_mode_param], indirect=True) # === df_type Parameterize === if ("df_type" in metafunc.fixturenames): @@ -208,47 +188,6 @@ def pytest_runtest_teardown(item, nextitem): reset_logging(logger_name=None) # Reset the root logger as well -def _get_gpu_mode(request: pytest.FixtureRequest) -> bool: - do_gpu_mode: bool = True - - # Check for the param if this was indirectly set - if (hasattr(request, "param") and isinstance(request.param, bool)): - do_gpu_mode = request.param - else: - # If not, check for the marker and use that - gpu_mode = request.node.get_closest_marker("gpu_mode") is not None - cpu_mode = request.node.get_closest_marker("cpu_mode") is not None - - if (gpu_mode and cpu_mode): - raise RuntimeError(f"Both markers (gpu_mode and cpu_mode) were added to function {request.node.nodeid}. " - "Remove markers to support both.") - - # This will default to True or follow gpu_mode - do_gpu_mode = not cpu_mode - - return do_gpu_mode - - -# This fixture will be used by all tests. -@pytest.fixture(scope="function", autouse=True) -def _set_gpu_mode(request: pytest.FixtureRequest): - do_gpu_mode = _get_gpu_mode(request) - - from morpheus.config import CppConfig - - CppConfig.set_should_use_cpp(do_gpu_mode) - - yield do_gpu_mode - - -# This fixture will be used by all tests. -@pytest.fixture(scope="function") -def gpu_mode(_set_gpu_mode: bool): - - # Just return the set value - yield _set_gpu_mode - - @pytest.fixture(scope="function") def config_only_cpp(): """ @@ -307,8 +246,51 @@ def df_type(request: pytest.FixtureRequest): yield df_type_str +def _get_execution_mode(request: pytest.FixtureRequest) -> "ExecutionMode": + do_gpu_mode: bool = True + + # Check for the param if this was indirectly set + if (hasattr(request, "param") and isinstance(request.param, bool)): + do_gpu_mode = request.param + else: + # If not, check for the marker and use that + gpu_mode = request.node.get_closest_marker("gpu_mode") is not None + cpu_mode = request.node.get_closest_marker("cpu_mode") is not None + + if (gpu_mode and cpu_mode): + raise RuntimeError(f"Both markers (gpu_mode and cpu_mode) were added to function {request.node.nodeid}. " + "Use the gpu_and_cpu_mode marker to test both.") + + # This will default to True or follow gpu_mode + do_gpu_mode = not cpu_mode + + from morpheus.config import ExecutionMode + if do_gpu_mode: + return ExecutionMode.GPU + + return ExecutionMode.CPU + + +@pytest.fixture(name="execution_mode", scope="function") +def execution_mode_fixture(request: pytest.FixtureRequest): + exec_mode = _get_execution_mode(request) + yield exec_mode + + +# This fixture will be used by all tests. +@pytest.fixture(scope="function", autouse=True) +def _set_use_cpp(request: pytest.FixtureRequest): + execution_mode = _get_execution_mode(request) + from morpheus.config import CppConfig + + do_use_cpp: bool = (execution_mode.value == "GPU") + CppConfig.set_should_use_cpp(do_use_cpp) + + yield do_use_cpp + + @pytest.fixture(scope="function") -def config(): +def config(execution_mode: "ExecutionMode"): """ For new pytest style tests, get the config by using this fixture. It will setup the config based on the marks set on the object. If no marks are added to the test, it will be parameterized for both C++ and python. For example, @@ -322,6 +304,7 @@ def my_python_test(config: Config): from morpheus.config import Config config = Config() + config.execution_mode = execution_mode yield config diff --git a/tests/test_conftest.py b/tests/test_conftest.py index 76d18b59d5..fcc33a6a82 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -21,11 +21,31 @@ import cudf from _utils.dataset_manager import DatasetManager +from morpheus.config import Config from morpheus.config import CppConfig +from morpheus.config import ExecutionMode -@pytest.fixture(name="cpp_from_marker", scope="function") -def cpp_from_marker_fixture(request: pytest.FixtureRequest) -> bool: +def exec_mode_to_cpp_mode(exec_mode: ExecutionMode) -> bool: + return exec_mode == ExecutionMode.GPU + + +@pytest.fixture(name="exec_mode_from_marker", scope="function") +def exec_mode_from_marker_fixture(request: pytest.FixtureRequest) -> ExecutionMode: + + gpu_mode = len([x for x in request.node.iter_markers("gpu_mode") if "added_by" in x.kwargs]) > 0 + cpu_mode = len([x for x in request.node.iter_markers("cpu_mode") if "added_by" in x.kwargs]) > 0 + + assert gpu_mode != cpu_mode + + if gpu_mode: + return ExecutionMode.GPU + + return ExecutionMode.CPU + + +@pytest.fixture(name="cpp_mode_from_marker", scope="function") +def cpp_mode_from_marker_fixture(request: pytest.FixtureRequest) -> bool: gpu_mode = len([x for x in request.node.iter_markers("gpu_mode") if "added_by" in x.kwargs]) > 0 cpu_mode = len([x for x in request.node.iter_markers("cpu_mode") if "added_by" in x.kwargs]) > 0 @@ -127,12 +147,6 @@ def test_mark_cpu_mode(): assert not CppConfig.get_should_use_cpp() -@pytest.mark.gpu_mode -@pytest.mark.cpu_mode -def test_mark_both(cpp_from_marker: bool): - assert CppConfig.get_should_use_cpp() == cpp_from_marker - - # === Marks and Config === @pytest.mark.gpu_mode @pytest.mark.usefixtures("config") @@ -141,44 +155,37 @@ def test_mark_and_config_gpu_mode(): @pytest.mark.cpu_mode -@pytest.mark.usefixtures("config") -def test_mark_and_config_cpu_mode(): +def test_mark_and_config_cpu_mode(config: Config): assert not CppConfig.get_should_use_cpp() + assert config.execution_mode == ExecutionMode.CPU -@pytest.mark.gpu_mode -@pytest.mark.cpu_mode -@pytest.mark.usefixtures("config") -def test_mark_and_config_both(cpp_from_marker: bool): - assert CppConfig.get_should_use_cpp() == cpp_from_marker +@pytest.mark.gpu_and_cpu_mode +def test_gpu_and_cpu_mode(config: Config, exec_mode_from_marker: ExecutionMode): + assert config.execution_mode == exec_mode_from_marker -@pytest.mark.usefixtures("config") -def test_mark_and_config_neither(cpp_from_marker: bool): - assert CppConfig.get_should_use_cpp() == cpp_from_marker +def test_mark_and_config_neither(config: Config): + assert CppConfig.get_should_use_cpp() + assert config.execution_mode == ExecutionMode.GPU # === Fixture === @pytest.mark.gpu_mode -def test_fixture_gpu_mode(gpu_mode: bool): - assert gpu_mode +def test_fixture_gpu_mode(execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.GPU assert CppConfig.get_should_use_cpp() @pytest.mark.cpu_mode -def test_fixture_cpu_mode(gpu_mode: bool): - assert not gpu_mode +def test_fixture_cpu_mode(execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.CPU assert not CppConfig.get_should_use_cpp() -@pytest.mark.gpu_mode -@pytest.mark.cpu_mode -def test_fixture_both(gpu_mode: bool): - assert CppConfig.get_should_use_cpp() == gpu_mode - - -def test_fixture_neither(gpu_mode: bool): - assert CppConfig.get_should_use_cpp() == gpu_mode +def test_fixture_neither(execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.GPU + assert CppConfig.get_should_use_cpp() # === Config Fixture === @@ -198,21 +205,19 @@ def test_no_marker(self): assert CppConfig.get_should_use_cpp() @pytest.mark.cpu_mode - def test_python_marker(self): + def test_python_marker(self, execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.CPU assert not CppConfig.get_should_use_cpp() @pytest.mark.gpu_mode - def test_cpp_marker(self): + def test_cpp_marker(self, execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.GPU assert CppConfig.get_should_use_cpp() - @pytest.mark.gpu_mode - @pytest.mark.cpu_mode - def test_marker_both(self, cpp_from_marker: bool): - assert CppConfig.get_should_use_cpp() == cpp_from_marker - @pytest.mark.slow - def test_other_marker(self, gpu_mode: bool): - assert CppConfig.get_should_use_cpp() == gpu_mode + def test_other_marker(self, execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.GPU + assert CppConfig.get_should_use_cpp() @pytest.mark.cpu_mode @@ -221,18 +226,15 @@ class TestPythonMarkerClass: def test_no_marker(self): assert not CppConfig.get_should_use_cpp() - def test_with_fixture(self, gpu_mode: bool): - assert not gpu_mode + def test_with_fixture(self, execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.CPU assert not CppConfig.get_should_use_cpp() @pytest.mark.cpu_mode - def test_extra_marker(self): + def test_extra_marker(self, execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.CPU assert not CppConfig.get_should_use_cpp() - @pytest.mark.gpu_mode - def test_add_marker(self, cpp_from_marker: bool): - assert CppConfig.get_should_use_cpp() == cpp_from_marker - @pytest.mark.gpu_mode class TestCppMarkerClass: @@ -240,18 +242,14 @@ class TestCppMarkerClass: def test_no_marker(self): assert CppConfig.get_should_use_cpp() - def test_with_fixture(self, gpu_mode: bool): - assert gpu_mode + def test_with_fixture(self, execution_mode: ExecutionMode): + assert execution_mode == ExecutionMode.GPU assert CppConfig.get_should_use_cpp() @pytest.mark.gpu_mode def test_extra_marker(self): assert CppConfig.get_should_use_cpp() - @pytest.mark.cpu_mode - def test_add_marker(self, cpp_from_marker: bool): - assert CppConfig.get_should_use_cpp() == cpp_from_marker - # === DF Type === def test_df_type_no_marks(df_type, df_type_from_marker): From 794e52e3ca3b77c60fdfc792ed7dd8caa53085dc Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 15:46:27 -0700 Subject: [PATCH 131/347] Support CPU mode in in-mem --- .../morpheus/stages/input/in_memory_data_generation_stage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/stages/input/in_memory_data_generation_stage.py b/python/morpheus/morpheus/stages/input/in_memory_data_generation_stage.py index a9f5d763ae..87e5e70a15 100644 --- a/python/morpheus/morpheus/stages/input/in_memory_data_generation_stage.py +++ b/python/morpheus/morpheus/stages/input/in_memory_data_generation_stage.py @@ -21,13 +21,14 @@ import mrc from morpheus.config import Config +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema logger = logging.getLogger(f"morpheus.{__name__}") -class InMemoryDataGenStage(SingleOutputSource): +class InMemoryDataGenStage(GpuAndCpuMixin, SingleOutputSource): """ Source stage that generates data in-memory using a provided iterable or generator function. @@ -54,7 +55,7 @@ def compute_schema(self, schema: StageSchema): # Set the output schema based on the OutputDataType schema.output_schema.set_type(self._output_data_type) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return False def _generate_data(self) -> Iterable[Any]: From 2962f0ab1799fd3bdee6cd80a92bc481c526a7f4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 16:15:13 -0700 Subject: [PATCH 132/347] Fix filter_probs_df fixture --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4561ba1e46..4922eab57c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -966,7 +966,7 @@ def test_something(dataset_cudf: DatasetManager): @pytest.fixture(scope="function") -def filter_probs_df(dataset, gpu_mode: bool): +def filter_probs_df(dataset): """ Shortcut fixture for loading the filter_probs.csv dataset. From 1c6bf902335f32cdb36e902f2337b52b1ce6f39e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 16:15:42 -0700 Subject: [PATCH 133/347] Mark tests for both CPU & GPU modes --- tests/test_kafka_source_stage_pipe.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_kafka_source_stage_pipe.py b/tests/test_kafka_source_stage_pipe.py index cb5adda659..92d93a6c6a 100644 --- a/tests/test_kafka_source_stage_pipe.py +++ b/tests/test_kafka_source_stage_pipe.py @@ -39,6 +39,7 @@ from kafka import KafkaConsumer +@pytest.mark.gpu_and_cpu_mode @pytest.mark.kafka def test_kafka_source_stage_pipe(config: Config, kafka_bootstrap_servers: str, kafka_topics: KafkaTopics) -> None: input_file = os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.jsonlines") @@ -63,6 +64,7 @@ def test_kafka_source_stage_pipe(config: Config, kafka_bootstrap_servers: str, k assert_results(comp_stage.get_results()) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.kafka def test_multi_topic_kafka_source_stage_pipe(config: Config, kafka_bootstrap_servers: str) -> None: input_file = os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.jsonlines") @@ -95,6 +97,7 @@ def test_multi_topic_kafka_source_stage_pipe(config: Config, kafka_bootstrap_ser assert_results(comp_stage.get_results()) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.kafka @pytest.mark.parametrize('async_commits', [True, False]) @pytest.mark.parametrize('num_records', [10, 100, 1000]) @@ -150,6 +153,7 @@ def test_kafka_source_commit(num_records: int, assert actual_offset == expected_offset +@pytest.mark.gpu_and_cpu_mode @pytest.mark.kafka @pytest.mark.parametrize('num_records', [1000]) def test_kafka_source_batch_pipe(config: Config, From 58cd456fc2346f3814ccb25d35551b65c91e13e1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 16:22:23 -0700 Subject: [PATCH 134/347] Mark tests for both CPU & GPU modes --- tests/test_error_pipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_error_pipe.py b/tests/test_error_pipe.py index e130b72951..9806ede797 100755 --- a/tests/test_error_pipe.py +++ b/tests/test_error_pipe.py @@ -28,6 +28,7 @@ from morpheus.stages.output.in_memory_sink_stage import InMemorySinkStage +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("exception_cls", [RuntimeError, ValueError, NotImplementedError]) def test_stage_raises_exception(config: Config, filter_probs_df: pd.DataFrame, exception_cls: type[Exception]): pipe = LinearPipeline(config) @@ -43,8 +44,7 @@ def test_stage_raises_exception(config: Config, filter_probs_df: pd.DataFrame, e assert len(sink_stage.get_messages()) == 0 -# TODO should work in both GPU and CPU -@pytest.mark.cpu_mode +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("delayed_start", [False, True]) def test_monitor_not_impl(config: Config, delayed_start: bool): From 0356c9ccb2d7deeaac79bb360ba104f8cb063b1b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 16:22:44 -0700 Subject: [PATCH 135/347] Mark test stages as supporting both CPU & GPU modes --- tests/_utils/stages/check_pre_alloc.py | 3 +- tests/_utils/stages/conv_msg.py | 31 ++++++++++++------- tests/_utils/stages/dfp_length_checker.py | 3 +- tests/_utils/stages/error_raiser.py | 3 +- .../stages/in_memory_multi_source_stage.py | 3 +- .../_utils/stages/in_memory_source_x_stage.py | 3 +- .../_utils/stages/multi_message_pass_thru.py | 3 +- tests/_utils/stages/multi_port_pass_thru.py | 3 +- tests/_utils/stages/record_thread_id_stage.py | 3 +- tests/_utils/stages/split_stage.py | 3 +- 10 files changed, 38 insertions(+), 20 deletions(-) diff --git a/tests/_utils/stages/check_pre_alloc.py b/tests/_utils/stages/check_pre_alloc.py index 674b73daae..3a02cbe3dc 100644 --- a/tests/_utils/stages/check_pre_alloc.py +++ b/tests/_utils/stages/check_pre_alloc.py @@ -21,11 +21,12 @@ from morpheus.common import typeid_to_numpy_str from morpheus.messages import MultiMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage -class CheckPreAlloc(PassThruTypeMixin, SinglePortStage): +class CheckPreAlloc(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Acts like add-class/add-scores in that it requests a preallocation, the node will assert that the preallocation occurred with the correct type. diff --git a/tests/_utils/stages/conv_msg.py b/tests/_utils/stages/conv_msg.py index aa5886c4f7..88d03429fc 100755 --- a/tests/_utils/stages/conv_msg.py +++ b/tests/_utils/stages/conv_msg.py @@ -22,19 +22,24 @@ import cudf -import morpheus._lib.messages as _messages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages import ControlMessage from morpheus.messages import MultiMessage from morpheus.messages import MultiResponseMessage from morpheus.messages import ResponseMemory +from morpheus.messages import TensorMemory +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import get_array_pkg +from morpheus.utils.type_utils import get_df_pkg +from morpheus.utils.type_utils import get_df_pkg_from_obj @register_stage("unittest-conv-msg", ignore_args=["expected_data", "message_type"]) -class ConvMsg(SinglePortStage): +class ConvMsg(GpuAndCpuMixin, SinglePortStage): """ Simple test stage to convert a MultiMessage to a MultiResponseProbsMessage, or a ControlMessage to a ControlMessage with probs tensor. @@ -50,7 +55,7 @@ class ConvMsg(SinglePortStage): def __init__(self, c: Config, - expected_data: typing.Union[pd.DataFrame, cudf.DataFrame] = None, + expected_data: DataFrameType = None, columns: typing.List[str] = None, order: str = 'K', probs_type: str = 'f4', @@ -58,11 +63,14 @@ def __init__(self, message_type: type[MultiResponseMessage] | type[ControlMessage] = MultiResponseMessage): super().__init__(c) + self._df_pkg = get_df_pkg(c.execution_mode) + self._array_pkg = get_array_pkg(c.execution_mode) + if expected_data is not None: - assert isinstance(expected_data, (pd.DataFrame, cudf.DataFrame)) + assert isinstance(expected_data, self._df_pkg.DataFrame) self._message_type = message_type - self._expected_data = expected_data + self._expected_data: DataFrameType | None = expected_data self._columns = columns self._order = order self._probs_type = probs_type @@ -86,10 +94,11 @@ def supports_cpp_node(self) -> bool: def _conv_message(self, message: MultiMessage | ControlMessage) -> MultiResponseMessage | ControlMessage: if self._expected_data is not None: - if (isinstance(self._expected_data, cudf.DataFrame)): + df_pkg = get_df_pkg_from_obj(self._expected_data) + if (isinstance(self._expected_data, self._df_pkg.DataFrame)): df = self._expected_data.copy(deep=True) else: - df = cudf.DataFrame(self._expected_data) + df = df_pkg.DataFrame(self._expected_data) else: if (isinstance(message, MultiMessage)): @@ -98,15 +107,15 @@ def _conv_message(self, message: MultiMessage | ControlMessage) -> MultiResponse else: df = message.get_meta(self._columns) else: - df: cudf.DataFrame = message.payload().get_data(self._columns) # type: ignore + df = message.payload().get_data(self._columns) if self._empty_probs: - probs = cp.zeros([len(df), 3], 'float') + probs = self._array_pkg.zeros([len(df), 3], 'float') else: - probs = cp.array(df.values, dtype=self._probs_type, copy=True, order=self._order) + probs = self._array_pkg.array(df.values, dtype=self._probs_type, copy=True, order=self._order) if (isinstance(message, ControlMessage)): - message.tensors(_messages.TensorMemory(count=len(probs), tensors={'probs': probs})) + message.tensors(TensorMemory(count=len(probs), tensors={'probs': probs})) return message memory = ResponseMemory(count=len(probs), tensors={'probs': probs}) diff --git a/tests/_utils/stages/dfp_length_checker.py b/tests/_utils/stages/dfp_length_checker.py index 1162a647f7..659c8c81ec 100755 --- a/tests/_utils/stages/dfp_length_checker.py +++ b/tests/_utils/stages/dfp_length_checker.py @@ -21,13 +21,14 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils.atomic_integer import AtomicInteger @register_stage("unittest-dfp-length-check") -class DFPLengthChecker(PassThruTypeMixin, SinglePortStage): +class DFPLengthChecker(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Verifies that the incoming MessageMeta classes are of a specific length diff --git a/tests/_utils/stages/error_raiser.py b/tests/_utils/stages/error_raiser.py index 8923229ab2..f3e0d8b5e6 100644 --- a/tests/_utils/stages/error_raiser.py +++ b/tests/_utils/stages/error_raiser.py @@ -19,12 +19,13 @@ from mrc.core import operators as ops from morpheus.config import Config +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils.atomic_integer import AtomicInteger -class ErrorRaiserStage(PassThruTypeMixin, SinglePortStage): +class ErrorRaiserStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Stage that raises an exception in the on_data method """ diff --git a/tests/_utils/stages/in_memory_multi_source_stage.py b/tests/_utils/stages/in_memory_multi_source_stage.py index 1eb2d46092..7497aff7ed 100644 --- a/tests/_utils/stages/in_memory_multi_source_stage.py +++ b/tests/_utils/stages/in_memory_multi_source_stage.py @@ -18,11 +18,12 @@ import mrc from morpheus.config import Config +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.source_stage import SourceStage from morpheus.pipeline.stage_schema import StageSchema -class InMemoryMultiSourceStage(SourceStage): +class InMemoryMultiSourceStage(GpuAndCpuMixin, SourceStage): """ In memory multi-source stage for testing purposes, accepts a 2d array `data`. The first dimenion represents the number of output ports, and the second represents the data for each port, and diff --git a/tests/_utils/stages/in_memory_source_x_stage.py b/tests/_utils/stages/in_memory_source_x_stage.py index bd1256c07a..229e8e1a3a 100644 --- a/tests/_utils/stages/in_memory_source_x_stage.py +++ b/tests/_utils/stages/in_memory_source_x_stage.py @@ -18,11 +18,12 @@ import mrc from morpheus.config import Config +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema -class InMemSourceXStage(SingleOutputSource): +class InMemSourceXStage(GpuAndCpuMixin, SingleOutputSource): """ InMemorySourceStage subclass that emits whatever you give it and doesn't assume the source data is a dataframe. diff --git a/tests/_utils/stages/multi_message_pass_thru.py b/tests/_utils/stages/multi_message_pass_thru.py index eb0b5c3789..8ae9b14c07 100644 --- a/tests/_utils/stages/multi_message_pass_thru.py +++ b/tests/_utils/stages/multi_message_pass_thru.py @@ -18,11 +18,12 @@ from mrc.core import operators as ops from morpheus.messages import MultiMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage -class MultiMessagePassThruStage(PassThruTypeMixin, SinglePortStage): +class MultiMessagePassThruStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): @property def name(self) -> str: diff --git a/tests/_utils/stages/multi_port_pass_thru.py b/tests/_utils/stages/multi_port_pass_thru.py index 5454974870..5cffb47b2b 100644 --- a/tests/_utils/stages/multi_port_pass_thru.py +++ b/tests/_utils/stages/multi_port_pass_thru.py @@ -20,11 +20,12 @@ import mrc.core.operators as ops from morpheus.config import Config +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.stage import Stage -class MultiPortPassThruStage(PassThruTypeMixin, Stage): +class MultiPortPassThruStage(GpuAndCpuMixin, PassThruTypeMixin, Stage): def __init__(self, c: Config, num_ports: int): super().__init__(c) diff --git a/tests/_utils/stages/record_thread_id_stage.py b/tests/_utils/stages/record_thread_id_stage.py index d2d9a12a82..0a991c1706 100644 --- a/tests/_utils/stages/record_thread_id_stage.py +++ b/tests/_utils/stages/record_thread_id_stage.py @@ -19,11 +19,12 @@ import mrc from morpheus.config import Config +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage -class RecordThreadIdStage(PassThruTypeMixin, SinglePortStage): +class RecordThreadIdStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Forwarding stage that records the thread id of the progress engine """ diff --git a/tests/_utils/stages/split_stage.py b/tests/_utils/stages/split_stage.py index 4e816de6c0..c03db636fa 100644 --- a/tests/_utils/stages/split_stage.py +++ b/tests/_utils/stages/split_stage.py @@ -20,11 +20,12 @@ from morpheus.config import Config from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.stage import Stage from morpheus.pipeline.stage_schema import StageSchema -class SplitStage(Stage): +class SplitStage(GpuAndCpuMixin, Stage): def __init__(self, c: Config): super().__init__(c) From aa6bea9ec4693ab07702bab3094131f95ade89a9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 16:24:19 -0700 Subject: [PATCH 136/347] Cleanup type-hint --- tests/test_error_pipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_error_pipe.py b/tests/test_error_pipe.py index 9806ede797..cb264f2231 100755 --- a/tests/test_error_pipe.py +++ b/tests/test_error_pipe.py @@ -16,7 +16,6 @@ import logging -import pandas as pd import pytest from _utils.stages.error_raiser import ErrorRaiserStage @@ -26,11 +25,12 @@ from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.input.in_memory_source_stage import InMemorySourceStage from morpheus.stages.output.in_memory_sink_stage import InMemorySinkStage +from morpheus.utils.type_aliases import DataFrameType @pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("exception_cls", [RuntimeError, ValueError, NotImplementedError]) -def test_stage_raises_exception(config: Config, filter_probs_df: pd.DataFrame, exception_cls: type[Exception]): +def test_stage_raises_exception(config: Config, filter_probs_df: DataFrameType, exception_cls: type[Exception]): pipe = LinearPipeline(config) pipe.set_source(InMemorySourceStage(config, [filter_probs_df])) error_raiser_stage = pipe.add_stage(ErrorRaiserStage(config, exception_cls=exception_cls)) From b7ff46e8fd159c24059084b6c6f713b3f126d39b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 4 Sep 2024 16:43:17 -0700 Subject: [PATCH 137/347] Set execution_mode fixture to autouse this allows tests to use the gpu_and_cpu_mode mark without explicitly using the execution_mode fixture --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4922eab57c..c4eea28cb6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -271,7 +271,7 @@ def _get_execution_mode(request: pytest.FixtureRequest) -> "ExecutionMode": return ExecutionMode.CPU -@pytest.fixture(name="execution_mode", scope="function") +@pytest.fixture(name="execution_mode", scope="function", autouse=True) def execution_mode_fixture(request: pytest.FixtureRequest): exec_mode = _get_execution_mode(request) yield exec_mode From 8cb873db0dccce210dc8966e0be76758212323c8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 09:15:06 -0700 Subject: [PATCH 138/347] Ensure that the df_type fixture implies setting the execution mode --- tests/conftest.py | 5 +++++ tests/test_conftest.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index c4eea28cb6..7fa9a866e2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -261,6 +261,11 @@ def _get_execution_mode(request: pytest.FixtureRequest) -> "ExecutionMode": raise RuntimeError(f"Both markers (gpu_mode and cpu_mode) were added to function {request.node.nodeid}. " "Use the gpu_and_cpu_mode marker to test both.") + # if both are undefined, infer based on the df_type + if (not gpu_mode and not cpu_mode): + gpu_mode = request.node.get_closest_marker("use_cudf") is not None + cpu_mode = request.node.get_closest_marker("use_pandas") is not None + # This will default to True or follow gpu_mode do_gpu_mode = not cpu_mode diff --git a/tests/test_conftest.py b/tests/test_conftest.py index fcc33a6a82..9752803496 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -24,6 +24,8 @@ from morpheus.config import Config from morpheus.config import CppConfig from morpheus.config import ExecutionMode +from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_utils import exec_mode_to_df_type_str def exec_mode_to_cpp_mode(exec_mode: ExecutionMode) -> bool: @@ -256,6 +258,10 @@ def test_df_type_no_marks(df_type, df_type_from_marker): assert df_type == df_type_from_marker +def test_df_type_matches_execution_mode(df_type: DataFrameTypeStr, execution_mode: ExecutionMode): + assert df_type == exec_mode_to_df_type_str(execution_mode) + + @pytest.mark.use_pandas def test_df_type_pandas_marker(df_type): assert df_type == "pandas" From 7f8fedca68366b5f5c3641096a85e57e401b82d2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 10:10:19 -0700 Subject: [PATCH 139/347] Update tests, gpu_mode is a marker not a fixture --- tests/test_multi_message.py | 109 ++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/tests/test_multi_message.py b/tests/test_multi_message.py index da823e4e6b..fe83f76d31 100644 --- a/tests/test_multi_message.py +++ b/tests/test_multi_message.py @@ -18,7 +18,6 @@ import dataclasses import string -import typing from unittest.mock import patch import cupy as cp @@ -29,6 +28,7 @@ import cudf from _utils.dataset_manager import DatasetManager +from morpheus.config import ExecutionMode from morpheus.messages import ControlMessage from morpheus.messages.memory.inference_memory import InferenceMemory from morpheus.messages.memory.response_memory import ResponseMemory @@ -45,6 +45,10 @@ from morpheus.messages.multi_response_message import MultiResponseProbsMessage from morpheus.messages.multi_tensor_message import MultiTensorMessage from morpheus.utils import logger as morpheus_logger +from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_utils import get_array_pkg +from morpheus.utils.type_utils import get_df_pkg_from_obj @pytest.mark.cpu_mode @@ -60,7 +64,8 @@ class BadMultiMessage(MultiMessage): BadMultiMessage(meta=None, value=5) -def test_constructor_empty(filter_probs_df: cudf.DataFrame): +@pytest.mark.gpu_and_cpu_mode +def test_constructor_empty(filter_probs_df: DataFrameType): meta = MessageMeta(filter_probs_df) @@ -71,7 +76,8 @@ def test_constructor_empty(filter_probs_df: cudf.DataFrame): assert multi.mess_count == meta.count -def test_constructor_values(filter_probs_df: cudf.DataFrame): +@pytest.mark.gpu_and_cpu_mode +def test_constructor_values(filter_probs_df: DataFrameType): meta = MessageMeta(filter_probs_df) @@ -94,7 +100,8 @@ def test_constructor_values(filter_probs_df: cudf.DataFrame): assert multi.mess_count == 5 -def test_constructor_invalid(filter_probs_df: cudf.DataFrame): +@pytest.mark.gpu_and_cpu_mode +def test_constructor_invalid(filter_probs_df: DataFrameType): meta = MessageMeta(filter_probs_df) @@ -115,13 +122,13 @@ def test_constructor_invalid(filter_probs_df: cudf.DataFrame): MultiMessage(meta=meta, mess_offset=5, mess_count=(meta.count - 5) + 1) -def _test_get_meta(df: typing.Union[cudf.DataFrame, pd.DataFrame]): +def _test_get_meta(df: DataFrameType): meta = MessageMeta(df) multi = MultiMessage(meta=meta, mess_offset=3, mess_count=5) # Manually slice the dataframe according to the multi settings - df_sliced: cudf.DataFrame = df.iloc[multi.mess_offset:multi.mess_offset + multi.mess_count, :] + df_sliced: DataFrameType = df.iloc[multi.mess_offset:multi.mess_offset + multi.mess_count, :] DatasetManager.assert_df_equal(multi.get_meta(), df_sliced) @@ -145,15 +152,11 @@ def _test_get_meta(df: typing.Union[cudf.DataFrame, pd.DataFrame]): DatasetManager.assert_df_equal(multi.get_meta(col_name), df_sliced[col_name]) -def test_get_meta(filter_probs_df: typing.Union[cudf.DataFrame, pd.DataFrame]): +def test_get_meta(filter_probs_df: DataFrameType): _test_get_meta(filter_probs_df) -# Ignore unused arguments warnigns due to using the `gpu_mode` fixture -# pylint:disable=unused-argument - - -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_get_meta_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta @@ -163,7 +166,7 @@ def test_get_meta_dup_index(dataset: DatasetManager): _test_get_meta(df) -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_set_meta(dataset: DatasetManager): df_saved = dataset.pandas["filter_probs.csv"] @@ -171,6 +174,7 @@ def test_set_meta(dataset: DatasetManager): multi = MultiMessage(meta=meta, mess_offset=3, mess_count=5) + # Although we are working with cuDF, this mask needs to be made with numpy saved_mask = np.ones(len(df_saved), bool) saved_mask[multi.mess_offset:multi.mess_offset + multi.mess_count] = False @@ -206,8 +210,7 @@ def test_value(columns, value): test_value(multi_columns, np.random.randn(multi.mess_count, len(multi_columns))) -def _test_set_meta_new_column(df: typing.Union[cudf.DataFrame, pd.DataFrame], df_type: typing.Literal['cudf', - 'pandas']): +def _test_set_meta_new_column(df: DataFrameType, df_type: DataFrameTypeStr): meta = MessageMeta(df) @@ -238,12 +241,12 @@ def _test_set_meta_new_column(df: typing.Union[cudf.DataFrame, pd.DataFrame], df DatasetManager.assert_df_equal(multi.get_meta(["v2", "new_column2"]), val_to_set) -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_set_meta_new_column(dataset: DatasetManager): _test_set_meta_new_column(dataset["filter_probs.csv"], dataset.default_df_type) -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_set_meta_new_column_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta df = dataset.replace_index(dataset["filter_probs.csv"], replace_ids={3: 4, 5: 4}) @@ -270,7 +273,7 @@ def test_set_meta_issue_286(filter_probs_df: cudf.DataFrame, use_series: bool): mm2.set_meta('letters', values[5:10]) -def _test_copy_ranges(df: typing.Union[cudf.DataFrame, pd.DataFrame]): +def _test_copy_ranges(df: DataFrameType): meta = MessageMeta(df) mm1 = MultiMessage(meta=meta) @@ -297,21 +300,17 @@ def _test_copy_ranges(df: typing.Union[cudf.DataFrame, pd.DataFrame]): assert mm3.mess_offset == 0 assert mm3.mess_count == (6 - 2) + (15 - 12) - if isinstance(df, pd.DataFrame): - concat_fn = pd.concat - else: - concat_fn = cudf.concat - - expected_df = concat_fn([df.iloc[2:6], df.iloc[12:15]]) + df_pkg = get_df_pkg_from_obj(df) + expected_df = df_pkg.concat([df.iloc[2:6], df.iloc[12:15]]) DatasetManager.assert_df_equal(mm3.get_meta(), expected_df) -def test_copy_ranges(filter_probs_df: typing.Union[cudf.DataFrame, pd.DataFrame]): +def test_copy_ranges(filter_probs_df: DataFrameType): _test_copy_ranges(filter_probs_df) -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_copy_ranges_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta @@ -321,7 +320,7 @@ def test_copy_ranges_dup_index(dataset: DatasetManager): _test_copy_ranges(df) -def test_get_slice_ranges(filter_probs_df: cudf.DataFrame): +def test_get_slice_ranges(filter_probs_df: DataFrameType): meta = MessageMeta(filter_probs_df) @@ -372,7 +371,7 @@ def test_get_slice_ranges(filter_probs_df: cudf.DataFrame): multi_full.get_slice(13, 16).get_slice(4, 5) -def _test_get_slice_values(df: typing.Union[cudf.DataFrame, pd.DataFrame]): +def _test_get_slice_values(df: DataFrameType): meta = MessageMeta(df) @@ -431,11 +430,11 @@ def _test_get_slice_values(df: typing.Union[cudf.DataFrame, pd.DataFrame]): df.iloc[10 + 5:(10 + 5) + (8 - 5)][["v4", "v1", "v2"]]) -def test_get_slice_values(filter_probs_df: cudf.DataFrame): +def test_get_slice_values(filter_probs_df: DataFrameType): _test_get_slice_values(filter_probs_df) -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_get_slice_values_dup_index(dataset: DatasetManager): # Duplicate some indices before creating the meta @@ -445,14 +444,15 @@ def test_get_slice_values_dup_index(dataset: DatasetManager): _test_get_slice_values(df) -def test_get_slice_derived(filter_probs_df: cudf.DataFrame): +def test_get_slice_derived(filter_probs_df: DataFrameType, execution_mode: ExecutionMode): + array_pkg = get_array_pkg(execution_mode) multi_tensor_message_tensors = { - "input_ids": cp.zeros((20, 2)), - "input_mask": cp.zeros((20, 2)), - "seq_ids": cp.expand_dims(cp.arange(0, 20, dtype=int), axis=1), - "input__0": cp.zeros((20, 2)), - "probs": cp.zeros((20, 2)), + "input_ids": array_pkg.zeros((20, 2)), + "input_mask": array_pkg.zeros((20, 2)), + "seq_ids": array_pkg.expand_dims(array_pkg.arange(0, 20, dtype=int), axis=1), + "input__0": array_pkg.zeros((20, 2)), + "probs": array_pkg.zeros((20, 2)), } def compare_slice(message_class, **kwargs): @@ -491,7 +491,7 @@ def compare_slice(message_class, **kwargs): memory=ResponseMemoryProbs(count=20, probs=multi_tensor_message_tensors["probs"])) -def test_from_message(filter_probs_df: cudf.DataFrame): +def test_from_message(filter_probs_df: DataFrameType): # Pylint currently fails to work with classmethod: https://github.com/pylint-dev/pylint/issues/981 # pylint: disable=no-member @@ -614,7 +614,7 @@ def test_from_message(filter_probs_df: cudf.DataFrame): MultiAEMessage.from_message(multi) -def test_tensor_constructor(filter_probs_df: cudf.DataFrame): +def test_tensor_constructor(filter_probs_df: DataFrameType, execution_mode: ExecutionMode): mess_len = len(filter_probs_df) ten_len = mess_len * 2 @@ -670,41 +670,42 @@ def test_tensor_constructor(filter_probs_df: cudf.DataFrame): MultiTensorMessage(meta=meta, mess_count=10, memory=memory, count=9) # === ID Tensors === - id_tensor = cp.expand_dims(cp.arange(0, mess_len, dtype=int), axis=1) + array_pkg = get_array_pkg(execution_mode) + id_tensor = array_pkg.expand_dims(array_pkg.arange(0, mess_len, dtype=int), axis=1) # With valid ID tensor multi_tensor = MultiTensorMessage(meta=meta, memory=TensorMemory(count=mess_len, tensors={"seq_ids": id_tensor})) - assert cp.all(multi_tensor.get_id_tensor() == id_tensor) + assert array_pkg.all(multi_tensor.get_id_tensor() == id_tensor) # With different ID name multi_tensor = MultiTensorMessage(meta=meta, memory=TensorMemory(count=mess_len, tensors={"other_seq_ids": id_tensor}), id_tensor_name="other_seq_ids") - assert cp.all(multi_tensor.get_id_tensor() == id_tensor) + assert array_pkg.all(multi_tensor.get_id_tensor() == id_tensor) # With message offset multi_tensor = MultiTensorMessage(meta=meta, mess_offset=4, memory=TensorMemory(count=mess_len, tensors={"seq_ids": id_tensor}), offset=4) - assert cp.all(multi_tensor.get_id_tensor() == id_tensor[4:]) + assert array_pkg.all(multi_tensor.get_id_tensor() == id_tensor[4:]) # Incorrect start ID - invalid_id_tensor = cp.copy(id_tensor) + invalid_id_tensor = array_pkg.copy(id_tensor) invalid_id_tensor[0] = -1 with pytest.raises(RuntimeError): multi_tensor = MultiTensorMessage(meta=meta, memory=TensorMemory(count=mess_len, tensors={"seq_ids": invalid_id_tensor})) # Incorrect end ID - invalid_id_tensor = cp.copy(id_tensor) + invalid_id_tensor = array_pkg.copy(id_tensor) invalid_id_tensor[-1] = invalid_id_tensor[-1] + 1 with pytest.raises(RuntimeError): multi_tensor = MultiTensorMessage(meta=meta, memory=TensorMemory(count=mess_len, tensors={"seq_ids": invalid_id_tensor})) # Incorrect end ID, different id tensor name - invalid_id_tensor = cp.copy(id_tensor) + invalid_id_tensor = array_pkg.copy(id_tensor) invalid_id_tensor[-1] = invalid_id_tensor[-1] + 1 with pytest.raises(RuntimeError): multi_tensor = MultiTensorMessage(meta=meta, @@ -716,7 +717,7 @@ def test_tensor_constructor(filter_probs_df: cudf.DataFrame): memory=TensorMemory(count=mess_len, tensors={"id_tensor": invalid_id_tensor})) -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_tensor_slicing(dataset: DatasetManager): # Pylint currently fails to work with classmethod: https://github.com/pylint-dev/pylint/issues/981 @@ -805,18 +806,18 @@ def test_tensor_slicing(dataset: DatasetManager): dataset.assert_df_equal(double_slice.get_meta(), single_slice.get_meta()) -@pytest.mark.usefixtures("gpu_mode") -@pytest.mark.cpu_mode -def test_deprecation_message(filter_probs_df: cudf.DataFrame, caplog): +@pytest.mark.gpu_and_cpu_mode +def test_deprecation_message(filter_probs_df: DataFrameType, execution_mode: ExecutionMode): meta = MessageMeta(filter_probs_df) + array_pkg = get_array_pkg(execution_mode) multi_tensor_message_tensors = { - "input_ids": cp.zeros((20, 2)), - "input_mask": cp.zeros((20, 2)), - "seq_ids": cp.expand_dims(cp.arange(0, 20, dtype=int), axis=1), - "input__0": cp.zeros((20, 2)), - "probs": cp.zeros((20, 2)), + "input_ids": array_pkg.zeros((20, 2)), + "input_mask": array_pkg.zeros((20, 2)), + "seq_ids": array_pkg.expand_dims(array_pkg.arange(0, 20, dtype=int), axis=1), + "input__0": array_pkg.zeros((20, 2)), + "probs": array_pkg.zeros((20, 2)), } def generate_deprecation_warning(deprecated_class, new_class): From 696f8a9109e2598f1ffe328593abe041c188ba36 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 10:11:26 -0700 Subject: [PATCH 140/347] Update markers to indicate that these tests need to execute in gpu and cpu mode --- tests/test_file_in_out.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_file_in_out.py b/tests/test_file_in_out.py index 40909b7557..f59794302b 100755 --- a/tests/test_file_in_out.py +++ b/tests/test_file_in_out.py @@ -41,6 +41,7 @@ @pytest.mark.slow +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("input_type", ["csv", "jsonlines", "parquet"]) @pytest.mark.parametrize("use_pathlib", [False, True]) @pytest.mark.parametrize("output_type", ["csv", "json", "jsonlines"]) @@ -91,6 +92,7 @@ def test_file_rw_pipe(tmp_path: pathlib.Path, assert output_data.tolist() == validation_data.tolist() +@pytest.mark.gpu_and_cpu_mode def test_file_read_json(config: Config): src_file = os.path.join(TEST_DIRS.tests_data_dir, "simple.json") @@ -110,7 +112,7 @@ def test_file_read_json(config: Config): @pytest.mark.slow -@pytest.mark.cpu_mode +@pytest.mark.gpu_and_cpu_mode @pytest.mark.usefixtures("chdir_tmpdir") def test_to_file_no_path(tmp_path: pathlib.Path, config: Config): """ @@ -131,6 +133,7 @@ def test_to_file_no_path(tmp_path: pathlib.Path, config: Config): @pytest.mark.slow +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("input_type", ["csv", "jsonlines", "parquet"]) @pytest.mark.parametrize("output_type", ["csv", "json", "jsonlines"]) def test_file_rw_multi_segment_pipe(tmp_path: pathlib.Path, config: Config, input_type: str, output_type: str): @@ -165,6 +168,7 @@ def test_file_rw_multi_segment_pipe(tmp_path: pathlib.Path, config: Config, inpu @pytest.mark.slow +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("input_file", [ os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv"), @@ -189,6 +193,7 @@ def test_file_rw_index_pipe(tmp_path: pathlib.Path, config: Config, input_file: assert output_data.tolist() == validation_data.tolist() +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("input_file,extra_kwargs", [(os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv"), { "include_header": True, "include_index_col": False @@ -196,7 +201,6 @@ def test_file_rw_index_pipe(tmp_path: pathlib.Path, config: Config, input_file: "include_header": True }), (os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.jsonlines"), {})], ids=["CSV", "CSV_ID", "JSON"]) -@pytest.mark.usefixtures("gpu_mode") def test_file_roundtrip(tmp_path: pathlib.Path, input_file: str, extra_kwargs: dict[str, typing.Any]): # Output file should be same type as input @@ -235,6 +239,7 @@ def test_read_cpp_compare(input_file: str): @pytest.mark.slow +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("output_type", ["csv", "json", "jsonlines"]) def test_file_rw_serialize_deserialize_pipe(tmp_path: pathlib.Path, config: Config, output_type: str): input_file = os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv") From b0216c35d32fd3765159e5ab880ca2b6a35a1753 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 10:17:31 -0700 Subject: [PATCH 141/347] Update markers to indicate that these tests need to execute in gpu and cpu mode --- tests/test_http_server_sink_stage.py | 2 +- tests/test_linear_modules_stage.py | 26 ++++++++++++++------------ tests/test_monitor_stage.py | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/test_http_server_sink_stage.py b/tests/test_http_server_sink_stage.py index 220ba43a92..1f9359dbf2 100644 --- a/tests/test_http_server_sink_stage.py +++ b/tests/test_http_server_sink_stage.py @@ -89,7 +89,7 @@ def _custom_serializer(df: DataFrameType) -> str: @pytest.mark.slow -@pytest.mark.cpu_mode +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("lines", [False, True]) @pytest.mark.parametrize("max_rows_per_response", [10000, 10]) @pytest.mark.parametrize("df_serializer_fn", [None, _custom_serializer]) diff --git a/tests/test_linear_modules_stage.py b/tests/test_linear_modules_stage.py index da3dc8979e..b0c16de59d 100755 --- a/tests/test_linear_modules_stage.py +++ b/tests/test_linear_modules_stage.py @@ -23,13 +23,14 @@ from morpheus.stages.general.linear_modules_stage import LinearModulesStage from morpheus.utils.module_utils import mrc_version -module_config = { - "module_id": "TestSimpleModule", "module_name": "test_simple_module", "namespace": "test_morpheus_modules" -} +@pytest.fixture(name="module_config") +def module_config_fixture(): + return {"module_id": "TestSimpleModule", "module_name": "test_simple_module", "namespace": "test_morpheus_modules"} -@pytest.mark.cpu_mode -def test_constructor(config): + +@pytest.mark.gpu_and_cpu_mode +def test_constructor(config, module_config: dict): mod_stage = LinearModulesStage(config, module_config, input_port_name="test_in", output_port_name="test_out") @@ -44,8 +45,8 @@ def test_constructor(config): pytest.raises(NotImplementedError, mod_stage._get_cpp_module_node, None) -@pytest.mark.cpu_mode -def test_build_single_before_module_registration(config): +@pytest.mark.gpu_and_cpu_mode +def test_build_single_before_module_registration(config, module_config: dict): mock_node = mock.MagicMock() mock_segment = mock.MagicMock() @@ -61,19 +62,20 @@ def test_build_single_before_module_registration(config): mod_stage._build_single(mock_segment, mock_input_stream) -def register_test_module(): +def register_test_module(id_postfix: str): registry = mrc.ModuleRegistry def module_init_fn(_: mrc.Builder): pass - registry.register_module("TestSimpleModule", "test_morpheus_modules", mrc_version, module_init_fn) + registry.register_module(f"TestSimpleModule_{id_postfix}", "test_morpheus_modules", mrc_version, module_init_fn) -@pytest.mark.cpu_mode -def test_build_single_after_module_registration(config): +@pytest.mark.gpu_and_cpu_mode +def test_build_single_after_module_registration(config, module_config: dict): - register_test_module() + register_test_module(config.execution_mode.value) + module_config["module_id"] = f"{module_config['module_id']}_{config.execution_mode.value}" mock_node = mock.MagicMock() mock_segment = mock.MagicMock() diff --git a/tests/test_monitor_stage.py b/tests/test_monitor_stage.py index 148a3f53cc..0491e3054f 100755 --- a/tests/test_monitor_stage.py +++ b/tests/test_monitor_stage.py @@ -178,7 +178,7 @@ def test_log_level(mock_progress_sink: mock.MagicMock, assert mock_sink_on_completed.call_count == expected_call_count -@pytest.mark.cpu_mode +@pytest.mark.gpu_and_cpu_mode def test_thread(config: Config, morpheus_log_level: int): """ Test ensures the monitor stage executes on the same thread as the parent stage From d81a1ded1a80894e9e3de4589c7885186bffd129 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 10:36:25 -0700 Subject: [PATCH 142/347] Fix assumption that ConvMsg is GPU-only --- tests/pipeline/test_execution_modes.py | 50 +++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/tests/pipeline/test_execution_modes.py b/tests/pipeline/test_execution_modes.py index 28499d24ab..d740235a1b 100755 --- a/tests/pipeline/test_execution_modes.py +++ b/tests/pipeline/test_execution_modes.py @@ -17,13 +17,17 @@ import collections.abc import typing +import mrc import pytest +from mrc.core import operators as ops from _utils.stages.conv_msg import ConvMsg from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.pipeline.execution_mode_mixins import CpuOnlyMixin from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin +from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin +from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_decorator import source from morpheus.pipeline.stage_decorator import stage @@ -61,17 +65,48 @@ def gpu_cpu_stage(message: typing.Any) -> typing.Any: return message -class CpuOnlyStage(CpuOnlyMixin, ConvMsg): - pass +class BaseStage(PassThruTypeMixin, SinglePortStage): + def accepted_types(self) -> typing.Tuple: + return (typing.Any, ) -class GpuAndCpuStage(GpuAndCpuMixin, ConvMsg): - pass + def supports_cpp_node(self) -> bool: + return False + + def on_data(self, data: typing.Any) -> typing.Any: + return data + + def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: + node = builder.make_node(self.unique_name, ops.map(self.on_data)) + builder.make_edge(input_node, node) + + return node + + +class CpuOnlyStage(CpuOnlyMixin, BaseStage): + + @property + def name(self) -> str: + return "test-cpu-only-stage" + + +class GpuOnlyStage(BaseStage): + + @property + def name(self) -> str: + return "test-gpu-only-stage" + + +class GpuAndCpuStage(GpuAndCpuMixin, BaseStage): + + @property + def name(self) -> str: + return "test-gpu-and-cpu-stage" @pytest.mark.parametrize("stage_cls, expected_modes", [ - (ConvMsg, {ExecutionMode.GPU}), + (GpuOnlyStage, {ExecutionMode.GPU}), (CpuOnlyStage, {ExecutionMode.CPU}), (GpuAndCpuStage, {ExecutionMode.GPU, ExecutionMode.CPU}), (gpu_only_source, {ExecutionMode.GPU}), @@ -82,8 +117,7 @@ class GpuAndCpuStage(GpuAndCpuMixin, ConvMsg): (gpu_cpu_stage, {ExecutionMode.GPU, ExecutionMode.CPU}), ]) def test_execution_mode_mixins(stage_cls: type[ConvMsg], expected_modes: set): - # intentionally not using the config fixture so that we can set the execution mode and avoid iterating over - # python/C++ execution modes + # intentionally not using the config fixture so that we can set the execution mode manually config = Config() if ExecutionMode.CPU in expected_modes: config.execution_mode = ExecutionMode.CPU @@ -96,7 +130,7 @@ def test_execution_mode_mixins(stage_cls: type[ConvMsg], expected_modes: set): @pytest.mark.parametrize("stage_cls, execution_mode", [ - (ConvMsg, ExecutionMode.CPU), + (GpuOnlyStage, ExecutionMode.CPU), (gpu_only_source, ExecutionMode.CPU), (gpu_only_stage, ExecutionMode.CPU), (CpuOnlyStage, ExecutionMode.GPU), From d0d3d2b54d2c367716ec17e8d627368964b139cb Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 10:44:12 -0700 Subject: [PATCH 143/347] Update docstring --- tests/conftest.py | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7fa9a866e2..c9d5761cd8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -895,35 +895,11 @@ def test_something(dataset: DatasetManager): ``` A test that requests this fixture will parameterize on the type of DataFrame returned by the DatasetManager. - If a test requests both this fixture and the `gpu_mode` fixture, or indirectly via the `config` fixture, then - the test will parameterize over both df_type:[cudf, pandas] and gpu_mode[True, False]. However it will remove the - df_type=pandas & gpu_mode=True and df_type=cudf & gpu_mode=False combinations as this will cause an unsupported usage - of Pandas dataframes with the C++ implementation of message classes, and cuDF with CPU-only implementations. + If a test requests both this fixture and is marked either `gpu_mode` or `cpu_mode` then only cudf or pandas will be + used to prevent an unsupported usage of Pandas dataframes with the C++ implementation of message classes, and cuDF + with CPU-only implementations. - This behavior can also be overridden by using the `use_cudf`, `use_pandas`, `gpu_mode` or `use_pandas` marks ex: - ``` - # This test will only run once with C++ enabled and cudf dataframes - @pytest.mark.gpu_mode - def test something(dataset: DatasetManager): - ... - # This test will run once for with pandas and C++ disabled - @pytest.mark.cpu_mode - def test something(dataset: DatasetManager): - ... - # This test will run once with C++ mode enabled, using cudf dataframes - @pytest.mark.use_cudf - def test something(gpu_mode: bool, dataset: DatasetManager): - ... - # This test creates an incompatible combination and will raise a RuntimeError without being executed - @pytest.mark.use_cudf - @pytest.mark.cpu_mode - def test something(dataset: DatasetManager): - ... - # This test creates an incompatible combination and will raise a RuntimeError without being executed - @pytest.mark.use_pandas - @pytest.mark.gpu_mode - def test something(dataset: DatasetManager): - ``` + Similarly the `use_cudf`, `use_pandas` marks will also prevent parametarization over the DataFrame type. Users who don't want to parametarize over the DataFrame should use the `dataset_pandas` or `dataset_cudf` fixtures. """ From 9684d74a422eeaa2db50b26d38441e2a2e2196df Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 10:46:44 -0700 Subject: [PATCH 144/347] Remove the gpu_mode fixture --- tests/examples/gnn_fraud_detection_pipeline/conftest.py | 2 +- tests/examples/log_parsing/conftest.py | 2 +- tests/examples/ransomware_detection/conftest.py | 2 +- tests/messages/test_message_meta.py | 6 ++---- tests/stages/test_preprocess_fil_stage.py | 2 +- tests/stages/test_preprocess_nlp_stage.py | 2 +- tests/test_add_classifications_stage.py | 2 +- tests/test_add_scores_stage.py | 2 +- 8 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/examples/gnn_fraud_detection_pipeline/conftest.py b/tests/examples/gnn_fraud_detection_pipeline/conftest.py index 2354949be7..a625d51862 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/conftest.py +++ b/tests/examples/gnn_fraud_detection_pipeline/conftest.py @@ -44,7 +44,7 @@ def cuml_fixture(fail_missing: bool): @pytest.fixture(name="config") -def config_fixture(config, gpu_mode: bool): # pylint: disable=unused-argument +def config_fixture(config): """ The GNN fraud detection pipeline utilizes the "other" pipeline mode. """ diff --git a/tests/examples/log_parsing/conftest.py b/tests/examples/log_parsing/conftest.py index e63ceac2f6..d31891873a 100644 --- a/tests/examples/log_parsing/conftest.py +++ b/tests/examples/log_parsing/conftest.py @@ -17,7 +17,7 @@ @pytest.fixture(name="config") -def config_fixture(config, gpu_mode: bool): # pylint: disable=unused-argument +def config_fixture(config): """ The log_parsing pipelie requires NLP mode. Set this here so all the tests don't need to set it themselves. """ diff --git a/tests/examples/ransomware_detection/conftest.py b/tests/examples/ransomware_detection/conftest.py index b24f321a29..e1c5e2541d 100644 --- a/tests/examples/ransomware_detection/conftest.py +++ b/tests/examples/ransomware_detection/conftest.py @@ -39,7 +39,7 @@ def dask_distributed(fail_missing: bool): @pytest.fixture(name="config") -def config_fixture(config, gpu_mode: bool): # pylint: disable=unused-argument +def config_fixture(config): """ The ransomware detection pipeline utilizes the FIL pipeline mode. """ diff --git a/tests/messages/test_message_meta.py b/tests/messages/test_message_meta.py index 1e738b0118..db28ea80d7 100644 --- a/tests/messages/test_message_meta.py +++ b/tests/messages/test_message_meta.py @@ -37,10 +37,8 @@ def fixture_index_type(request: pytest.FixtureRequest) -> typing.Literal["normal @pytest.fixture(name="df", scope="function") def fixture_df( - gpu_mode: bool, # pylint: disable=unused-argument - dataset: DatasetManager, - index_type: typing.Literal['normal', 'skip', 'dup', 'down', - 'updown']) -> typing.Union[cudf.DataFrame, pd.DataFrame]: + dataset: DatasetManager, index_type: typing.Literal['normal', 'skip', 'dup', 'down', + 'updown']) -> typing.Union[cudf.DataFrame, pd.DataFrame]: test_df = dataset["test_dataframe.jsonlines"] if (index_type == "normal"): diff --git a/tests/stages/test_preprocess_fil_stage.py b/tests/stages/test_preprocess_fil_stage.py index 85cb84347d..30695349e7 100644 --- a/tests/stages/test_preprocess_fil_stage.py +++ b/tests/stages/test_preprocess_fil_stage.py @@ -30,7 +30,7 @@ @pytest.fixture(name='config') -def fixture_config(config: Config, gpu_mode: bool): # pylint: disable=unused-argument +def fixture_config(config: Config): config.feature_length = 1 config.fil = ConfigFIL() config.fil.feature_columns = ["data"] diff --git a/tests/stages/test_preprocess_nlp_stage.py b/tests/stages/test_preprocess_nlp_stage.py index bdf667fa7a..9acfdd4ff1 100644 --- a/tests/stages/test_preprocess_nlp_stage.py +++ b/tests/stages/test_preprocess_nlp_stage.py @@ -31,7 +31,7 @@ @pytest.fixture(name='config') -def fixture_config(config: Config, gpu_mode: bool): # pylint: disable=unused-argument +def fixture_config(config: Config): config.class_labels = [ "address", "bank_acct", diff --git a/tests/test_add_classifications_stage.py b/tests/test_add_classifications_stage.py index 2d6a99b15f..9aea29ecc4 100755 --- a/tests/test_add_classifications_stage.py +++ b/tests/test_add_classifications_stage.py @@ -32,7 +32,7 @@ @pytest.fixture(name="config") -def config_fixture(config: Config, gpu_mode: bool): # pylint: disable=unused-argument +def config_fixture(config: Config): config.class_labels = ['frogs', 'lizards', 'toads'] yield config diff --git a/tests/test_add_scores_stage.py b/tests/test_add_scores_stage.py index 1d026c8d4e..c3d9082eee 100755 --- a/tests/test_add_scores_stage.py +++ b/tests/test_add_scores_stage.py @@ -36,7 +36,7 @@ @pytest.fixture(name='config') -def fixture_config(config: Config, gpu_mode: bool): # pylint: disable=unused-argument +def fixture_config(config: Config): config.class_labels = ['frogs', 'lizards', 'toads'] config.feature_length = 12 yield config From 85e08a707328e23e3e09b420a575ea214d615e87 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 10:53:31 -0700 Subject: [PATCH 145/347] Update test --- tests/test_filter_detections_stage_pipe.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_filter_detections_stage_pipe.py b/tests/test_filter_detections_stage_pipe.py index fa4fa0dea7..2d5db13534 100755 --- a/tests/test_filter_detections_stage_pipe.py +++ b/tests/test_filter_detections_stage_pipe.py @@ -131,13 +131,15 @@ def test_filter_detections_stage_pipe(config: Config, return _test_filter_detections_stage_pipe(config, dataset_pandas, do_copy, order, pipeline_batch_size, repeat) +@pytest.mark.slow @pytest.mark.parametrize('do_copy', [True, False]) def test_filter_detections_stage_multi_segment_pipe(config: Config, dataset_pandas: DatasetManager, do_copy: bool): return _test_filter_detections_stage_multi_segment_pipe(config, dataset_pandas, do_copy) +@pytest.mark.slow @pytest.mark.parametrize('do_copy', [True, False]) def test_filter_detections_control_message_stage_multi_segment_pipe(config: Config, - dataset: DatasetManager, + dataset_pandas: DatasetManager, do_copy: bool): - return _test_filter_detections_control_message_stage_multi_segment_pipe(config, dataset, do_copy) + return _test_filter_detections_control_message_stage_multi_segment_pipe(config, dataset_pandas, do_copy) From 362376ae124d0b53ed2c5e2257847dd9a792b1a4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 11:25:40 -0700 Subject: [PATCH 146/347] WIP --- tests/stages/test_deserialize_stage_pipe.py | 2 +- tests/test_messages.py | 2 +- tests/test_rss_source_stage_pipe.py | 2 -- tests/test_write_to_elasticsearch_stage_pipe.py | 14 +++++--------- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/stages/test_deserialize_stage_pipe.py b/tests/stages/test_deserialize_stage_pipe.py index 4044789de5..e4a5fd18b0 100755 --- a/tests/stages/test_deserialize_stage_pipe.py +++ b/tests/stages/test_deserialize_stage_pipe.py @@ -32,7 +32,7 @@ @pytest.mark.use_cudf -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_fixing_non_unique_indexes(dataset: DatasetManager): # Set 2 ids equal to others df = dataset.dup_index(dataset["filter_probs.csv"], count=2) diff --git a/tests/test_messages.py b/tests/test_messages.py index 48080aa75e..cfbe02823c 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -168,6 +168,6 @@ def check_all_messages(should_be_cpp: bool, no_cpp_class: bool): ) -@pytest.mark.usefixtures("gpu_mode") +@pytest.mark.gpu_mode def test_constructor_cpp(): check_all_messages(morpheus.config.CppConfig.get_should_use_cpp(), False) diff --git a/tests/test_rss_source_stage_pipe.py b/tests/test_rss_source_stage_pipe.py index 80772dca9d..84beb5d636 100644 --- a/tests/test_rss_source_stage_pipe.py +++ b/tests/test_rss_source_stage_pipe.py @@ -35,7 +35,6 @@ def test_support_cpp_node(config): assert rss_source_stage.supports_cpp_node() is False -@pytest.mark.cpu_mode @pytest.mark.parametrize( "feed_input, batch_size, expected_count, enable_cache", [([valid_feed_input], 30, 1, False), ([valid_feed_input], 12, 3, True), @@ -60,7 +59,6 @@ def test_rss_source_stage_pipe(config: Config, assert len(sink_stage.get_messages()) == expected_count -@pytest.mark.cpu_mode def test_invalid_input_rss_source_stage(config: Config): with pytest.raises(ValueError, match=f"Invalid URL or file path: {invalid_feed_input}"): diff --git a/tests/test_write_to_elasticsearch_stage_pipe.py b/tests/test_write_to_elasticsearch_stage_pipe.py index 7660c5a4e5..d63dbee4ca 100644 --- a/tests/test_write_to_elasticsearch_stage_pipe.py +++ b/tests/test_write_to_elasticsearch_stage_pipe.py @@ -17,16 +17,14 @@ import typing from unittest.mock import patch -import pandas as pd import pytest import yaml -import cudf - from morpheus.config import Config from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.input.in_memory_source_stage import InMemorySourceStage from morpheus.stages.output.write_to_elasticsearch_stage import WriteToElasticsearchStage +from morpheus.utils.type_aliases import DataFrameType def connection_kwargs_func(kwargs): @@ -71,11 +69,12 @@ def test_constructor_with_custom_func(config: Config, connection_conf_file: str) assert stage._controller._connection_kwargs == expected_connection_kwargs +@pytest.mark.use_cudf @patch("morpheus.stages.output.write_to_elasticsearch_stage.ElasticsearchController") def test_write_to_elasticsearch_stage_pipe(mock_controller: typing.Any, connection_conf_file: str, config: Config, - filter_probs_df: typing.Union[cudf.DataFrame, pd.DataFrame]): + filter_probs_df: DataFrameType): mock_df_to_parallel_bulk_write = mock_controller.return_value.df_to_parallel_bulk_write mock_refresh_client = mock_controller.return_value.refresh_client @@ -89,14 +88,11 @@ def test_write_to_elasticsearch_stage_pipe(mock_controller: typing.Any, # Run the pipeline pipe.run() - if isinstance(filter_probs_df, cudf.DataFrame): - filter_probs_df = filter_probs_df.to_pandas() - expected_index = mock_df_to_parallel_bulk_write.call_args[1]["index"] - expected_df = mock_df_to_parallel_bulk_write.call_args[1]["df"] + actual_df = mock_df_to_parallel_bulk_write.call_args[1]["df"] mock_refresh_client.assert_called_once() mock_df_to_parallel_bulk_write.assert_called_once() assert expected_index == "t_index" - assert expected_df.equals(filter_probs_df) + assert actual_df.equals(filter_probs_df.to_pandas()) From 0ef9ba589241a3993fb2473f5f766f2e317e4604 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 11:35:33 -0700 Subject: [PATCH 147/347] Fix test --- tests/pipeline/test_pipe_viz.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/pipeline/test_pipe_viz.py b/tests/pipeline/test_pipe_viz.py index da2b245886..156496ab56 100755 --- a/tests/pipeline/test_pipe_viz.py +++ b/tests/pipeline/test_pipe_viz.py @@ -25,6 +25,7 @@ from _utils.dataset_manager import DatasetManager from _utils.stages.conv_msg import ConvMsg from morpheus.cli.commands import RANKDIR_CHOICES +from morpheus.config import Config from morpheus.pipeline import LinearPipeline from morpheus.pipeline.pipeline import Pipeline from morpheus.pipeline.pipeline import PipelineState @@ -35,10 +36,8 @@ from morpheus.stages.preprocess.deserialize_stage import DeserializeStage -# pylint: disable=redefined-outer-name -@pytest.mark.use_cudf @pytest.fixture(name="viz_pipeline", scope="function") -def viz_pipeline_fixture(config, filter_probs_df): +def viz_pipeline_fixture(config: Config, dataset_cudf: DatasetManager): """ Creates a quick pipeline. """ @@ -46,9 +45,9 @@ def viz_pipeline_fixture(config, filter_probs_df): config.num_threads = 1 pipe = LinearPipeline(config) - pipe.set_source(InMemorySourceStage(config, [filter_probs_df])) + pipe.set_source(InMemorySourceStage(config, [dataset_cudf["filter_probs.csv"]])) pipe.add_stage(DeserializeStage(config)) - pipe.add_stage(ConvMsg(config, filter_probs_df)) + pipe.add_stage(ConvMsg(config, dataset_cudf["filter_probs.csv"])) pipe.add_stage(AddClassificationsStage(config)) pipe.add_stage(SerializeStage(config, include=[f"^{c}$" for c in config.class_labels])) pipe.add_stage(InMemorySinkStage(config)) From c064833519260860f33d524264f0e18bcf69bdc5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 11:47:46 -0700 Subject: [PATCH 148/347] Remove unnescesary cpu_mode mark --- tests/pipeline/test_stage_decorator.py | 30 ++++++-------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/tests/pipeline/test_stage_decorator.py b/tests/pipeline/test_stage_decorator.py index 91812b824f..89bfc52088 100644 --- a/tests/pipeline/test_stage_decorator.py +++ b/tests/pipeline/test_stage_decorator.py @@ -27,6 +27,7 @@ from _utils import assert_results from morpheus.common import TypeId from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.pipeline import LinearPipeline @@ -38,6 +39,7 @@ from morpheus.pipeline.stage_decorator import stage from morpheus.pipeline.stage_schema import StageSchema from morpheus.stages.output.compare_dataframe_stage import CompareDataFrameStage +from morpheus.utils.type_aliases import DataFrameType def _get_annotation(type_: type, generator_type: type) -> type: @@ -56,7 +58,6 @@ def _mk_compute_schema_fn(return_type: type) -> ComputeSchemaType: return lambda schema: schema.output_schema.set_type(return_type) -@pytest.mark.cpu_mode @pytest.mark.parametrize("generator_type", [None, typing.Iterator, typing.Generator, collections.abc.Iterator, collections.abc.Generator]) @pytest.mark.parametrize("return_type, is_prealloc", @@ -94,7 +95,6 @@ def test_source_gen() -> return_annotation: mock_compute_schema_fn.assert_called_once_with(schema) -@pytest.mark.cpu_mode @pytest.mark.parametrize("src_cls", [WrappedFunctionSourceStage, PreAllocatedWrappedFunctionStage]) def test_wrapped_function_source_stage_not_generator_error(config: Config, src_cls: type): @@ -108,7 +108,6 @@ def test_source_gen() -> MessageMeta: compute_schema_fn=_mk_compute_schema_fn(MessageMeta)) -@pytest.mark.cpu_mode @pytest.mark.parametrize("generator_type", [None, typing.Iterator, typing.Generator, collections.abc.Iterator, collections.abc.Generator]) @pytest.mark.parametrize("return_type, is_prealloc", @@ -132,7 +131,6 @@ def test_source_gen() -> return_annotation: assert schema.output_schema.get_type() is return_type -@pytest.mark.cpu_mode def test_source_decorator_name(config: Config): @source @@ -143,7 +141,6 @@ def test_source_gen(value: int) -> int: assert source_stage.name == 'test_source_gen' # pylint: disable=no-member -@pytest.mark.cpu_mode def test_source_decorator_explicit_name(config: Config): @source(name="source_gen") @@ -154,7 +151,6 @@ def test_source_gen(value: int) -> int: assert source_stage.name == 'source_gen' # pylint: disable=no-member -@pytest.mark.cpu_mode def test_source_decorator_explicit_compute_schema(config: Config): mock_compute_schema_fn = mock.MagicMock() mock_compute_schema_fn.side_effect = _mk_compute_schema_fn(int) @@ -170,7 +166,6 @@ def test_source_gen(value: int) -> int: mock_compute_schema_fn.assert_called_once_with(schema) -@pytest.mark.cpu_mode def test_source_decorator_no_annoation_error(config: Config): @source @@ -181,7 +176,6 @@ def test_source_gen(): test_source_gen(config) # pylint: disable=too-many-function-args -@pytest.mark.cpu_mode def test_not_generator_error(config: Config): @source @@ -192,7 +186,6 @@ def test_fn() -> int: test_fn(config) # pylint: disable=too-many-function-args -@pytest.mark.cpu_mode def test_source_stage_arg_no_value_error(config: Config): @source @@ -203,7 +196,6 @@ def test_source_gen(value: int) -> int: test_source_gen(config) -@pytest.mark.cpu_mode @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), (typing.Union[float, int], float), (float, typing.Any), (typing.Any, float), @@ -219,7 +211,6 @@ def test_wrapped_function_stage_constructor(config: Config, accept_type: type, r assert wrapped_stage.accepted_types() == (accept_type, ) -@pytest.mark.cpu_mode @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), (typing.Union[float, int], float), (float, float), (typing.Any, float), @@ -255,7 +246,6 @@ def source_fn(): assert schema.output_schema.get_type() is return_type -@pytest.mark.cpu_mode def test_wrapped_function_stage_name(config: Config): def multiplier(message: MessageMeta, column: str, value: int | float) -> MessageMeta: @@ -272,7 +262,6 @@ def multiplier(message: MessageMeta, column: str, value: int | float) -> Message assert wrapped_stage.name == 'multiplier' -@pytest.mark.cpu_mode @pytest.mark.parametrize("needed_columns", [None, { 'result': TypeId.INT64 @@ -294,7 +283,6 @@ def test_fn(message: MessageMeta) -> MessageMeta: assert wrapped_stage._needed_columns == expected_needed_columns -@pytest.mark.cpu_mode @pytest.mark.parametrize("use_accept_type_annotation", [True, False]) @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), @@ -319,7 +307,6 @@ def test_fn(message) -> return_type: assert wrapped_stage.accepted_types() == (accept_type, ) -@pytest.mark.cpu_mode @pytest.mark.parametrize("name", [None, "unittest-stage"]) def test_stage_decorator_name(config: Config, name: str): if name is None: @@ -335,7 +322,6 @@ def test_fn(message: float, value: float) -> float: assert wrapped_stage.name == expected_name -@pytest.mark.cpu_mode @pytest.mark.parametrize("explicit_compute_schema_fn", [True, False]) @pytest.mark.parametrize("accept_type, return_type", [(pd.DataFrame, MessageMeta), (int, int), (MessageMeta, MessageMeta), (typing.Any, bool), @@ -376,7 +362,6 @@ def test_stage(message: accept_type) -> return_type: assert schema.output_schema.get_type() is return_type -@pytest.mark.cpu_mode def test_stage_decorator_no_annotation_error(config: Config): @stage @@ -387,7 +372,6 @@ def test_fn(message): test_fn(config) -@pytest.mark.cpu_mode def test_stage_arg_no_value_error(config: Config): @stage @@ -398,7 +382,6 @@ def test_fn(message: float, value: float) -> float: test_fn(config) # pylint: disable=no-value-for-parameter -@pytest.mark.cpu_mode @pytest.mark.parametrize("needed_columns", [None, { 'result': TypeId.INT64 @@ -416,14 +399,15 @@ def test_fn(message: MessageMeta) -> MessageMeta: assert wrapped_stage._needed_columns == expected_needed_columns -def test_end_to_end_pipe(config: Config, filter_probs_df: cudf.DataFrame): +@pytest.mark.gpu_and_cpu_mode +def test_end_to_end_pipe(config: Config, filter_probs_df: DataFrameType): - @source + @source(execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) def source_gen(dataframes: list[cudf.DataFrame]) -> collections.abc.Iterator[MessageMeta]: for df in dataframes: yield MessageMeta(df) - @stage + @stage(execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) def multiplier(message: MessageMeta, column: str, value: int | float = 2.0) -> MessageMeta: with message.mutable_dataframe() as df: df[column] = df[column] * value @@ -435,7 +419,7 @@ def multiplier(message: MessageMeta, column: str, value: int | float = 2.0) -> M expected_df['v2'] = expected_df['v2'] * multipy_by * 2.0 pipe = LinearPipeline(config) - pipe.set_source(source_gen(config, dataframes=[filter_probs_df])) # pylint: disable=redundant-keyword-arg + pipe.set_source(source_gen(config, dataframes=[filter_probs_df])) pipe.add_stage(multiplier(config, column='v2', value=multipy_by)) pipe.add_stage(multiplier(config, column='v2')) sink = pipe.add_stage(CompareDataFrameStage(config, expected_df)) From 622f962cc7db2370c0fb1f2ad5713baf9d7d27ee Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 12:34:27 -0700 Subject: [PATCH 149/347] Remove test_write_to_file_stage.py there is an identical test in test_file_in_out.py --- tests/test_write_to_file_stage.py | 61 ------------------------------- 1 file changed, 61 deletions(-) delete mode 100755 tests/test_write_to_file_stage.py diff --git a/tests/test_write_to_file_stage.py b/tests/test_write_to_file_stage.py deleted file mode 100755 index 12a07ef76b..0000000000 --- a/tests/test_write_to_file_stage.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest - -from _utils import TEST_DIRS -from _utils.dataset_manager import DatasetManager -from morpheus.config import Config -from morpheus.pipeline import LinearPipeline -from morpheus.stages.input.file_source_stage import FileSourceStage -from morpheus.stages.output.write_to_file_stage import WriteToFileStage -from morpheus.stages.postprocess.serialize_stage import SerializeStage -from morpheus.stages.preprocess.deserialize_stage import DeserializeStage - - -@pytest.mark.cpu_mode -@pytest.mark.parametrize("use_deserialize", [False, True]) -@pytest.mark.parametrize("flush", [False, True]) -@pytest.mark.parametrize("output_type", ["csv", "json", "jsonlines"]) -def test_file_rw_pipe(tmp_path: str, - config: Config, - dataset: DatasetManager, - output_type: str, - flush: bool, - use_deserialize: bool): - """ - Test the flush functionality of the WriteToFileStage. - """ - input_file = os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv") - out_file = os.path.join(tmp_path, f'results.{output_type}') - - pipe = LinearPipeline(config) - pipe.set_source(FileSourceStage(config, filename=input_file)) - - if use_deserialize: - pipe.add_stage(DeserializeStage(config)) - pipe.add_stage(SerializeStage(config)) - - pipe.add_stage(WriteToFileStage(config, filename=out_file, overwrite=False, flush=flush)) - pipe.run() - - assert os.path.exists(out_file) - - expected_df = dataset['filter_probs.csv'] - actual_df = dataset.get_df(out_file, no_cache=True) - dataset.assert_compare_df(expected_df, actual_df) From 52282ee40345a4d651eb9b70b3e995f0d76a563a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 12:46:08 -0700 Subject: [PATCH 150/347] Cleanup usage of cpu_mode mark --- tests/conftest.py | 12 +++ .../common/test_content_extractor_module.py | 2 - .../llm/common/test_web_scraper_module.py | 2 - .../llm/common/test_web_scraper_stage.py | 2 - .../test_schema_transform_module.py | 2 - .../test_create_features.py | 1 - .../test_preprocessing.py | 1 - tests/llm/test_vdb_upload_pipe.py | 2 +- tests/stages/test_llm_engine_stage_pipe.py | 2 - tests/stages/test_ml_flow_drift_stage.py | 1 - tests/stages/test_timeseries_stage.py | 1 - tests/test_add_classifications_stage.py | 2 +- tests/test_add_scores_stage.py | 2 +- tests/test_multi_port_modules_stage.py | 2 +- tests/test_serialize_stage.py | 11 ++- tests/test_tensor_memory.py | 84 +++++++++++-------- 16 files changed, 73 insertions(+), 56 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c9d5761cd8..56b8b3fab3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1099,3 +1099,15 @@ def load_cudf_helper_fixture(): if os.environ.get("MORPHEUS_CPU_ONLY") is None: from morpheus.common import load_cudf_helper load_cudf_helper() + + +@pytest.fixture(name="array_pkg") +def array_pkg_fixture(execution_mode: "ExecutionMode") -> types.ModuleType: + from morpheus.utils.type_utils import get_array_pkg + return get_array_pkg(execution_mode) + + +@pytest.fixture(name="df_pkg") +def df_pkg_fixture(execution_mode: "ExecutionMode") -> types.ModuleType: + from morpheus.utils.type_utils import get_df_pkg + return get_df_pkg(execution_mode) diff --git a/tests/examples/llm/common/test_content_extractor_module.py b/tests/examples/llm/common/test_content_extractor_module.py index 0cd4e78ed9..805f9b2b1f 100644 --- a/tests/examples/llm/common/test_content_extractor_module.py +++ b/tests/examples/llm/common/test_content_extractor_module.py @@ -88,8 +88,6 @@ def generate_random_string(length: int) -> str: return ''.join(random.choices(string.ascii_letters + string.digits, k=length)) -@pytest.mark.cpu_mode -@pytest.mark.use_cudf @pytest.mark.parametrize("data_len, num_rows_per_file, batch_size", [(40, 5, 2), (51, 3, 1), (150, 10, 5), (500, 3, 2), (1000, 5, 3), (50, 10, 2), (100, 20, 3), (50, 5, 1), (100, 10, 1), (49, 5, 2), (99, 5, 2), (60, 7, 2), (120, 6, 3), (1000, 50, 10), diff --git a/tests/examples/llm/common/test_web_scraper_module.py b/tests/examples/llm/common/test_web_scraper_module.py index c79026e7e1..012cb45fa3 100644 --- a/tests/examples/llm/common/test_web_scraper_module.py +++ b/tests/examples/llm/common/test_web_scraper_module.py @@ -30,8 +30,6 @@ @pytest.mark.slow -@pytest.mark.cpu_mode -@pytest.mark.use_cudf @pytest.mark.import_mod(os.path.join(TEST_DIRS.examples_dir, 'llm/vdb_upload/module/web_scraper_module.py')) def test_web_scraper_module(config: Config, mock_rest_server: str, import_mod: types.ModuleType): url = f"{mock_rest_server}/www/index" diff --git a/tests/examples/llm/common/test_web_scraper_stage.py b/tests/examples/llm/common/test_web_scraper_stage.py index eac1604ff2..6526c00df1 100644 --- a/tests/examples/llm/common/test_web_scraper_stage.py +++ b/tests/examples/llm/common/test_web_scraper_stage.py @@ -28,8 +28,6 @@ @pytest.mark.slow -@pytest.mark.cpu_mode -@pytest.mark.use_cudf @pytest.mark.import_mod(os.path.join(TEST_DIRS.examples_dir, 'llm/vdb_upload/module/web_scraper_stage.py')) def test_http_client_source_stage_pipe(config: Config, mock_rest_server: str, import_mod: types.ModuleType): url = f"{mock_rest_server}/www/index" diff --git a/tests/examples/llm/vdb_upload/test_schema_transform_module.py b/tests/examples/llm/vdb_upload/test_schema_transform_module.py index 7aed8642f8..75dc7178a5 100644 --- a/tests/examples/llm/vdb_upload/test_schema_transform_module.py +++ b/tests/examples/llm/vdb_upload/test_schema_transform_module.py @@ -27,8 +27,6 @@ from morpheus.stages.output.compare_dataframe_stage import CompareDataFrameStage -@pytest.mark.cpu_mode -@pytest.mark.use_cudf @pytest.mark.parametrize("num_select, num_renames", [(1, 0), (0, 1), (1, 1), (6, 6), (13, 10), (10, 13)]) def test_schema_transform_module(num_select, num_renames, diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index f27ea725bb..fbb03f941c 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -30,7 +30,6 @@ from morpheus.stages.input.appshield_source_stage import AppShieldSourceStage -@pytest.mark.cpu_mode class TestCreateFeaturesRWStage: # pylint: disable=no-name-in-module diff --git a/tests/examples/ransomware_detection/test_preprocessing.py b/tests/examples/ransomware_detection/test_preprocessing.py index 35f5eaf425..bcb25b3950 100644 --- a/tests/examples/ransomware_detection/test_preprocessing.py +++ b/tests/examples/ransomware_detection/test_preprocessing.py @@ -25,7 +25,6 @@ from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage -@pytest.mark.cpu_mode class TestPreprocessingRWStage: # pylint: disable=no-name-in-module diff --git a/tests/llm/test_vdb_upload_pipe.py b/tests/llm/test_vdb_upload_pipe.py index 9487d5322d..5d5a335fc1 100755 --- a/tests/llm/test_vdb_upload_pipe.py +++ b/tests/llm/test_vdb_upload_pipe.py @@ -30,8 +30,8 @@ from morpheus.service.vdb.milvus_vector_db_service import MilvusVectorDBService +@pytest.mark.xfail(reason="This is a GPU centric test, but depends on the Python impl of the Triton client") @pytest.mark.milvus -@pytest.mark.cpu_mode @pytest.mark.use_pandas @pytest.mark.import_mod([ os.path.join(TEST_DIRS.examples_dir, 'llm/common'), diff --git a/tests/stages/test_llm_engine_stage_pipe.py b/tests/stages/test_llm_engine_stage_pipe.py index 5f445d853b..7d0fe2e92f 100644 --- a/tests/stages/test_llm_engine_stage_pipe.py +++ b/tests/stages/test_llm_engine_stage_pipe.py @@ -40,8 +40,6 @@ def _build_engine() -> LLMEngine: return engine -@pytest.mark.use_cudf -@pytest.mark.cpu_mode def test_pipeline(config: Config, dataset_cudf: DatasetManager): test_data = os.path.join(TEST_DIRS.validation_data_dir, 'root-cause-validation-data-input.jsonlines') input_df = dataset_cudf[test_data] diff --git a/tests/stages/test_ml_flow_drift_stage.py b/tests/stages/test_ml_flow_drift_stage.py index 501bb3eb1a..ec72f59b09 100644 --- a/tests/stages/test_ml_flow_drift_stage.py +++ b/tests/stages/test_ml_flow_drift_stage.py @@ -56,7 +56,6 @@ def test_constructor(config): @pytest.mark.use_cudf -@pytest.mark.cpu_mode def test_calc_drift(config, filter_probs_df): with patch("morpheus.stages.postprocess.ml_flow_drift_stage.mlflow.start_run"): labels = ["a", "b", "c"] diff --git a/tests/stages/test_timeseries_stage.py b/tests/stages/test_timeseries_stage.py index 30265ed865..8973e6f88d 100644 --- a/tests/stages/test_timeseries_stage.py +++ b/tests/stages/test_timeseries_stage.py @@ -67,7 +67,6 @@ def test_constructor(config): assert typing_utils.issubtype(ControlMessage, accepted_union) -@pytest.mark.use_cudf @pytest.mark.cpu_mode def test_call_timeseries_user(config): stage = TimeSeriesStage(config) diff --git a/tests/test_add_classifications_stage.py b/tests/test_add_classifications_stage.py index 9aea29ecc4..c48083a890 100755 --- a/tests/test_add_classifications_stage.py +++ b/tests/test_add_classifications_stage.py @@ -61,7 +61,7 @@ def test_constructor_errors(config: Config): AddClassificationsStage(config, labels=['missing']) -@pytest.mark.cpu_mode +@pytest.mark.skip(reason="TODO: determine python impls for gpu only stages") def test_add_labels_with_multi_response_message_and_contgrol_message(): class_labels = {0: "frogs", 1: "lizards", 2: "toads"} diff --git a/tests/test_add_scores_stage.py b/tests/test_add_scores_stage.py index c3d9082eee..cedb29ce81 100755 --- a/tests/test_add_scores_stage.py +++ b/tests/test_add_scores_stage.py @@ -66,7 +66,7 @@ def test_constructor_errors(config: Config): AddScoresStage(config, labels=['missing']) -@pytest.mark.cpu_mode +@pytest.mark.skip(reason="TODO: determine python impls for gpu only stages") def test_add_labels_with_multi_response_message_and_control_message(): class_labels = {0: "frogs", 1: "lizards", 2: "toads"} diff --git a/tests/test_multi_port_modules_stage.py b/tests/test_multi_port_modules_stage.py index fdeef14a44..77a6b08444 100755 --- a/tests/test_multi_port_modules_stage.py +++ b/tests/test_multi_port_modules_stage.py @@ -50,7 +50,7 @@ def registered_module_conf(): yield registered_module_conf -@pytest.mark.cpu_mode +@pytest.mark.gpu_and_cpu_mode def test_constructor(config, unregistered_module_conf): mod_stage = MultiPortModulesStage(config, diff --git a/tests/test_serialize_stage.py b/tests/test_serialize_stage.py index 12fdd42710..14610986d6 100755 --- a/tests/test_serialize_stage.py +++ b/tests/test_serialize_stage.py @@ -16,10 +16,9 @@ import re +import pandas as pd import pytest -import cudf - from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.stages.postprocess.serialize_stage import SerializeStage @@ -27,13 +26,17 @@ @pytest.mark.cpu_mode def test_fixed_columns(config): - df1 = cudf.DataFrame() + """ + The serialize stage works in both GPU and CPU mode, however this test is only for CPU mode since it is testing the + CPU implementation of the stage. + """ + df1 = pd.DataFrame() df1['apples'] = range(0, 4) df1['pears'] = range(5, 9) df1['apple_sauce'] = range(4, 0, -1) mm1 = MultiMessage(meta=MessageMeta(df1)) - df2 = cudf.DataFrame() + df2 = pd.DataFrame() df2['apples'] = range(4, 7) df2['applause'] = range(9, 6, -1) df2['pears'] = range(7, 10) diff --git a/tests/test_tensor_memory.py b/tests/test_tensor_memory.py index 631f6493b2..19b7f5bb3e 100644 --- a/tests/test_tensor_memory.py +++ b/tests/test_tensor_memory.py @@ -16,14 +16,15 @@ import os import string +import types import typing -import cupy as cp import numpy as np import pytest from _utils import TEST_DIRS from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages.memory.inference_memory import InferenceMemory from morpheus.messages.memory.inference_memory import InferenceMemoryAE from morpheus.messages.memory.inference_memory import InferenceMemoryFIL @@ -33,6 +34,7 @@ from morpheus.messages.memory.response_memory import ResponseMemoryProbs from morpheus.messages.memory.tensor_memory import TensorMemory from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import NDArrayType INPUT_FILE = os.path.join(TEST_DIRS.tests_data_dir, 'filter_probs.csv') @@ -40,14 +42,14 @@ # pylint: disable=unused-argument -def compare_tensors(tensors1: typing.Dict[str, cp.ndarray], tensors2: typing.Dict[str, cp.ndarray]): +def compare_tensors(tensors1: typing.Dict[str, NDArrayType], tensors2: typing.Dict[str, NDArrayType]): assert sorted(tensors1.keys()) == sorted(tensors2.keys()) for (k, val1) in tensors1.items(): assert (val1 == tensors2[k]).all() -def check_tensor_memory(cls: type, count: int, tensors: typing.Dict[str, cp.ndarray]): - other_tensors = {'ones': cp.ones(count), 'zeros': cp.zeros(count)} +def check_tensor_memory(cls: type, count: int, tensors: typing.Dict[str, NDArrayType], array_pkg: types.ModuleType): + other_tensors = {'ones': array_pkg.ones(count), 'zeros': array_pkg.zeros(count)} mem = cls(count=count) assert mem.count == count @@ -73,18 +75,19 @@ def check_tensor_memory(cls: type, count: int, tensors: typing.Dict[str, cp.ndar cls(count, tensors) -def test_tensor_memory(config: Config): - test_data = cp.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) +@pytest.mark.gpu_and_cpu_mode +def test_tensor_memory(array_pkg: types.ModuleType): + test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) count = test_data.shape[0] # TensorMemory expects a dictionary of { : } # Convert each column into a 1d cupy array tensors = {} for col in range(test_data.shape[1]): - tensors[string.ascii_lowercase[col]] = cp.array(test_data[:, col]) + tensors[string.ascii_lowercase[col]] = array_pkg.array(test_data[:, col]) for cls in (TensorMemory, InferenceMemory, ResponseMemory): - check_tensor_memory(cls, count, tensors) + check_tensor_memory(cls=cls, count=count, tensors=tensors, array_pkg=array_pkg) @pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") @@ -106,12 +109,14 @@ def test_inference_memory_ae(config: Config): InferenceMemoryAE(count, input_tensor, seq_ids) # pylint: disable=too-many-function-args,missing-kwoa -def test_inference_memory_fil(config: Config): - test_data = cp.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) +# TODO: Determine what to do about the Python impls for GPU based messages +@pytest.mark.gpu_and_cpu_mode +def test_inference_memory_fil(array_pkg: types.ModuleType): + test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) count = test_data.shape[0] - input_0 = cp.array(test_data[:, 0]) - seq_ids = cp.array(test_data[:, 1]) + input_0 = array_pkg.array(test_data[:, 0]) + seq_ids = array_pkg.array(test_data[:, 1]) mem = InferenceMemoryFIL(count=count, input__0=input_0, seq_ids=seq_ids) assert mem.count == count @@ -123,13 +128,14 @@ def test_inference_memory_fil(config: Config): InferenceMemoryFIL(count, input_0, seq_ids) # pylint: disable=too-many-function-args,missing-kwoa -def test_inference_memory_nlp(config: Config): - test_data = cp.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) +@pytest.mark.gpu_and_cpu_mode +def test_inference_memory_nlp(array_pkg: types.ModuleType): + test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) count = test_data.shape[0] - input_ids = cp.array(test_data[:, 0]) - input_mask = cp.array(test_data[:, 1]) - seq_ids = cp.array(test_data[:, 2]) + input_ids = array_pkg.array(test_data[:, 0]) + input_mask = array_pkg.array(test_data[:, 1]) + seq_ids = array_pkg.array(test_data[:, 2]) mem = InferenceMemoryNLP(count=count, input_ids=input_ids, input_mask=input_mask, seq_ids=seq_ids) assert mem.count == count @@ -142,8 +148,8 @@ def test_inference_memory_nlp(config: Config): InferenceMemoryNLP(count, input_ids, input_mask, seq_ids) # pylint: disable=too-many-function-args,missing-kwoa -def check_response_memory_probs_and_ae(cls: type): - test_data = cp.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) +def check_response_memory_probs(cls: type, array_pkg: types.ModuleType): + test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) count = test_data.shape[0] mem = cls(count=count, probs=test_data) @@ -160,7 +166,7 @@ def check_response_memory_probs_and_ae(cls: type): @pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") @pytest.mark.cpu_mode def test_response_memory_ae(config: Config, filter_probs_df: DataFrameType): - mem = check_response_memory_probs_and_ae(ResponseMemoryAE) + mem = check_response_memory_probs(ResponseMemoryAE) assert mem.user_id == "" assert mem.explain_df is None @@ -172,38 +178,43 @@ def test_response_memory_ae(config: Config, filter_probs_df: DataFrameType): assert (mem.explain_df.values == filter_probs_df.values).all() -def test_response_memory_probs(config: Config): - check_response_memory_probs_and_ae(ResponseMemoryProbs) +@pytest.mark.gpu_and_cpu_mode +def test_response_memory_probs(array_pkg: types.ModuleType): + check_response_memory_probs(ResponseMemoryProbs, array_pkg) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("tensor_cls", [TensorMemory, InferenceMemory, ResponseMemory]) -def test_constructor_length_error(config: Config, tensor_cls: type): +def test_constructor_length_error(array_pkg: types.ModuleType, tensor_cls: type): count = 10 - tensors = {"a": cp.zeros(count), "b": cp.ones(count)} + tensors = {"a": array_pkg.zeros(count), "b": array_pkg.ones(count)} with pytest.raises(ValueError): tensor_cls(count=count - 1, tensors=tensors) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("tensor_cls", [TensorMemory, InferenceMemory, ResponseMemory]) -def test_set_tensor_length_error(config: Config, tensor_cls: type): +def test_set_tensor_length_error(array_pkg: types.ModuleType, tensor_cls: type): count = 10 mem = tensor_cls(count=count) with pytest.raises(ValueError): - mem.set_tensor('a', cp.zeros(count + 1)) + mem.set_tensor('a', array_pkg.zeros(count + 1)) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("tensor_cls", [TensorMemory, InferenceMemory, ResponseMemory]) -def test_set_tensors_length_error(config: Config, tensor_cls: type): +def test_set_tensors_length_error(array_pkg: types.ModuleType, tensor_cls: type): count = 10 - tensors = {"a": cp.zeros(count), "b": cp.ones(count)} + tensors = {"a": array_pkg.zeros(count), "b": array_pkg.ones(count)} mem = tensor_cls(count=count + 1) with pytest.raises(ValueError): mem.set_tensors(tensors) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("tensor_cls", [TensorMemory, InferenceMemory, ResponseMemory]) @pytest.mark.parametrize( "shape", @@ -211,12 +222,12 @@ def test_set_tensors_length_error(config: Config, tensor_cls: type): (536870912, 1), # bytesize > 2**31 (134217728, 4) # bytesize > 2**31 and element count > 2**31 ]) -def test_tensorindex_bug(config: Config, tensor_cls: type, shape: typing.Tuple[int, int]): +def test_tensorindex_bug(array_pkg: types.ModuleType, tensor_cls: type, shape: typing.Tuple[int, int]): """ Test for issue #1004. We use a 32bit signed integer for shape and strides, but we shouldn't for element counts and byte sizes. """ - tensors = {"a": cp.zeros(shape, dtype=np.float32)} + tensors = {"a": array_pkg.zeros(shape, dtype=np.float32)} mem = tensor_cls(count=shape[0], tensors=tensors) tensor_a = mem.get_tensor('a') @@ -224,19 +235,24 @@ def test_tensorindex_bug(config: Config, tensor_cls: type, shape: typing.Tuple[i assert tensor_a.nbytes == shape[0] * shape[1] * 4 -def test_tensor_update(config: Config): +@pytest.mark.gpu_and_cpu_mode +def test_tensor_update(array_pkg: types.ModuleType): tensor_data = { - "input_ids": cp.array([1, 2, 3]), "input_mask": cp.array([1, 1, 1]), "segment_ids": cp.array([0, 0, 1]) + "input_ids": array_pkg.array([1, 2, 3]), + "input_mask": array_pkg.array([1, 1, 1]), + "segment_ids": array_pkg.array([0, 0, 1]) } tensor_memory = TensorMemory(count=3, tensors=tensor_data) # Update tensors with new data new_tensors = { - "input_ids": cp.array([4, 5, 6]), "input_mask": cp.array([1, 0, 1]), "segment_ids": cp.array([1, 1, 0]) + "input_ids": array_pkg.array([4, 5, 6]), + "input_mask": array_pkg.array([1, 0, 1]), + "segment_ids": array_pkg.array([1, 1, 0]) } tensor_memory.set_tensors(new_tensors) for (key, cp_arr) in new_tensors.items(): tensor = tensor_memory.get_tensor(key) - cp.allclose(tensor, cp_arr) + array_pkg.allclose(tensor, cp_arr) From 87a0f883edc4687914f9026d9be4955ee6d059c4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 13:10:23 -0700 Subject: [PATCH 151/347] WIP --- tests/stages/test_timeseries_stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/stages/test_timeseries_stage.py b/tests/stages/test_timeseries_stage.py index 8973e6f88d..fe9d129565 100644 --- a/tests/stages/test_timeseries_stage.py +++ b/tests/stages/test_timeseries_stage.py @@ -21,13 +21,13 @@ import pytest import typing_utils -import morpheus._lib.messages as _messages from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.messages import ControlMessage from morpheus.messages import MultiResponseAEMessage from morpheus.messages import MultiResponseMessage from morpheus.messages import ResponseMemory +from morpheus.messages import TensorMemory from morpheus.messages.message_meta import MessageMeta from morpheus.stages.postprocess.timeseries_stage import TimeSeriesStage @@ -52,7 +52,7 @@ def _make_control_message(df, probs): df_ = df[0:len(probs)] cm = ControlMessage() cm.payload(MessageMeta(df_)) - cm.tensors(_messages.TensorMemory(count=len(df_), tensors={'probs': probs})) + cm.tensors(TensorMemory(count=len(df_), tensors={'probs': probs})) cm.set_metadata("user_id", "test_user_id") return cm From 9760cdc65fa2d076b29d72d83b078430ea84851e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 13:17:39 -0700 Subject: [PATCH 152/347] Mark ransomeware tests as xfail, the pipeline depends on the AppshieldMeta message which is currently Python only --- tests/examples/ransomware_detection/test_create_features.py | 2 ++ tests/examples/ransomware_detection/test_preprocessing.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index fbb03f941c..879bd719f3 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -124,6 +124,8 @@ def test_on_next(self, expected_df.reset_index(drop=True, inplace=True) dataset_pandas.assert_compare_df(meta.copy_dataframe(), expected_df) + @pytest.mark.xfail(reason="The ransomeware pipeline depends on the AppShieldMeta class, which forces the pipeline " + "to be python (CPU) only") @mock.patch('stages.create_features.Client') def test_create_multi_messages(self, mock_dask_client, diff --git a/tests/examples/ransomware_detection/test_preprocessing.py b/tests/examples/ransomware_detection/test_preprocessing.py index bcb25b3950..99e102e8fb 100644 --- a/tests/examples/ransomware_detection/test_preprocessing.py +++ b/tests/examples/ransomware_detection/test_preprocessing.py @@ -147,6 +147,8 @@ def test_merge_curr_and_prev_snapshots(self, config: Config, rwd_conf: dict, dat stage._merge_curr_and_prev_snapshots(df, source_pid_process) dataset_pandas.assert_compare_df(df.fillna(''), expected_df) + @pytest.mark.xfail(reason="The ransomeware pipeline depends on the AppShieldMeta class, which forces the pipeline " + "to be python (CPU) only") def test_pre_process_batch(self, config: Config, rwd_conf: dict, dataset_pandas: DatasetManager): # Pylint currently fails to work with classmethod: https://github.com/pylint-dev/pylint/issues/981 From 2f4b33f82400536caaae7796cdd672dc253fe216 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 13:28:05 -0700 Subject: [PATCH 153/347] Mark the http client and kafka sink stages as supporting both GPU & CPU mode --- .../morpheus/stages/output/http_client_sink_stage.py | 3 ++- .../morpheus/stages/output/write_to_kafka_stage.py | 3 ++- tests/test_write_to_kafka_stage_pipe.py | 12 ++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/morpheus/morpheus/stages/output/http_client_sink_stage.py b/python/morpheus/morpheus/stages/output/http_client_sink_stage.py index a9cb872b4c..083a97b9ce 100644 --- a/python/morpheus/morpheus/stages/output/http_client_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_client_sink_stage.py @@ -25,6 +25,7 @@ from morpheus.config import Config from morpheus.io import serializers from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils import http_utils @@ -36,7 +37,7 @@ @register_stage("to-http", ignore_args=["query_params", "headers", "df_to_request_kwargs_fn", "**request_kwargs"]) -class HttpClientSinkStage(PassThruTypeMixin, SinglePortStage): +class HttpClientSinkStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Write all messages to an HTTP endpoint. diff --git a/python/morpheus/morpheus/stages/output/write_to_kafka_stage.py b/python/morpheus/morpheus/stages/output/write_to_kafka_stage.py index 3546a14563..ad7954f977 100644 --- a/python/morpheus/morpheus/stages/output/write_to_kafka_stage.py +++ b/python/morpheus/morpheus/stages/output/write_to_kafka_stage.py @@ -24,6 +24,7 @@ from morpheus.config import Config from morpheus.io import serializers from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @@ -31,7 +32,7 @@ @register_stage("to-kafka") -class WriteToKafkaStage(PassThruTypeMixin, SinglePortStage): +class WriteToKafkaStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Write all messages to a Kafka cluster. diff --git a/tests/test_write_to_kafka_stage_pipe.py b/tests/test_write_to_kafka_stage_pipe.py index 94b17a196c..0677128d7c 100644 --- a/tests/test_write_to_kafka_stage_pipe.py +++ b/tests/test_write_to_kafka_stage_pipe.py @@ -33,9 +33,9 @@ @pytest.mark.kafka -@pytest.mark.use_cudf +@pytest.mark.gpu_and_cpu_mode def test_write_to_kafka_stage_pipe(config, - dataset_cudf: DatasetManager, + dataset: DatasetManager, kafka_bootstrap_servers: str, kafka_consumer: "KafkaConsumer", kafka_topics: KafkaTopics) -> None: @@ -44,7 +44,7 @@ def test_write_to_kafka_stage_pipe(config, to ensure it works just as well with the C++ impls of the message classes. """ - filter_probs_df = dataset_cudf['filter_probs.csv'] + filter_probs_df = dataset['filter_probs.csv'] pipe = LinearPipeline(config) pipe.set_source(InMemorySourceStage(config, [filter_probs_df])) pipe.add_stage(DeserializeStage(config)) @@ -59,9 +59,9 @@ def test_write_to_kafka_stage_pipe(config, kafka_messages = list(kafka_consumer) assert len(kafka_messages) == len(filter_probs_df) - output_df = cudf.io.read_json("\n".join(rec.value.decode("utf-8") for rec in kafka_messages), - lines=True).to_pandas() + # TODO determine if we need to use cudf here rather than matching the df package of the test + output_df = cudf.io.read_json("\n".join(rec.value.decode("utf-8") for rec in kafka_messages), lines=True) assert len(output_df) == len(filter_probs_df) - dataset_cudf.assert_compare_df(filter_probs_df, output_df) + dataset.assert_compare_df(filter_probs_df, output_df) From e586501f3f667556f4ea1c8cbea8c856ef8e8929 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 13:36:30 -0700 Subject: [PATCH 154/347] Switch to using the array_pkg fixture --- tests/test_multi_message.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_multi_message.py b/tests/test_multi_message.py index fe83f76d31..6f0e989dab 100644 --- a/tests/test_multi_message.py +++ b/tests/test_multi_message.py @@ -18,6 +18,7 @@ import dataclasses import string +import types from unittest.mock import patch import cupy as cp @@ -47,7 +48,6 @@ from morpheus.utils import logger as morpheus_logger from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameTypeStr -from morpheus.utils.type_utils import get_array_pkg from morpheus.utils.type_utils import get_df_pkg_from_obj @@ -444,9 +444,7 @@ def test_get_slice_values_dup_index(dataset: DatasetManager): _test_get_slice_values(df) -def test_get_slice_derived(filter_probs_df: DataFrameType, execution_mode: ExecutionMode): - array_pkg = get_array_pkg(execution_mode) - +def test_get_slice_derived(filter_probs_df: DataFrameType, array_pkg: types.ModuleType): multi_tensor_message_tensors = { "input_ids": array_pkg.zeros((20, 2)), "input_mask": array_pkg.zeros((20, 2)), @@ -614,7 +612,7 @@ def test_from_message(filter_probs_df: DataFrameType): MultiAEMessage.from_message(multi) -def test_tensor_constructor(filter_probs_df: DataFrameType, execution_mode: ExecutionMode): +def test_tensor_constructor(filter_probs_df: DataFrameType, array_pkg: types.ModuleType): mess_len = len(filter_probs_df) ten_len = mess_len * 2 @@ -670,7 +668,6 @@ def test_tensor_constructor(filter_probs_df: DataFrameType, execution_mode: Exec MultiTensorMessage(meta=meta, mess_count=10, memory=memory, count=9) # === ID Tensors === - array_pkg = get_array_pkg(execution_mode) id_tensor = array_pkg.expand_dims(array_pkg.arange(0, mess_len, dtype=int), axis=1) # With valid ID tensor @@ -807,11 +804,10 @@ def test_tensor_slicing(dataset: DatasetManager): @pytest.mark.gpu_and_cpu_mode -def test_deprecation_message(filter_probs_df: DataFrameType, execution_mode: ExecutionMode): +def test_deprecation_message(filter_probs_df: DataFrameType, array_pkg: types.ModuleType): meta = MessageMeta(filter_probs_df) - array_pkg = get_array_pkg(execution_mode) multi_tensor_message_tensors = { "input_ids": array_pkg.zeros((20, 2)), "input_mask": array_pkg.zeros((20, 2)), From f859ca0c0076812e4eee46c8c309bcd6e81b628d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 5 Sep 2024 15:08:53 -0700 Subject: [PATCH 155/347] Lint fixes --- tests/_utils/dataset_manager.py | 42 +++++++++------------- tests/_utils/stages/conv_msg.py | 4 --- tests/conftest.py | 2 +- tests/pipeline/test_stage_decorator.py | 6 ++-- tests/stages/test_llm_engine_stage_pipe.py | 2 -- tests/stages/test_preprocess_fil_stage.py | 4 --- tests/stages/test_preprocess_nlp_stage.py | 6 ---- tests/test_abp.py | 3 -- tests/test_abp_kafka.py | 3 -- tests/test_add_classifications_stage.py | 1 - tests/test_add_scores_stage.py | 3 -- tests/test_messages.py | 3 -- tests/test_multi_message.py | 1 - tests/test_phishing.py | 3 -- tests/test_phishing_kafka.py | 3 -- tests/test_sid_kafka.py | 3 -- tests/test_tensor_memory.py | 1 - tests/test_triton_inference_stage.py | 12 ------- tests/test_write_to_kafka_stage_pipe.py | 7 ++-- 19 files changed, 24 insertions(+), 85 deletions(-) diff --git a/tests/_utils/dataset_manager.py b/tests/_utils/dataset_manager.py index da389d49c6..404462a3ce 100644 --- a/tests/_utils/dataset_manager.py +++ b/tests/_utils/dataset_manager.py @@ -30,6 +30,8 @@ from morpheus.io.deserializers import read_file_to_df from morpheus.utils import compare_df from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import SeriesType class DatasetManager: @@ -38,19 +40,19 @@ class DatasetManager: Parameters ---------- - df_type : typing.Literal['cudf', 'pandas'] + df_type : DataFrameTypeStr Type of DataFrame to return unless otherwise explicitly specified. """ - __df_cache: typing.Dict[typing.Tuple[typing.Literal['cudf', 'pandas'], str], DataFrameType] = {} + __df_cache: dict[tuple[DataFrameTypeStr, str], DataFrameType] = {} # Values in `__instances` are instances of `DatasetLoader` - __instances: typing.Dict[typing.Literal['cudf', 'pandas'], typing.Any] = {} + __instances: dict[DataFrameTypeStr, "DatasetManager"] = {} # Explicitly using __new__ instead of of an __init__ to implement this as a singleton for each dataframe type. # Initialization is also being performed here instead of an __init__ method as an __init__ method would be re-run # the __init__ on the singleton instance for each cache hit. - def __new__(cls, df_type: typing.Literal['cudf', 'pandas']): + def __new__(cls, df_type: DataFrameTypeStr): """Returns the singleton instance of `DatasetManager` for the specified `df_type`.""" try: return cls.__instances[df_type] @@ -61,7 +63,7 @@ def __new__(cls, df_type: typing.Literal['cudf', 'pandas']): return instance @staticmethod - def get_alt_df_type(df_type: typing.Literal['cudf', 'pandas']) -> typing.Literal['cudf', 'pandas']: + def get_alt_df_type(df_type: DataFrameTypeStr) -> DataFrameTypeStr: """Returns the other possible df type.""" return 'cudf' if df_type == 'pandas' else 'pandas' @@ -71,7 +73,7 @@ def clear(self): def get_df(self, file_path: str, - df_type: typing.Literal['cudf', 'pandas'] = None, + df_type: DataFrameTypeStr = None, no_cache: bool = False, **reader_kwargs) -> DataFrameType: """ @@ -123,9 +125,7 @@ def get_df(self, return df.copy(deep=True) - def __getitem__( - self, item: typing.Union[str, typing.Tuple[str], typing.Tuple[str, typing.Literal['cudf', - 'pandas']]]) -> DataFrameType: + def __getitem__(self, item: str | tuple[str] | tuple[str, DataFrameTypeStr]) -> DataFrameType: """Implements `__getitem__` to allow for fetching DataFrames using the `[]` operator.""" if not isinstance(item, tuple): item = (item, ) @@ -172,7 +172,7 @@ def repeat(df: DataFrameType, repeat_count: int = 2, reset_index: bool = True) - return repeated_df @staticmethod - def replace_index(df: DataFrameType, replace_ids: typing.Dict[int, int]) -> DataFrameType: + def replace_index(df: DataFrameType, replace_ids: dict[int, int]) -> DataFrameType: """Return a new DataFrame's where we replace some index values with others.""" return df.rename(index=replace_ids) @@ -192,7 +192,7 @@ def dup_index(cls, df: DataFrameType, count: int = 1) -> DataFrameType: return cls.replace_index(df, replace_dict) @staticmethod - def _value_as_pandas(val: typing.Union[pd.DataFrame, pd.Series, cdf.DataFrame, cdf.Series], assert_is_pandas=True): + def _value_as_pandas(val: DataFrameType | SeriesType, assert_is_pandas=True): if (isinstance(val, (cdf.DataFrame, cdf.Series))): return val.to_pandas() @@ -202,17 +202,15 @@ def _value_as_pandas(val: typing.Union[pd.DataFrame, pd.Series, cdf.DataFrame, c return val @classmethod - def _value_as_pandas_df(cls, - val: typing.Union[pd.DataFrame, pd.Series, cdf.DataFrame, cdf.Series], - assert_is_pandas=True): - pval = cls._value_as_pandas(val) + def _value_as_pandas_df(cls, val: DataFrameType | SeriesType, assert_is_pandas=True): + pval = cls._value_as_pandas(val, assert_is_pandas=assert_is_pandas) if isinstance(pval, pd.Series): pval = pval.to_frame() return pval @classmethod - def df_equal(cls, df_to_check: typing.Union[pd.DataFrame, cdf.DataFrame], val_to_check: typing.Any): + def df_equal(cls, df_to_check: DataFrameType, val_to_check: typing.Any): """ Compare a DataFrame against a validation dataset which can either be a DataFrame, Series or CuPy array. Returns True if they are equal. @@ -234,7 +232,7 @@ def df_equal(cls, df_to_check: typing.Union[pd.DataFrame, cdf.DataFrame], val_to @classmethod def assert_df_equal(cls, - df_to_check: typing.Union[pd.DataFrame, cdf.DataFrame], + df_to_check: DataFrameType, val_to_check: typing.Any, assert_msg="Dataframes are not equal."): """ @@ -244,10 +242,7 @@ def assert_df_equal(cls, assert cls.df_equal(df_to_check=df_to_check, val_to_check=val_to_check), assert_msg @classmethod - def compare_df(cls, - dfa: typing.Union[pd.DataFrame, cdf.DataFrame], - dfb: typing.Union[pd.DataFrame, cdf.DataFrame], - **compare_args): + def compare_df(cls, dfa: DataFrameType, dfb: DataFrameType, **compare_args): """Wrapper for `morpheus.utils.compare_df.compare_df`.""" with warnings.catch_warnings(): # Ignore performance warnings from pandas triggered by the comparison @@ -255,9 +250,6 @@ def compare_df(cls, return compare_df.compare_df(cls._value_as_pandas_df(dfa), cls._value_as_pandas_df(dfb), **compare_args) @classmethod - def assert_compare_df(cls, - dfa: typing.Union[pd.DataFrame, cdf.DataFrame], - dfb: typing.Union[pd.DataFrame, cdf.DataFrame], - **compare_args): + def assert_compare_df(cls, dfa: DataFrameType, dfb: DataFrameType, **compare_args): """Convenience method for calling `compare_df` and asserting that the results are equivalent.""" assert_results(cls.compare_df(dfa, dfb, **compare_args)) diff --git a/tests/_utils/stages/conv_msg.py b/tests/_utils/stages/conv_msg.py index 88d03429fc..74e1087bed 100755 --- a/tests/_utils/stages/conv_msg.py +++ b/tests/_utils/stages/conv_msg.py @@ -15,13 +15,9 @@ import typing -import cupy as cp import mrc -import pandas as pd from mrc.core import operators as ops -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages import ControlMessage diff --git a/tests/conftest.py b/tests/conftest.py index 56b8b3fab3..548ab6b13f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,7 +46,7 @@ (PYTEST_KAFKA_AVAIL, PYTEST_KAFKA_ERROR) = _init_pytest_kafka() if PYTEST_KAFKA_AVAIL: # Pull out the fixtures into this namespace - from _utils.kafka import _kafka_consumer # noqa: F401 pylint:disable=unused-import + from _utils.kafka import _kafka_consumer # noqa: F401 pylint:disable=unused-import,ungrouped-imports from _utils.kafka import kafka_server # noqa: F401 pylint:disable=unused-import from _utils.kafka import zookeeper_proc # noqa: F401 pylint:disable=unused-import diff --git a/tests/pipeline/test_stage_decorator.py b/tests/pipeline/test_stage_decorator.py index 89bfc52088..fe391a0707 100644 --- a/tests/pipeline/test_stage_decorator.py +++ b/tests/pipeline/test_stage_decorator.py @@ -403,12 +403,12 @@ def test_fn(message: MessageMeta) -> MessageMeta: def test_end_to_end_pipe(config: Config, filter_probs_df: DataFrameType): @source(execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) - def source_gen(dataframes: list[cudf.DataFrame]) -> collections.abc.Iterator[MessageMeta]: + def source_gen(*, dataframes: list[DataFrameType]) -> collections.abc.Iterator[MessageMeta]: for df in dataframes: yield MessageMeta(df) @stage(execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) - def multiplier(message: MessageMeta, column: str, value: int | float = 2.0) -> MessageMeta: + def multiplier(message: MessageMeta, *, column: str, value: int | float = 2.0) -> MessageMeta: with message.mutable_dataframe() as df: df[column] = df[column] * value @@ -419,7 +419,7 @@ def multiplier(message: MessageMeta, column: str, value: int | float = 2.0) -> M expected_df['v2'] = expected_df['v2'] * multipy_by * 2.0 pipe = LinearPipeline(config) - pipe.set_source(source_gen(config, dataframes=[filter_probs_df])) + pipe.set_source(source_gen(config, dataframes=[filter_probs_df])) # pylint: disable=too-many-function-args pipe.add_stage(multiplier(config, column='v2', value=multipy_by)) pipe.add_stage(multiplier(config, column='v2')) sink = pipe.add_stage(CompareDataFrameStage(config, expected_df)) diff --git a/tests/stages/test_llm_engine_stage_pipe.py b/tests/stages/test_llm_engine_stage_pipe.py index 7d0fe2e92f..c21fc33d72 100644 --- a/tests/stages/test_llm_engine_stage_pipe.py +++ b/tests/stages/test_llm_engine_stage_pipe.py @@ -16,8 +16,6 @@ import os -import pytest - from _utils import TEST_DIRS from _utils import assert_results from _utils.dataset_manager import DatasetManager diff --git a/tests/stages/test_preprocess_fil_stage.py b/tests/stages/test_preprocess_fil_stage.py index 30695349e7..a66e98cd1e 100644 --- a/tests/stages/test_preprocess_fil_stage.py +++ b/tests/stages/test_preprocess_fil_stage.py @@ -15,16 +15,12 @@ import typing -import cupy as cp import pytest import typing_utils -import cudf - from morpheus.config import Config from morpheus.config import ConfigFIL from morpheus.messages import ControlMessage -from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.stages.preprocess.preprocess_fil_stage import PreprocessFILStage diff --git a/tests/stages/test_preprocess_nlp_stage.py b/tests/stages/test_preprocess_nlp_stage.py index 9acfdd4ff1..b237cda590 100644 --- a/tests/stages/test_preprocess_nlp_stage.py +++ b/tests/stages/test_preprocess_nlp_stage.py @@ -14,18 +14,12 @@ # limitations under the License. import typing -from unittest.mock import Mock -from unittest.mock import patch -import cupy as cp import pytest import typing_utils -import cudf - from morpheus.config import Config from morpheus.messages import ControlMessage -from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.stages.preprocess.preprocess_nlp_stage import PreprocessNLPStage diff --git a/tests/test_abp.py b/tests/test_abp.py index ea55530094..f39a41c684 100755 --- a/tests/test_abp.py +++ b/tests/test_abp.py @@ -15,15 +15,12 @@ # limitations under the License. import os -from unittest import mock -import numpy as np import pytest from _utils import TEST_DIRS from _utils import calc_error_val from _utils import compare_class_to_scores -from _utils import mk_async_infer from morpheus.config import Config from morpheus.config import ConfigFIL from morpheus.config import PipelineModes diff --git a/tests/test_abp_kafka.py b/tests/test_abp_kafka.py index 0a13444f74..7c241b8a5d 100755 --- a/tests/test_abp_kafka.py +++ b/tests/test_abp_kafka.py @@ -17,14 +17,11 @@ import os import typing from io import StringIO -from unittest import mock -import numpy as np import pandas import pytest from _utils import TEST_DIRS -from _utils import mk_async_infer from _utils.dataset_manager import DatasetManager from _utils.kafka import KafkaTopics from _utils.kafka import write_file_to_kafka diff --git a/tests/test_add_classifications_stage.py b/tests/test_add_classifications_stage.py index c48083a890..928743f637 100755 --- a/tests/test_add_classifications_stage.py +++ b/tests/test_add_classifications_stage.py @@ -25,7 +25,6 @@ from morpheus.config import Config from morpheus.messages import ControlMessage from morpheus.messages import TensorMemory -from morpheus.messages.memory.tensor_memory import TensorMemory from morpheus.messages.message_meta import MessageMeta from morpheus.messages.multi_response_message import MultiResponseMessage from morpheus.stages.postprocess.add_classifications_stage import AddClassificationsStage diff --git a/tests/test_add_scores_stage.py b/tests/test_add_scores_stage.py index cedb29ce81..964c9918ba 100755 --- a/tests/test_add_scores_stage.py +++ b/tests/test_add_scores_stage.py @@ -16,14 +16,11 @@ import typing -import cupy as cp import numpy as np import pandas as pd import pytest import typing_utils -import cudf - import morpheus._lib.messages as _messages from _utils.dataset_manager import DatasetManager from morpheus.config import Config diff --git a/tests/test_messages.py b/tests/test_messages.py index cfbe02823c..19c2cf8276 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -13,9 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import importlib -import os - import cupy as cp import pytest diff --git a/tests/test_multi_message.py b/tests/test_multi_message.py index 6f0e989dab..87a6de5644 100644 --- a/tests/test_multi_message.py +++ b/tests/test_multi_message.py @@ -29,7 +29,6 @@ import cudf from _utils.dataset_manager import DatasetManager -from morpheus.config import ExecutionMode from morpheus.messages import ControlMessage from morpheus.messages.memory.inference_memory import InferenceMemory from morpheus.messages.memory.response_memory import ResponseMemory diff --git a/tests/test_phishing.py b/tests/test_phishing.py index f81a65095a..41f3d9c7c2 100755 --- a/tests/test_phishing.py +++ b/tests/test_phishing.py @@ -15,14 +15,11 @@ # limitations under the License. import os -from unittest import mock -import numpy as np import pytest from _utils import TEST_DIRS from _utils import calc_error_val -from _utils import mk_async_infer from morpheus.config import Config from morpheus.config import PipelineModes from morpheus.pipeline import LinearPipeline diff --git a/tests/test_phishing_kafka.py b/tests/test_phishing_kafka.py index dd12af6dd9..3524cc62f4 100755 --- a/tests/test_phishing_kafka.py +++ b/tests/test_phishing_kafka.py @@ -17,14 +17,11 @@ import os import typing from io import StringIO -from unittest import mock -import numpy as np import pandas import pytest from _utils import TEST_DIRS -from _utils import mk_async_infer from _utils.dataset_manager import DatasetManager from _utils.kafka import KafkaTopics from _utils.kafka import write_file_to_kafka diff --git a/tests/test_sid_kafka.py b/tests/test_sid_kafka.py index 57c8f6edb2..e1708b1ffb 100755 --- a/tests/test_sid_kafka.py +++ b/tests/test_sid_kafka.py @@ -17,14 +17,11 @@ import os import typing from io import StringIO -from unittest import mock -import numpy as np import pandas import pytest from _utils import TEST_DIRS -from _utils import mk_async_infer from _utils.dataset_manager import DatasetManager from _utils.kafka import KafkaTopics from morpheus.config import Config diff --git a/tests/test_tensor_memory.py b/tests/test_tensor_memory.py index 19b7f5bb3e..c045275436 100644 --- a/tests/test_tensor_memory.py +++ b/tests/test_tensor_memory.py @@ -24,7 +24,6 @@ from _utils import TEST_DIRS from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.messages.memory.inference_memory import InferenceMemory from morpheus.messages.memory.inference_memory import InferenceMemoryAE from morpheus.messages.memory.inference_memory import InferenceMemoryFIL diff --git a/tests/test_triton_inference_stage.py b/tests/test_triton_inference_stage.py index c91811693c..0b075a0bcd 100644 --- a/tests/test_triton_inference_stage.py +++ b/tests/test_triton_inference_stage.py @@ -17,28 +17,16 @@ import queue from unittest import mock -import numpy as np -import pandas as pd import pytest -import cudf - from _utils import assert_results from _utils import mk_async_infer from morpheus.config import Config -from morpheus.config import ConfigFIL from morpheus.config import PipelineModes -from morpheus.pipeline import LinearPipeline from morpheus.stages.inference.triton_inference_stage import ProducerConsumerQueue from morpheus.stages.inference.triton_inference_stage import ResourcePool from morpheus.stages.inference.triton_inference_stage import TritonInferenceStage from morpheus.stages.inference.triton_inference_stage import TritonInferenceWorker -from morpheus.stages.input.in_memory_source_stage import InMemorySourceStage -from morpheus.stages.output.compare_dataframe_stage import CompareDataFrameStage -from morpheus.stages.postprocess.add_scores_stage import AddScoresStage -from morpheus.stages.postprocess.serialize_stage import SerializeStage -from morpheus.stages.preprocess.deserialize_stage import DeserializeStage -from morpheus.stages.preprocess.preprocess_fil_stage import PreprocessFILStage MODEL_MAX_BATCH_SIZE = 1024 diff --git a/tests/test_write_to_kafka_stage_pipe.py b/tests/test_write_to_kafka_stage_pipe.py index 0677128d7c..56f9a7dcff 100644 --- a/tests/test_write_to_kafka_stage_pipe.py +++ b/tests/test_write_to_kafka_stage_pipe.py @@ -14,12 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import types import typing import pytest -import cudf - from _utils.dataset_manager import DatasetManager from _utils.kafka import KafkaTopics from morpheus.pipeline.linear_pipeline import LinearPipeline @@ -35,6 +34,7 @@ @pytest.mark.kafka @pytest.mark.gpu_and_cpu_mode def test_write_to_kafka_stage_pipe(config, + df_pkg: types.ModuleType, dataset: DatasetManager, kafka_bootstrap_servers: str, kafka_consumer: "KafkaConsumer", @@ -59,8 +59,7 @@ def test_write_to_kafka_stage_pipe(config, kafka_messages = list(kafka_consumer) assert len(kafka_messages) == len(filter_probs_df) - # TODO determine if we need to use cudf here rather than matching the df package of the test - output_df = cudf.io.read_json("\n".join(rec.value.decode("utf-8") for rec in kafka_messages), lines=True) + output_df = df_pkg.read_json("\n".join(rec.value.decode("utf-8") for rec in kafka_messages), lines=True) assert len(output_df) == len(filter_probs_df) From bc6ac3db4c8357630af90117950137d0d07bc137 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 6 Sep 2024 09:26:43 -0700 Subject: [PATCH 156/347] Add a few vocabulary entries --- ci/vale/styles/config/vocabularies/morpheus/accept.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/vale/styles/config/vocabularies/morpheus/accept.txt b/ci/vale/styles/config/vocabularies/morpheus/accept.txt index 157edebd18..285a85c7d8 100644 --- a/ci/vale/styles/config/vocabularies/morpheus/accept.txt +++ b/ci/vale/styles/config/vocabularies/morpheus/accept.txt @@ -18,6 +18,9 @@ CMake Conda CPython [Cc]ryptocurrenc[y|ies] +cuDF +cuML +CuPy [Cc]yber [Cc]ybersecurity Cython @@ -51,7 +54,9 @@ NeMo nginx NIC NIM(s?) +NumPy NVIDIA +pandas [Pp]arallelization [Pp]arsable PCIe From e898922acad76f132250ec97c8a3569aba2370e0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 6 Sep 2024 09:27:34 -0700 Subject: [PATCH 157/347] WIP --- python/morpheus/morpheus/_lib/common/__init__.pyi | 6 ++---- .../morpheus/morpheus/_lib/cudf_helpers/__init__.pyi | 7 ++----- python/morpheus/morpheus/_lib/messages/__init__.pyi | 7 ++----- python/morpheus/morpheus/_lib/modules/__init__.pyi | 4 +--- python/morpheus/morpheus/_lib/stages/__init__.pyi | 11 ++++------- .../morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi | 7 ++----- 6 files changed, 13 insertions(+), 29 deletions(-) diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi index bc7af1a803..bba480d220 100644 --- a/python/morpheus/morpheus/_lib/common/__init__.pyi +++ b/python/morpheus/morpheus/_lib/common/__init__.pyi @@ -5,11 +5,9 @@ :toctree: _generate """ from __future__ import annotations - -import os -import typing - import morpheus._lib.common +import typing +import os __all__ = [ "FiberQueue", diff --git a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi index 027b74ecc0..4ce3dd3269 100644 --- a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi +++ b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi @@ -1,11 +1,8 @@ from __future__ import annotations - +import morpheus._lib.cudf_helpers import typing - -import cudf from cudf.core.dtypes import StructDtype - -import morpheus._lib.cudf_helpers +import cudf __all__ = [ "StructDtype", diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi index baae7c8c3d..7a5ac78488 100644 --- a/python/morpheus/morpheus/_lib/messages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/messages/__init__.pyi @@ -6,14 +6,11 @@ """ from __future__ import annotations - +import morpheus._lib.messages import typing - import cupy -import mrc.core.node - import morpheus._lib.common -import morpheus._lib.messages +import mrc.core.node __all__ = [ "ControlMessage", diff --git a/python/morpheus/morpheus/_lib/modules/__init__.pyi b/python/morpheus/morpheus/_lib/modules/__init__.pyi index ff105f6d3b..0ec21dfaad 100644 --- a/python/morpheus/morpheus/_lib/modules/__init__.pyi +++ b/python/morpheus/morpheus/_lib/modules/__init__.pyi @@ -6,10 +6,8 @@ """ from __future__ import annotations - -import typing - import morpheus._lib.modules +import typing __all__ = [ diff --git a/python/morpheus/morpheus/_lib/stages/__init__.pyi b/python/morpheus/morpheus/_lib/stages/__init__.pyi index ffd9a46c91..86a91e9088 100644 --- a/python/morpheus/morpheus/_lib/stages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/stages/__init__.pyi @@ -6,16 +6,13 @@ """ from __future__ import annotations - -import os +import morpheus._lib.stages import typing - +from morpheus._lib.common import FilterSource +import morpheus._lib.common import mrc.core.coro import mrc.core.segment - -import morpheus._lib.common -import morpheus._lib.stages -from morpheus._lib.common import FilterSource +import os __all__ = [ "AddClassificationsControlMessageStage", diff --git a/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi b/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi index 351790410d..36c202e16c 100644 --- a/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi +++ b/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi @@ -6,15 +6,12 @@ """ from __future__ import annotations - +import morpheus_llm._lib.llm import typing - +import morpheus._lib.messages import mrc.core.coro import mrc.core.segment -import morpheus._lib.llm -import morpheus._lib.messages - __all__ = [ "InputMap", "LLMContext", From 16930df51b353446d88457587581172660c67ca7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 6 Sep 2024 09:27:49 -0700 Subject: [PATCH 158/347] WIP --- examples/cpu_only/run.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index 2d23a20da5..c0d0f966d2 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -45,7 +45,6 @@ type=bool, is_flag=True, help=("Whether or not to run in CPU only mode, setting this to True will disable C++ mode.")) -@click.option("--use_python", is_flag=True, default=False, show_default=True) @click.option("--log_level", default="DEBUG", type=click.Choice(get_log_levels(), case_sensitive=False), @@ -65,20 +64,16 @@ default="output.csv", required=True, ) -def run_pipeline(log_level: int, use_python: bool, use_cpu_only: bool, in_file: pathlib.Path, out_file: pathlib.Path): +def run_pipeline(log_level: int, use_cpu_only: bool, in_file: pathlib.Path, out_file: pathlib.Path): # Enable the default logger configure_logging(log_level=log_level) if use_cpu_only: execution_mode = ExecutionMode.CPU - if not use_python: - logging.warning("C++ mode is disabled when running in CPU only mode.") - use_python = True - else: execution_mode = ExecutionMode.GPU - CppConfig.set_should_use_cpp(not use_python) + CppConfig.set_should_use_cpp(not use_cpu_only) config = Config() config.execution_mode = execution_mode From 68c5f83a6ee9c2fabaeb68b9573545891650ebf9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 6 Sep 2024 09:28:17 -0700 Subject: [PATCH 159/347] Passthru stage should support GPU & CPU modes --- .../1_simple_python_stage/pass_thru.py | 3 +- .../1_simple_python_stage/pass_thru_deco.py | 3 +- .../rabbitmq_source_stage_deco.py | 6 +-- .../src/simple_cpp_stage/pass_thru.py | 3 +- .../developer_guide/test_pass_thru.py | 39 ++++++++++++------- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/examples/developer_guide/1_simple_python_stage/pass_thru.py b/examples/developer_guide/1_simple_python_stage/pass_thru.py index 7e6a8e125c..52edba71e7 100644 --- a/examples/developer_guide/1_simple_python_stage/pass_thru.py +++ b/examples/developer_guide/1_simple_python_stage/pass_thru.py @@ -19,12 +19,13 @@ from mrc.core import operators as ops from morpheus.cli.register_stage import register_stage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @register_stage("pass-thru") -class PassThruStage(PassThruTypeMixin, SinglePortStage): +class PassThruStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ A Simple Pass Through Stage """ diff --git a/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py b/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py index da9c51fa9a..9755f63765 100644 --- a/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py +++ b/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py @@ -15,10 +15,11 @@ import typing +from morpheus.config import ExecutionMode from morpheus.pipeline.stage_decorator import stage -@stage +@stage(execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) def pass_thru_stage(message: typing.Any) -> typing.Any: # Return the message for the next stage return message diff --git a/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py b/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py index 39d7883188..33260a94f7 100644 --- a/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py +++ b/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py @@ -21,15 +21,15 @@ import pandas as pd import pika -import cudf - +from morpheus.config import ExecutionMode from morpheus.messages.message_meta import MessageMeta from morpheus.pipeline.stage_decorator import source +from morpheus.utils.type_aliases import DataFrameType logger = logging.getLogger(__name__) -@source(name="from-rabbitmq") +@source(name="from-rabbitmq", execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) def rabbitmq_source(host: str, exchange: str, exchange_type: str = 'fanout', diff --git a/examples/developer_guide/3_simple_cpp_stage/src/simple_cpp_stage/pass_thru.py b/examples/developer_guide/3_simple_cpp_stage/src/simple_cpp_stage/pass_thru.py index 3d22d25b8a..1631d56e60 100644 --- a/examples/developer_guide/3_simple_cpp_stage/src/simple_cpp_stage/pass_thru.py +++ b/examples/developer_guide/3_simple_cpp_stage/src/simple_cpp_stage/pass_thru.py @@ -21,13 +21,14 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages import MultiMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema @register_stage("pass-thru") -class PassThruStage(PassThruTypeMixin, SinglePortStage): +class PassThruStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): def __init__(self, config: Config): super().__init__(config) diff --git a/tests/examples/developer_guide/test_pass_thru.py b/tests/examples/developer_guide/test_pass_thru.py index e8ae2d0086..426b30eae2 100644 --- a/tests/examples/developer_guide/test_pass_thru.py +++ b/tests/examples/developer_guide/test_pass_thru.py @@ -19,39 +19,52 @@ import pytest from _utils import TEST_DIRS +from _utils import assert_results from morpheus.config import Config -from morpheus.messages import MessageMeta -from morpheus.messages import MultiMessage +from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.pipeline.single_port_stage import SinglePortStage +from morpheus.stages.input.in_memory_source_stage import InMemorySourceStage +from morpheus.stages.output.compare_dataframe_stage import CompareDataFrameStage +from morpheus.stages.output.in_memory_sink_stage import InMemorySinkStage from morpheus.utils.type_aliases import DataFrameType -def _check_pass_thru(config: Config, - filter_probs_df: DataFrameType, - pass_thru_stage_cls: SinglePortStage, - on_data_fn_name: str = 'on_data'): - stage = pass_thru_stage_cls(config) - assert isinstance(stage, SinglePortStage) +def _check_pass_thru(config: Config, filter_probs_df: DataFrameType, pass_thru_stage_cls: SinglePortStage): + pass_thru_stage = pass_thru_stage_cls(config) + assert isinstance(pass_thru_stage, SinglePortStage) - meta = MessageMeta(filter_probs_df) - multi = MultiMessage(meta=meta) + pipe = LinearPipeline(config) + pipe.set_source(InMemorySourceStage(config, dataframes=[filter_probs_df.copy(deep=True)])) + sink_1 = pipe.add_stage(InMemorySinkStage(config)) + pipe.add_stage(pass_thru_stage) + sink_2 = pipe.add_stage(InMemorySinkStage(config)) + comp_stage = pipe.add_stage(CompareDataFrameStage(config, filter_probs_df.copy(deep=True))) + pipe.run() - on_data_fn = getattr(stage, on_data_fn_name) - assert on_data_fn(multi) is multi + assert_results(comp_stage.get_results()) + in_messages = sink_1.get_messages() + assert len(in_messages) == 1 + out_messages = sink_2.get_messages() + assert len(out_messages) == 1 + assert in_messages[0] is out_messages[0] + +@pytest.mark.gpu_and_cpu_mode @pytest.mark.import_mod(os.path.join(TEST_DIRS.examples_dir, 'developer_guide/1_simple_python_stage/pass_thru.py')) def test_pass_thru_ex1(config: Config, filter_probs_df: DataFrameType, import_mod: types.ModuleType): pass_thru = import_mod _check_pass_thru(config, filter_probs_df, pass_thru.PassThruStage) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.import_mod(os.path.join(TEST_DIRS.examples_dir, 'developer_guide/1_simple_python_stage/pass_thru_deco.py')) def test_pass_thru_ex1_deco(config: Config, filter_probs_df: DataFrameType, import_mod: types.ModuleType): pass_thru = import_mod - _check_pass_thru(config, filter_probs_df, pass_thru.pass_thru_stage, on_data_fn_name='_on_data_fn') + _check_pass_thru(config, filter_probs_df, pass_thru.pass_thru_stage) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.import_mod( os.path.join(TEST_DIRS.examples_dir, 'developer_guide/3_simple_cpp_stage/src/simple_cpp_stage/pass_thru.py')) def test_pass_thru_ex3(config: Config, filter_probs_df: DataFrameType, import_mod: types.ModuleType): From 4f92ccbff49d92b072cd167d056f786b263cdfba Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 6 Sep 2024 12:33:58 -0700 Subject: [PATCH 160/347] First pass at updating AppShieldSourceStage to emit ControlMessages --- .../stages/input/appshield_source_stage.py | 70 ++++++++++--------- .../test_create_features.py | 2 +- tests/test_appshield_source_stage.py | 2 +- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/python/morpheus/morpheus/stages/input/appshield_source_stage.py b/python/morpheus/morpheus/stages/input/appshield_source_stage.py index acd22a54fa..cb2d768a3b 100644 --- a/python/morpheus/morpheus/stages/input/appshield_source_stage.py +++ b/python/morpheus/morpheus/stages/input/appshield_source_stage.py @@ -16,7 +16,6 @@ import json import logging import re -import typing from functools import partial from json.decoder import JSONDecodeError @@ -27,7 +26,8 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes -from morpheus.messages.message_meta import AppShieldMessageMeta +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.pipeline import SingleOutputSource from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.stage_schema import StageSchema @@ -77,9 +77,9 @@ class AppShieldSourceStage(PreallocatorMixin, SingleOutputSource): def __init__(self, c: Config, input_glob: str, - plugins_include: typing.List[str], - cols_include: typing.List[str], - cols_exclude: typing.List[str] = None, + plugins_include: list[str], + cols_include: list[str], + cols_exclude: list[str] = None, watch_directory: bool = False, max_files: int = -1, sort_glob: bool = False, @@ -124,10 +124,10 @@ def supports_cpp_node(self): return False def compute_schema(self, schema: StageSchema): - schema.output_schema.set_type(AppShieldMessageMeta) + schema.output_schema.set_type(ControlMessage) @staticmethod - def fill_interested_cols(plugin_df: pd.DataFrame, cols_include: typing.List[str]): + def fill_interested_cols(plugin_df: pd.DataFrame, cols_include: list[str]): """ Fill missing interested plugin columns. @@ -135,7 +135,7 @@ def fill_interested_cols(plugin_df: pd.DataFrame, cols_include: typing.List[str] ---------- plugin_df : pandas.DataFrame Snapshot plugin dataframe - cols_include : typing.List[str] + cols_include : list[str] Columns that needs to be included. Returns @@ -152,7 +152,7 @@ def fill_interested_cols(plugin_df: pd.DataFrame, cols_include: typing.List[str] return plugin_df @staticmethod - def read_file_to_df(file: io.TextIOWrapper, cols_exclude: typing.List[str]): + def read_file_to_df(file: io.TextIOWrapper, cols_exclude: list[str]): """ Read file content to dataframe. @@ -160,7 +160,7 @@ def read_file_to_df(file: io.TextIOWrapper, cols_exclude: typing.List[str]): ---------- file : `io.TextIOWrapper` Input file object - cols_exclude : typing.List[str] + cols_exclude : list[str] Dropping columns from a dataframe. Returns @@ -185,7 +185,7 @@ def read_file_to_df(file: io.TextIOWrapper, cols_exclude: typing.List[str]): return plugin_df @staticmethod - def load_df(filepath: str, cols_exclude: typing.List[str], encoding: str) -> pd.DataFrame: + def load_df(filepath: str, cols_exclude: list[str], encoding: str) -> pd.DataFrame: """ Reads a file into a dataframe. @@ -193,7 +193,7 @@ def load_df(filepath: str, cols_exclude: typing.List[str], encoding: str) -> pd. ---------- filepath : str Path to a file. - cols_exclude : typing.List[str] + cols_exclude : list[str] Columns that needs to exclude. encoding : str Encoding to read a file. @@ -228,13 +228,13 @@ def load_df(filepath: str, cols_exclude: typing.List[str], encoding: str) -> pd. return plugin_df @staticmethod - def load_meta_cols(filepath_split: typing.List[str], plugin: str, plugin_df: pd.DataFrame) -> pd.DataFrame: + def load_meta_cols(filepath_split: list[str], plugin: str, plugin_df: pd.DataFrame) -> pd.DataFrame: """ Loads meta columns to dataframe. Parameters ---------- - filepath_split : typing.List[str] + filepath_split : list[str] Splits of file path. plugin : str Plugin name to which the data belongs to. @@ -268,20 +268,20 @@ def load_meta_cols(filepath_split: typing.List[str], plugin: str, plugin_df: pd. return plugin_df @staticmethod - def batch_source_split(x: typing.List[pd.DataFrame], source: str) -> typing.Dict[str, pd.DataFrame]: + def batch_source_split(x: list[pd.DataFrame], source: str) -> dict[str, pd.DataFrame]: """ Combines plugin dataframes from multiple snapshot and split dataframe per source. Parameters ---------- - x : typing.List[pd.DataFrame] + x : list[pd.DataFrame] Dataframes from multiple sources. source : str source column name to group it. Returns ------- - typing.Dict[str, pandas.DataFrame] + dict[str, pandas.DataFrame] Grouped dataframes by source. """ @@ -301,30 +301,30 @@ def batch_source_split(x: typing.List[pd.DataFrame], source: str) -> typing.Dict return source_dfs @staticmethod - def files_to_dfs(x: typing.List[str], - cols_include: typing.List[str], - cols_exclude: typing.List[str], - plugins_include: typing.List[str], - encoding: str) -> typing.Dict[str, pd.DataFrame]: + def files_to_dfs(x: list[str], + cols_include: list[str], + cols_exclude: list[str], + plugins_include: list[str], + encoding: str) -> dict[str, pd.DataFrame]: """ Load plugin files into a dataframe, then segment the dataframe by source. Parameters ---------- - x : typing.List[str] + x : list[str] List of file paths. - cols_include : typing.List[str] + cols_include : list[str] Columns that needs to include. - cols_exclude : typing.List[str] + cols_exclude : list[str] Columns that needs to exclude. - plugins_include: typing.List[str] + plugins_include: list[str] For each path in `x`, a list of plugins to load additional meta cols from. encoding : str Encoding to read a file. Returns ------- - typing.Dict[str, pandas.DataFrame] + dict[str, pandas.DataFrame] Grouped dataframes by source. """ # Using pandas to parse nested JSON until cuDF adds support @@ -349,17 +349,19 @@ def files_to_dfs(x: typing.List[str], return df_per_source @staticmethod - def _build_metadata(x: typing.Dict[str, pd.DataFrame]): + def _build_messages(x: dict[str, pd.DataFrame]): - metas = [] + messages = [] for source, df in x.items(): - # Now make a AppShieldMessageMeta with the source name - meta = AppShieldMessageMeta(df, source) - metas.append(meta) + # Now make a message with the source name + cm = ControlMessage() + cm.payload(MessageMeta(df)) + cm.set_metadata("source", source) + messages.append(cm) - return metas + return messages def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: # The first source just produces filenames @@ -376,7 +378,7 @@ def _post_build_single(self, builder: mrc.Builder, out_node: mrc.SegmentObject) cols_exclude=self._cols_exclude, plugins_include=self._plugins_include, encoding=self._encoding)), - ops.map(self._build_metadata), + ops.map(self._build_messages), # Finally flatten to single meta ops.flatten()) builder.make_edge(out_node, post_node) diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index 879bd719f3..b3cd1c922c 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -97,7 +97,7 @@ def test_on_next(self, plugins_include=interested_plugins, encoding='latin1') - input_metas = AppShieldSourceStage._build_metadata(input_data) + input_metas = AppShieldSourceStage._build_messages(input_data) # Make sure the input test date looks the way we expect it assert len(input_metas) == 1 diff --git a/tests/test_appshield_source_stage.py b/tests/test_appshield_source_stage.py index f69983b2ea..95e89d4255 100755 --- a/tests/test_appshield_source_stage.py +++ b/tests/test_appshield_source_stage.py @@ -307,7 +307,7 @@ def test_files_to_dfs(cols_include, cols_exclude, plugins_include, meta_columns, ] }]) def test_build_metadata(input_df_per_source): - appshield_message_metas = AppShieldSourceStage._build_metadata(input_df_per_source) + appshield_message_metas = AppShieldSourceStage._build_messages(input_df_per_source) assert len(appshield_message_metas) == 2 assert isinstance(appshield_message_metas[0], AppShieldMessageMeta) From 1bc26a03de2d33cbde71988a22c5f08a2421bfa3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 6 Sep 2024 12:45:03 -0700 Subject: [PATCH 161/347] update tests --- tests/test_appshield_source_stage.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_appshield_source_stage.py b/tests/test_appshield_source_stage.py index 95e89d4255..52526ea5ca 100755 --- a/tests/test_appshield_source_stage.py +++ b/tests/test_appshield_source_stage.py @@ -23,7 +23,7 @@ from pandas.testing import assert_frame_equal from _utils import TEST_DIRS -from morpheus.messages.message_meta import AppShieldMessageMeta +from morpheus.messages import ControlMessage from morpheus.stages.input.appshield_source_stage import AppShieldSourceStage from morpheus.utils.directory_watcher import DirectoryWatcher @@ -279,7 +279,7 @@ def test_files_to_dfs(cols_include, cols_exclude, plugins_include, meta_columns, @pytest.mark.parametrize( 'input_df_per_source', [{ - 'appshield': [ + 'appshield': pd.DataFrame({ 'PID': pd.Series(['304', '304', '444', '350', '360', '563'], index=[0, 1, 3, 0, 1, 3]), @@ -290,8 +290,7 @@ def test_files_to_dfs(cols_include, cols_exclude, plugins_include, meta_columns, pd.Series(['appshield', 'appshield', 'appshield', 'appshield', 'appshield', 'appshield'], index=[0, 1, 3, 0, 1, 3]) }), - ], - 'appshield-v2': [ + 'appshield-v2': pd.DataFrame({ 'PID': pd.Series(['304', '304', '444', '350', '360', '563'], index=[0, 1, 3, 0, 1, 3]), @@ -303,11 +302,17 @@ def test_files_to_dfs(cols_include, cols_exclude, plugins_include, meta_columns, 'appshield-v2', 'appshield-v2', 'appshield-v2', 'appshield-v2', 'appshield-v2', 'appshield-v2' ], index=[0, 1, 3, 0, 1, 3]) - }), - ] + }) }]) -def test_build_metadata(input_df_per_source): +def test_build_metadata(input_df_per_source: dict): + expected_sources = sorted(input_df_per_source.keys()) appshield_message_metas = AppShieldSourceStage._build_messages(input_df_per_source) - assert len(appshield_message_metas) == 2 - assert isinstance(appshield_message_metas[0], AppShieldMessageMeta) + assert len(appshield_message_metas) == len(expected_sources) + + actual_sources = [] + for message in appshield_message_metas: + assert isinstance(message, ControlMessage) + actual_sources.append(message.get_metadata('source')) + + assert sorted(actual_sources) == expected_sources From 0b4b7ba8120a2ba22763b5649fe06764dbc1c135 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 6 Sep 2024 13:46:18 -0700 Subject: [PATCH 162/347] WIP --- .../stages/create_features.py | 77 ++++++++++--------- .../stages/preprocessing.py | 39 ++++------ 2 files changed, 59 insertions(+), 57 deletions(-) diff --git a/examples/ransomware_detection/stages/create_features.py b/examples/ransomware_detection/stages/create_features.py index 03a4f61a04..3b1352c31b 100644 --- a/examples/ransomware_detection/stages/create_features.py +++ b/examples/ransomware_detection/stages/create_features.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing - import mrc from mrc.core import operators as ops @@ -24,25 +22,26 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes -from morpheus.messages import MultiMessage -from morpheus.pipeline.multi_message_stage import MultiMessageStage -from morpheus.stages.input.appshield_source_stage import AppShieldMessageMeta +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta +from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin +from morpheus.pipeline.single_port_stage import SinglePortStage @register_stage("create-features", modes=[PipelineModes.FIL]) -class CreateFeaturesRWStage(MultiMessageStage): +class CreateFeaturesRWStage(PassThruTypeMixin, SinglePortStage): """ - This class extends MultiMessageStage to deal with scenario specific features from Appshiled plugins data. + Stage creates features from Appshiled plugins data. Parameters ---------- c : morpheus.config.Config Pipeline configuration instance - interested_plugins : typing.List[str] + interested_plugins : list[str] Only intrested plugins files will be read from Appshield snapshots - feature_columns : typing.List[str] + feature_columns : list[str] List of features needed to be extracted. - file_extns : typing.List[str] + file_extns : list[str] File extensions. n_workers: int, default = 2 Number of dask workers. @@ -53,9 +52,9 @@ class CreateFeaturesRWStage(MultiMessageStage): def __init__( self, c: Config, - interested_plugins: typing.List[str], - feature_columns: typing.List[str], - file_extns: typing.List[str], + interested_plugins: list[str], + feature_columns: list[str], + file_extns: list[str], n_workers: int = 2, threads_per_worker: int = 2, ): @@ -72,20 +71,21 @@ def __init__( def name(self) -> str: return "create-features-rw" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> tuple: """ Returns accepted input types for this stage. """ - return (AppShieldMessageMeta, ) + return (ControlMessage, ) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return False - def on_next(self, x: AppShieldMessageMeta): + def on_next(self, msg: ControlMessage) -> ControlMessage: snapshot_fea_dfs = [] - df = x.df + df = msg.payload().copy_dataframe() + msg_source = msg.get_metadata("source") # Type cast CommitCharge. df["CommitCharge"] = df["CommitCharge"].astype("float").astype("Int32") @@ -117,37 +117,44 @@ def on_next(self, x: AppShieldMessageMeta): # Snapshot sequence will be generated using `source_pid_process`. # Determines which source generated the snapshot messages. # There's a chance of receiving the same snapshots names from multiple sources(hosts) - features_df['source_pid_process'] = x.source + '_' + features_df.pid_process + features_df['source_pid_process'] = msg_source + '_' + features_df.pid_process # Sort entries by pid_process and snapshot_id features_df = features_df.sort_values(by=["pid_process", "snapshot_id"]).reset_index(drop=True) # Create AppShieldMessageMeta with extracted features information. - meta = AppShieldMessageMeta(features_df, x.source) + meta = MessageMeta(features_df) + cm_msg = ControlMessage() + cm_msg.payload(meta) + cm_msg.set_metadata("source", msg_source) + + return cm_msg - return meta + def split_messages(self, msg: ControlMessage) -> list[ControlMessage]: - def create_multi_messages(self, x: AppShieldMessageMeta) -> typing.List[MultiMessage]: + output_messages = [] - multi_messages = [] + msg_source = msg.get_metadata("source") + meta: MessageMeta = msg.payload() + with meta.mutable_dataframe() as df: - df = x.df + pid_processes = df.pid_process.unique() - pid_processes = df.pid_process.unique() + # Create a unique messaage per pid_process, this assumes the DF has been sorted by the `pid_process` column + for pid_process in pid_processes: - # Create multi messaage per pid_process, this assumes that the DF has been sorted by the `pid_process` column - for pid_process in pid_processes: + pid_process_index = df[df.pid_process == pid_process].index - pid_process_index = df[df.pid_process == pid_process].index + start = pid_process_index.min() + stop = pid_process_index.max() + 1 - start = pid_process_index.min() - stop = pid_process_index.max() + 1 - mess_count = stop - start + out_msg = ControlMessage() + out_msg.payload(MessageMeta(df.iloc[start:stop])) + out_msg.set_metadata("source", msg_source) - multi_message = MultiMessage(meta=x, mess_offset=start, mess_count=mess_count) - multi_messages.append(multi_message) + output_messages.append(out_msg) - return multi_messages + return output_messages def on_completed(self): # Close dask client when pipeline initiates shutdown @@ -156,7 +163,7 @@ def on_completed(self): def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: node = builder.make_node(self.unique_name, ops.map(self.on_next), - ops.map(self.create_multi_messages), + ops.map(self.split_messages), ops.on_completed(self.on_completed), ops.flatten()) builder.make_edge(input_node, node) diff --git a/examples/ransomware_detection/stages/preprocessing.py b/examples/ransomware_detection/stages/preprocessing.py index 5c6afd3b87..719b9ac674 100644 --- a/examples/ransomware_detection/stages/preprocessing.py +++ b/examples/ransomware_detection/stages/preprocessing.py @@ -23,10 +23,8 @@ from morpheus.common import TypeId from morpheus.config import Config from morpheus.config import PipelineModes +from morpheus.messages import ControlMessage from morpheus.messages import InferenceMemoryFIL -from morpheus.messages import MultiInferenceFILMessage -from morpheus.messages import MultiInferenceMessage -from morpheus.messages import MultiMessage from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage @@ -40,13 +38,13 @@ class PreprocessingRWStage(PreprocessBaseStage): ---------- c : morpheus.config.Config Pipeline configuration instance - feature_columns : typing.List[str] + feature_columns : list[str] List of features needed to be extracted. sliding_window: int, default = 3 Window size to arrange the sanpshots in seequential order. """ - def __init__(self, c: Config, feature_columns: typing.List[str], sliding_window: int = 3): + def __init__(self, c: Config, feature_columns: list[str], sliding_window: int = 3): super().__init__(c) @@ -55,7 +53,7 @@ def __init__(self, c: Config, feature_columns: typing.List[str], sliding_window: self._features_len = len(self._feature_columns) # Stateful member to hold unprocessed snapshots. - self._snapshot_dict: typing.Dict[str, typing.List[SnapshotData]] = {} + self._snapshot_dict: dict[str, list[SnapshotData]] = {} # Padding data to map inference response with input messages. self._padding_data = [0 for i in range(self._features_len * sliding_window)] @@ -65,11 +63,10 @@ def __init__(self, c: Config, feature_columns: typing.List[str], sliding_window: def name(self) -> str: return "preprocess-rw" - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return False - def _sliding_window_offsets(self, ids: typing.List[int], ids_len: int, - window: int) -> typing.List[typing.Tuple[int]]: + def _sliding_window_offsets(self, ids: list[int], ids_len: int, window: int) -> list[tuple[int]]: """ Create snapshot_id's sliding sequence for a given window """ @@ -87,10 +84,7 @@ def _sliding_window_offsets(self, ids: typing.List[int], ids_len: int, return sliding_window_offsets - def _rollover_pending_snapshots(self, - snapshot_ids: typing.List[int], - source_pid_process: str, - snapshot_df: pd.DataFrame): + def _rollover_pending_snapshots(self, snapshot_ids: list[int], source_pid_process: str, snapshot_df: pd.DataFrame): """ Store the unprocessed snapshots from current run to a stateful member to process them in the next run. """ @@ -121,7 +115,7 @@ def _merge_curr_and_prev_snapshots(self, snapshot_df: pd.DataFrame, source_pid_p return snapshot_df - def _pre_process_batch(self, x: MultiMessage) -> MultiInferenceFILMessage: + def _pre_process_batch(self, msg: ControlMessage) -> ControlMessage: """ This function is invoked for every source_pid_process. It looks for any pending snapshots related to the source and pid process in the memory. @@ -131,7 +125,9 @@ def _pre_process_batch(self, x: MultiMessage) -> MultiInferenceFILMessage: Current run's unprocessed snapshots will be rolled over to the next. """ - snapshot_df = x.get_meta() + meta = msg.payload() + snapshot_df = meta.copy_dataframe() + curr_snapshots_size = len(snapshot_df) # Set snapshot_id as index this is used to get ordered snapshots based on sliding window. @@ -171,22 +167,21 @@ def _pre_process_batch(self, x: MultiMessage) -> MultiInferenceFILMessage: # Rollover pending snapshots self._rollover_pending_snapshots(snapshot_ids, source_pid_process, snapshot_df) - # This column is used to identify whether sequence is genuine or dummy - x.set_meta('sequence', sequence) + with meta.mutable_dataframe() as df: + df['sequence'] = sequence # Convert data to cupy array data = cp.asarray(data) seg_ids = cp.zeros((curr_snapshots_size, 3), dtype=cp.uint32) - seg_ids[:, 0] = cp.arange(x.mess_offset, x.mess_offset + curr_snapshots_size, dtype=cp.uint32) + seg_ids[:, 0] = cp.arange(0, curr_snapshots_size, dtype=cp.uint32) seg_ids[:, 2] = self._features_len * 3 memory = InferenceMemoryFIL(count=curr_snapshots_size, input__0=data, seq_ids=seg_ids) + msg.tensors(memory) + return msg - infer_message = MultiInferenceFILMessage.from_message(x, memory=memory) - return infer_message - - def _get_preprocess_fn(self) -> typing.Callable[[MultiMessage], MultiInferenceMessage]: + def _get_preprocess_fn(self) -> typing.Callable[[ControlMessage], ControlMessage]: pre_process_batch_fn = self._pre_process_batch return pre_process_batch_fn From 76c16ceb24c0d0f5837ae3fb6963917b5c6391a0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 9 Sep 2024 12:41:54 -0700 Subject: [PATCH 163/347] Set the http port to be the default since we are running in C++ mode --- examples/ransomware_detection/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ransomware_detection/README.md b/examples/ransomware_detection/README.md index 0388140227..0619af26ec 100644 --- a/examples/ransomware_detection/README.md +++ b/examples/ransomware_detection/README.md @@ -68,7 +68,7 @@ Once Triton server finishes starting up, it will display the status of all loade Run the following from the root of the Morpheus repo to start the ransomware detection pipeline: ```bash -python examples/ransomware_detection/run.py --server_url=localhost:8001 \ +python examples/ransomware_detection/run.py --server_url=localhost:8000 \ --sliding_window=3 \ --model_name=ransomw-model-short-rf \ --input_glob=./examples/data/appshield/*/snapshot-*/*.json \ From a16bc904945afb02ddb79de758e351a50ab47a93 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 9 Sep 2024 12:42:16 -0700 Subject: [PATCH 164/347] Fix spelling error --- python/morpheus/morpheus/pipeline/preallocator_mixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/pipeline/preallocator_mixin.py b/python/morpheus/morpheus/pipeline/preallocator_mixin.py index 3627bbf5cf..204ab5c15c 100644 --- a/python/morpheus/morpheus/pipeline/preallocator_mixin.py +++ b/python/morpheus/morpheus/pipeline/preallocator_mixin.py @@ -39,7 +39,7 @@ class PreallocatorMixin(ABC): """ Mixin intented to be added to stages, typically source stages, which are emitting newly constructed DataFrame or - MessageMeta instances into the segment. During segment build, if the `_needed_columns` addtribut is not empty an + MessageMeta instances into the segment. During segment build, if the `_needed_columns` addtribute is not empty an additional node will be inserted into the graph after the derived class' node which will perform the allocation. The exceptions would be non-source stages like DFP's `DFPFileToDataFrameStage` which are not sources but are From be15217d206efb4c030962a6fe9280fd60732340 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 9 Sep 2024 12:42:51 -0700 Subject: [PATCH 165/347] Remove final decorator as this conflicts with using the preallocator mixin --- python/morpheus/morpheus/pipeline/single_port_stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/morpheus/morpheus/pipeline/single_port_stage.py b/python/morpheus/morpheus/pipeline/single_port_stage.py index b9ea20aeeb..49687afc96 100644 --- a/python/morpheus/morpheus/pipeline/single_port_stage.py +++ b/python/morpheus/morpheus/pipeline/single_port_stage.py @@ -83,7 +83,6 @@ def _build(self, builder: mrc.Builder, input_nodes: list[mrc.SegmentObject]) -> def _post_build_single(self, _: mrc.Builder, out_node: mrc.SegmentObject) -> mrc.SegmentObject: return out_node - @typing.final def _post_build(self, builder: mrc.Builder, out_ports_nodes: list[mrc.SegmentObject]) -> list[mrc.SegmentObject]: ret_val = self._post_build_single(builder, out_ports_nodes[0]) From 36b01900d7ff9084a2e70f6ccba2ca15a1d96bad Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 9 Sep 2024 12:43:13 -0700 Subject: [PATCH 166/347] Deprecate AppShieldMessageMeta --- python/morpheus/morpheus/messages/message_meta.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/morpheus/morpheus/messages/message_meta.py b/python/morpheus/morpheus/messages/message_meta.py index d2fd18480e..29b6b7c737 100644 --- a/python/morpheus/morpheus/messages/message_meta.py +++ b/python/morpheus/morpheus/messages/message_meta.py @@ -23,6 +23,7 @@ import morpheus._lib.messages as _messages from morpheus.messages.message_base import MessageBase +from morpheus.utils import logger as morpheus_logger from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import SeriesType @@ -380,5 +381,7 @@ class AppShieldMessageMeta(MessageMeta, cpp_class=None): source: str = dataclasses.field(init=False) def __init__(self, df: pd.DataFrame, source: str) -> None: + from morpheus.messages.control_message import ControlMessage + morpheus_logger.deprecated_message_warning(AppShieldMessageMeta, ControlMessage) super().__init__(df) self.source = source From 5bd82f2a112f5e117ed7aff480930bc5fc64cb2a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 9 Sep 2024 15:43:16 -0700 Subject: [PATCH 167/347] Explicitly convert source dataframes to cudf --- .../stages/input/appshield_source_stage.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/python/morpheus/morpheus/stages/input/appshield_source_stage.py b/python/morpheus/morpheus/stages/input/appshield_source_stage.py index cb2d768a3b..2fb8b11975 100644 --- a/python/morpheus/morpheus/stages/input/appshield_source_stage.py +++ b/python/morpheus/morpheus/stages/input/appshield_source_stage.py @@ -102,6 +102,9 @@ def __init__(self, self._input_count = None + import cudf + self._cudf = cudf + self._watcher = DirectoryWatcher(input_glob=input_glob, watch_directory=watch_directory, max_files=max_files, @@ -348,20 +351,19 @@ def files_to_dfs(x: list[str], return df_per_source - @staticmethod - def _build_messages(x: dict[str, pd.DataFrame]): + def _build_messages(self, source_dfs: dict[str, pd.DataFrame]): - messages = [] + output_messages = [] - for source, df in x.items(): + for source, df in source_dfs.items(): # Now make a message with the source name cm = ControlMessage() - cm.payload(MessageMeta(df)) + cm.payload(MessageMeta(self._cudf.DataFrame(df))) cm.set_metadata("source", source) - messages.append(cm) + output_messages.append(cm) - return messages + return output_messages def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: # The first source just produces filenames @@ -379,7 +381,7 @@ def _post_build_single(self, builder: mrc.Builder, out_node: mrc.SegmentObject) plugins_include=self._plugins_include, encoding=self._encoding)), ops.map(self._build_messages), - # Finally flatten to single meta + # Emit each message individually ops.flatten()) builder.make_edge(out_node, post_node) From 9835e5f73e3f093f62adc7372a37d888e6c6eae0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 9 Sep 2024 16:26:07 -0700 Subject: [PATCH 168/347] Refactor to use control messages, and limit pandas<->cudf conversions --- .../stages/create_features.py | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/examples/ransomware_detection/stages/create_features.py b/examples/ransomware_detection/stages/create_features.py index 3b1352c31b..58055936c6 100644 --- a/examples/ransomware_detection/stages/create_features.py +++ b/examples/ransomware_detection/stages/create_features.py @@ -13,10 +13,13 @@ # limitations under the License. import mrc +import pandas as pd from mrc.core import operators as ops from dask.distributed import Client +import cudf + from common.data_models import FeatureConfig # pylint: disable=no-name-in-module from common.feature_extractor import FeatureExtractor # pylint: disable=no-name-in-module from morpheus.cli.register_stage import register_stage @@ -80,11 +83,13 @@ def accepted_types(self) -> tuple: def supports_cpp_node(self) -> bool: return False - def on_next(self, msg: ControlMessage) -> ControlMessage: + def on_next(self, msg: ControlMessage) -> list[ControlMessage]: snapshot_fea_dfs = [] - df = msg.payload().copy_dataframe() + with msg.payload().mutable_dataframe() as cdf: + df = cdf.to_pandas() + msg_source = msg.get_metadata("source") # Type cast CommitCharge. @@ -119,42 +124,37 @@ def on_next(self, msg: ControlMessage) -> ControlMessage: # There's a chance of receiving the same snapshots names from multiple sources(hosts) features_df['source_pid_process'] = msg_source + '_' + features_df.pid_process + # Cast int values to string preventing the df from converting to cuDF. + features_df['ldrmodules_df_path'] = features_df['ldrmodules_df_path'].astype(str) + # Sort entries by pid_process and snapshot_id features_df = features_df.sort_values(by=["pid_process", "snapshot_id"]).reset_index(drop=True) - # Create AppShieldMessageMeta with extracted features information. - meta = MessageMeta(features_df) - cm_msg = ControlMessage() - cm_msg.payload(meta) - cm_msg.set_metadata("source", msg_source) + return self.split_messages(msg_source, features_df) - return cm_msg - - def split_messages(self, msg: ControlMessage) -> list[ControlMessage]: + def split_messages(self, msg_source: str, df: pd.DataFrame) -> list[ControlMessage]: output_messages = [] - msg_source = msg.get_metadata("source") - meta: MessageMeta = msg.payload() - with meta.mutable_dataframe() as df: + pid_processes = df.pid_process.unique() - pid_processes = df.pid_process.unique() + # Create a unique messaage per pid_process, this assumes the DF has been sorted by the `pid_process` column + for pid_process in pid_processes: - # Create a unique messaage per pid_process, this assumes the DF has been sorted by the `pid_process` column - for pid_process in pid_processes: + pid_process_index = df[df.pid_process == pid_process].index - pid_process_index = df[df.pid_process == pid_process].index + start = pid_process_index.min() + stop = pid_process_index.max() + 1 - start = pid_process_index.min() - stop = pid_process_index.max() + 1 + cdf = cudf.DataFrame(df.iloc[start:stop]) - out_msg = ControlMessage() - out_msg.payload(MessageMeta(df.iloc[start:stop])) - out_msg.set_metadata("source", msg_source) + out_msg = ControlMessage() + out_msg.payload(MessageMeta(cdf)) + out_msg.set_metadata("source", msg_source) - output_messages.append(out_msg) + output_messages.append(out_msg) - return output_messages + return output_messages def on_completed(self): # Close dask client when pipeline initiates shutdown @@ -163,7 +163,6 @@ def on_completed(self): def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: node = builder.make_node(self.unique_name, ops.map(self.on_next), - ops.map(self.split_messages), ops.on_completed(self.on_completed), ops.flatten()) builder.make_edge(input_node, node) From 2ad1c999ace8c53b197b0693aa8f714f628d0e4b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 10:07:22 -0700 Subject: [PATCH 169/347] wip --- examples/ransomware_detection/stages/preprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ransomware_detection/stages/preprocessing.py b/examples/ransomware_detection/stages/preprocessing.py index 719b9ac674..d4703aad1c 100644 --- a/examples/ransomware_detection/stages/preprocessing.py +++ b/examples/ransomware_detection/stages/preprocessing.py @@ -126,7 +126,7 @@ def _pre_process_batch(self, msg: ControlMessage) -> ControlMessage: """ meta = msg.payload() - snapshot_df = meta.copy_dataframe() + snapshot_df = meta.copy_dataframe().to_pandas() curr_snapshots_size = len(snapshot_df) From 8d2f424cafb745c1c7c35ac7b2e3534fe9b9d571 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 10:37:46 -0700 Subject: [PATCH 170/347] Use PreallocatorMixin in create features since its emitting new dataframes --- examples/ransomware_detection/stages/create_features.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/ransomware_detection/stages/create_features.py b/examples/ransomware_detection/stages/create_features.py index 58055936c6..e7655de92f 100644 --- a/examples/ransomware_detection/stages/create_features.py +++ b/examples/ransomware_detection/stages/create_features.py @@ -28,11 +28,12 @@ from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin +from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_port_stage import SinglePortStage @register_stage("create-features", modes=[PipelineModes.FIL]) -class CreateFeaturesRWStage(PassThruTypeMixin, SinglePortStage): +class CreateFeaturesRWStage(PassThruTypeMixin, PreallocatorMixin, SinglePortStage): """ Stage creates features from Appshiled plugins data. From 206bc4e825109ba9aa43d81cb72df4f95afa88f7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 10:38:05 -0700 Subject: [PATCH 171/347] Use C++ mode --- examples/ransomware_detection/run.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/ransomware_detection/run.py b/examples/ransomware_detection/run.py index 0c06f21f95..ec3ff4e015 100644 --- a/examples/ransomware_detection/run.py +++ b/examples/ransomware_detection/run.py @@ -20,7 +20,6 @@ import yaml from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage @@ -38,7 +37,6 @@ @click.command() @click.option('--debug', default=False) -@click.option('--use_cpp', default=False, help="Enable C++ execution for this pipeline, currently this is unsupported.") @click.option( "--num_threads", default=len(os.sched_getaffinity(0)), @@ -101,7 +99,6 @@ help="The path to the file where the inference output will be saved.", ) def run_pipeline(debug, - use_cpp, num_threads, n_dask_workers, threads_per_dask_worker, @@ -121,8 +118,6 @@ def run_pipeline(debug, snapshot_fea_length = 99 - CppConfig.set_should_use_cpp(use_cpp) - # Its necessary to get the global config object and configure it for FIL mode. config = Config() config.mode = PipelineModes.FIL From 45291ea2c48a394d9485b136c639446118697b3e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 10:54:13 -0700 Subject: [PATCH 172/347] Relocate test and update --- tests/{ => stages}/test_appshield_source_stage.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) rename tests/{ => stages}/test_appshield_source_stage.py (96%) diff --git a/tests/test_appshield_source_stage.py b/tests/stages/test_appshield_source_stage.py similarity index 96% rename from tests/test_appshield_source_stage.py rename to tests/stages/test_appshield_source_stage.py index 52526ea5ca..03920bef8b 100755 --- a/tests/test_appshield_source_stage.py +++ b/tests/stages/test_appshield_source_stage.py @@ -23,6 +23,7 @@ from pandas.testing import assert_frame_equal from _utils import TEST_DIRS +from morpheus.config import Config from morpheus.messages import ControlMessage from morpheus.stages.input.appshield_source_stage import AppShieldSourceStage from morpheus.utils.directory_watcher import DirectoryWatcher @@ -304,14 +305,18 @@ def test_files_to_dfs(cols_include, cols_exclude, plugins_include, meta_columns, index=[0, 1, 3, 0, 1, 3]) }) }]) -def test_build_metadata(input_df_per_source: dict): +def test_build_messages(config: Config, tmp_path: str, input_df_per_source: dict): expected_sources = sorted(input_df_per_source.keys()) - appshield_message_metas = AppShieldSourceStage._build_messages(input_df_per_source) - assert len(appshield_message_metas) == len(expected_sources) + input_glob = os.path.join(tmp_path, '*.json') + # These constructor arguments are not used by the _build_messages method + stage = AppShieldSourceStage(config, input_glob, ['unused'], ['unused']) + appshield_messages = stage._build_messages(input_df_per_source) + + assert len(appshield_messages) == len(expected_sources) actual_sources = [] - for message in appshield_message_metas: + for message in appshield_messages: assert isinstance(message, ControlMessage) actual_sources.append(message.get_metadata('source')) From cf4aa841bb87180c1b70b697a59d4027fe3cda84 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 11:33:56 -0700 Subject: [PATCH 173/347] Update tests --- .../test_create_features.py | 104 +++++------------- .../test_preprocessing.py | 34 +++--- 2 files changed, 46 insertions(+), 92 deletions(-) diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index b3cd1c922c..015cc323a3 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -19,14 +19,14 @@ import typing from unittest import mock +import pandas as pd import pytest from _utils import TEST_DIRS from _utils.dataset_manager import DatasetManager from morpheus.config import Config -from morpheus.messages import MultiMessage +from morpheus.messages import ControlMessage from morpheus.messages.message_meta import AppShieldMessageMeta -from morpheus.pipeline.multi_message_stage import MultiMessageStage from morpheus.stages.input.appshield_source_stage import AppShieldSourceStage @@ -55,7 +55,6 @@ def test_constructor( n_workers=n_workers, threads_per_worker=threads_per_worker) - assert isinstance(stage, MultiMessageStage) assert stage._client is mock_dask_client scheduler_info = stage._client.scheduler_info() for worker in scheduler_info['workers'].values(): @@ -91,18 +90,25 @@ def test_on_next(self, mock_dask_client.submit.return_value = mock_dask_future input_glob = os.path.join(TEST_DIRS.tests_data_dir, 'appshield', 'snapshot-1', '*.json') - input_data = AppShieldSourceStage.files_to_dfs(glob.glob(input_glob), - cols_include=rwd_conf['raw_columns'], - cols_exclude=["SHA256"], - plugins_include=interested_plugins, - encoding='latin1') + appshield_source_stage = AppShieldSourceStage(config, + input_glob, + plugins_include=interested_plugins, + cols_include=rwd_conf['raw_columns'], + cols_exclude=["SHA256"], + encoding='latin1') - input_metas = AppShieldSourceStage._build_messages(input_data) + input_data = appshield_source_stage.files_to_dfs(glob.glob(input_glob), + cols_include=rwd_conf['raw_columns'], + cols_exclude=["SHA256"], + plugins_include=interested_plugins, + encoding='latin1') + + input_messages = appshield_source_stage._build_messages(input_data) # Make sure the input test date looks the way we expect it - assert len(input_metas) == 1 - input_meta = input_metas[0] - assert input_meta.source == 'appshield' + assert len(input_messages) == 1 + input_message = input_messages[0] + assert input_message.get_metadata('source') == 'appshield' stage = CreateFeaturesRWStage(config, interested_plugins=interested_plugins, @@ -114,74 +120,24 @@ def test_on_next(self, # make sure we have a mocked dask client assert stage._client is mock_dask_client - meta = stage.on_next(input_meta) - assert isinstance(meta, AppShieldMessageMeta) - assert meta.source == input_meta.source + messages = stage.on_next(input_message) + + dataframes = [] + for message in messages: + assert message.get_metadata('source') == input_message.get_metadata('source') + dataframes.append(message.payload().copy_dataframe().to_pandas()) + + actual_df = pd.concat(dataframes, ignore_index=True) + actual_df.sort_values(by=["pid_process", "snapshot_id"], inplace=True) + actual_df.reset_index(drop=True, inplace=True) expected_df = dataset_pandas[os.path.join(test_data_dir, 'dask_results.csv')] expected_df['source_pid_process'] = 'appshield_' + expected_df.pid_process + expected_df['ldrmodules_df_path'] = expected_df['ldrmodules_df_path'].astype(str) # convert to string expected_df.sort_values(by=["pid_process", "snapshot_id"], inplace=True) expected_df.reset_index(drop=True, inplace=True) - dataset_pandas.assert_compare_df(meta.copy_dataframe(), expected_df) - - @pytest.mark.xfail(reason="The ransomeware pipeline depends on the AppShieldMeta class, which forces the pipeline " - "to be python (CPU) only") - @mock.patch('stages.create_features.Client') - def test_create_multi_messages(self, - mock_dask_client, - config: Config, - rwd_conf: dict, - interested_plugins: typing.List[str], - dataset_pandas: DatasetManager): - from stages.create_features import CreateFeaturesRWStage - mock_dask_client.return_value = mock_dask_client - - pids = [75956, 118469, 1348612, 2698363, 2721362, 2788672] - df = dataset_pandas["filter_probs.csv"] - df['pid_process'] = [ - 2788672, - 75956, - 75956, - 2788672, - 2788672, - 2698363, - 2721362, - 118469, - 1348612, - 2698363, - 118469, - 2698363, - 1348612, - 118469, - 75956, - 2721362, - 75956, - 118469, - 118469, - 118469 - ] - df = df.sort_values(by=["pid_process"]).reset_index(drop=True) - - stage = CreateFeaturesRWStage(config, - interested_plugins=interested_plugins, - feature_columns=rwd_conf['model_features'], - file_extns=rwd_conf['file_extensions'], - n_workers=5, - threads_per_worker=6) - - meta = AppShieldMessageMeta(df, source='tests') - multi_messages = stage.create_multi_messages(meta) - assert len(multi_messages) == len(pids) - - prev_loc = 0 - for (i, _multi_message) in enumerate(multi_messages): - assert isinstance(_multi_message, MultiMessage) - pid = pids[i] - (_multi_message.get_meta(['pid_process']) == pid).all() - assert _multi_message.mess_offset == prev_loc - prev_loc = _multi_message.mess_offset + _multi_message.mess_count - assert prev_loc == len(df) + dataset_pandas.assert_compare_df(actual_df, expected_df) @mock.patch('stages.create_features.Client') def test_on_completed(self, mock_dask_client, config: Config, rwd_conf: dict, interested_plugins: typing.List[str]): diff --git a/tests/examples/ransomware_detection/test_preprocessing.py b/tests/examples/ransomware_detection/test_preprocessing.py index 99e102e8fb..9d1f8e81ef 100644 --- a/tests/examples/ransomware_detection/test_preprocessing.py +++ b/tests/examples/ransomware_detection/test_preprocessing.py @@ -19,9 +19,8 @@ from _utils.dataset_manager import DatasetManager from morpheus.config import Config -from morpheus.messages import MultiMessage -from morpheus.messages.message_meta import AppShieldMessageMeta -from morpheus.messages.multi_inference_message import MultiInferenceFILMessage +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage @@ -147,24 +146,20 @@ def test_merge_curr_and_prev_snapshots(self, config: Config, rwd_conf: dict, dat stage._merge_curr_and_prev_snapshots(df, source_pid_process) dataset_pandas.assert_compare_df(df.fillna(''), expected_df) - @pytest.mark.xfail(reason="The ransomeware pipeline depends on the AppShieldMeta class, which forces the pipeline " - "to be python (CPU) only") - def test_pre_process_batch(self, config: Config, rwd_conf: dict, dataset_pandas: DatasetManager): - - # Pylint currently fails to work with classmethod: https://github.com/pylint-dev/pylint/issues/981 - # pylint: disable=no-member - + def test_pre_process_batch(self, config: Config, rwd_conf: dict, dataset_cudf: DatasetManager): from stages.preprocessing import PreprocessingRWStage - df = dataset_pandas['examples/ransomware_detection/dask_results.csv'] + df = dataset_cudf['examples/ransomware_detection/dask_results.csv'] df['source_pid_process'] = 'appshield_' + df.pid_process expected_df = df.copy(deep=True).fillna('') - meta = AppShieldMessageMeta(df=df, source='tests') - multi = MultiMessage(meta=meta) + meta = MessageMeta(df) + cm = ControlMessage() + cm.payload(meta) + cm.set_metadata('source', 'tests') sliding_window = 4 stage = PreprocessingRWStage(config, feature_columns=rwd_conf['model_features'], sliding_window=sliding_window) - results: MultiInferenceFILMessage = stage._pre_process_batch(multi) - assert isinstance(results, MultiInferenceFILMessage) + results: ControlMessage = stage._pre_process_batch(cm) + assert isinstance(results, ControlMessage) expected_df['sequence'] = ['dummy' for _ in range(len(expected_df))] expected_input__0 = cp.asarray([0 for i in range(len(rwd_conf['model_features']) * sliding_window)]) @@ -172,6 +167,9 @@ def test_pre_process_batch(self, config: Config, rwd_conf: dict, dataset_pandas: expected_seq_ids[:, 0] = cp.arange(0, len(expected_df), dtype=cp.uint32) expected_seq_ids[:, 2] = len(rwd_conf['model_features']) * 3 - dataset_pandas.assert_compare_df(results.get_meta().fillna(''), expected_df) - assert (results.get_tensor('input__0') == expected_input__0).all() - assert (results.get_tensor('seq_ids') == expected_seq_ids).all() + actual_df = results.payload().copy_dataframe().to_pandas().fillna('') + dataset_cudf.assert_compare_df(actual_df, expected_df) + + actual_tensors = results.tensors() + assert (actual_tensors.get_tensor('input__0') == expected_input__0).all() + assert (actual_tensors.get_tensor('seq_ids') == expected_seq_ids).all() From fc9e66b0e2ae2d2f264eaf20428cb5048a29aa8d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 12:10:06 -0700 Subject: [PATCH 174/347] Config.freeze enforces a compatible value for CppConfig on the first call, raises a ValueError on mismatch on subsequent calls --- python/morpheus/morpheus/config.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/python/morpheus/morpheus/config.py b/python/morpheus/morpheus/config.py index 7d1b6ca9f6..ed71bcf991 100644 --- a/python/morpheus/morpheus/config.py +++ b/python/morpheus/morpheus/config.py @@ -227,10 +227,6 @@ class Config(ConfigBase): fil: ConfigFIL = dataclasses.field(default=None) frozen: bool = False - def __post_init__(self): - if self.execution_mode == ExecutionMode.CPU: - CppConfig.set_should_use_cpp(False) - def freeze(self): """ Freeze the Config object, making it immutable. This method will be invoked when the config object is passed to @@ -238,9 +234,26 @@ def freeze(self): Calling `freeze` on a frozen instance will not have any effect. """ + self._check_cpp_mode(fix_mis_match=not self.frozen) if not self.frozen: self.frozen = True + def _check_cpp_mode(self, fix_mis_match: bool = False): + """ + Check if C++ mode matched the execution mode. If ` + + Parameters + ---------- + fix_mis_match : bool + If True, set the C++ mode to the correct value. If False, raise an exception if the value is incorrect. + """ + should_use_cpp: bool = (self.execution_mode == ExecutionMode.GPU) + if fix_mis_match: + CppConfig.set_should_use_cpp(should_use_cpp) + elif CppConfig.get_should_use_cpp() != should_use_cpp: + raise ValueError( + f"Execution mode {self.execution_mode} does not match C++ mode {CppConfig.get_should_use_cpp()}") + def __setattr__(self, name, value): # Since __frozen is defined in the __post_init__, the attribute won't exist in the __init__ method. if self.frozen: From f6d5513ba804620103230d4d9bed1cdadbd6d025 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 12:11:10 -0700 Subject: [PATCH 175/347] Ensure the empty config created for boundary stages have a matching execution mode --- python/morpheus/morpheus/pipeline/linear_pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/morpheus/morpheus/pipeline/linear_pipeline.py b/python/morpheus/morpheus/pipeline/linear_pipeline.py index 7b8a4a767c..add998fa4c 100644 --- a/python/morpheus/morpheus/pipeline/linear_pipeline.py +++ b/python/morpheus/morpheus/pipeline/linear_pipeline.py @@ -148,6 +148,7 @@ def add_segment_boundary(self, data_type=None, as_shared_pointer=False): raise RuntimeError("Cannot create a segment boundary, current segment is empty.") empty_config = Config() + empty_config.execution_mode = self._execution_mode boundary_egress = LinearBoundaryEgressStage(empty_config, boundary_port_id=self._current_segment_id, data_type=data_type) From 45dc8026d6c7f5d3d1cd32f523326f48289e6700 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 12:11:48 -0700 Subject: [PATCH 176/347] Boundary stages should support CPU mode --- .../morpheus/stages/boundary/linear_boundary_stage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/stages/boundary/linear_boundary_stage.py b/python/morpheus/morpheus/stages/boundary/linear_boundary_stage.py index ad8db9ebc2..c1e42169c2 100644 --- a/python/morpheus/morpheus/stages/boundary/linear_boundary_stage.py +++ b/python/morpheus/morpheus/stages/boundary/linear_boundary_stage.py @@ -20,6 +20,7 @@ from morpheus.config import Config from morpheus.pipeline.boundary_stage_mixin import BoundaryStageMixin +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource @@ -29,7 +30,7 @@ logger = logging.getLogger(__name__) -class LinearBoundaryEgressStage(BoundaryStageMixin, PassThruTypeMixin, SinglePortStage): +class LinearBoundaryEgressStage(BoundaryStageMixin, PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ The LinearBoundaryEgressStage acts as an egress point from one linear segment to another. Given an existing linear pipeline that we want to connect to another segment, a linear boundary egress stage would be added, in conjunction @@ -82,7 +83,7 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> return input_node -class LinearBoundaryIngressStage(BoundaryStageMixin, PreallocatorMixin, SingleOutputSource): +class LinearBoundaryIngressStage(BoundaryStageMixin, PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): """ The LinearBoundaryIngressStage acts as source ingress point from a corresponding egress in another linear segment. Given an existing linear pipeline that we want to connect to another segment, a linear boundary egress stage would From ca81a8dae2b7d13ff81669437b307a26f88ebae2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 12:12:47 -0700 Subject: [PATCH 177/347] Remove setting CppConfig as the Config class does this now --- python/morpheus/morpheus/pipeline/pipeline.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index 2bfe9b9bd8..0c8a66708a 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -31,7 +31,6 @@ import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import from morpheus.common import load_cudf_helper from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import ExecutionMode from morpheus.utils.type_utils import pretty_print_type_name @@ -65,15 +64,6 @@ class Pipeline(): def __init__(self, config: Config): config.freeze() - # Ensure we have a valid configuration - if config.execution_mode == ExecutionMode.CPU and CppConfig.get_should_use_cpp(): - logger.warning("CPU execution mode requires disabling C++ execution.") - CppConfig.set_should_use_cpp(False) - - elif config.execution_mode == ExecutionMode.GPU and not CppConfig.get_should_use_cpp(): - logger.warning("GPU mode requires C++ execution mode.") - CppConfig.set_should_use_cpp(True) - self._mutex = threading.RLock() self._source_count: int = None # Maximum number of iterations for progress reporting. None = Unknown/Unlimited From 89e2add342cabae2fcd521e028260f92477569de Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 12:13:37 -0700 Subject: [PATCH 178/347] Add type hints --- tests/pipeline/test_preallocation_pipe.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/pipeline/test_preallocation_pipe.py b/tests/pipeline/test_preallocation_pipe.py index afd1239be3..89cb8195f2 100755 --- a/tests/pipeline/test_preallocation_pipe.py +++ b/tests/pipeline/test_preallocation_pipe.py @@ -23,6 +23,7 @@ from _utils.stages.conv_msg import ConvMsg from morpheus.common import TypeId from morpheus.common import typeid_to_numpy_str +from morpheus.config import Config from morpheus.messages import MessageMeta from morpheus.messages import MultiMessage from morpheus.messages import MultiResponseMessage @@ -33,10 +34,12 @@ from morpheus.stages.postprocess.add_scores_stage import AddScoresStage from morpheus.stages.postprocess.serialize_stage import SerializeStage from morpheus.stages.preprocess.deserialize_stage import DeserializeStage +from morpheus.utils.type_aliases import DataFrameType +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize('probs_type', [TypeId.FLOAT32, TypeId.FLOAT64]) -def test_preallocation(config, filter_probs_df, probs_type): +def test_preallocation(config: Config, filter_probs_df: DataFrameType, probs_type: TypeId): config.class_labels = ['frogs', 'lizards', 'toads', 'turtles'] probs_np_type = typeid_to_numpy_str(probs_type) expected_df = pd.DataFrame( @@ -62,8 +65,9 @@ def test_preallocation(config, filter_probs_df, probs_type): assert_results(comp_stage.get_results()) +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize('probs_type', [TypeId.FLOAT32, TypeId.FLOAT64]) -def test_preallocation_multi_segment_pipe(config, filter_probs_df, probs_type): +def test_preallocation_multi_segment_pipe(config: Config, filter_probs_df: DataFrameType, probs_type: TypeId): """ Test ensures that when columns are needed for preallocation in a multi-segment pipeline, the preallocagtion will always be performed on the closest source to the stage that requested preallocation. Which in cases where the From 51b4833f68274a275f19ecf44a67638f4a64fcdf Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 13:07:06 -0700 Subject: [PATCH 179/347] First pass at deprecating UserMessageMeta and updating AutoencoderSourceStage to emit ControlMessages --- .../morpheus/messages/message_meta.py | 2 + .../stages/input/autoencoder_source_stage.py | 73 +++++++++++-------- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/python/morpheus/morpheus/messages/message_meta.py b/python/morpheus/morpheus/messages/message_meta.py index 29b6b7c737..4a3507fdf6 100644 --- a/python/morpheus/morpheus/messages/message_meta.py +++ b/python/morpheus/morpheus/messages/message_meta.py @@ -362,6 +362,8 @@ class UserMessageMeta(MessageMeta, cpp_class=None): user_id: str = dataclasses.field(init=False) def __init__(self, df: pd.DataFrame, user_id: str) -> None: + from morpheus.messages.control_message import ControlMessage + morpheus_logger.deprecated_message_warning(UserMessageMeta, ControlMessage) super().__init__(df) self.user_id = user_id diff --git a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py index 6675b3eacd..6a7fa4c40f 100644 --- a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py +++ b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import typing from abc import abstractmethod from functools import partial @@ -23,14 +22,17 @@ from morpheus.common import FileTypes from morpheus.config import Config -from morpheus.messages import UserMessageMeta +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema from morpheus.utils.directory_watcher import DirectoryWatcher +from morpheus.utils.type_utils import get_df_class -class AutoencoderSourceStage(PreallocatorMixin, SingleOutputSource): +class AutoencoderSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): """ All AutoEncoder source stages must extend this class and implement the `files_to_dfs_per_user` abstract method. Feature columns can be managed by overriding the `derive_features` method. Otherwise, all columns from input @@ -91,12 +93,14 @@ def __init__(self, self._input_count = None # Hold the max index we have seen to ensure sequential and increasing indexes - self._rows_per_user: typing.Dict[str, int] = {} + self._rows_per_user: dict[str, int] = {} # Iterative mode will emit dataframes one at a time. Otherwise a list of dataframes is emitted. Iterative mode # is good for interleaving source stages. self._repeat_count = repeat + self._df_class = get_df_class(c.execution_mode) + self._watcher = DirectoryWatcher(input_glob=input_glob, watch_directory=watch_directory, max_files=max_files, @@ -112,7 +116,7 @@ def input_count(self) -> int: return self._input_count if self._input_count is not None else 0 def compute_schema(self, schema: StageSchema): - schema.output_schema.set_type(UserMessageMeta) + schema.output_schema.set_type(ControlMessage) def get_match_pattern(self, glob_split): """Return a file match pattern""" @@ -122,7 +126,7 @@ def get_match_pattern(self, glob_split): return match_pattern @staticmethod - def repeat_df(df: pd.DataFrame, repeat_count: int) -> typing.List[pd.DataFrame]: + def repeat_df(df: pd.DataFrame, repeat_count: int) -> list[pd.DataFrame]: """ This function iterates over the same dataframe to extending small datasets in debugging with incremental updates to the `event_dt` and `eventTime` columns. @@ -136,7 +140,7 @@ def repeat_df(df: pd.DataFrame, repeat_count: int) -> typing.List[pd.DataFrame]: Returns ------- - df_array : typing.List[pd.DataFrame] + df_array : list[pd.DataFrame] List of repeated dataframes. """ @@ -159,7 +163,7 @@ def repeat_df(df: pd.DataFrame, repeat_count: int) -> typing.List[pd.DataFrame]: return df_array @staticmethod - def batch_user_split(x: typing.List[pd.DataFrame], + def batch_user_split(x: list[pd.DataFrame], userid_column_name: str, userid_filter: str, datetime_column_name="event_dt"): @@ -168,7 +172,7 @@ def batch_user_split(x: typing.List[pd.DataFrame], Parameters ---------- - x : typing.List[pd.DataFrame] + x : list[pd.DataFrame] List of dataframes. userid_column_name : str Name of a dataframe column used for categorization. @@ -179,7 +183,7 @@ def batch_user_split(x: typing.List[pd.DataFrame], Returns ------- - user_dfs : typing.Dict[str, pd.DataFrame] + user_dfs : dict[str, pd.DataFrame] Dataframes, each of which is associated with a single userid. """ @@ -220,22 +224,22 @@ def batch_user_split(x: typing.List[pd.DataFrame], @staticmethod @abstractmethod - def files_to_dfs_per_user(x: typing.List[str], + def files_to_dfs_per_user(x: list[str], userid_column_name: str, - feature_columns: typing.List[str], + feature_columns: list[str], userid_filter: str = None, - repeat_count: int = 1) -> typing.Dict[str, pd.DataFrame]: + repeat_count: int = 1) -> dict[str, pd.DataFrame]: """ Stages that extend `AutoencoderSourceStage` must implement this abstract function in order to convert messages in the files to dataframes per userid. Parameters ---------- - x : typing.List[str] + x : list[str] List of messages. userid_column_name : str Name of the column used for categorization. - feature_columns : typing.List[str] + feature_columns : list[str] Feature column names. userid_filter : str Only rows with the supplied userid are filtered. @@ -244,14 +248,14 @@ def files_to_dfs_per_user(x: typing.List[str], Returns ------- - : typing.Dict[str, pd.DataFrame] + : dict[str, pd.DataFrame] Dataframe per userid. """ pass @staticmethod - def derive_features(df: pd.DataFrame, feature_columns: typing.List[str]): # pylint: disable=unused-argument + def derive_features(df: pd.DataFrame, feature_columns: list[str] | None): # pylint: disable=unused-argument """ If any features are available to be derived, can be implemented by overriding this function. @@ -259,28 +263,28 @@ def derive_features(df: pd.DataFrame, feature_columns: typing.List[str]): # pyl ---------- df : pd.DataFrame A dataframe. - feature_columns : typing.List[str] + feature_columns : list[str] Names of columns that are need to be derived. Returns ------- - df : typing.List[pd.DataFrame] + df : list[pd.DataFrame] Dataframe with actual and derived columns. """ return df - def _add_derived_features(self, x: typing.Dict[str, pd.DataFrame]): + def _add_derived_features(self, user_dataframes: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]: - for user_name in x.keys(): - x[user_name] = self.derive_features(x[user_name], None) + for user_name in user_dataframes.keys(): + user_dataframes[user_name] = self.derive_features(user_dataframes[user_name], None) - return x + return user_dataframes - def _build_user_metadata(self, x: typing.Dict[str, pd.DataFrame]): + def _build_message(self, user_dataframes: dict[str, pd.DataFrame]) -> list[ControlMessage]: - user_metas = [] + messages = [] - for user_name, user_df in x.items(): + for user_name, user_df in user_dataframes.items(): # See if we have seen this user before if (user_name not in self._rows_per_user): @@ -294,12 +298,19 @@ def _build_user_metadata(self, x: typing.Dict[str, pd.DataFrame]): user_df.index = range(self._rows_per_user[user_name], self._rows_per_user[user_name] + len(user_df)) self._rows_per_user[user_name] += len(user_df) - # Now make a UserMessageMeta with the user name - meta = UserMessageMeta(user_df, user_name) + # If we're in GPU mode we need to convert to cuDF + if not isinstance(user_df, self._df_class): + user_df = self._df_class(user_df) + + # Now make a message with the user name in metadata + meta = MessageMeta(user_df) + message = ControlMessage() + message.payload(meta) + message.set_metadata("user_id", user_name) - user_metas.append(meta) + messages.append(message) - return user_metas + return messages def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: # The first source just produces filenames @@ -321,7 +332,7 @@ def _post_build_single(self, builder: mrc.Builder, out_node: mrc.SegmentObject) ops.map(self._add_derived_features), # Now group the batch of dataframes into a single df, split by user, and send a single UserMessageMeta # per user - ops.map(self._build_user_metadata), + ops.map(self._build_message), # Finally flatten to single meta ops.flatten()) builder.make_edge(out_node, post_node) From 3370623eb4bd371b6fe807a82f37a223fa4c3274 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 15:33:38 -0700 Subject: [PATCH 180/347] AppShieldSourceStage should support both cpu and gpu modes --- .../morpheus/morpheus/stages/input/appshield_source_stage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/stages/input/appshield_source_stage.py b/python/morpheus/morpheus/stages/input/appshield_source_stage.py index 2fb8b11975..e1e76b4023 100644 --- a/python/morpheus/morpheus/stages/input/appshield_source_stage.py +++ b/python/morpheus/morpheus/stages/input/appshield_source_stage.py @@ -29,6 +29,7 @@ from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.pipeline import SingleOutputSource +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.stage_schema import StageSchema from morpheus.utils.directory_watcher import DirectoryWatcher @@ -37,7 +38,7 @@ @register_stage("from-appshield", modes=[PipelineModes.FIL]) -class AppShieldSourceStage(PreallocatorMixin, SingleOutputSource): +class AppShieldSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): """ Source stage is used to load Appshield messages from one or more plugins into a dataframe. It normalizes nested json messages and arranges them into a dataframe by snapshot From 8baf123527fedad87fb74df371edc479226ca06a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 15:34:03 -0700 Subject: [PATCH 181/347] Cleanup comments --- .../morpheus/morpheus/stages/input/autoencoder_source_stage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py index 6a7fa4c40f..5c734f700d 100644 --- a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py +++ b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py @@ -330,10 +330,9 @@ def _post_build_single(self, builder: mrc.Builder, out_node: mrc.SegmentObject) userid_filter=self._userid_filter, repeat_count=self._repeat_count)), ops.map(self._add_derived_features), - # Now group the batch of dataframes into a single df, split by user, and send a single UserMessageMeta + # Now group the batch of dataframes into a single df, split by user, and send a single ControlMessage # per user ops.map(self._build_message), - # Finally flatten to single meta ops.flatten()) builder.make_edge(out_node, post_node) From 8bf6bc49a5004145a11d3f8e47e3ba288b711b3c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 15:34:37 -0700 Subject: [PATCH 182/347] Update TrainAE to accept incoming ControlMessages --- .../stages/preprocess/train_ae_stage.py | 74 ++++++++++--------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py b/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py index d757a086bd..c7cde34110 100644 --- a/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py @@ -16,7 +16,6 @@ import importlib import logging import pathlib -import typing import dill import mrc @@ -26,11 +25,10 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes -from morpheus.messages.message_meta import UserMessageMeta -from morpheus.messages.multi_ae_message import MultiAEMessage +from morpheus.messages import ControlMessage from morpheus.models.dfencoder import AutoEncoder -from morpheus.pipeline.multi_message_stage import MultiMessageStage -from morpheus.pipeline.stage_schema import StageSchema +from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin +from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils.seed import manual_seed logger = logging.getLogger(__name__) @@ -123,7 +121,7 @@ def train(self, df: pd.DataFrame) -> AutoEncoder: @register_stage("train-ae", modes=[PipelineModes.AE]) -class TrainAEStage(MultiMessageStage): +class TrainAEStage(PassThruTypeMixin, SinglePortStage): """ Train an Autoencoder model on incoming data. @@ -196,34 +194,33 @@ def __init__(self, self._pretrained_model: AutoEncoder = None # Per user model data - self._user_models: typing.Dict[str, _UserModelManager] = {} + self._user_models: dict[str, _UserModelManager] = {} @property def name(self) -> str: return "train-ae" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> tuple: """ Returns accepted input types for this stage. """ - return (UserMessageMeta, ) - - def compute_schema(self, schema: StageSchema): - schema.output_schema.set_type(MultiAEMessage) + return (ControlMessage, ) def supports_cpp_node(self): return False - def _get_per_user_model(self, x: UserMessageMeta): + def _get_per_user_model(self, msg: ControlMessage): model = None train_scores_mean = None train_scores_std = None user_model = None - if x.user_id in self._user_models: - user_model = self._user_models[x.user_id] + user_id = msg.get_metadata("user_id") + + if user_id in self._user_models: + user_model = self._user_models[user_id] elif self._use_generic_model and "generic" in self._user_models.keys(): user_model = self._user_models["generic"] @@ -234,17 +231,21 @@ def _get_per_user_model(self, x: UserMessageMeta): return model, train_scores_mean, train_scores_std - def _train_model(self, x: UserMessageMeta) -> typing.List[MultiAEMessage]: + def _train_model(self, msg: ControlMessage) -> list[ControlMessage]: + user_id = msg.get_metadata("user_id") + + if (user_id not in self._user_models): + self._user_models[user_id] = _UserModelManager(self._config, + user_id, + False, + self._train_epochs, + self._train_max_history, + self._seed) - if (x.user_id not in self._user_models): - self._user_models[x.user_id] = _UserModelManager(self._config, - x.user_id, - False, - self._train_epochs, - self._train_max_history, - self._seed) + with msg.payload().mutable_dataframe() as cdf: + pdf = cdf.to_pandas() - return self._user_models[x.user_id].train(x.df) + return self._user_models[user_id].train(pdf) def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: get_model_fn = None @@ -312,21 +313,28 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> else: get_model_fn = self._train_model - def on_next(x: UserMessageMeta): + def on_next(full_message: ControlMessage): - model, scores_mean, scores_std = get_model_fn(x) + model, scores_mean, scores_std = get_model_fn(full_message) - full_message = MultiAEMessage(meta=x, - model=model, - train_scores_mean=scores_mean, - train_scores_std=scores_std) + full_message.set_metadata("model", model) + full_message.set_metadata("train_scores_mean", scores_mean) + full_message.set_metadata("train_scores_std", scores_std) + + # cuDF does not yet support timezone-aware datetimes + # Remove timezone information from pd.DatetimeTZDtype columns + meta = full_message.payload() + with meta.mutable_dataframe() as df: + for col in [col for col in df.columns if isinstance(df[col].dtype, pd.DatetimeTZDtype)]: + df[col] = df[col].dt.tz_convert(None) to_send = [] # Now split into batches - for i in range(0, full_message.mess_count, self._batch_size): - - to_send.append(full_message.get_slice(i, min(i + self._batch_size, full_message.mess_count))) + for i in range(0, meta.count, self._batch_size): + output_message = ControlMessage(full_message) + output_message.payload(meta.get_slice(i, min(i + self._batch_size, meta.count))) + to_send.append(output_message) return to_send From 2949820a358e6cc217203826ad44ed95e82b31a5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 16:20:04 -0700 Subject: [PATCH 183/347] typehint cleanups --- .../morpheus/stages/preprocess/drop_null_stage.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py b/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py index a0aac5deb5..7926aeb8d4 100644 --- a/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/drop_null_stage.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing - import mrc from mrc.core import operators as ops @@ -51,20 +49,19 @@ def __init__(self, c: Config, column: str): def name(self) -> str: return "dropna" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> tuple: """ Accepted input types for this stage are returned. Returns ------- - typing.Tuple + tuple Accepted input types. """ return (MessageMeta, ) - def supports_cpp_node(self): - # Enable support by default + def supports_cpp_node(self) -> bool: return False def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: From 642dcd95253f7a29432815a0f784e7ae63df4a21 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 16:20:35 -0700 Subject: [PATCH 184/347] Fix type-o in exception string --- .../morpheus/stages/preprocess/preprocess_base_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py index 71a2c4e595..920762997b 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py @@ -78,7 +78,7 @@ def _get_preprocess_node(self, builder: mrc.Builder) -> mrc.SegmentObject: """ This method should be implemented by any subclasses with a C++ implementation. """ - raise NotImplementedError("No Python implementation provided by this stage") + raise NotImplementedError("No C++ implementation provided by this stage") def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: if self._build_cpp_node(): From e3b3f8664f3acfda78a0c72b48f6ed2afe89fa53 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 16:21:27 -0700 Subject: [PATCH 185/347] Minor type hint cleanups --- .../morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py | 2 +- .../morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py index 069d3bee6d..47432b81a8 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_fil_stage.py @@ -52,7 +52,7 @@ def __init__(self, c: Config): def name(self) -> str: return "preprocess-fil" - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return True def _get_preprocess_node(self, builder: mrc.Builder): diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py index b0bd5417db..ae725863be 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py @@ -93,7 +93,7 @@ def __init__(self, def name(self) -> str: return "preprocess-nlp" - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: return True def _get_preprocess_node(self, builder: mrc.Builder): From ac8ef0d19df8e7f76442db606b3dedff34f0619b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 16:38:14 -0700 Subject: [PATCH 186/347] Replace abstractmethod decorators with NotImplementedError exceptions, allowing for a subclass without a Python impl --- .../stages/inference/inference_stage.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/python/morpheus/morpheus/stages/inference/inference_stage.py b/python/morpheus/morpheus/stages/inference/inference_stage.py index a00e4741b4..b801ce0ef6 100644 --- a/python/morpheus/morpheus/stages/inference/inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/inference_stage.py @@ -14,7 +14,6 @@ import logging import typing -from abc import abstractmethod from functools import partial import cupy as cp @@ -94,8 +93,7 @@ def build_output_message(self, x: MultiInferenceMessage) -> MultiResponseMessage return output_message - @abstractmethod - def calc_output_dims(self, x: MultiInferenceMessage) -> typing.Tuple: + def calc_output_dims(self, x: MultiInferenceMessage) -> tuple: """ Calculates the dimensions of the inference output message data given an input message. @@ -106,12 +104,11 @@ def calc_output_dims(self, x: MultiInferenceMessage) -> typing.Tuple: Returns ------- - typing.Tuple + tuple Output dimensions of response. """ - pass + raise NotImplementedError("No Python implementation provided by this stage") - @abstractmethod def process(self, batch: MultiInferenceMessage, callback: typing.Callable[[TensorMemory], None]): """ Main inference processing function. This function will be called once for each mini-batch. Once the inference is @@ -126,7 +123,7 @@ def process(self, batch: MultiInferenceMessage, callback: typing.Callable[[Tenso Callback to set the values for the inference response. """ - pass + raise NotImplementedError("No Python implementation provided by this stage") class InferenceStage(MultiMessageStage): @@ -182,13 +179,13 @@ def __init__(self, c: Config): def name(self) -> str: return "inference" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> tuple: """ Accepted input types to this stage. Returns ------- - typing.Tuple + tuple Tuple of input types. """ return (MultiInferenceMessage, ControlMessage) @@ -199,11 +196,10 @@ def compute_schema(self, schema: StageSchema): else: schema.output_schema.set_type(MultiResponseMessage) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: # Default to False unless derived classes override this value return False - @abstractmethod def _get_inference_worker(self, inf_queue: ProducerConsumerQueue) -> InferenceWorker: """ Returns the main inference worker which manages requests possibly in another thread depending on which mode the @@ -221,7 +217,7 @@ def _get_inference_worker(self, inf_queue: ProducerConsumerQueue) -> InferenceWo `InferenceWorker` Inference worker implementation for stage. """ - pass + raise NotImplementedError("No Python implementation provided by this stage") def _get_cpp_inference_node(self, builder: mrc.Builder) -> mrc.SegmentObject: raise NotImplementedError("No C++ node is available for this inference type") From 630adf954603e4f0c9f1801a3ad012f96d824453 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Sep 2024 16:43:36 -0700 Subject: [PATCH 187/347] Add CPU support --- .../morpheus/stages/postprocess/add_classifications_stage.py | 2 +- .../morpheus/stages/postprocess/add_scores_stage_base.py | 3 ++- .../morpheus/stages/postprocess/filter_detections_stage.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/morpheus/morpheus/stages/postprocess/add_classifications_stage.py b/python/morpheus/morpheus/stages/postprocess/add_classifications_stage.py index 40e37f264f..6cdca321e1 100644 --- a/python/morpheus/morpheus/stages/postprocess/add_classifications_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/add_classifications_stage.py @@ -64,7 +64,7 @@ def __init__(self, def name(self) -> str: return "add-class" - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: # Enable support by default return True diff --git a/python/morpheus/morpheus/stages/postprocess/add_scores_stage_base.py b/python/morpheus/morpheus/stages/postprocess/add_scores_stage_base.py index 40320d26a1..95bc7d392a 100644 --- a/python/morpheus/morpheus/stages/postprocess/add_scores_stage_base.py +++ b/python/morpheus/morpheus/stages/postprocess/add_scores_stage_base.py @@ -24,13 +24,14 @@ from morpheus.config import Config from morpheus.messages import ControlMessage from morpheus.messages import MultiResponseMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage logger = logging.getLogger(__name__) -class AddScoresStageBase(PassThruTypeMixin, SinglePortStage): +class AddScoresStageBase(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Base class for the `AddScoresStage` and `AddClassificationStage` diff --git a/python/morpheus/morpheus/stages/postprocess/filter_detections_stage.py b/python/morpheus/morpheus/stages/postprocess/filter_detections_stage.py index 68e36053b0..25fd7e5f41 100644 --- a/python/morpheus/morpheus/stages/postprocess/filter_detections_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/filter_detections_stage.py @@ -25,6 +25,7 @@ from morpheus.messages import ControlMessage from morpheus.messages import MultiMessage from morpheus.messages import MultiResponseMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema @@ -32,7 +33,7 @@ @register_stage("filter") -class FilterDetectionsStage(SinglePortStage): +class FilterDetectionsStage(GpuAndCpuMixin, SinglePortStage): """ Filter message by a classification threshold. From 0853542dd722c745328eca199533c7d28b3f32c5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 11:17:08 -0700 Subject: [PATCH 188/347] WIP --- .../morpheus/_lib/common/__init__.pyi | 273 +++------ .../morpheus/_lib/messages/__init__.pyi | 532 ++++-------------- .../morpheus/_lib/modules/__init__.pyi | 5 +- .../morpheus/_lib/stages/__init__.pyi | 209 +------ 4 files changed, 190 insertions(+), 829 deletions(-) diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi index 2462c95f9f..bba480d220 100644 --- a/python/morpheus/morpheus/_lib/common/__init__.pyi +++ b/python/morpheus/morpheus/_lib/common/__init__.pyi @@ -5,11 +5,9 @@ :toctree: _generate """ from __future__ import annotations - -import os -import typing - import morpheus._lib.common +import typing +import os __all__ = [ "FiberQueue", @@ -20,6 +18,7 @@ __all__ = [ "Tensor", "TypeId", "determine_file_type", + "load_cudf_helper", "read_file_to_df", "typeid_is_fully_supported", "typeid_to_numpy_str", @@ -28,31 +27,14 @@ __all__ = [ class FiberQueue(): - - def __enter__(self) -> FiberQueue: - ... - - def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: - ... - - def __init__(self, max_size: int) -> None: - ... - - def close(self) -> None: - ... - - def get(self, block: bool = True, timeout: float = 0.0) -> object: - ... - - def is_closed(self) -> bool: - ... - - def put(self, item: object, block: bool = True, timeout: float = 0.0) -> None: - ... - + def __enter__(self) -> FiberQueue: ... + def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: ... + def __init__(self, max_size: int) -> None: ... + def close(self) -> None: ... + def get(self, block: bool = True, timeout: float = 0.0) -> object: ... + def is_closed(self) -> bool: ... + def put(self, item: object, block: bool = True, timeout: float = 0.0) -> None: ... pass - - class FileTypes(): """ The type of files that the `FileSourceStage` can read and `WriteToFileStage` can write. Use 'auto' to determine from the file extension. @@ -67,54 +49,31 @@ class FileTypes(): PARQUET """ - - def __eq__(self, other: object) -> bool: - ... - - def __getstate__(self) -> int: - ... - - def __hash__(self) -> int: - ... - - def __index__(self) -> int: - ... - - def __init__(self, value: int) -> None: - ... - - def __int__(self) -> int: - ... - - def __ne__(self, other: object) -> bool: - ... - - def __repr__(self) -> str: - ... - - def __setstate__(self, state: int) -> None: - ... - + def __eq__(self, other: object) -> bool: ... + def __getstate__(self) -> int: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __init__(self, value: int) -> None: ... + def __int__(self) -> int: ... + def __ne__(self, other: object) -> bool: ... + def __repr__(self) -> str: ... + def __setstate__(self, state: int) -> None: ... @property def name(self) -> str: """ :type: str """ - @property def value(self) -> int: """ :type: int """ - - Auto: morpheus._lib.common.FileTypes # value = - CSV: morpheus._lib.common.FileTypes # value = - JSON: morpheus._lib.common.FileTypes # value = - PARQUET: morpheus._lib.common.FileTypes # value = - __members__: dict # value = {'Auto': , 'JSON': , 'CSV': , 'PARQUET': } + Auto: morpheus._lib.common.FileTypes # value = + CSV: morpheus._lib.common.FileTypes # value = + JSON: morpheus._lib.common.FileTypes # value = + PARQUET: morpheus._lib.common.FileTypes # value = + __members__: dict # value = {'Auto': , 'JSON': , 'CSV': , 'PARQUET': } pass - - class FilterSource(): """ Enum to indicate which source the FilterDetectionsStage should operate on. @@ -127,108 +86,51 @@ class FilterSource(): DATAFRAME """ - - def __eq__(self, other: object) -> bool: - ... - - def __getstate__(self) -> int: - ... - - def __hash__(self) -> int: - ... - - def __index__(self) -> int: - ... - - def __init__(self, value: int) -> None: - ... - - def __int__(self) -> int: - ... - - def __ne__(self, other: object) -> bool: - ... - - def __repr__(self) -> str: - ... - - def __setstate__(self, state: int) -> None: - ... - + def __eq__(self, other: object) -> bool: ... + def __getstate__(self) -> int: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __init__(self, value: int) -> None: ... + def __int__(self) -> int: ... + def __ne__(self, other: object) -> bool: ... + def __repr__(self) -> str: ... + def __setstate__(self, state: int) -> None: ... @property def name(self) -> str: """ :type: str """ - @property def value(self) -> int: """ :type: int """ - - Auto: morpheus._lib.common.FilterSource # value = - DATAFRAME: morpheus._lib.common.FilterSource # value = - TENSOR: morpheus._lib.common.FilterSource # value = - __members__: dict # value = {'Auto': , 'TENSOR': , 'DATAFRAME': } + Auto: morpheus._lib.common.FilterSource # value = + DATAFRAME: morpheus._lib.common.FilterSource # value = + TENSOR: morpheus._lib.common.FilterSource # value = + __members__: dict # value = {'Auto': , 'TENSOR': , 'DATAFRAME': } pass - - class HttpEndpoint(): - - def __init__(self, py_parse_fn: function, url: str, method: str) -> None: - ... - + def __init__(self, py_parse_fn: function, url: str, method: str) -> None: ... pass - - class HttpServer(): - - def __enter__(self) -> HttpServer: - ... - - def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: - ... - - def __init__(self, - endpoints: typing.List[HttpEndpoint], - bind_address: str = '127.0.0.1', - port: int = 8080, - num_threads: int = 1, - max_payload_size: int = 10485760, - request_timeout: int = 30) -> None: - ... - - def is_running(self) -> bool: - ... - - def start(self) -> None: - ... - - def stop(self) -> None: - ... - + def __enter__(self) -> HttpServer: ... + def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: ... + def __init__(self, endpoints: typing.List[HttpEndpoint], bind_address: str = '127.0.0.1', port: int = 8080, num_threads: int = 1, max_payload_size: int = 10485760, request_timeout: int = 30) -> None: ... + def is_running(self) -> bool: ... + def start(self) -> None: ... + def stop(self) -> None: ... pass - - class Tensor(): - @staticmethod - def from_cupy(arg0: object) -> Tensor: - ... - - def to_cupy(self) -> object: - ... - + def from_cupy(arg0: object) -> Tensor: ... + def to_cupy(self) -> object: ... @property def __cuda_array_interface__(self) -> dict: """ :type: dict """ - pass - - class TypeId(): """ Supported Morpheus types @@ -261,87 +163,54 @@ class TypeId(): STRING """ - - def __eq__(self, other: object) -> bool: - ... - - def __getstate__(self) -> int: - ... - - def __hash__(self) -> int: - ... - - def __index__(self) -> int: - ... - - def __init__(self, value: int) -> None: - ... - - def __int__(self) -> int: - ... - - def __ne__(self, other: object) -> bool: - ... - - def __repr__(self) -> str: - ... - - def __setstate__(self, state: int) -> None: - ... - + def __eq__(self, other: object) -> bool: ... + def __getstate__(self) -> int: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __init__(self, value: int) -> None: ... + def __int__(self) -> int: ... + def __ne__(self, other: object) -> bool: ... + def __repr__(self) -> str: ... + def __setstate__(self, state: int) -> None: ... @property def name(self) -> str: """ :type: str """ - @property def value(self) -> int: """ :type: int """ - - BOOL8: morpheus._lib.common.TypeId # value = - EMPTY: morpheus._lib.common.TypeId # value = - FLOAT32: morpheus._lib.common.TypeId # value = - FLOAT64: morpheus._lib.common.TypeId # value = - INT16: morpheus._lib.common.TypeId # value = - INT32: morpheus._lib.common.TypeId # value = - INT64: morpheus._lib.common.TypeId # value = - INT8: morpheus._lib.common.TypeId # value = - STRING: morpheus._lib.common.TypeId # value = - UINT16: morpheus._lib.common.TypeId # value = - UINT32: morpheus._lib.common.TypeId # value = - UINT64: morpheus._lib.common.TypeId # value = - UINT8: morpheus._lib.common.TypeId # value = - __members__: dict # value = {'EMPTY': , 'INT8': , 'INT16': , 'INT32': , 'INT64': , 'UINT8': , 'UINT16': , 'UINT32': , 'UINT64': , 'FLOAT32': , 'FLOAT64': , 'BOOL8': , 'STRING': } + BOOL8: morpheus._lib.common.TypeId # value = + EMPTY: morpheus._lib.common.TypeId # value = + FLOAT32: morpheus._lib.common.TypeId # value = + FLOAT64: morpheus._lib.common.TypeId # value = + INT16: morpheus._lib.common.TypeId # value = + INT32: morpheus._lib.common.TypeId # value = + INT64: morpheus._lib.common.TypeId # value = + INT8: morpheus._lib.common.TypeId # value = + STRING: morpheus._lib.common.TypeId # value = + UINT16: morpheus._lib.common.TypeId # value = + UINT32: morpheus._lib.common.TypeId # value = + UINT64: morpheus._lib.common.TypeId # value = + UINT8: morpheus._lib.common.TypeId # value = + __members__: dict # value = {'EMPTY': , 'INT8': , 'INT16': , 'INT32': , 'INT64': , 'UINT8': , 'UINT16': , 'UINT32': , 'UINT64': , 'FLOAT32': , 'FLOAT64': , 'BOOL8': , 'STRING': } pass - - @typing.overload def determine_file_type(filename: os.PathLike) -> FileTypes: pass - - @typing.overload def determine_file_type(filename: str) -> FileTypes: pass - - +def load_cudf_helper() -> None: + pass def read_file_to_df(filename: str, file_type: FileTypes = FileTypes.Auto) -> object: pass - - def typeid_is_fully_supported(arg0: TypeId) -> bool: pass - - def typeid_to_numpy_str(arg0: TypeId) -> str: pass - - def write_df_to_file(df: object, filename: str, file_type: FileTypes = FileTypes.Auto, **kwargs) -> None: pass - - __version__ = '24.10.0' diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi index 83aa5211e3..7a5ac78488 100644 --- a/python/morpheus/morpheus/_lib/messages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/messages/__init__.pyi @@ -6,14 +6,11 @@ """ from __future__ import annotations - +import morpheus._lib.messages import typing - import cupy -import mrc.core.node - import morpheus._lib.common -import morpheus._lib.messages +import mrc.core.node __all__ = [ "ControlMessage", @@ -41,100 +38,52 @@ __all__ = [ class ControlMessage(): - @typing.overload - def __init__(self) -> None: - ... - + def __init__(self) -> None: ... @typing.overload - def __init__(self, arg0: ControlMessage) -> None: - ... - + def __init__(self, arg0: ControlMessage) -> None: ... @typing.overload - def __init__(self, arg0: dict) -> None: - ... - - def add_task(self, task_type: str, task: object) -> None: - ... - + def __init__(self, arg0: object) -> None: ... + def add_task(self, task_type: str, task: object) -> None: ... @typing.overload - def config(self) -> object: - ... - + def config(self) -> object: ... @typing.overload - def config(self, config: object) -> None: - ... - - def copy(self) -> ControlMessage: - ... - - def filter_timestamp(self, regex_filter: str) -> dict: + def config(self, config: object) -> None: ... + def copy(self) -> ControlMessage: ... + def filter_timestamp(self, regex_filter: str) -> dict: """ Retrieve timestamps matching a regex filter within a given group. """ - - def get_metadata(self, key: object = None, default_value: object = None) -> object: - ... - - def get_tasks(self) -> object: - ... - - def get_timestamp(self, key: str, fail_if_nonexist: bool = False) -> object: + def get_metadata(self, key: object = None, default_value: object = None) -> object: ... + def get_tasks(self) -> object: ... + def get_timestamp(self, key: str, fail_if_nonexist: bool = False) -> object: """ Retrieve the timestamp for a given group and key. Returns None if the timestamp does not exist and fail_if_nonexist is False. """ - - def has_metadata(self, key: str) -> bool: - ... - - def has_task(self, task_type: str) -> bool: - ... - - def list_metadata(self) -> list: - ... - + def has_metadata(self, key: str) -> bool: ... + def has_task(self, task_type: str) -> bool: ... + def list_metadata(self) -> list: ... @typing.overload - def payload(self) -> MessageMeta: - ... - + def payload(self) -> MessageMeta: ... @typing.overload - def payload(self, arg0: MessageMeta) -> None: - ... - + def payload(self, arg0: MessageMeta) -> None: ... @typing.overload - def payload(self, meta: object) -> None: - ... - - def remove_task(self, task_type: str) -> object: - ... - - def set_metadata(self, key: str, value: object) -> None: - ... - - def set_timestamp(self, key: str, timestamp: object) -> None: + def payload(self, meta: object) -> None: ... + def remove_task(self, task_type: str) -> object: ... + def set_metadata(self, key: str, value: object) -> None: ... + def set_timestamp(self, key: str, timestamp: object) -> None: """ Set a timestamp for a given key and group. """ - @typing.overload - def task_type(self) -> ControlMessageType: - ... - + def task_type(self) -> ControlMessageType: ... @typing.overload - def task_type(self, task_type: ControlMessageType) -> None: - ... - + def task_type(self, task_type: ControlMessageType) -> None: ... @typing.overload - def tensors(self) -> TensorMemory: - ... - + def tensors(self) -> TensorMemory: ... @typing.overload - def tensors(self, arg0: TensorMemory) -> None: - ... - + def tensors(self, arg0: TensorMemory) -> None: ... pass - - class ControlMessageType(): """ Members: @@ -145,571 +94,288 @@ class ControlMessageType(): TRAINING """ - - def __eq__(self, other: object) -> bool: - ... - - def __getstate__(self) -> int: - ... - - def __hash__(self) -> int: - ... - - def __index__(self) -> int: - ... - - def __init__(self, value: int) -> None: - ... - - def __int__(self) -> int: - ... - - def __ne__(self, other: object) -> bool: - ... - - def __repr__(self) -> str: - ... - - def __setstate__(self, state: int) -> None: - ... - + def __eq__(self, other: object) -> bool: ... + def __getstate__(self) -> int: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __init__(self, value: int) -> None: ... + def __int__(self) -> int: ... + def __ne__(self, other: object) -> bool: ... + def __repr__(self) -> str: ... + def __setstate__(self, state: int) -> None: ... @property def name(self) -> str: """ :type: str """ - @property def value(self) -> int: """ :type: int """ - - INFERENCE: morpheus._lib.messages.ControlMessageType # value = - NONE: morpheus._lib.messages.ControlMessageType # value = - TRAINING: morpheus._lib.messages.ControlMessageType # value = - __members__: dict # value = {'INFERENCE': , 'NONE': , 'TRAINING': } + INFERENCE: morpheus._lib.messages.ControlMessageType # value = + NONE: morpheus._lib.messages.ControlMessageType # value = + TRAINING: morpheus._lib.messages.ControlMessageType # value = + __members__: dict # value = {'INFERENCE': , 'NONE': , 'TRAINING': } pass - - class DataLoaderRegistry(): - @staticmethod - def contains(name: str) -> bool: - ... - + def contains(name: str) -> bool: ... @staticmethod - def list() -> typing.List[str]: - ... - + def list() -> typing.List[str]: ... @staticmethod - def register_loader(name: str, - loader: typing.Callable[[ControlMessage, dict], ControlMessage], - throw_if_exists: bool = True) -> None: - ... - + def register_loader(name: str, loader: typing.Callable[[ControlMessage, dict], ControlMessage], throw_if_exists: bool = True) -> None: ... @staticmethod - def unregister_loader(name: str, throw_if_not_exists: bool = True) -> None: - ... - + def unregister_loader(name: str, throw_if_not_exists: bool = True) -> None: ... pass - - class DataTable(): pass - - class TensorMemory(): - - def __init__(self, *, count: int, tensors: object = None) -> None: - ... - - def get_tensor(self, name: str) -> object: - ... - - def get_tensors(self) -> typing.Dict[str, object]: - ... - - def has_tensor(self, arg0: str) -> bool: - ... - - def set_tensor(self, name: str, tensor: object) -> None: - ... - - def set_tensors(self, tensors: typing.Dict[str, object]) -> None: - ... - + def __init__(self, *, count: int, tensors: object = None) -> None: ... + def get_tensor(self, name: str) -> object: ... + def get_tensors(self) -> typing.Dict[str, object]: ... + def has_tensor(self, arg0: str) -> bool: ... + def set_tensor(self, name: str, tensor: object) -> None: ... + def set_tensors(self, tensors: typing.Dict[str, object]) -> None: ... @property def count(self) -> int: """ :type: int """ - @property def tensor_names(self) -> typing.List[str]: """ :type: typing.List[str] """ - pass - - class InferenceMemory(TensorMemory): - - def __init__(self, *, count: int, tensors: object = None) -> None: - ... - - def get_input(self, name: str) -> object: - ... - - def set_input(self, name: str, tensor: object) -> None: - ... - + def __init__(self, *, count: int, tensors: object = None) -> None: ... + def get_input(self, name: str) -> object: ... + def set_input(self, name: str, tensor: object) -> None: ... pass - - class InferenceMemoryNLP(InferenceMemory, TensorMemory): - - def __init__(self, *, count: int, input_ids: object, input_mask: object, seq_ids: object) -> None: - ... - + def __init__(self, *, count: int, input_ids: object, input_mask: object, seq_ids: object) -> None: ... @property def input_ids(self) -> object: """ :type: object """ - @input_ids.setter def input_ids(self, arg1: object) -> None: pass - @property def input_mask(self) -> object: """ :type: object """ - @input_mask.setter def input_mask(self, arg1: object) -> None: pass - @property def seq_ids(self) -> object: """ :type: object """ - @seq_ids.setter def seq_ids(self, arg1: object) -> None: pass - pass - - class MessageMeta(): - - def __init__(self, df: object) -> None: - ... - - def copy_dataframe(self) -> object: - ... - - def copy_ranges(self, ranges: typing.List[typing.Tuple[int, int]]) -> MessageMeta: - ... - - def ensure_sliceable_index(self) -> typing.Optional[str]: - ... - - def get_column_names(self) -> typing.List[str]: - ... - + def __init__(self, df: object) -> None: ... + def copy_dataframe(self) -> object: ... + def copy_ranges(self, ranges: typing.List[typing.Tuple[int, int]]) -> MessageMeta: ... + def ensure_sliceable_index(self) -> typing.Optional[str]: ... + def get_column_names(self) -> typing.List[str]: ... @typing.overload - def get_data(self) -> object: - ... - + def get_data(self) -> object: ... @typing.overload - def get_data(self, columns: None) -> object: - ... - + def get_data(self, columns: None) -> object: ... @typing.overload - def get_data(self, columns: str) -> object: - ... - + def get_data(self, columns: str) -> object: ... @typing.overload - def get_data(self, columns: typing.List[str]) -> object: - ... - - def get_slice(self, start: int, stop: int) -> MessageMeta: - ... - - def has_sliceable_index(self) -> bool: - ... - + def get_data(self, columns: typing.List[str]) -> object: ... + def get_slice(self, start: int, stop: int) -> MessageMeta: ... + def has_sliceable_index(self) -> bool: ... @staticmethod - def make_from_file(arg0: str) -> MessageMeta: - ... - - def mutable_dataframe(self) -> MutableTableCtxMgr: - ... - - def set_data(self, arg0: object, arg1: object) -> None: - ... - + def make_from_file(arg0: str) -> MessageMeta: ... + def mutable_dataframe(self) -> MutableTableCtxMgr: ... + def set_data(self, arg0: object, arg1: object) -> None: ... @property def count(self) -> int: """ :type: int """ - @property def df(self) -> object: """ :type: object """ - pass - - class MultiMessage(): - - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1) -> None: - ... - - def copy_ranges(self, - ranges: typing.List[typing.Tuple[int, int]], - num_selected_rows: object = None) -> MultiMessage: - ... - + def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1) -> None: ... + def copy_ranges(self, ranges: typing.List[typing.Tuple[int, int]], num_selected_rows: object = None) -> MultiMessage: ... @typing.overload - def get_meta(self) -> object: - ... - + def get_meta(self) -> object: ... @typing.overload - def get_meta(self, columns: None) -> object: - ... - + def get_meta(self, columns: None) -> object: ... @typing.overload - def get_meta(self, columns: str) -> object: - ... - + def get_meta(self, columns: str) -> object: ... @typing.overload - def get_meta(self, columns: typing.List[str]) -> object: - ... - - def get_meta_column_names(self) -> typing.List[str]: - ... - - def get_meta_list(self, arg0: object) -> object: - ... - - def get_slice(self, arg0: int, arg1: int) -> MultiMessage: - ... - - def set_meta(self, arg0: object, arg1: object) -> None: - ... - + def get_meta(self, columns: typing.List[str]) -> object: ... + def get_meta_column_names(self) -> typing.List[str]: ... + def get_meta_list(self, arg0: object) -> object: ... + def get_slice(self, arg0: int, arg1: int) -> MultiMessage: ... + def set_meta(self, arg0: object, arg1: object) -> None: ... @property def mess_count(self) -> int: """ :type: int """ - @property def mess_offset(self) -> int: """ :type: int """ - @property def meta(self) -> MessageMeta: """ :type: MessageMeta """ - pass - - class MultiTensorMessage(MultiMessage): - - def __init__(self, - *, - meta: MessageMeta, - mess_offset: int = 0, - mess_count: int = -1, - memory: TensorMemory, - offset: int = 0, - count: int = -1, - id_tensor_name: str = 'seq_ids') -> None: - ... - - def get_id_tensor(self) -> object: - ... - - def get_tensor(self, arg0: str) -> object: - ... - + def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... + def get_id_tensor(self) -> object: ... + def get_tensor(self, arg0: str) -> object: ... @property def count(self) -> int: """ :type: int """ - @property def memory(self) -> TensorMemory: """ :type: TensorMemory """ - @property def offset(self) -> int: """ :type: int """ - pass - - class MultiInferenceMessage(MultiTensorMessage, MultiMessage): - - def __init__(self, - *, - meta: MessageMeta, - mess_offset: int = 0, - mess_count: int = -1, - memory: TensorMemory, - offset: int = 0, - count: int = -1, - id_tensor_name: str = 'seq_ids') -> None: - ... - - def get_input(self, arg0: str) -> object: - ... - + def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... + def get_input(self, arg0: str) -> object: ... pass - - class MultiInferenceFILMessage(MultiInferenceMessage, MultiTensorMessage, MultiMessage): - - def __init__(self, - *, - meta: MessageMeta, - mess_offset: int = 0, - mess_count: int = -1, - memory: TensorMemory, - offset: int = 0, - count: int = -1, - id_tensor_name: str = 'seq_ids') -> None: - ... - + def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... @property def input__0(self) -> object: """ :type: object """ - @property def seq_ids(self) -> object: """ :type: object """ - pass - - class MultiResponseMessage(MultiTensorMessage, MultiMessage): - - def __init__(self, - *, - meta: MessageMeta, - mess_offset: int = 0, - mess_count: int = -1, - memory: TensorMemory, - offset: int = 0, - count: int = -1, - id_tensor_name: str = 'seq_ids', - probs_tensor_name: str = 'probs') -> None: - ... - - def get_output(self, arg0: str) -> object: - ... - - def get_probs_tensor(self) -> object: - ... - + def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids', probs_tensor_name: str = 'probs') -> None: ... + def get_output(self, arg0: str) -> object: ... + def get_probs_tensor(self) -> object: ... @property def probs_tensor_name(self) -> str: """ :type: str """ - @probs_tensor_name.setter def probs_tensor_name(self, arg1: str) -> None: pass - pass - - class MultiResponseProbsMessage(MultiResponseMessage, MultiTensorMessage, MultiMessage): - - def __init__(self, - *, - meta: MessageMeta, - mess_offset: int = 0, - mess_count: int = -1, - memory: TensorMemory, - offset: int = 0, - count: int = -1, - id_tensor_name: str = 'seq_ids', - probs_tensor_name: str = 'probs') -> None: - ... - + def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids', probs_tensor_name: str = 'probs') -> None: ... @property def probs(self) -> object: """ :type: object """ - pass - - class MultiInferenceNLPMessage(MultiInferenceMessage, MultiTensorMessage, MultiMessage): - - def __init__(self, - *, - meta: MessageMeta, - mess_offset: int = 0, - mess_count: int = -1, - memory: TensorMemory, - offset: int = 0, - count: int = -1, - id_tensor_name: str = 'seq_ids') -> None: - ... - + def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... @property def input_ids(self) -> object: """ :type: object """ - @property def input_mask(self) -> object: """ :type: object """ - @property def seq_ids(self) -> object: """ :type: object """ - pass - - class MutableTableCtxMgr(): - - def __enter__(self) -> object: - ... - - def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: - ... - - def __getattr__(self, *args, **kwargs) -> None: - ... - - def __getitem__(self, *args, **kwargs) -> None: - ... - - def __setattr__(self, *args, **kwargs) -> None: - ... - - def __setitem__(self, *args, **kwargs) -> None: - ... - + def __enter__(self) -> object: ... + def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: ... + def __getattr__(self, *args, **kwargs) -> None: ... + def __getitem__(self, *args, **kwargs) -> None: ... + def __setattr__(self, *args, **kwargs) -> None: ... + def __setitem__(self, *args, **kwargs) -> None: ... pass - - class RawPacketMessage(): - @property def gpu_mem(self) -> bool: """ :type: bool """ - @property def max_size(self) -> int: """ :type: int """ - @property def num(self) -> int: """ :type: int """ - pass - - class ResponseMemory(TensorMemory): - - def __init__(self, *, count: int, tensors: object = None) -> None: - ... - - def get_output(self, name: str) -> object: - ... - - def set_output(self, name: str, tensor: object) -> None: - ... - + def __init__(self, *, count: int, tensors: object = None) -> None: ... + def get_output(self, name: str) -> object: ... + def set_output(self, name: str, tensor: object) -> None: ... pass - - class ResponseMemoryProbs(ResponseMemory, TensorMemory): - - def __init__(self, *, count: int, probs: object) -> None: - ... - + def __init__(self, *, count: int, probs: object) -> None: ... @property def probs(self) -> object: """ :type: object """ - @probs.setter def probs(self, arg1: object) -> None: pass - pass - - class InferenceMemoryFIL(InferenceMemory, TensorMemory): - - def __init__(self, *, count: int, input__0: object, seq_ids: object) -> None: - ... - + def __init__(self, *, count: int, input__0: object, seq_ids: object) -> None: ... @property def input__0(self) -> object: """ :type: object """ - @input__0.setter def input__0(self, arg1: object) -> None: pass - @property def seq_ids(self) -> object: """ :type: object """ - @seq_ids.setter def seq_ids(self, arg1: object) -> None: pass - pass - - __version__ = '24.10.0' diff --git a/python/morpheus/morpheus/_lib/modules/__init__.pyi b/python/morpheus/morpheus/_lib/modules/__init__.pyi index 1cc0c44c22..0ec21dfaad 100644 --- a/python/morpheus/morpheus/_lib/modules/__init__.pyi +++ b/python/morpheus/morpheus/_lib/modules/__init__.pyi @@ -9,6 +9,9 @@ from __future__ import annotations import morpheus._lib.modules import typing -__all__ = [] +__all__ = [ + +] + __version__ = '24.10.0' diff --git a/python/morpheus/morpheus/_lib/stages/__init__.pyi b/python/morpheus/morpheus/_lib/stages/__init__.pyi index 99ce8eaacf..bb40f3916b 100644 --- a/python/morpheus/morpheus/_lib/stages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/stages/__init__.pyi @@ -34,228 +34,51 @@ __all__ = [ class AddClassificationsStage(mrc.core.segment.SegmentObject): - - def __init__(self, builder: mrc.core.segment.Builder, name: str, idx2label: typing.Dict[int, str], - threshold: float) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, idx2label: typing.Dict[int, str], threshold: float) -> None: ... pass - - class AddScoresStage(mrc.core.segment.SegmentObject): - - def __init__(self, builder: mrc.core.segment.Builder, name: str, idx2label: typing.Dict[int, str]) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, idx2label: typing.Dict[int, str]) -> None: ... pass - - class DeserializeStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - batch_size: int, - ensure_sliceable_index: bool = True, - task_type: object = None, - task_payload: object = None) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, batch_size: int, ensure_sliceable_index: bool = True, task_type: object = None, task_payload: object = None) -> None: ... pass - - class FileSourceStage(mrc.core.segment.SegmentObject): - @typing.overload - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - filename: os.PathLike, - repeat: int, - filter_null: bool, - filter_null_columns: typing.List[str], - parser_kwargs: dict) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, filename: os.PathLike, repeat: int, filter_null: bool, filter_null_columns: typing.List[str], parser_kwargs: dict) -> None: ... @typing.overload - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - filename: str, - repeat: int, - filter_null: bool, - filter_null_columns: typing.List[str], - parser_kwargs: dict) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, filename: str, repeat: int, filter_null: bool, filter_null_columns: typing.List[str], parser_kwargs: dict) -> None: ... pass - - class FilterDetectionsStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - threshold: float, - copy: bool, - filter_source: morpheus._lib.common.FilterSource, - field_name: str = 'probs') -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, threshold: float, copy: bool, filter_source: morpheus._lib.common.FilterSource, field_name: str = 'probs') -> None: ... pass - - class HttpServerSourceStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - bind_address: str = '127.0.0.1', - port: int = 8080, - endpoint: str = '/message', - live_endpoint: str = '/live', - ready_endpoint: str = '/ready', - method: str = 'POST', - live_method: str = 'GET', - ready_method: str = 'GET', - accept_status: int = 201, - sleep_time: float = 0.10000000149011612, - queue_timeout: int = 5, - max_queue_size: int = 1024, - num_server_threads: int = 1, - max_payload_size: int = 10485760, - request_timeout: int = 30, - lines: bool = False, - stop_after: int = 0) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, bind_address: str = '127.0.0.1', port: int = 8080, endpoint: str = '/message', live_endpoint: str = '/live', ready_endpoint: str = '/ready', method: str = 'POST', live_method: str = 'GET', ready_method: str = 'GET', accept_status: int = 201, sleep_time: float = 0.10000000149011612, queue_timeout: int = 5, max_queue_size: int = 1024, num_server_threads: int = 1, max_payload_size: int = 10485760, request_timeout: int = 30, lines: bool = False, stop_after: int = 0) -> None: ... pass - - class InferenceClientStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - server_url: str, - model_name: str, - needs_logits: bool, - force_convert_inputs: bool, - input_mapping: typing.Dict[str, str] = {}, - output_mapping: typing.Dict[str, str] = {}) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, server_url: str, model_name: str, needs_logits: bool, force_convert_inputs: bool, input_mapping: typing.Dict[str, str] = {}, output_mapping: typing.Dict[str, str] = {}) -> None: ... pass - - class KafkaSourceStage(mrc.core.segment.SegmentObject): - @typing.overload - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - max_batch_size: int, - topic: str, - batch_timeout_ms: int, - config: typing.Dict[str, str], - disable_commits: bool = False, - disable_pre_filtering: bool = False, - stop_after: int = 0, - async_commits: bool = True, - oauth_callback: typing.Optional[function] = None) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, max_batch_size: int, topic: str, batch_timeout_ms: int, config: typing.Dict[str, str], disable_commits: bool = False, disable_pre_filtering: bool = False, stop_after: int = 0, async_commits: bool = True, oauth_callback: typing.Optional[function] = None) -> None: ... @typing.overload - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - max_batch_size: int, - topics: typing.List[str], - batch_timeout_ms: int, - config: typing.Dict[str, str], - disable_commits: bool = False, - disable_pre_filtering: bool = False, - stop_after: int = 0, - async_commits: bool = True, - oauth_callback: typing.Optional[function] = None) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, max_batch_size: int, topics: typing.List[str], batch_timeout_ms: int, config: typing.Dict[str, str], disable_commits: bool = False, disable_pre_filtering: bool = False, stop_after: int = 0, async_commits: bool = True, oauth_callback: typing.Optional[function] = None) -> None: ... pass - - class PreallocateControlMessageStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - needed_columns: typing.List[typing.Tuple[str, morpheus._lib.common.TypeId]]) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, needed_columns: typing.List[typing.Tuple[str, morpheus._lib.common.TypeId]]) -> None: ... pass - - class PreallocateMessageMetaStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - needed_columns: typing.List[typing.Tuple[str, morpheus._lib.common.TypeId]]) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, needed_columns: typing.List[typing.Tuple[str, morpheus._lib.common.TypeId]]) -> None: ... pass - - class PreprocessFILStage(mrc.core.segment.SegmentObject): - - def __init__(self, builder: mrc.core.segment.Builder, name: str, features: typing.List[str]) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, features: typing.List[str]) -> None: ... pass - - class PreprocessNLPStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - vocab_hash_file: str, - sequence_length: int, - truncation: bool, - do_lower_case: bool, - add_special_token: bool, - stride: int, - column: str) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, vocab_hash_file: str, sequence_length: int, truncation: bool, do_lower_case: bool, add_special_token: bool, stride: int, column: str) -> None: ... pass - - class SerializeStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - include: typing.List[str], - exclude: typing.List[str], - fixed_columns: bool = True) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, include: typing.List[str], exclude: typing.List[str], fixed_columns: bool = True) -> None: ... pass - - class WriteToFileStage(mrc.core.segment.SegmentObject): - - def __init__(self, - builder: mrc.core.segment.Builder, - name: str, - filename: str, - mode: str = 'w', - file_type: morpheus._lib.common.FileTypes = FileTypes.Auto, - include_index_col: bool = True, - flush: bool = False) -> None: - ... - + def __init__(self, builder: mrc.core.segment.Builder, name: str, filename: str, mode: str = 'w', file_type: morpheus._lib.common.FileTypes = FileTypes.Auto, include_index_col: bool = True, flush: bool = False) -> None: ... pass - - __version__ = '24.10.0' From 785218413a81ebf6d8c7647e108da57e33e52966 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 11:27:13 -0700 Subject: [PATCH 189/347] Fix merge error --- .../stages/preprocess/preprocess_base_stage.py | 13 +------------ .../ransomware_detection/test_create_features.py | 8 -------- tests/stages/test_filter_detections_stage.py | 8 +++----- tests/test_add_classifications_stage.py | 3 --- 4 files changed, 4 insertions(+), 28 deletions(-) diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py index b00f8db35d..d8f5debf28 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_base_stage.py @@ -19,14 +19,7 @@ from morpheus.config import Config from morpheus.messages import ControlMessage -<<<<<<< HEAD -from morpheus.messages import MessageBase -from morpheus.messages import MultiInferenceMessage -from morpheus.messages import MultiMessage -from morpheus.pipeline.multi_message_stage import MultiMessageStage -======= from morpheus.pipeline.control_message_stage import ControlMessageStage ->>>>>>> 1d02332d6a6fa57a1198565d036ca646b2c6e50e from morpheus.pipeline.stage_schema import StageSchema @@ -46,9 +39,6 @@ def __init__(self, c: Config): self._should_log_timestamps = True - # only used when not using control message - self._fallback_output_type: type[MessageBase] = None - def accepted_types(self) -> typing.Tuple: """ Returns accepted input types for this stage. @@ -59,8 +49,7 @@ def accepted_types(self) -> typing.Tuple: def compute_schema(self, schema: StageSchema): schema.output_schema.set_type(ControlMessage) - def _get_preprocess_fn( - self) -> typing.Callable[[ControlMessage], ControlMessage]: + def _get_preprocess_fn(self) -> typing.Callable[[ControlMessage], ControlMessage]: """ This method should be implemented by any subclasses with a Python implementation. """ diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index 77a6a68cc6..ac6cd74311 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -25,12 +25,7 @@ from _utils import TEST_DIRS from _utils.dataset_manager import DatasetManager from morpheus.config import Config -from morpheus.messages import ControlMessage -from morpheus.messages.message_meta import AppShieldMessageMeta -<<<<<<< HEAD -======= from morpheus.pipeline.control_message_stage import ControlMessageStage ->>>>>>> 1d02332d6a6fa57a1198565d036ca646b2c6e50e from morpheus.stages.input.appshield_source_stage import AppShieldSourceStage @@ -59,10 +54,7 @@ def test_constructor( n_workers=n_workers, threads_per_worker=threads_per_worker) -<<<<<<< HEAD -======= assert isinstance(stage, ControlMessageStage) ->>>>>>> 1d02332d6a6fa57a1198565d036ca646b2c6e50e assert stage._client is mock_dask_client scheduler_info = stage._client.scheduler_info() for worker in scheduler_info['workers'].values(): diff --git a/tests/stages/test_filter_detections_stage.py b/tests/stages/test_filter_detections_stage.py index 7b3765787c..3f43c45183 100644 --- a/tests/stages/test_filter_detections_stage.py +++ b/tests/stages/test_filter_detections_stage.py @@ -140,8 +140,7 @@ def test_filter_slice(config, filter_probs_df): mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_slice(mock_control_message) assert len(output_control_message) == len(output_multi_response_messages) - assert output_control_message[0].payload().get_data().to_numpy().tolist( - ) == filter_probs_df.loc[ + assert output_control_message[0].payload().get_data().to_numpy().tolist() == filter_probs_df.loc[ 1:1, :].to_numpy().tolist() # Two adjacent rows have a value above the threashold @@ -156,9 +155,8 @@ def test_filter_slice(config, filter_probs_df): mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_slice(mock_control_message) assert len(output_control_message) == len(output_multi_response_messages) - assert output_control_message[0].payload().get_data().to_numpy().tolist( - ) == filter_probs_df.loc[ - 2:3, :]..to_numpy().tolist() + assert output_control_message[0].payload().get_data().to_numpy().tolist() == filter_probs_df.loc[ + 2:3, :].to_numpy().tolist() # Two non-adjacent rows have a value above the threashold probs = np.array([ diff --git a/tests/test_add_classifications_stage.py b/tests/test_add_classifications_stage.py index 41a410d6f1..0dd9bd8461 100755 --- a/tests/test_add_classifications_stage.py +++ b/tests/test_add_classifications_stage.py @@ -24,10 +24,7 @@ from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.messages import ControlMessage -<<<<<<< HEAD from morpheus.messages import TensorMemory -======= ->>>>>>> 1d02332d6a6fa57a1198565d036ca646b2c6e50e from morpheus.messages.message_meta import MessageMeta from morpheus.stages.postprocess.add_classifications_stage import AddClassificationsStage From 88c08fe2d6164951bde012956e4ef4e8fc4d4ee3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 11:33:15 -0700 Subject: [PATCH 190/347] merge cleanups --- examples/abp_pcap_detection/abp_pcap_preprocessing.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/abp_pcap_detection/abp_pcap_preprocessing.py b/examples/abp_pcap_detection/abp_pcap_preprocessing.py index f4ebdfbb04..62c1518629 100644 --- a/examples/abp_pcap_detection/abp_pcap_preprocessing.py +++ b/examples/abp_pcap_detection/abp_pcap_preprocessing.py @@ -21,12 +21,12 @@ import cudf -import morpheus._lib.messages as _messages from morpheus.cli.register_stage import register_stage from morpheus.common import TypeId from morpheus.config import Config from morpheus.config import PipelineModes from morpheus.messages import ControlMessage +from morpheus.messages import InferenceMemoryFIL from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage @@ -184,7 +184,7 @@ def round_time_kernel(timestamp, rollup_time, secs): seq_ids[:, 2] = fea_len - 1 # Create the inference memory. Keep in mind count here could be > than input count - memory = _messages.InferenceMemoryFIL(count=count, input__0=data, seq_ids=seq_ids) + memory = InferenceMemoryFIL(count=count, input__0=data, seq_ids=seq_ids) infer_message = ControlMessage(msg) infer_message.payload(meta) @@ -197,6 +197,3 @@ def _get_preprocess_fn(self) -> typing.Callable[[ControlMessage], ControlMessage fea_len=self._fea_length, fea_cols=self.features, req_cols=self.req_cols) - - def _get_preprocess_node(self, builder: mrc.Builder): - raise NotImplementedError("C++ node not implemented for this stage") From 25df4e164560aa167fea97fcf3ca8c49991563e4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 11:34:48 -0700 Subject: [PATCH 191/347] merge cleanups --- examples/log_parsing/inference.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/log_parsing/inference.py b/examples/log_parsing/inference.py index 67f4062409..099928cff9 100644 --- a/examples/log_parsing/inference.py +++ b/examples/log_parsing/inference.py @@ -19,7 +19,6 @@ import tritonclient.grpc as tritonclient from scipy.special import softmax -import morpheus._lib.messages as _messages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes @@ -62,7 +61,7 @@ def build_output_message(self, msg: ControlMessage) -> ControlMessage: seq_ids[:, 0] = cp.arange(0, msg.tensors().count, dtype=cp.uint32) seq_ids[:, 2] = msg.tensors().get_tensor('seq_ids')[:, 2] - memory = _messages.TensorMemory( + memory = TensorMemory( count=msg.tensors().count, tensors={ 'confidences': cp.zeros((msg.tensors().count, self._inputs[list(self._inputs.keys())[0]].shape[1])), From d6beeaf1ae5851983fac17de97e0d52c0a3ce5f1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 11:43:33 -0700 Subject: [PATCH 192/347] merge cleanups --- python/morpheus/morpheus/controllers/monitor_controller.py | 2 +- .../morpheus/stages/inference/auto_encoder_inference_stage.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/morpheus/morpheus/controllers/monitor_controller.py b/python/morpheus/morpheus/controllers/monitor_controller.py index d8994366aa..74e5b829ba 100644 --- a/python/morpheus/morpheus/controllers/monitor_controller.py +++ b/python/morpheus/morpheus/controllers/monitor_controller.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) -SupportedTypes = typing.Union[DataFrameType, MessageMeta, ControlMessage, typing.List] +SupportedTypes = typing.Union[DataFrameType, MessageMeta, ControlMessage, list] class MonitorController: diff --git a/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py b/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py index 32f3c569ad..81e6684a55 100644 --- a/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py @@ -18,7 +18,6 @@ import numpy as np import pandas as pd -import morpheus._lib.messages as _messages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes @@ -65,7 +64,7 @@ def build_output_message(self, msg: ControlMessage) -> ControlMessage: output_message = ControlMessage(msg) output_message.payload(msg.payload()) - output_message.tensors(_messages.TensorMemory(count=output_dims[0], tensors={"probs": cp.zeros(output_dims)})) + output_message.tensors(TensorMemory(count=output_dims[0], tensors={"probs": cp.zeros(output_dims)})) return output_message From f12b564336ecead0a98f431ab1aaa4156a369a3f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 11:52:35 -0700 Subject: [PATCH 193/347] merge cleanups --- python/morpheus/morpheus/messages/__init__.py | 2 +- .../morpheus/stages/preprocess/preprocess_ae_stage.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/morpheus/morpheus/messages/__init__.py b/python/morpheus/morpheus/messages/__init__.py index a0e4e92953..7186b35fb7 100644 --- a/python/morpheus/morpheus/messages/__init__.py +++ b/python/morpheus/morpheus/messages/__init__.py @@ -18,7 +18,6 @@ # Import order is very important here. Import base classes before child ones # isort: off -from morpheus._lib.messages import ControlMessage from morpheus._lib.messages import DataLoaderRegistry from morpheus._lib.messages import RawPacketMessage from morpheus.messages.memory.tensor_memory import TensorMemory @@ -33,6 +32,7 @@ from morpheus.messages.message_meta import MessageMeta from morpheus.messages.multi_message import MultiMessage from morpheus.messages.message_meta import UserMessageMeta +from morpheus.messages.control_message import ControlMessage __all__ = [ "ControlMessage", diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py index 6b29195682..9163d5491f 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py @@ -18,11 +18,11 @@ import cupy as cp -import morpheus._lib.messages as _messages from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.config import PipelineModes from morpheus.messages import ControlMessage +from morpheus.messages import TensorMemory from morpheus.stages.preprocess.preprocess_base_stage import PreprocessBaseStage logger = logging.getLogger(__name__) @@ -101,7 +101,7 @@ def pre_process_batch(msg: ControlMessage, fea_len: int, feature_columns: typing msg.set_metadata("model", autoencoder) msg.set_metadata("train_scores_mean", scores_mean) msg.set_metadata("train_scores_std", scores_std) - msg.tensors(_messages.TensorMemory(count=count, tensors={"input": inputs, "seq_ids": seg_ids})) + msg.tensors(TensorMemory(count=count, tensors={"input": inputs, "seq_ids": seg_ids})) return msg def _get_preprocess_fn(self) -> typing.Callable[[ControlMessage], ControlMessage]: From 021189fc4c5587072a0d9b42876c1746d524ac6c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 13:03:58 -0700 Subject: [PATCH 194/347] WIP --- .../stages/inference/inference_stage.py | 2 +- .../stages/inference/triton_inference_stage.py | 1 + .../stages/preprocess/preprocess_ae_stage.py | 1 - .../stages/preprocess/preprocess_nlp_stage.py | 18 +++++++++--------- tests/stages/test_filter_detections_stage.py | 4 ++-- tests/test_cli.py | 2 ++ tests/test_triton_inference_stage.py | 13 +++++++++++++ 7 files changed, 28 insertions(+), 13 deletions(-) diff --git a/python/morpheus/morpheus/stages/inference/inference_stage.py b/python/morpheus/morpheus/stages/inference/inference_stage.py index d969c97f39..6cc1763d33 100644 --- a/python/morpheus/morpheus/stages/inference/inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/inference_stage.py @@ -80,7 +80,7 @@ def build_output_message(self, msg: ControlMessage) -> ControlMessage: dims = self.calc_output_dims(msg) output_dims = (msg.payload().count, *dims[1:]) - memory = _messages.TensorMemory(count=output_dims[0], tensors={'probs': cp.zeros(output_dims)}) + memory = TensorMemory(count=output_dims[0], tensors={'probs': cp.zeros(output_dims)}) output_message = ControlMessage(msg) output_message.payload(msg.payload()) output_message.tensors(memory) diff --git a/python/morpheus/morpheus/stages/inference/triton_inference_stage.py b/python/morpheus/morpheus/stages/inference/triton_inference_stage.py index f7fc8d6027..2dc31925f7 100644 --- a/python/morpheus/morpheus/stages/inference/triton_inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/triton_inference_stage.py @@ -780,6 +780,7 @@ def _get_inference_worker(self, inf_queue: ProducerConsumerQueue) -> TritonInfer needs_logits=self._needs_logits) def _get_cpp_inference_node(self, builder: mrc.Builder) -> mrc.SegmentObject: + import morpheus._lib.stages as _stages return _stages.InferenceClientStage(builder, self.unique_name, self._server_url, diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py index 9163d5491f..9f61dafe51 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py @@ -45,7 +45,6 @@ def __init__(self, c: Config): self._fea_length = c.feature_length self._feature_columns = c.ae.feature_columns - self._fallback_output_type = MultiInferenceAEMessage @property def name(self) -> str: diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py index 3e98c2887f..3a85af54cb 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_nlp_stage.py @@ -96,12 +96,12 @@ def supports_cpp_node(self) -> bool: def _get_preprocess_node(self, builder: mrc.Builder): import morpheus._lib.stages as _stages - _stages.PreprocessNLPStage(builder, - self.unique_name, - self._vocab_hash_file, - self._seq_length, - self._truncation, - self._do_lower_case, - self._add_special_tokens, - self._stride, - self._column) + return _stages.PreprocessNLPStage(builder, + self.unique_name, + self._vocab_hash_file, + self._seq_length, + self._truncation, + self._do_lower_case, + self._add_special_tokens, + self._stride, + self._column) diff --git a/tests/stages/test_filter_detections_stage.py b/tests/stages/test_filter_detections_stage.py index 3f43c45183..d0bd75d622 100644 --- a/tests/stages/test_filter_detections_stage.py +++ b/tests/stages/test_filter_detections_stage.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing + import numpy as np import pytest import typing_utils @@ -139,7 +141,6 @@ def test_filter_slice(config, filter_probs_df): mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_slice(mock_control_message) - assert len(output_control_message) == len(output_multi_response_messages) assert output_control_message[0].payload().get_data().to_numpy().tolist() == filter_probs_df.loc[ 1:1, :].to_numpy().tolist() @@ -154,7 +155,6 @@ def test_filter_slice(config, filter_probs_df): mock_control_message = _make_control_message(filter_probs_df, probs) output_control_message = fds._controller.filter_slice(mock_control_message) - assert len(output_control_message) == len(output_multi_response_messages) assert output_control_message[0].payload().get_data().to_numpy().tolist() == filter_probs_df.loc[ 2:3, :].to_numpy().tolist() diff --git a/tests/test_cli.py b/tests/test_cli.py index 0e35b4a364..ed1a22bf80 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -266,6 +266,7 @@ def test_pipeline_ae(self, config, callback_values): assert isinstance(to_file, WriteToFileStage) assert to_file._controller._output_file == 'out.csv' + @pytest.mark.xfail(reason="TODO: Fix this") @pytest.mark.replace_callback('pipeline_ae') def test_pipeline_ae_all(self, callback_values): """ @@ -1030,6 +1031,7 @@ def test_pipeline_fil_relative_path_precedence(self, config: Config, tmp_path: s assert config.fil.feature_columns == test_columns # pylint: disable=unused-argument + @pytest.mark.xfail(reason="TODO: Fix this") @pytest.mark.replace_callback('pipeline_ae') def test_pipeline_ae_relative_path_precedence(self, config: Config, tmp_path: str, callback_values: dict): """ diff --git a/tests/test_triton_inference_stage.py b/tests/test_triton_inference_stage.py index 67931a9cb4..24270e5de7 100644 --- a/tests/test_triton_inference_stage.py +++ b/tests/test_triton_inference_stage.py @@ -17,16 +17,28 @@ import queue from unittest import mock +import numpy as np +import pandas as pd import pytest +import cudf + from _utils import assert_results from _utils import mk_async_infer from morpheus.config import Config +from morpheus.config import ConfigFIL from morpheus.config import PipelineModes +from morpheus.pipeline import LinearPipeline from morpheus.stages.inference.triton_inference_stage import ProducerConsumerQueue from morpheus.stages.inference.triton_inference_stage import ResourcePool from morpheus.stages.inference.triton_inference_stage import TritonInferenceStage from morpheus.stages.inference.triton_inference_stage import TritonInferenceWorker +from morpheus.stages.input.in_memory_source_stage import InMemorySourceStage +from morpheus.stages.output.compare_dataframe_stage import CompareDataFrameStage +from morpheus.stages.postprocess.add_scores_stage import AddScoresStage +from morpheus.stages.postprocess.serialize_stage import SerializeStage +from morpheus.stages.preprocess.deserialize_stage import DeserializeStage +from morpheus.stages.preprocess.preprocess_fil_stage import PreprocessFILStage MODEL_MAX_BATCH_SIZE = 1024 @@ -141,6 +153,7 @@ def test_stage_get_inference_worker(config: Config, pipeline_mode: PipelineModes assert worker.needs_logits == expexted_needs_logits +@pytest.mark.skip(reason="TODO: fix this currently failing an assertion in meta.cpp") @pytest.mark.slow @pytest.mark.gpu_mode @pytest.mark.parametrize('num_records', [10]) From 100e3ae8d72f5ab5f6de79c6c6cf816f987b6b62 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 15:03:59 -0700 Subject: [PATCH 195/347] Fix broken tests --- tests/test_add_classifications_stage.py | 4 ++-- tests/test_add_scores_stage.py | 6 +++--- tests/test_inference_stage.py | 3 +-- tests/test_inference_worker.py | 7 +++---- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/test_add_classifications_stage.py b/tests/test_add_classifications_stage.py index 0dd9bd8461..98eff9e698 100755 --- a/tests/test_add_classifications_stage.py +++ b/tests/test_add_classifications_stage.py @@ -58,8 +58,8 @@ def test_constructor_errors(config: Config): AddClassificationsStage(config, labels=['missing']) -@pytest.mark.skip(reason="TODO: determine python impls for gpu only stages") -def test_add_labels_with_multi_response_message_and_contgrol_message(): +@pytest.mark.cpu_mode +def test_add_labels_with_contgrol_message(): class_labels = {0: "frogs", 1: "lizards", 2: "toads"} diff --git a/tests/test_add_scores_stage.py b/tests/test_add_scores_stage.py index 151784f9fb..8694632abe 100755 --- a/tests/test_add_scores_stage.py +++ b/tests/test_add_scores_stage.py @@ -21,10 +21,10 @@ import pytest import typing_utils -import morpheus._lib.messages as _messages from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.messages import ControlMessage +from morpheus.messages import TensorMemory from morpheus.messages.message_meta import MessageMeta from morpheus.stages.postprocess.add_classifications_stage import AddClassificationsStage from morpheus.stages.postprocess.add_scores_stage import AddScoresStage @@ -60,8 +60,8 @@ def test_constructor_errors(config: Config): AddScoresStage(config, labels=['missing']) -@pytest.mark.skip(reason="TODO: determine python impls for gpu only stages") -def test_add_labels_with_multi_response_message_and_control_message(): +@pytest.mark.cpu_mode +def test_add_labels_with_control_message(): class_labels = {0: "frogs", 1: "lizards", 2: "toads"} df = pd.DataFrame([0, 1], columns=["dummy"]) diff --git a/tests/test_inference_stage.py b/tests/test_inference_stage.py index 269dd1cf6d..89fc5bfdfe 100755 --- a/tests/test_inference_stage.py +++ b/tests/test_inference_stage.py @@ -95,8 +95,7 @@ def test_join(config): worker.join.assert_awaited_once() -@pytest.mark.skip(reason="Test is passing, but python only impls for inference remains TBD") -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode def test_convert_one_response(): # Test ControlMessage # Test first branch where `inf.mess_count == inf.count` diff --git a/tests/test_inference_worker.py b/tests/test_inference_worker.py index 27c67e04bb..cfbefe821c 100755 --- a/tests/test_inference_worker.py +++ b/tests/test_inference_worker.py @@ -19,10 +19,10 @@ import cudf -import morpheus._lib.messages as _messages from _utils.inference_worker import IW from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta +from morpheus.messages import TensorMemory from morpheus.stages.inference import inference_stage from morpheus.utils.producer_consumer_queue import ProducerConsumerQueue @@ -37,8 +37,7 @@ def test_constructor(): worker.stop() -@pytest.mark.skip(reason="Test is passing, but python only impls for inference remains TBD") -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode @pytest.mark.usefixtures("config") def test_build_output_message(): @@ -59,7 +58,7 @@ def test_build_output_message(): input__0 = cp.array([[0.], [2.], [4.], [6.], [8.], [10.], [12.], [14.], [16.], [18.]]) seq_ids = cp.array([[0, 0, 0], [1, 0, 0], [2, 0, 0], [3, 0, 0], [4, 0, 0], [5, 0, 0], [6, 0, 0], [7, 0, 0], [8, 0, 0], [9, 0, 0]]) - msg.tensors(_messages.TensorMemory(count=num_records, tensors={'input__0': input__0, 'seq_ids': seq_ids})) + msg.tensors(TensorMemory(count=num_records, tensors={'input__0': input__0, 'seq_ids': seq_ids})) output_message = worker.build_output_message(msg) From 5628e48e1e382568b54be817cabe8207579829c4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 15:22:10 -0700 Subject: [PATCH 196/347] Fix tests --- tests/_utils/inference_worker.py | 2 -- tests/examples/log_parsing/test_inference.py | 24 +++++++++---------- .../log_parsing/test_postprocessing.py | 4 ++-- .../stages/test_generate_viz_frames_stage.py | 4 ++-- tests/stages/test_ml_flow_drift_stage.py | 6 ++--- tests/test_inference_stage.py | 23 +++++++++--------- 6 files changed, 30 insertions(+), 33 deletions(-) diff --git a/tests/_utils/inference_worker.py b/tests/_utils/inference_worker.py index 7470e474d5..29af5a0c02 100644 --- a/tests/_utils/inference_worker.py +++ b/tests/_utils/inference_worker.py @@ -26,8 +26,6 @@ class IW(inference_stage.InferenceWorker): """ def calc_output_dims(self, _): - # Intentionally calling the abc empty method for coverage - super().calc_output_dims(_) return (1, 2) def process(self, _: ControlMessage, __: typing.Callable[[TensorMemory], None]): diff --git a/tests/examples/log_parsing/test_inference.py b/tests/examples/log_parsing/test_inference.py index f4a7aac660..a721d8afc7 100644 --- a/tests/examples/log_parsing/test_inference.py +++ b/tests/examples/log_parsing/test_inference.py @@ -22,10 +22,10 @@ import numpy as np import pytest -import morpheus._lib.messages as _messages from _utils import TEST_DIRS from morpheus.config import Config from morpheus.messages import ControlMessage +from morpheus.messages import InferenceMemoryNLP from morpheus.messages import MessageMeta from morpheus.messages import TensorMemory from morpheus.stages.inference.triton_inference_stage import TritonInferenceWorker @@ -52,13 +52,13 @@ def build_resp_message(df: DataFrameType, num_cols: int = 2) -> ControlMessage: seq_ids[:, 2] = 42 meta = MessageMeta(df) - mem = _messages.TensorMemory(count=count, - tensors={ - 'confidences': cp.zeros((count, num_cols)), - 'labels': cp.zeros((count, num_cols)), - 'input_ids': cp.zeros((count, num_cols), dtype=cp.float32), - 'seq_ids': seq_ids - }) + mem = TensorMemory(count=count, + tensors={ + 'confidences': cp.zeros((count, num_cols)), + 'labels': cp.zeros((count, num_cols)), + 'input_ids': cp.zeros((count, num_cols), dtype=cp.float32), + 'seq_ids': seq_ids + }) cm = ControlMessage() cm.payload(meta) cm.tensors(mem) @@ -78,10 +78,10 @@ def build_inf_message(df: DataFrameType, mess_count: int, count: int, num_cols: seq_ids[:, 2] = 42 meta = MessageMeta(df) - mem = _messages.InferenceMemoryNLP(count=tensor_length, - input_ids=cp.zeros((tensor_length, num_cols), dtype=cp.float32), - input_mask=cp.zeros((tensor_length, num_cols), dtype=cp.float32), - seq_ids=seq_ids) + mem = InferenceMemoryNLP(count=tensor_length, + input_ids=cp.zeros((tensor_length, num_cols), dtype=cp.float32), + input_mask=cp.zeros((tensor_length, num_cols), dtype=cp.float32), + seq_ids=seq_ids) cm = ControlMessage() cm.payload(meta) cm.tensors(mem) diff --git a/tests/examples/log_parsing/test_postprocessing.py b/tests/examples/log_parsing/test_postprocessing.py index 48baeaddc1..e6271d8a42 100644 --- a/tests/examples/log_parsing/test_postprocessing.py +++ b/tests/examples/log_parsing/test_postprocessing.py @@ -23,12 +23,12 @@ import numpy as np import pytest -import morpheus._lib.messages as _messages from _utils import TEST_DIRS from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta +from morpheus.messages import TensorMemory @pytest.fixture(scope='module', name="model_config_file") @@ -55,7 +55,7 @@ def build_post_proc_message(dataset_cudf: DatasetManager, log_test_data_dir: str seq_ids[:, 2] = cp.asarray(host__seq_data)[:, 2] tensors['seq_ids'] = seq_ids - memory = _messages.TensorMemory(count=5, tensors=tensors) + memory = TensorMemory(count=5, tensors=tensors) msg = ControlMessage() msg.payload(meta) diff --git a/tests/stages/test_generate_viz_frames_stage.py b/tests/stages/test_generate_viz_frames_stage.py index 30ddece85d..c4e1e36ee2 100644 --- a/tests/stages/test_generate_viz_frames_stage.py +++ b/tests/stages/test_generate_viz_frames_stage.py @@ -21,10 +21,10 @@ import cudf -import morpheus._lib.messages as _messages from morpheus.config import Config from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta +from morpheus.messages import TensorMemory from morpheus.stages.postprocess.generate_viz_frames_stage import GenerateVizFramesStage @@ -32,7 +32,7 @@ def _make_control_message(df, probs): df_ = df[0:len(probs)] cm = ControlMessage() cm.payload(MessageMeta(df_)) - cm.tensors(_messages.TensorMemory(count=len(df_), tensors={'probs': probs})) + cm.tensors(TensorMemory(count=len(df_), tensors={'probs': probs})) return cm diff --git a/tests/stages/test_ml_flow_drift_stage.py b/tests/stages/test_ml_flow_drift_stage.py index ae4235834e..ff0f4d2a92 100644 --- a/tests/stages/test_ml_flow_drift_stage.py +++ b/tests/stages/test_ml_flow_drift_stage.py @@ -21,9 +21,9 @@ import pytest import typing_utils -import morpheus._lib.messages as _messages from morpheus.messages import ControlMessage -from morpheus.messages.message_meta import MessageMeta +from morpheus.messages import MessageMeta +from morpheus.messages import TensorMemory from morpheus.stages.postprocess.ml_flow_drift_stage import MLFlowDriftStage @@ -31,7 +31,7 @@ def _make_control_message(df, probs): df_ = df[0:len(probs)] cm = ControlMessage() cm.payload(MessageMeta(df_)) - cm.tensors(_messages.TensorMemory(count=len(df_), tensors={'probs': probs})) + cm.tensors(TensorMemory(count=len(df_), tensors={'probs': probs})) return cm diff --git a/tests/test_inference_stage.py b/tests/test_inference_stage.py index 89fc5bfdfe..10370210a1 100755 --- a/tests/test_inference_stage.py +++ b/tests/test_inference_stage.py @@ -22,9 +22,10 @@ import cudf -import morpheus._lib.messages as _messages from _utils.inference_worker import IW from morpheus.messages import ControlMessage +from morpheus.messages import InferenceMemory +from morpheus.messages import ResponseMemory from morpheus.messages.message_meta import MessageMeta from morpheus.stages.inference.inference_stage import InferenceStage @@ -45,12 +46,11 @@ def _mk_control_message(mess_count=1, count=1): msg = ControlMessage() msg.payload(MessageMeta(df)) msg.tensors( - _messages.InferenceMemory( - count=total_tensor_count, - tensors={ - "probs": cp.random.rand(total_tensor_count, 2), - "seq_ids": cp.tile(cp.expand_dims(cp.arange(0, total_tensor_count), axis=1), (1, 3)) - })) + InferenceMemory(count=total_tensor_count, + tensors={ + "probs": cp.random.rand(total_tensor_count, 2), + "seq_ids": cp.tile(cp.expand_dims(cp.arange(0, total_tensor_count), axis=1), (1, 3)) + })) return msg @@ -99,10 +99,10 @@ def test_join(config): def test_convert_one_response(): # Test ControlMessage # Test first branch where `inf.mess_count == inf.count` - mem = _messages.ResponseMemory(count=4, tensors={"probs": cp.zeros((4, 3))}) + mem = ResponseMemory(count=4, tensors={"probs": cp.zeros((4, 3))}) inf = _mk_control_message(mess_count=4, count=4) - res = _messages.ResponseMemory(count=4, tensors={"probs": cp.random.rand(4, 3)}) + res = ResponseMemory(count=4, tensors={"probs": cp.random.rand(4, 3)}) output = _mk_control_message(mess_count=4, count=4) output.tensors(mem) @@ -115,10 +115,9 @@ def test_convert_one_response(): # Test for the second branch inf = _mk_control_message(mess_count=2, count=3) inf.tensors().set_tensor("seq_ids", cp.array([[0], [1], [1]])) - res = _messages.ResponseMemory(count=3, - tensors={"probs": cp.array([[0, 0.6, 0.7], [5.6, 4.4, 9.2], [4.5, 6.7, 8.9]])}) + res = ResponseMemory(count=3, tensors={"probs": cp.array([[0, 0.6, 0.7], [5.6, 4.4, 9.2], [4.5, 6.7, 8.9]])}) - mem = _messages.ResponseMemory(count=2, tensors={"probs": cp.zeros((2, 3))}) + mem = ResponseMemory(count=2, tensors={"probs": cp.zeros((2, 3))}) output = _mk_control_message(mess_count=2, count=3) output.tensors(mem) cm = InferenceStageT._convert_one_response(output, inf, res) From 2d3a0d058e2f9dae507921172b4fb9ca3e734747 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 15:49:14 -0700 Subject: [PATCH 197/347] Fix broken test --- tests/test_triton_inference_stage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_triton_inference_stage.py b/tests/test_triton_inference_stage.py index 24270e5de7..abbd7ab262 100644 --- a/tests/test_triton_inference_stage.py +++ b/tests/test_triton_inference_stage.py @@ -25,6 +25,7 @@ from _utils import assert_results from _utils import mk_async_infer +from morpheus.common import TypeId from morpheus.config import Config from morpheus.config import ConfigFIL from morpheus.config import PipelineModes @@ -153,7 +154,6 @@ def test_stage_get_inference_worker(config: Config, pipeline_mode: PipelineModes assert worker.needs_logits == expexted_needs_logits -@pytest.mark.skip(reason="TODO: fix this currently failing an assertion in meta.cpp") @pytest.mark.slow @pytest.mark.gpu_mode @pytest.mark.parametrize('num_records', [10]) @@ -200,12 +200,13 @@ def test_triton_stage_pipe(mock_triton_client, config, num_records): pipe_cm.add_stage(DeserializeStage(config)) pipe_cm.add_stage(PreprocessFILStage(config)) pipe_cm.add_stage( + # Intentionally using use_shared_memory=True as this is the only way to use the Python impl TritonInferenceStage(config, model_name='abp-nvsmi-xgb', server_url='test:0000', force_convert_inputs=True, use_shared_memory=True)) - pipe_cm.add_stage(AddScoresStage(config, prefix="score_")) + pipe_cm.add_stage(AddScoresStage(config, prefix="score_", probs_type=TypeId.FLOAT64)) pipe_cm.add_stage(SerializeStage(config)) comp_stage = pipe_cm.add_stage(CompareDataFrameStage(config, expected_df)) From 6a6f5eb6ea00b33263291deb557d281b8ac60c0c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 16:05:23 -0700 Subject: [PATCH 198/347] AE pipelines no longer need Python mode --- python/morpheus/morpheus/cli/commands.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/python/morpheus/morpheus/cli/commands.py b/python/morpheus/morpheus/cli/commands.py index 79577ee544..74e245a6ef 100644 --- a/python/morpheus/morpheus/cli/commands.py +++ b/python/morpheus/morpheus/cli/commands.py @@ -536,13 +536,6 @@ def pipeline_ae(ctx: click.Context, **kwargs): config = get_config_from_ctx(ctx) config.mode = PipelineModes.AE - - # TODO: Need to determine if we can enable C++ for AE pipelines, or if we can get this working in CPUY mode - if CppConfig.get_should_use_cpp(): - logger.warning("C++ is disabled for AutoEncoder pipelines at this time.") - CppConfig.set_should_use_cpp(False) - config.execution_mode = ExecutionMode.CPU - config.ae = ConfigAutoEncoder() config.ae.userid_column_name = kwargs["userid_column_name"] config.ae.timestamp_column_name = kwargs["timestamp_column_name"] From a0014408ec7ff911a571fe3d765d584ff8ab8815 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 16:24:16 -0700 Subject: [PATCH 199/347] Enable AE tests --- tests/test_tensor_memory.py | 49 +++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/tests/test_tensor_memory.py b/tests/test_tensor_memory.py index c045275436..230a082796 100644 --- a/tests/test_tensor_memory.py +++ b/tests/test_tensor_memory.py @@ -74,6 +74,21 @@ def check_tensor_memory(cls: type, count: int, tensors: typing.Dict[str, NDArray cls(count, tensors) +def check_response_memory_probs(cls: type, array_pkg: types.ModuleType): + test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) + count = test_data.shape[0] + + mem = cls(count=count, probs=test_data) + assert mem.count == count + compare_tensors(mem.get_tensors(), {'probs': test_data}) + assert (mem.get_output('probs') == test_data).all() + + with pytest.raises(TypeError): + cls(count, test_data) + + return mem + + @pytest.mark.gpu_and_cpu_mode def test_tensor_memory(array_pkg: types.ModuleType): test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) @@ -89,14 +104,13 @@ def test_tensor_memory(array_pkg: types.ModuleType): check_tensor_memory(cls=cls, count=count, tensors=tensors, array_pkg=array_pkg) -@pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") -@pytest.mark.cpu_mode -def test_inference_memory_ae(config: Config): - test_data = cp.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) +@pytest.mark.gpu_and_cpu_mode +def test_inference_memory_ae(array_pkg: types.ModuleType): + test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) count = test_data.shape[0] - input_tensor = cp.array(test_data[:, 0]) - seq_ids = cp.array(test_data[:, 1]) + input_tensor = array_pkg.array(test_data[:, 0]) + seq_ids = array_pkg.array(test_data[:, 1]) mem = InferenceMemoryAE(count=count, inputs=input_tensor, seq_ids=seq_ids) assert mem.count == count @@ -108,7 +122,6 @@ def test_inference_memory_ae(config: Config): InferenceMemoryAE(count, input_tensor, seq_ids) # pylint: disable=too-many-function-args,missing-kwoa -# TODO: Determine what to do about the Python impls for GPU based messages @pytest.mark.gpu_and_cpu_mode def test_inference_memory_fil(array_pkg: types.ModuleType): test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) @@ -147,25 +160,9 @@ def test_inference_memory_nlp(array_pkg: types.ModuleType): InferenceMemoryNLP(count, input_ids, input_mask, seq_ids) # pylint: disable=too-many-function-args,missing-kwoa -def check_response_memory_probs(cls: type, array_pkg: types.ModuleType): - test_data = array_pkg.array(np.loadtxt(INPUT_FILE, delimiter=",", skiprows=1)) - count = test_data.shape[0] - - mem = cls(count=count, probs=test_data) - assert mem.count == count - compare_tensors(mem.get_tensors(), {'probs': test_data}) - assert (mem.get_output('probs') == test_data).all() - - with pytest.raises(TypeError): - cls(count, test_data) - - return mem - - -@pytest.mark.skip(reason="TODO: determine what to do about AE pipelines") -@pytest.mark.cpu_mode -def test_response_memory_ae(config: Config, filter_probs_df: DataFrameType): - mem = check_response_memory_probs(ResponseMemoryAE) +@pytest.mark.gpu_and_cpu_mode +def test_response_memory_ae(array_pkg: types.ModuleType, filter_probs_df: DataFrameType): + mem = check_response_memory_probs(ResponseMemoryAE, array_pkg) assert mem.user_id == "" assert mem.explain_df is None From aff5dc8e7d5089d49b84ea8dd7a4b66a601cede7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 16:28:44 -0700 Subject: [PATCH 200/347] Fix AE tests --- tests/test_cli.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index ed1a22bf80..d06c9a4dfb 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -176,7 +176,6 @@ def test_manual_seed(self, mock_manual_seed: mock.MagicMock, value: int, use_env assert result.exit_code == 0, result.output mock_manual_seed.assert_called_once_with(value) - @pytest.mark.xfail(reason="TODO: Fix this") @pytest.mark.replace_callback('pipeline_ae') def test_pipeline_ae(self, config, callback_values): """ @@ -211,7 +210,6 @@ def test_pipeline_ae(self, config, callback_values): config = obj["config"] assert config.mode == PipelineModes.AE - assert not CppConfig.get_should_use_cpp() assert config.class_labels == ["reconstruct_loss", "zscore"] assert config.model_max_batch_size == 1024 assert config.pipeline_batch_size == 1024 @@ -266,7 +264,6 @@ def test_pipeline_ae(self, config, callback_values): assert isinstance(to_file, WriteToFileStage) assert to_file._controller._output_file == 'out.csv' - @pytest.mark.xfail(reason="TODO: Fix this") @pytest.mark.replace_callback('pipeline_ae') def test_pipeline_ae_all(self, callback_values): """ @@ -1031,7 +1028,6 @@ def test_pipeline_fil_relative_path_precedence(self, config: Config, tmp_path: s assert config.fil.feature_columns == test_columns # pylint: disable=unused-argument - @pytest.mark.xfail(reason="TODO: Fix this") @pytest.mark.replace_callback('pipeline_ae') def test_pipeline_ae_relative_path_precedence(self, config: Config, tmp_path: str, callback_values: dict): """ From bf2822198fa03d393bf8e775f4df29a76b55bdfc Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 16:35:03 -0700 Subject: [PATCH 201/347] Remove redundant xfail --- tests/llm/test_vdb_upload_pipe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/llm/test_vdb_upload_pipe.py b/tests/llm/test_vdb_upload_pipe.py index 5d5a335fc1..ecfc5ff5ef 100755 --- a/tests/llm/test_vdb_upload_pipe.py +++ b/tests/llm/test_vdb_upload_pipe.py @@ -30,7 +30,6 @@ from morpheus.service.vdb.milvus_vector_db_service import MilvusVectorDBService -@pytest.mark.xfail(reason="This is a GPU centric test, but depends on the Python impl of the Triton client") @pytest.mark.milvus @pytest.mark.use_pandas @pytest.mark.import_mod([ From 832b8d0fd4142cb99eeeaa3bf6449b4d10020c86 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 16:45:26 -0700 Subject: [PATCH 202/347] Remove broken test, because it lives inside a directory named 'dfencoder' it will always find a mode named 'dfencode', however this test doesn't provide much anyways --- tests/dfencoder/test_pkg.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100755 tests/dfencoder/test_pkg.py diff --git a/tests/dfencoder/test_pkg.py b/tests/dfencoder/test_pkg.py deleted file mode 100755 index 3b5d39585c..0000000000 --- a/tests/dfencoder/test_pkg.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - - -@pytest.mark.skip -def test_old_dfencoder_not_in_env(): - """ - Verify the old external dfencoder doesn't exist in the current env - """ - with pytest.raises(ModuleNotFoundError): - import dfencoder # noqa: F401 #pylint:disable=unused-import From 25c8b88760c8d869ca188f718af675bc6414d0d3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Sep 2024 16:52:28 -0700 Subject: [PATCH 203/347] Enable gnn tests - not currently passing --- .../gnn_fraud_detection_pipeline/test_classification_stage.py | 3 +-- .../gnn_fraud_detection_pipeline/test_graph_sage_stage.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py index 0271782e65..e65e71e56d 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_classification_stage.py @@ -24,8 +24,7 @@ # pylint: disable=no-name-in-module -@pytest.mark.xfail(reason="TODO: Need to determine what to do with GPU pipelines which only run in Python mode") -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode class TestClassificationStage: def test_constructor(self, config: Config, xgb_model: str, cuml: types.ModuleType): diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py index 2e717efe67..00ab36edf0 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_graph_sage_stage.py @@ -24,9 +24,8 @@ # pylint: disable=no-name-in-module -@pytest.mark.xfail(reason="TODO: Need to determine what to do with GPU pipelines which only run in Python mode") @pytest.mark.usefixtures("manual_seed") -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode class TestGraphSageStage: def test_constructor(self, config: Config, model_dir: str): From b63bd8700ae75cd7f41d4c003ae8d32a9696e700 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 16 Sep 2024 14:08:39 -0700 Subject: [PATCH 204/347] WIP --- .../morpheus/_lib/messages/__init__.pyi | 114 ------------------ .../digital_fingerprinting/conftest.py | 1 + 2 files changed, 1 insertion(+), 114 deletions(-) diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi index 7a5ac78488..9cd844d4d2 100644 --- a/python/morpheus/morpheus/_lib/messages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/messages/__init__.pyi @@ -21,13 +21,6 @@ __all__ = [ "InferenceMemoryFIL", "InferenceMemoryNLP", "MessageMeta", - "MultiInferenceFILMessage", - "MultiInferenceMessage", - "MultiInferenceNLPMessage", - "MultiMessage", - "MultiResponseMessage", - "MultiResponseProbsMessage", - "MultiTensorMessage", "MutableTableCtxMgr", "RawPacketMessage", "ResponseMemory", @@ -211,113 +204,6 @@ class MessageMeta(): :type: object """ pass -class MultiMessage(): - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1) -> None: ... - def copy_ranges(self, ranges: typing.List[typing.Tuple[int, int]], num_selected_rows: object = None) -> MultiMessage: ... - @typing.overload - def get_meta(self) -> object: ... - @typing.overload - def get_meta(self, columns: None) -> object: ... - @typing.overload - def get_meta(self, columns: str) -> object: ... - @typing.overload - def get_meta(self, columns: typing.List[str]) -> object: ... - def get_meta_column_names(self) -> typing.List[str]: ... - def get_meta_list(self, arg0: object) -> object: ... - def get_slice(self, arg0: int, arg1: int) -> MultiMessage: ... - def set_meta(self, arg0: object, arg1: object) -> None: ... - @property - def mess_count(self) -> int: - """ - :type: int - """ - @property - def mess_offset(self) -> int: - """ - :type: int - """ - @property - def meta(self) -> MessageMeta: - """ - :type: MessageMeta - """ - pass -class MultiTensorMessage(MultiMessage): - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... - def get_id_tensor(self) -> object: ... - def get_tensor(self, arg0: str) -> object: ... - @property - def count(self) -> int: - """ - :type: int - """ - @property - def memory(self) -> TensorMemory: - """ - :type: TensorMemory - """ - @property - def offset(self) -> int: - """ - :type: int - """ - pass -class MultiInferenceMessage(MultiTensorMessage, MultiMessage): - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... - def get_input(self, arg0: str) -> object: ... - pass -class MultiInferenceFILMessage(MultiInferenceMessage, MultiTensorMessage, MultiMessage): - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... - @property - def input__0(self) -> object: - """ - :type: object - """ - @property - def seq_ids(self) -> object: - """ - :type: object - """ - pass -class MultiResponseMessage(MultiTensorMessage, MultiMessage): - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids', probs_tensor_name: str = 'probs') -> None: ... - def get_output(self, arg0: str) -> object: ... - def get_probs_tensor(self) -> object: ... - @property - def probs_tensor_name(self) -> str: - """ - :type: str - """ - @probs_tensor_name.setter - def probs_tensor_name(self, arg1: str) -> None: - pass - pass -class MultiResponseProbsMessage(MultiResponseMessage, MultiTensorMessage, MultiMessage): - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids', probs_tensor_name: str = 'probs') -> None: ... - @property - def probs(self) -> object: - """ - :type: object - """ - pass -class MultiInferenceNLPMessage(MultiInferenceMessage, MultiTensorMessage, MultiMessage): - def __init__(self, *, meta: MessageMeta, mess_offset: int = 0, mess_count: int = -1, memory: TensorMemory, offset: int = 0, count: int = -1, id_tensor_name: str = 'seq_ids') -> None: ... - @property - def input_ids(self) -> object: - """ - :type: object - """ - @property - def input_mask(self) -> object: - """ - :type: object - """ - @property - def seq_ids(self) -> object: - """ - :type: object - """ - pass class MutableTableCtxMgr(): def __enter__(self) -> object: ... def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: ... diff --git a/tests/examples/digital_fingerprinting/conftest.py b/tests/examples/digital_fingerprinting/conftest.py index 25fb2b54c0..098c7bff37 100644 --- a/tests/examples/digital_fingerprinting/conftest.py +++ b/tests/examples/digital_fingerprinting/conftest.py @@ -90,6 +90,7 @@ def dfp_prod_in_sys_path( @pytest.fixture(name="dfp_message_meta") def dfp_message_meta_fixture(config, dataset_pandas: DatasetManager): + # TODO: This should be a cudf DataFrame, and emit a control message import pandas as pd from dfp.messages.dfp_message_meta import DFPMessageMeta From 8ed491a6ae7f38e2beaec89d18ba7ca56796f12f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 16 Sep 2024 14:18:03 -0700 Subject: [PATCH 205/347] Set pipeline to run in GPU mode --- examples/gnn_fraud_detection_pipeline/run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/gnn_fraud_detection_pipeline/run.py b/examples/gnn_fraud_detection_pipeline/run.py index 80cd9d5d0b..78b9811bdd 100644 --- a/examples/gnn_fraud_detection_pipeline/run.py +++ b/examples/gnn_fraud_detection_pipeline/run.py @@ -99,8 +99,6 @@ def run_pipeline(num_threads, # Enable the default logger. configure_logging(log_level=logging.INFO) - CppConfig.set_should_use_cpp(False) - # Its necessary to get the global config object and configure it for FIL mode. config = Config() config.mode = PipelineModes.OTHER From 6a837af9e9bbc0a5e82265d072f5d52cab5b527e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 16 Sep 2024 14:20:34 -0700 Subject: [PATCH 206/347] Fix mode marker --- .../test_graph_construction_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py b/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py index 8853d3854c..d7a8f47e8e 100644 --- a/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py +++ b/tests/examples/gnn_fraud_detection_pipeline/test_graph_construction_stage.py @@ -28,7 +28,7 @@ # pylint: disable=no-name-in-module -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode class TestGraphConstructionStage: def test_constructor(self, config: Config, training_file: str): From 364fb6254504f095c1271b46d649adb67317088c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 16 Sep 2024 14:22:19 -0700 Subject: [PATCH 207/347] Remove unused import --- examples/gnn_fraud_detection_pipeline/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/gnn_fraud_detection_pipeline/run.py b/examples/gnn_fraud_detection_pipeline/run.py index 78b9811bdd..4e64dd3a3a 100644 --- a/examples/gnn_fraud_detection_pipeline/run.py +++ b/examples/gnn_fraud_detection_pipeline/run.py @@ -18,7 +18,6 @@ import click from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage From faab0c539cc05c4799618fd9fa28f5ed4b09730d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 16 Sep 2024 14:28:24 -0700 Subject: [PATCH 208/347] WIP --- examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py | 3 --- examples/doca/run_tcp.py | 3 --- examples/doca/run_udp_convert.py | 3 --- examples/doca/run_udp_raw.py | 3 --- examples/doca/vdb_realtime/vdb.py | 3 --- examples/sid_visualization/run.py | 3 --- 6 files changed, 18 deletions(-) diff --git a/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py b/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py index 835b3e2809..0c4968b74b 100644 --- a/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py +++ b/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py @@ -21,7 +21,6 @@ from morpheus.config import AEFeatureScalar from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage @@ -101,8 +100,6 @@ def run_pipeline(num_threads, """Configure and run the pipeline.""" configure_logging(log_level=logging.DEBUG) - CppConfig.set_should_use_cpp(False) - config = Config() config.mode = PipelineModes.AE config.ae = ConfigAutoEncoder() diff --git a/examples/doca/run_tcp.py b/examples/doca/run_tcp.py index 5c4b4035a7..cf2e797efc 100644 --- a/examples/doca/run_tcp.py +++ b/examples/doca/run_tcp.py @@ -17,7 +17,6 @@ import click from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.doca.doca_convert_stage import DocaConvertStage @@ -71,8 +70,6 @@ def run_pipeline(pipeline_batch_size, model_max_batch_size, model_fea_length, ou # Enable the default logger configure_logging(log_level=logging.DEBUG) - CppConfig.set_should_use_cpp(True) - config = Config() config.mode = PipelineModes.NLP diff --git a/examples/doca/run_udp_convert.py b/examples/doca/run_udp_convert.py index 52c9b216b7..c88c80ac6c 100644 --- a/examples/doca/run_udp_convert.py +++ b/examples/doca/run_udp_convert.py @@ -19,7 +19,6 @@ from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import parse_log_level from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.messages import RawPacketMessage from morpheus.pipeline.linear_pipeline import LinearPipeline @@ -90,8 +89,6 @@ def run_pipeline(nic_addr: str, # Enable the default logger configure_logging(log_level=log_level) - CppConfig.set_should_use_cpp(True) - config = Config() config.mode = PipelineModes.NLP diff --git a/examples/doca/run_udp_raw.py b/examples/doca/run_udp_raw.py index 576ecff957..cb31c5bb6c 100644 --- a/examples/doca/run_udp_raw.py +++ b/examples/doca/run_udp_raw.py @@ -17,7 +17,6 @@ import click from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.messages import RawPacketMessage from morpheus.pipeline.linear_pipeline import LinearPipeline @@ -41,8 +40,6 @@ def run_pipeline(nic_addr, gpu_addr): # Enable the default logger configure_logging(log_level=logging.DEBUG) - CppConfig.set_should_use_cpp(True) - config = Config() config.mode = PipelineModes.NLP diff --git a/examples/doca/vdb_realtime/vdb.py b/examples/doca/vdb_realtime/vdb.py index 371c39cf34..7779b7ebf5 100644 --- a/examples/doca/vdb_realtime/vdb.py +++ b/examples/doca/vdb_realtime/vdb.py @@ -18,7 +18,6 @@ import pymilvus from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.doca.doca_convert_stage import DocaConvertStage @@ -119,8 +118,6 @@ def run_pipeline(nic_addr: str, # Enable the default logger configure_logging(log_level=logging.DEBUG) - CppConfig.set_should_use_cpp(True) - config = Config() config.mode = PipelineModes.NLP config.pipeline_batch_size = 1024 diff --git a/examples/sid_visualization/run.py b/examples/sid_visualization/run.py index 4db84fac11..17c85cbfbc 100644 --- a/examples/sid_visualization/run.py +++ b/examples/sid_visualization/run.py @@ -21,7 +21,6 @@ from morpheus.common import FileTypes from morpheus.config import Config -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.messages import MessageMeta @@ -156,8 +155,6 @@ def run_pipeline(debug, use_cpp, num_threads, input_file, max_batch_size, model_ else: configure_logging(log_level=logging.INFO) - CppConfig.set_should_use_cpp(use_cpp) - # Its necessary to get the global config object and configure it for FIL mode. config = Config() config.mode = PipelineModes.NLP From 1a5f0276a58e9128f9379cf8d30b3507e74985ce Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 16 Sep 2024 15:40:04 -0700 Subject: [PATCH 209/347] First pass at replacing DFPMessageMeta with MessageMeta+ControlMessage --- .../morpheus/dfp/messages/__init__.py | 13 ------ .../morpheus/dfp/messages/dfp_message_meta.py | 42 ------------------- .../morpheus/dfp/modules/dfp_inference.py | 10 ++--- .../morpheus/dfp/modules/dfp_training.py | 8 ++-- .../dfp/stages/dfp_rolling_window_stage.py | 24 ++++++----- .../dfp/stages/dfp_split_users_stage.py | 31 +++++++------- 6 files changed, 39 insertions(+), 89 deletions(-) delete mode 100644 examples/digital_fingerprinting/production/morpheus/dfp/messages/__init__.py delete mode 100644 examples/digital_fingerprinting/production/morpheus/dfp/messages/dfp_message_meta.py diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/messages/__init__.py b/examples/digital_fingerprinting/production/morpheus/dfp/messages/__init__.py deleted file mode 100644 index 66061e580b..0000000000 --- a/examples/digital_fingerprinting/production/morpheus/dfp/messages/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/messages/dfp_message_meta.py b/examples/digital_fingerprinting/production/morpheus/dfp/messages/dfp_message_meta.py deleted file mode 100644 index 49b8c98ba9..0000000000 --- a/examples/digital_fingerprinting/production/morpheus/dfp/messages/dfp_message_meta.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2021-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import dataclasses -import logging - -import pandas as pd - -from morpheus.messages.message_meta import MessageMeta - -logger = logging.getLogger(__name__) - - -@dataclasses.dataclass(init=False) -class DFPMessageMeta(MessageMeta, cpp_class=None): - """ - This class extends MessageMeta to also hold userid corresponding to batched metadata. - - Parameters - ---------- - df : pandas.DataFrame - Input rows in dataframe. - user_id : str - User id. - - """ - user_id: str - - def __init__(self, df: pd.DataFrame, user_id: str) -> None: - super().__init__(df) - self.user_id = user_id diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_inference.py b/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_inference.py index 48e6e03568..4ad65d501c 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_inference.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_inference.py @@ -24,10 +24,10 @@ import cudf from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.utils.module_ids import MORPHEUS_MODULE_NAMESPACE from morpheus.utils.module_utils import register_module -from ..messages.dfp_message_meta import DFPMessageMeta from ..utils.module_ids import DFP_INFERENCE logger = logging.getLogger(f"morpheus.{__name__}") @@ -112,11 +112,11 @@ def process_task(control_message: ControlMessage) -> ControlMessage: output_df = cudf.concat([payload.df, results_df[results_cols]], axis=1) # Create an output message to allow setting meta + meta = MessageMeta(output_df) + meta.set_data('model_version', f"{model_cache.reg_model_name}:{model_cache.reg_model_version}") output_message = ControlMessage() - output_message.payload(DFPMessageMeta(output_df, user_id=user_id)) - - output_message.payload().set_data('model_version', - f"{model_cache.reg_model_name}:{model_cache.reg_model_version}") + output_message.payload(meta) + output_message.set_metadata("user_id", user_id) if logger.isEnabledFor(logging.DEBUG): load_model_duration = (post_model_time - start_time) * 1000.0 diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py b/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py index 0ea7283d03..d77c8ecc95 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py @@ -21,11 +21,11 @@ import cudf from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.models.dfencoder import AutoEncoder from morpheus.utils.module_ids import MORPHEUS_MODULE_NAMESPACE from morpheus.utils.module_utils import register_module -from ..messages.dfp_message_meta import DFPMessageMeta from ..utils.module_ids import DFP_TRAINING logger = logging.getLogger(f"morpheus.{__name__}") @@ -98,13 +98,13 @@ def on_data(control_message: ControlMessage) -> list[ControlMessage]: model.fit(train_df, epochs=epochs, validation_data=validation_df, run_validation=run_validation) logger.debug("Training AE model for user: '%s'... Complete.", user_id) - dfp_mm = DFPMessageMeta(cudf.from_pandas(final_df), user_id=user_id) - + meta = MessageMeta(cudf.from_pandas(final_df)) output_message = ControlMessage() - output_message.payload(dfp_mm) + output_message.payload(meta) output_message.set_metadata("model", model) output_message.set_metadata("train_scores_mean", 0.0) output_message.set_metadata("train_scores_std", 1.0) + output_message.set_metadata("user_id", user_id) output_messages.append(output_message) diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py index 59b98a57df..3d8767f92e 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py @@ -22,12 +22,14 @@ import pandas as pd from mrc.core import operators as ops +import cudf + from morpheus.config import Config from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema -from ..messages.dfp_message_meta import DFPMessageMeta from ..utils.cached_user_window import CachedUserWindow from ..utils.logging_timer import log_time @@ -89,7 +91,7 @@ def supports_cpp_node(self): def accepted_types(self) -> typing.Tuple: """Input types accepted by this stage.""" - return (DFPMessageMeta, ) + return (ControlMessage, ) def compute_schema(self, schema: StageSchema): schema.output_schema.set_type(ControlMessage) @@ -116,13 +118,13 @@ def _get_user_cache(self, user_id: str) -> typing.Generator[CachedUserWindow, No # # When it returns, make sure to save # user_cache.save() - def _build_window(self, message: DFPMessageMeta) -> ControlMessage: + def _build_window(self, message: ControlMessage) -> ControlMessage: - user_id = message.user_id + user_id = message.get_metadata('user_id') with self._get_user_cache(user_id) as user_cache: - incoming_df = message.get_data() + incoming_df = message.payload().get_data() # existing_df = user_cache.df if (not user_cache.append_dataframe(incoming_df=incoming_df)): @@ -162,12 +164,12 @@ def _build_window(self, message: DFPMessageMeta) -> ControlMessage: # Otherwise return a new message response_msg = ControlMessage() - response_msg.payload(DFPMessageMeta(df=train_df, user_id=user_id)) + response_msg.payload(MessageMeta(df=cudf.DataFrame(train_df))) response_msg.set_metadata("user_id", user_id) return response_msg - def on_data(self, message: DFPMessageMeta) -> ControlMessage: + def on_data(self, message: ControlMessage) -> ControlMessage: """ Emits a new message containing the rolling window for the user if and only if the history requirments are met, returns `None` otherwise. @@ -181,10 +183,10 @@ def on_data(self, message: DFPMessageMeta) -> ControlMessage: log_info.set_log( ("Rolling window complete for %s in {duration:0.2f} ms. " "Input: %s rows from %s to %s. Output: %s rows from %s to %s"), - message.user_id, - len(message.df), - message.df[self._config.ae.timestamp_column_name].min(), - message.df[self._config.ae.timestamp_column_name].max(), + message.get_metadata('user_id'), + len(message.payload().df), + message.payload().df[self._config.ae.timestamp_column_name].min(), + message.payload().df[self._config.ae.timestamp_column_name].max(), result.payload().count, result.payload().get_data(self._config.ae.timestamp_column_name).min(), result.payload().get_data(self._config.ae.timestamp_column_name).max(), diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py index 9a6a448bd5..ba60f2da0a 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py @@ -14,7 +14,6 @@ """Split messages into individual users and generic messages.""" import logging -import typing import mrc import numpy as np @@ -24,11 +23,12 @@ import cudf from morpheus.config import Config +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema from morpheus.utils.type_aliases import DataFrameType -from ..messages.dfp_message_meta import DFPMessageMeta from ..utils.logging_timer import log_time logger = logging.getLogger(f"morpheus.{__name__}") @@ -60,8 +60,8 @@ def __init__(self, c: Config, include_generic: bool, include_individual: bool, - skip_users: typing.List[str] = None, - only_users: typing.List[str] = None): + skip_users: list[str] = None, + only_users: list[str] = None): super().__init__(c) self._include_generic = include_generic @@ -70,25 +70,25 @@ def __init__(self, self._only_users = only_users if only_users is not None else [] # Map of user ids to total number of messages. Keeps indexes monotonic and increasing per user - self._user_index_map: typing.Dict[str, int] = {} + self._user_index_map: dict[str, int] = {} @property def name(self) -> str: """Stage name.""" return "dfp-split-users" - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: """Whether this stage supports a C++ node.""" return False - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> tuple: """Input types accepted by this stage.""" return (cudf.DataFrame, pd.DataFrame) def compute_schema(self, schema: StageSchema): - schema.output_schema.set_type(DFPMessageMeta) + schema.output_schema.set_type(ControlMessage) - def extract_users(self, message: DataFrameType) -> typing.List[DFPMessageMeta]: + def extract_users(self, message: DataFrameType) -> list[ControlMessage]: """ Extract users from a message, splitting the incoming data into unique messages on a per-user basis, and potentially filtering data based on the user. @@ -102,7 +102,7 @@ def extract_users(self, message: DataFrameType) -> typing.List[DFPMessageMeta]: # Convert to pandas because cudf is slow at this message = message.to_pandas() - split_dataframes: typing.Dict[str, pd.DataFrame] = {} + split_dataframes: dict[str, pd.DataFrame] = {} # If we are skipping users, do that here if (len(self._skip_users) > 0): @@ -124,7 +124,7 @@ def extract_users(self, message: DataFrameType) -> typing.List[DFPMessageMeta]: user_df in message.groupby(self._config.ae.userid_column_name, sort=False) }) - output_messages: typing.List[DFPMessageMeta] = [] + output_messages: list[ControlMessage] = [] for user_id in sorted(split_dataframes.keys()): @@ -139,7 +139,11 @@ def extract_users(self, message: DataFrameType) -> typing.List[DFPMessageMeta]: user_df.index = range(current_user_count, current_user_count + len(user_df)) self._user_index_map[user_id] = current_user_count + len(user_df) - output_messages.append(DFPMessageMeta(df=user_df, user_id=user_id)) + meta = MessageMeta(cudf.DataFrame(user_df)) + cm_msg = ControlMessage() + cm_msg.payload(meta) + cm_msg.set_metadata("user_id", user_id) + output_messages.append(cm_msg) # logger.debug("Emitting dataframe for user '%s'. Start: %s, End: %s, Count: %s", # user, @@ -147,9 +151,8 @@ def extract_users(self, message: DataFrameType) -> typing.List[DFPMessageMeta]: # df_user[self._config.ae.timestamp_column_name].max(), # df_user[self._config.ae.timestamp_column_name].count()) - rows_per_user = [len(x.df) for x in output_messages] - if (len(output_messages) > 0): + rows_per_user = [len(x.payload().df) for x in output_messages] log_info.set_log( ("Batch split users complete. Input: %s rows from %s to %s. " "Output: %s users, rows/user min: %s, max: %s, avg: %.2f. Duration: {duration:.2f} ms"), From f18a81e6c73cb0abbdd48d8e5c3062f7e3229dec Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 16 Sep 2024 17:03:07 -0700 Subject: [PATCH 210/347] Set DFP pipelines to run in GPU mode --- .../morpheus/dfp/stages/dfp_rolling_window_stage.py | 2 +- .../production/morpheus/dfp/utils/config_generator.py | 4 ---- .../production/morpheus/dfp_azure_pipeline.py | 4 ---- .../production/morpheus/dfp_duo_pipeline.py | 4 ---- .../production/morpheus/notebooks/dfp_azure_inference.ipynb | 4 ---- .../production/morpheus/notebooks/dfp_duo_inference.ipynb | 4 ---- .../production/morpheus/notebooks/dfp_duo_training.ipynb | 4 ---- 7 files changed, 1 insertion(+), 25 deletions(-) diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py index 3d8767f92e..9acc1ea151 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_rolling_window_stage.py @@ -124,7 +124,7 @@ def _build_window(self, message: ControlMessage) -> ControlMessage: with self._get_user_cache(user_id) as user_cache: - incoming_df = message.payload().get_data() + incoming_df = message.payload().get_data().to_pandas() # existing_df = user_cache.df if (not user_cache.append_dataframe(incoming_df=incoming_df)): diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py b/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py index 927c9bfbb7..3b6d6a86c9 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py @@ -24,7 +24,6 @@ from morpheus.cli.utils import load_labels_file from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.messages import ControlMessage from morpheus.utils.module_ids import MORPHEUS_MODULE_NAMESPACE @@ -176,9 +175,6 @@ def generate_ae_config(source: str, use_cpp: bool = False, num_threads: int = len(os.sched_getaffinity(0))): config = Config() - - CppConfig.set_should_use_cpp(use_cpp) - config.num_threads = num_threads if pipeline_batch_size > 0: diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py index 81de60094d..cd7bb43105 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_azure_pipeline.py @@ -44,7 +44,6 @@ from morpheus.common import FilterSource from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage @@ -230,9 +229,6 @@ def run_pipeline(train_users, logger.info("Tracking URI: %s", mlflow.get_tracking_uri()) config = Config() - - CppConfig.set_should_use_cpp(False) - config.num_threads = len(os.sched_getaffinity(0)) config.ae = ConfigAutoEncoder() diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py index 4ba43aced1..f1153e9bb0 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py @@ -44,7 +44,6 @@ from morpheus.common import FilterSource from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage @@ -227,9 +226,6 @@ def run_pipeline(train_users, logger.info("Tracking URI: %s", mlflow.get_tracking_uri()) config = Config() - - CppConfig.set_should_use_cpp(False) - config.num_threads = len(os.sched_getaffinity(0)) config.ae = ConfigAutoEncoder() diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb index 8e5413f71c..9aaaa7c22d 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb @@ -65,7 +65,6 @@ "from morpheus.cli.utils import parse_log_level\n", "from morpheus.config import Config\n", "from morpheus.config import ConfigAutoEncoder\n", - "from morpheus.config import CppConfig\n", "from morpheus.pipeline import LinearPipeline\n", "from morpheus.stages.output.write_to_file_stage import WriteToFileStage\n", "from morpheus.utils.column_info import ColumnInfo\n", @@ -194,9 +193,6 @@ "configure_logging(log_level=logging.DEBUG)\n", "\n", "config = Config()\n", - "\n", - "CppConfig.set_should_use_cpp(False)\n", - "\n", "config.num_threads = len(os.sched_getaffinity(0))\n", "\n", "config.ae = ConfigAutoEncoder()\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb index 8bb35d5f78..e36820d869 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb @@ -63,7 +63,6 @@ "from morpheus.cli.utils import parse_log_level\n", "from morpheus.config import Config\n", "from morpheus.config import ConfigAutoEncoder\n", - "from morpheus.config import CppConfig\n", "from morpheus.pipeline import LinearPipeline\n", "from morpheus.stages.output.write_to_file_stage import WriteToFileStage\n", "from morpheus.utils.column_info import BoolColumn\n", @@ -193,9 +192,6 @@ "configure_logging(log_level=logging.DEBUG)\n", "\n", "config = Config()\n", - "\n", - "CppConfig.set_should_use_cpp(False)\n", - "\n", "config.num_threads = len(os.sched_getaffinity(0))\n", "\n", "config.ae = ConfigAutoEncoder()\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb index 1b1837d3e5..42b9773a4b 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb @@ -62,7 +62,6 @@ "from morpheus.cli.utils import parse_log_level\n", "from morpheus.config import Config\n", "from morpheus.config import ConfigAutoEncoder\n", - "from morpheus.config import CppConfig\n", "from morpheus.pipeline import LinearPipeline\n", "from morpheus.utils.column_info import BoolColumn\n", "from morpheus.utils.column_info import ColumnInfo\n", @@ -192,9 +191,6 @@ "configure_logging(log_level=logging.DEBUG)\n", "\n", "config = Config()\n", - "\n", - "CppConfig.set_should_use_cpp(False)\n", - "\n", "config.num_threads = len(os.sched_getaffinity(0))\n", "\n", "config.ae = ConfigAutoEncoder()\n", From b83f342160b96ff6124f815db0318187237fea32 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang <134643420+yczhang-nv@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:05:18 -0700 Subject: [PATCH 211/347] fix dfp_training --- .../production/morpheus/dfp/modules/dfp_training.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py b/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py index 0ea7283d03..388eba25ec 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/modules/dfp_training.py @@ -102,6 +102,7 @@ def on_data(control_message: ControlMessage) -> list[ControlMessage]: output_message = ControlMessage() output_message.payload(dfp_mm) + output_message.set_metadata("user_id", user_id) output_message.set_metadata("model", model) output_message.set_metadata("train_scores_mean", 0.0) output_message.set_metadata("train_scores_std", 1.0) From 169639d6550ceea4ee8b978a8969d147bab66834 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 08:58:20 -0700 Subject: [PATCH 212/347] Updating DFP tests --- .../digital_fingerprinting/conftest.py | 25 +++--- .../test_dfp_mlflow_model_writer.py | 7 +- .../test_dfp_rolling_window_stage.py | 79 ++++++++----------- .../test_dfp_training.py | 8 +- 4 files changed, 52 insertions(+), 67 deletions(-) diff --git a/tests/examples/digital_fingerprinting/conftest.py b/tests/examples/digital_fingerprinting/conftest.py index 098c7bff37..44992d0e06 100644 --- a/tests/examples/digital_fingerprinting/conftest.py +++ b/tests/examples/digital_fingerprinting/conftest.py @@ -88,25 +88,22 @@ def dfp_prod_in_sys_path( sys.path.append(example_dir) -@pytest.fixture(name="dfp_message_meta") -def dfp_message_meta_fixture(config, dataset_pandas: DatasetManager): - # TODO: This should be a cudf DataFrame, and emit a control message - import pandas as pd - from dfp.messages.dfp_message_meta import DFPMessageMeta +@pytest.fixture +def control_message(config, dataset_cudf: DatasetManager): + import cudf + + from morpheus.messages import ControlMessage + from morpheus.messages import MessageMeta user_id = 'test_user' - df = dataset_pandas['filter_probs.csv'] - df[config.ae.timestamp_column_name] = pd.to_datetime([1683054498 + i for i in range(0, len(df) * 30, 30)], unit='s') + df = dataset_cudf['filter_probs.csv'] + timestamps = [1683054498 + i for i in range(0, len(df) * 30, 30)] + df[config.ae.timestamp_column_name] = cudf.to_datetime(timestamps, unit='s') df[config.ae.userid_column_name] = user_id - yield DFPMessageMeta(df, user_id) - -@pytest.fixture -def control_message(dfp_message_meta): - from morpheus.messages import ControlMessage message = ControlMessage() - message.payload(dfp_message_meta) - message.set_metadata("user_id", dfp_message_meta.user_id) + message.payload(MessageMeta(df)) + message.set_metadata("user_id", user_id) message.set_metadata("model", mock.MagicMock()) yield message diff --git a/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py b/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py index 49fc093ba4..02508eb7b3 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py +++ b/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py @@ -27,6 +27,7 @@ from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.pipeline.single_port_stage import SinglePortStage MockedRequests = namedtuple("MockedRequests", ["get", "patch", "response"]) @@ -238,7 +239,6 @@ def test_on_data( databricks_env: dict, databricks_permissions: dict, tracking_uri: str): - from dfp.messages.dfp_message_meta import DFPMessageMeta from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage from dfp.stages.dfp_mlflow_model_writer import conda_env @@ -272,11 +272,10 @@ def test_on_data( mock_model.prepare_df.return_value = df mock_model.get_anomaly_score.return_value = pd.Series(float(i) for i in range(len(df))) - meta = DFPMessageMeta(df, 'Account-123456789') msg = ControlMessage() - msg.payload(meta) + msg.payload(MessageMeta(df)) msg.set_metadata("model", mock_model) - msg.set_metadata("user_id", meta.user_id) + msg.set_metadata("user_id", 'Account-123456789') stage = DFPMLFlowModelWriterStage(config, databricks_permissions=databricks_permissions, timeout=10) assert stage._controller.on_data(msg) is msg # Should be a pass-thru diff --git a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py index 01504d7d47..23c42ea727 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py @@ -21,9 +21,15 @@ from _utils.dataset_manager import DatasetManager from morpheus.config import Config +from morpheus.messages import ControlMessage from morpheus.pipeline.single_port_stage import SinglePortStage +@pytest.fixture(name="train_df") +def train_df_fixture(control_message: ControlMessage) -> pd.DataFrame: + return control_message.payload().copy_dataframe().to_pandas() + + def build_mock_user_cache(user_id: str = 'test_user', train_df: pd.DataFrame = None, count: int = 10, @@ -81,90 +87,74 @@ def test_get_user_cache_miss(config: Config): assert results2 is results -def test_build_window_no_new( - config: Config, - dfp_message_meta: "DFPMessageMeta" # noqa: F821 -): +def test_build_window_no_new(config: Config, control_message: ControlMessage): from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') mock_cache = build_mock_user_cache() mock_cache.append_dataframe.return_value = False - stage._user_cache_map[dfp_message_meta.user_id] = mock_cache - assert stage._build_window(dfp_message_meta) is None + stage._user_cache_map[control_message.get_metadata('user_id')] = mock_cache + assert stage._build_window(control_message) is None -def test_build_window_not_enough_data( - config: Config, - dfp_message_meta: "DFPMessageMeta" # noqa: F821 -): +def test_build_window_not_enough_data(config: Config, control_message: ControlMessage): from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') mock_cache = build_mock_user_cache(count=3) - stage._user_cache_map[dfp_message_meta.user_id] = mock_cache - assert stage._build_window(dfp_message_meta) is None + stage._user_cache_map[control_message.get_metadata('user_id')] = mock_cache + assert stage._build_window(control_message) is None -def test_build_window_min_increment( - config: Config, - dfp_message_meta: "DFPMessageMeta" # noqa: F821 -): +def test_build_window_min_increment(config: Config, control_message: ControlMessage): from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') mock_cache = build_mock_user_cache(count=5, total_count=30, last_train_count=25) - stage._user_cache_map[dfp_message_meta.user_id] = mock_cache - assert stage._build_window(dfp_message_meta) is None + stage._user_cache_map[control_message.get_metadata('user_id')] = mock_cache + assert stage._build_window(control_message) is None -def test_build_window_invalid( - config: Config, - dfp_message_meta: "DFPMessageMeta" # noqa: F821 -): +def test_build_window_invalid(config: Config, control_message: ControlMessage, train_df: pd.DataFrame): from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') - train_df = dfp_message_meta.copy_dataframe() # exact values not important so long as they don't match the actual hash train_df['_row_hash'] = [-1 for _ in range(len(train_df))] mock_cache = build_mock_user_cache(train_df=train_df) - stage._user_cache_map[dfp_message_meta.user_id] = mock_cache + stage._user_cache_map[control_message.get_metadata('user_id')] = mock_cache with pytest.raises(RuntimeError): - stage._build_window(dfp_message_meta) + stage._build_window(control_message) -def test_build_window_overlap( - config: Config, - dfp_message_meta: "DFPMessageMeta" # noqa: F821 -): +def test_build_window_overlap(config: Config, control_message: ControlMessage, train_df: pd.DataFrame): from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') # Create an overlap - train_df = dfp_message_meta.copy_dataframe()[-5:] + train_df = train_df[-5:] train_df['_row_hash'] = pd.util.hash_pandas_object(train_df, index=False) mock_cache = build_mock_user_cache(train_df=train_df) - stage._user_cache_map[dfp_message_meta.user_id] = mock_cache + stage._user_cache_map[control_message.get_metadata('user_id')] = mock_cache with pytest.raises(RuntimeError): - stage._build_window(dfp_message_meta) + stage._build_window(control_message) @pytest.mark.parametrize('use_on_data', [True, False]) -def test_build_window( - config: Config, - use_on_data: bool, - dfp_message_meta: "DFPMessageMeta", # noqa: F821 - dataset_pandas: DatasetManager): +def test_build_window(config: Config, + use_on_data: bool, + control_message: ControlMessage, + dataset_pandas: DatasetManager, + train_df: pd.DataFrame): from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage from morpheus.messages import ControlMessage @@ -172,19 +162,18 @@ def test_build_window( stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') # Create an overlap - train_df = dfp_message_meta.copy_dataframe() train_df['_row_hash'] = pd.util.hash_pandas_object(train_df, index=False) mock_cache = build_mock_user_cache(train_df=train_df) - stage._user_cache_map[dfp_message_meta.user_id] = mock_cache + stage._user_cache_map[control_message.get_metadata('user_id')] = mock_cache # on_data is a thin wrapper around _build_window, results should be the same if use_on_data: - msg = stage.on_data(dfp_message_meta) + out_msg = stage.on_data(control_message) else: - msg = stage._build_window(dfp_message_meta) + out_msg = stage._build_window(control_message) - assert isinstance(msg, ControlMessage) - assert msg.get_metadata("user_id") == dfp_message_meta.user_id - assert msg.payload().count == len(dataset_pandas['filter_probs.csv']) - dataset_pandas.assert_df_equal(msg.payload().df, train_df) + assert isinstance(out_msg, ControlMessage) + assert out_msg.get_metadata("user_id") == control_message.get_metadata('user_id') + assert out_msg.payload().count == len(dataset_pandas['filter_probs.csv']) + dataset_pandas.assert_df_equal(out_msg.payload().df, train_df) diff --git a/tests/examples/digital_fingerprinting/test_dfp_training.py b/tests/examples/digital_fingerprinting/test_dfp_training.py index 60cd545eab..d69fd315a6 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_training.py +++ b/tests/examples/digital_fingerprinting/test_dfp_training.py @@ -21,6 +21,8 @@ from _utils import TEST_DIRS from _utils.dataset_manager import DatasetManager from morpheus.config import Config +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus.pipeline.single_port_stage import SinglePortStage @@ -50,7 +52,6 @@ def test_on_data(mock_train_test_split: mock.MagicMock, config: Config, dataset_pandas: DatasetManager, validation_size: float): - from dfp.messages.dfp_message_meta import DFPMessageMeta from dfp.stages.dfp_training import DFPTraining from morpheus.messages import ControlMessage @@ -64,10 +65,9 @@ def test_on_data(mock_train_test_split: mock.MagicMock, mock_validation_df = mock.MagicMock() mock_train_test_split.return_value = (train_df, mock_validation_df) - meta = DFPMessageMeta(df, 'Account-123456789') msg = ControlMessage() - msg.payload(meta) - msg.set_metadata("user_id", meta.user_id) + msg.payload(MessageMeta(df)) + msg.set_metadata("user_id", 'Account-123456789') stage = DFPTraining(config, validation_size=validation_size) results = stage.on_data(msg) From b0c65e146a85f6fd4d449828709da594d6fef125 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 10:22:46 -0700 Subject: [PATCH 213/347] Refactor to avoid costly list comprehension --- .../production/morpheus/dfp/stages/dfp_split_users_stage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py index ba60f2da0a..cd687e0a2c 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_split_users_stage.py @@ -125,6 +125,7 @@ def extract_users(self, message: DataFrameType) -> list[ControlMessage]: }) output_messages: list[ControlMessage] = [] + rows_per_user: list[int] = [] for user_id in sorted(split_dataframes.keys()): @@ -139,7 +140,8 @@ def extract_users(self, message: DataFrameType) -> list[ControlMessage]: user_df.index = range(current_user_count, current_user_count + len(user_df)) self._user_index_map[user_id] = current_user_count + len(user_df) - meta = MessageMeta(cudf.DataFrame(user_df)) + rows_per_user.append(len(user_df)) + meta = MessageMeta(cudf.DataFrame.from_pandas(user_df)) cm_msg = ControlMessage() cm_msg.payload(meta) cm_msg.set_metadata("user_id", user_id) @@ -152,7 +154,6 @@ def extract_users(self, message: DataFrameType) -> list[ControlMessage]: # df_user[self._config.ae.timestamp_column_name].count()) if (len(output_messages) > 0): - rows_per_user = [len(x.payload().df) for x in output_messages] log_info.set_log( ("Batch split users complete. Input: %s rows from %s to %s. " "Output: %s users, rows/user min: %s, max: %s, avg: %.2f. Duration: {duration:.2f} ms"), From 7b01c2f4a5e58ba3693a49ec313e44891782edd9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 10:22:56 -0700 Subject: [PATCH 214/347] Fix test --- .../test_dfp_split_users_stage.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py b/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py index 8189df73fe..2cf9676a17 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py @@ -17,7 +17,9 @@ import os import typing import warnings +from collections import defaultdict +import pandas as pd import pytest from _utils import TEST_DIRS @@ -59,7 +61,7 @@ def test_constructor(config: Config): [[], ['WENDY.HUERTA@Brooklyn-dynamic.edu'], ['terrietahon@planner-viral.com', 'SAMUEL.DAVIS@transition-high-life.com']]) def test_extract_users(config: Config, - dataset: DatasetManager, + dataset_pandas: DatasetManager, include_generic: bool, include_individual: bool, skip_users: typing.List[str], @@ -71,10 +73,11 @@ def test_extract_users(config: Config, input_file = os.path.join(TEST_DIRS.tests_data_dir, "examples/developer_guide/email_with_addresses_first_10.jsonlines") - df = dataset[input_file] + df = dataset_pandas[input_file] all_data = [] - expected_data = {} + expected_data = defaultdict(list) + ts_col = config.ae.timestamp_column_name with open(input_file, encoding='UTF-8') as fh: for line in fh: json_data = json.loads(line) @@ -85,11 +88,13 @@ def test_extract_users(config: Config, if len(only_users) > 0 and user_id not in only_users: continue + json_data[ts_col] = pd.to_datetime(json_data[ts_col], unit='s') + if include_generic: all_data.append(json_data) if include_individual: - expected_data[user_id] = [json_data] + expected_data[user_id].append(json_data) if include_generic: expected_data[config.ae.fallback_username] = all_data @@ -114,9 +119,11 @@ def test_extract_users(config: Config, # Add one for the generic user assert len(results) == len(expected_data) for msg in results: - assert len(msg.df) == len(expected_data[msg.user_id]) - if msg.user_id != config.ae.fallback_username: - assert msg.df.iloc[0].to_dict() == expected_data[msg.user_id][0] + actual_df = msg.payload().df + user_id = msg.get_metadata('user_id') + assert len(actual_df) == len(expected_data[user_id]) + if user_id != config.ae.fallback_username: + assert actual_df.to_dict('records') == expected_data[user_id] def test_extract_users_none_to_empty(config: Config): From e8600d54a54a6c4ed5c8991cde364aa8722804bd Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 10:55:01 -0700 Subject: [PATCH 215/347] Fix test --- .../test_dfp_split_users_stage.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py b/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py index 2cf9676a17..e844c59c53 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py @@ -19,13 +19,13 @@ import warnings from collections import defaultdict -import pandas as pd import pytest from _utils import TEST_DIRS from _utils.dataset_manager import DatasetManager from morpheus.config import Config from morpheus.pipeline.single_port_stage import SinglePortStage +from morpheus.utils.type_utils import get_df_pkg_from_obj def test_constructor(config: Config): @@ -61,7 +61,7 @@ def test_constructor(config: Config): [[], ['WENDY.HUERTA@Brooklyn-dynamic.edu'], ['terrietahon@planner-viral.com', 'SAMUEL.DAVIS@transition-high-life.com']]) def test_extract_users(config: Config, - dataset_pandas: DatasetManager, + dataset: DatasetManager, include_generic: bool, include_individual: bool, skip_users: typing.List[str], @@ -69,15 +69,24 @@ def test_extract_users(config: Config, from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage config.ae.userid_column_name = "From" config.ae.fallback_username = "testy_testerson" + ts_col = config.ae.timestamp_column_name input_file = os.path.join(TEST_DIRS.tests_data_dir, "examples/developer_guide/email_with_addresses_first_10.jsonlines") - df = dataset_pandas[input_file] + df = dataset[input_file] + df_pkg = get_df_pkg_from_obj(df) + + # When the file is read using pandas (as is the case in the actual DFP pipeline), the timestamp column is + # automatically converted to datetime objects. However cuDF doesn't do this and the column will contain integers. + # When `dataset` is returning pandas DFs this might still be the case if `input_file` is first read using cuDF and + # cached by the DatasetManager and then converted to pandas. + if df[ts_col].dtype == 'int64': + df[ts_col] = df_pkg.to_datetime(df[ts_col], unit='s') all_data = [] expected_data = defaultdict(list) - ts_col = config.ae.timestamp_column_name + with open(input_file, encoding='UTF-8') as fh: for line in fh: json_data = json.loads(line) @@ -88,7 +97,7 @@ def test_extract_users(config: Config, if len(only_users) > 0 and user_id not in only_users: continue - json_data[ts_col] = pd.to_datetime(json_data[ts_col], unit='s') + json_data[ts_col] = df_pkg.to_datetime(json_data[ts_col], unit='s') if include_generic: all_data.append(json_data) From 511dbdbc4053abe99e508a4a339cb9d5880fbe5a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 11:44:42 -0700 Subject: [PATCH 216/347] Move timezone truncation from the TrainAEStage to AutoencoderSourceStage --- .../morpheus/stages/input/autoencoder_source_stage.py | 3 +++ python/morpheus/morpheus/stages/preprocess/train_ae_stage.py | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py index 5c734f700d..97f3afb1aa 100644 --- a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py +++ b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py @@ -300,6 +300,9 @@ def _build_message(self, user_dataframes: dict[str, pd.DataFrame]) -> list[Contr # If we're in GPU mode we need to convert to cuDF if not isinstance(user_df, self._df_class): + for col in [col for col in user_df.columns if isinstance(user_df[col].dtype, pd.DatetimeTZDtype)]: + user_df[col] = user_df[col].dt.tz_convert(None) + user_df = self._df_class(user_df) # Now make a message with the user name in metadata diff --git a/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py b/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py index c7cde34110..c765aa7f20 100644 --- a/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/train_ae_stage.py @@ -324,10 +324,6 @@ def on_next(full_message: ControlMessage): # cuDF does not yet support timezone-aware datetimes # Remove timezone information from pd.DatetimeTZDtype columns meta = full_message.payload() - with meta.mutable_dataframe() as df: - for col in [col for col in df.columns if isinstance(df[col].dtype, pd.DatetimeTZDtype)]: - df[col] = df[col].dt.tz_convert(None) - to_send = [] # Now split into batches From c7cf6873fd85357377f4fb49c8f4f117fd726afc Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 11:44:58 -0700 Subject: [PATCH 217/347] Fix dfp tests --- tests/test_dfp.py | 19 ++++++++----------- tests/test_dfp_kafka.py | 11 +++++------ 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/tests/test_dfp.py b/tests/test_dfp.py index 0253c362ff..fa51270930 100755 --- a/tests/test_dfp.py +++ b/tests/test_dfp.py @@ -23,12 +23,12 @@ from _utils import TEST_DIRS from _utils import calc_error_val +from morpheus.common import TypeId from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.config import PipelineModes from morpheus.messages import ControlMessage from morpheus.messages.message_meta import MessageMeta -from morpheus.messages.message_meta import UserMessageMeta from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.inference.auto_encoder_inference_stage import AutoEncoderInferenceStage @@ -44,9 +44,8 @@ # End-to-end test intended to imitate the DFP validation test -@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -98,7 +97,7 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, config: Config, tmp_path: str, morph sort_glob=True)) pipe.add_stage(preprocess_ae_stage.PreprocessAEStage(config)) pipe.add_stage(AutoEncoderInferenceStage(config)) - pipe.add_stage(AddScoresStage(config)) + pipe.add_stage(AddScoresStage(config, probs_type=TypeId.FLOAT64)) pipe.add_stage( TimeSeriesStage(config, resolution="1m", @@ -131,9 +130,8 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, config: Config, tmp_path: str, morph assert results.diff_rows == 0 -@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -183,7 +181,7 @@ def test_dfp_user123(mock_ae: mock.MagicMock, config: Config, tmp_path: str, mor sort_glob=True)) pipe.add_stage(preprocess_ae_stage.PreprocessAEStage(config)) pipe.add_stage(AutoEncoderInferenceStage(config)) - pipe.add_stage(AddScoresStage(config)) + pipe.add_stage(AddScoresStage(config, probs_type=TypeId.FLOAT64)) pipe.add_stage( TimeSeriesStage(config, resolution="1m", @@ -215,9 +213,8 @@ def test_dfp_user123(mock_ae: mock.MagicMock, config: Config, tmp_path: str, mor assert results.diff_rows == 0 -@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.slow -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -258,7 +255,7 @@ def test_dfp_user123_multi_segment(mock_ae: mock.MagicMock, config: Config, tmp_ pipe = LinearPipeline(config) pipe.set_source(CloudTrailSourceStage(config, input_glob=input_glob, sort_glob=True)) - pipe.add_segment_boundary(UserMessageMeta) # Boundary 1 + pipe.add_segment_boundary(ControlMessage) # Boundary 1 pipe.add_stage( train_ae_stage.TrainAEStage( config, @@ -271,7 +268,7 @@ def test_dfp_user123_multi_segment(mock_ae: mock.MagicMock, config: Config, tmp_ pipe.add_segment_boundary(ControlMessage) # Boundary 3 pipe.add_stage(AutoEncoderInferenceStage(config)) pipe.add_segment_boundary(ControlMessage) # Boundary 4 - pipe.add_stage(AddScoresStage(config)) + pipe.add_stage(AddScoresStage(config, probs_type=TypeId.FLOAT64)) pipe.add_segment_boundary(ControlMessage) # Boundary 5 pipe.add_stage( TimeSeriesStage(config, diff --git a/tests/test_dfp_kafka.py b/tests/test_dfp_kafka.py index 63571ebb17..98ef108350 100755 --- a/tests/test_dfp_kafka.py +++ b/tests/test_dfp_kafka.py @@ -27,6 +27,7 @@ from _utils.dataset_manager import DatasetManager from _utils.kafka import KafkaTopics from morpheus.cli import commands +from morpheus.common import TypeId from morpheus.config import Config from morpheus.config import ConfigAutoEncoder from morpheus.config import PipelineModes @@ -48,10 +49,9 @@ from kafka import KafkaConsumer -@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.kafka @pytest.mark.slow -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode @pytest.mark.reload_modules([commands, preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules", "loglevel_debug") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -104,7 +104,7 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, sort_glob=True)) pipe.add_stage(preprocess_ae_stage.PreprocessAEStage(config)) pipe.add_stage(AutoEncoderInferenceStage(config)) - pipe.add_stage(AddScoresStage(config)) + pipe.add_stage(AddScoresStage(config, probs_type=TypeId.FLOAT64)) pipe.add_stage( TimeSeriesStage(config, resolution="1m", @@ -153,10 +153,9 @@ def test_dfp_roleg(mock_ae: mock.MagicMock, assert results['diff_rows'] == 0 -@pytest.mark.skip(reason="Need to determine what to do with GPU pipelines that are Python only") @pytest.mark.kafka @pytest.mark.slow -@pytest.mark.cpu_mode +@pytest.mark.gpu_mode @pytest.mark.reload_modules([preprocess_ae_stage, train_ae_stage]) @pytest.mark.usefixtures("reload_modules", "loglevel_debug") @mock.patch('morpheus.stages.preprocess.train_ae_stage.AutoEncoder') @@ -208,7 +207,7 @@ def test_dfp_user123(mock_ae: mock.MagicMock, sort_glob=True)) pipe.add_stage(preprocess_ae_stage.PreprocessAEStage(config)) pipe.add_stage(AutoEncoderInferenceStage(config)) - pipe.add_stage(AddScoresStage(config)) + pipe.add_stage(AddScoresStage(config, probs_type=TypeId.FLOAT64)) pipe.add_stage( TimeSeriesStage(config, resolution="1m", From 379457e5df5b3157d97c65f79b0dfaf46b232ca7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 11:50:40 -0700 Subject: [PATCH 218/347] Remove message_type --- examples/cpu_only/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index c0d0f966d2..d87545c9c2 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -98,8 +98,7 @@ def print_msg(msg: typing.Any) -> typing.Any: pipeline.add_stage(print_msg(config)) - # TODO: Remove if PR #1803 is merged first - pipeline.add_stage(DeserializeStage(config, message_type=ControlMessage)) + pipeline.add_stage(DeserializeStage(config)) pipeline.add_stage(MonitorStage(config, description="deserialize")) From 53cbbef308d5c558669cbf347b5c1fa340df6e93 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 12:58:49 -0700 Subject: [PATCH 219/347] Lint cleanups --- .../abp_pcap_preprocessing.py | 1 - .../2_2_rabbitmq/rabbitmq_source_stage.py | 11 ++-- .../rabbitmq_source_stage_deco.py | 6 ++- python/morpheus/morpheus/utils/type_utils.py | 50 ++++++++++++------- .../test_dfp_training.py | 2 - .../test_create_features.py | 1 - tests/test_cli.py | 1 - tests/test_tensor_memory.py | 1 - 8 files changed, 44 insertions(+), 29 deletions(-) diff --git a/examples/abp_pcap_detection/abp_pcap_preprocessing.py b/examples/abp_pcap_detection/abp_pcap_preprocessing.py index 62c1518629..ebc0392217 100644 --- a/examples/abp_pcap_detection/abp_pcap_preprocessing.py +++ b/examples/abp_pcap_detection/abp_pcap_preprocessing.py @@ -16,7 +16,6 @@ from functools import partial import cupy as cp -import mrc import numpy as np import cudf diff --git a/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage.py b/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage.py index 3ac84b0db9..d4192dae90 100644 --- a/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage.py +++ b/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage.py @@ -22,20 +22,20 @@ import pandas as pd import pika -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages.message_meta import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) @register_stage("from-rabbitmq") -class RabbitMQSourceStage(PreallocatorMixin, SingleOutputSource): +class RabbitMQSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): """ Source stage used to load messages from a RabbitMQ queue. @@ -77,6 +77,9 @@ def __init__(self, self._poll_interval = pd.Timedelta(poll_interval) + # This will return either cudf.DataFrame or pandas.DataFrame depending on the execution mode + self._df_pkg = get_df_pkg(config.execution_mode) + @property def name(self) -> str: return "from-rabbitmq" @@ -97,7 +100,7 @@ def source_generator(self) -> collections.abc.Iterator[MessageMeta]: if method_frame is not None: try: buffer = StringIO(body.decode("utf-8")) - df = cudf.io.read_json(buffer, orient='records', lines=True) + df = self._df_pkg.read_json(buffer, orient='records', lines=True) yield MessageMeta(df=df) except Exception as ex: logger.exception("Error occurred converting RabbitMQ message to Dataframe: %s", ex) diff --git a/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py b/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py index 33260a94f7..2488a847e6 100644 --- a/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py +++ b/examples/developer_guide/2_2_rabbitmq/rabbitmq_source_stage_deco.py @@ -24,7 +24,7 @@ from morpheus.config import ExecutionMode from morpheus.messages.message_meta import MessageMeta from morpheus.pipeline.stage_decorator import source -from morpheus.utils.type_aliases import DataFrameType +from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) @@ -65,13 +65,15 @@ def rabbitmq_source(host: str, poll_interval = pd.Timedelta(poll_interval) + df_pkg = get_df_pkg() + try: while True: (method_frame, _, body) = channel.basic_get(queue_name) if method_frame is not None: try: buffer = StringIO(body.decode("utf-8")) - df = cudf.io.read_json(buffer, orient='records', lines=True) + df = df_pkg.read_json(buffer, orient='records', lines=True) yield MessageMeta(df=df) except Exception as ex: logger.exception("Error occurred converting RabbitMQ message to Dataframe: %s", ex) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index daacfa9b09..f1892074d3 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -20,6 +20,7 @@ import numpy as np import pandas as pd +from morpheus.config import CppConfig from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameTypeStr @@ -191,6 +192,12 @@ def exec_mode_to_df_type_str(execution_mode: ExecutionMode) -> DataFrameTypeStr: return "pandas" +def cpp_mode_to_exec_mode() -> ExecutionMode: + if CppConfig.get_should_use_cpp(): + return ExecutionMode.GPU + return ExecutionMode.CPU + + def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: """ Return the appropriate DataFrame package based on the DataFrame type string. @@ -206,20 +213,22 @@ def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: @typing.overload -def get_df_pkg(selector: DataFrameTypeStr) -> types.ModuleType: +def get_df_pkg(selector: DataFrameTypeStr = None) -> types.ModuleType: ... @typing.overload -def get_df_pkg(selector: ExecutionMode) -> types.ModuleType: +def get_df_pkg(selector: ExecutionMode = None) -> types.ModuleType: ... -def get_df_pkg(selector: ExecutionMode | DataFrameTypeStr) -> types.ModuleType: +def get_df_pkg(selector: ExecutionMode | DataFrameTypeStr = None) -> types.ModuleType: """ Return the appropriate DataFrame package based on the execution mode. """ - if not isinstance(selector, ExecutionMode): + if selector is None: + execution_mode = cpp_mode_to_exec_mode() + elif not isinstance(selector, ExecutionMode): execution_mode = df_type_str_to_exec_mode(selector) else: execution_mode = selector @@ -231,28 +240,21 @@ def get_df_pkg(selector: ExecutionMode | DataFrameTypeStr) -> types.ModuleType: return pd -def get_array_pkg(execution_mode: ExecutionMode) -> types.ModuleType: - """ - Return the appropriate array package (CuPy for GPU, NumPy for CPU) based on the execution mode. - """ - - if execution_mode == ExecutionMode.GPU: - import cupy - return cupy - - return np +@typing.overload +def get_df_class(selector: DataFrameTypeStr = None) -> type[DataFrameType]: + ... @typing.overload -def get_df_class(df_type_str: DataFrameTypeStr) -> type[DataFrameType]: +def get_df_class(selector: ExecutionMode = None) -> type[DataFrameType]: ... -def get_df_class(execution_mode: ExecutionMode) -> type[DataFrameType]: +def get_df_class(selector: ExecutionMode | DataFrameTypeStr = None) -> type[DataFrameType]: """ Return the appropriate DataFrame class based on the execution mode. """ - df_pkg = get_df_pkg(execution_mode) + df_pkg = get_df_pkg(selector) return df_pkg.DataFrame @@ -280,3 +282,17 @@ def is_dataframe(obj: typing.Any) -> bool: """ df_pkg = get_df_pkg_from_obj(obj) return isinstance(obj, df_pkg.DataFrame) + + +def get_array_pkg(execution_mode: ExecutionMode = None) -> types.ModuleType: + """ + Return the appropriate array package (CuPy for GPU, NumPy for CPU) based on the execution mode. + """ + if execution_mode is None: + execution_mode = cpp_mode_to_exec_mode() + + if execution_mode == ExecutionMode.GPU: + import cupy + return cupy + + return np diff --git a/tests/examples/digital_fingerprinting/test_dfp_training.py b/tests/examples/digital_fingerprinting/test_dfp_training.py index d69fd315a6..e5194f3718 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_training.py +++ b/tests/examples/digital_fingerprinting/test_dfp_training.py @@ -54,8 +54,6 @@ def test_on_data(mock_train_test_split: mock.MagicMock, validation_size: float): from dfp.stages.dfp_training import DFPTraining - from morpheus.messages import ControlMessage - mock_ae.return_value = mock_ae input_file = os.path.join(TEST_DIRS.validation_data_dir, "dfp-cloudtrail-role-g-validation-data-input.csv") diff --git a/tests/examples/ransomware_detection/test_create_features.py b/tests/examples/ransomware_detection/test_create_features.py index ac6cd74311..9f5ffa9218 100644 --- a/tests/examples/ransomware_detection/test_create_features.py +++ b/tests/examples/ransomware_detection/test_create_features.py @@ -20,7 +20,6 @@ from unittest import mock import pandas as pd -import pytest from _utils import TEST_DIRS from _utils.dataset_manager import DatasetManager diff --git a/tests/test_cli.py b/tests/test_cli.py index d06c9a4dfb..0d108e0623 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,7 +33,6 @@ from morpheus.common import FilterSource from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.config import PipelineModes from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.inference.auto_encoder_inference_stage import AutoEncoderInferenceStage diff --git a/tests/test_tensor_memory.py b/tests/test_tensor_memory.py index 230a082796..7e8d3be655 100644 --- a/tests/test_tensor_memory.py +++ b/tests/test_tensor_memory.py @@ -23,7 +23,6 @@ import pytest from _utils import TEST_DIRS -from morpheus.config import Config from morpheus.messages.memory.inference_memory import InferenceMemory from morpheus.messages.memory.inference_memory import InferenceMemoryAE from morpheus.messages.memory.inference_memory import InferenceMemoryFIL From 56ac198f2b7b342ef148b67f4e393152af6cb6dd Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 13:00:35 -0700 Subject: [PATCH 220/347] WIP --- python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi index 89a097d83b..4ce3dd3269 100644 --- a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi +++ b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi @@ -4,7 +4,11 @@ import typing from cudf.core.dtypes import StructDtype import cudf -__all__ = ["StructDtype", "cudf"] +__all__ = [ + "StructDtype", + "cudf" +] -__pyx_capi__: dict # value = {'make_table_from_table_with_metadata': , 'make_table_from_table_info_data': , 'make_table_info_data_from_table': , 'data_from_table_view_indexed': } + +__pyx_capi__: dict # value = {'make_table_from_table_with_metadata': , 'make_table_from_table_info_data': , 'make_table_info_data_from_table': , 'data_from_table_view_indexed': } __test__ = {} From 073cac1e20aa89387da6ac22614ada94d6be54de Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 13:18:14 -0700 Subject: [PATCH 221/347] Lint fixes --- .../morpheus/morpheus/_lib/messages/module.cpp | 3 +-- .../morpheus/_lib/src/messages/control.cpp | 18 ++++++++++-------- .../morpheus/_lib/src/utilities/cudf_util.cpp | 2 +- .../morpheus/morpheus/_lib/stages/module.cpp | 1 - .../morpheus_llm/_lib/llm/module.cpp | 1 - 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index 1ebaa2d4cf..af559bde58 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -29,8 +29,7 @@ #include "morpheus/messages/raw_packet.hpp" #include "morpheus/objects/data_table.hpp" #include "morpheus/objects/mutable_table_ctx_mgr.hpp" -#include "morpheus/pybind11/json.hpp" // IWYU pragma: keep -#include "morpheus/utilities/cudf_util.hpp" +#include "morpheus/pybind11/json.hpp" // IWYU pragma: keep #include "morpheus/utilities/json_types.hpp" // for json_t #include "morpheus/utilities/string_util.hpp" #include "morpheus/version.hpp" diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index ecedcdfdcf..d20334c35a 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -19,21 +19,23 @@ #include "morpheus/messages/memory/tensor_memory.hpp" // for TensorMemory, TensorMemoryInterfaceProxy #include "morpheus/messages/meta.hpp" // for MessageMeta, MessageMetaInterfaceProxy +#include "morpheus/types.hpp" // for TensorIndex -#include // for to_lower_copy -#include // for COMPACT_GOOGLE_LOG_INFO, LogMessage, VLOG -#include // for basic_json, json_ref, iter_impl, operator<< -#include // IWYU pragma: keep -#include // for cast, object::cast -#include // for object, none, dict, isinstance, list, str, value_error, generic_item -#include // for casting to std::map -#include // for cast_from_pyobject +#include // for to_lower_copy +#include // for COMPACT_GOOGLE_LOG_INFO, LogMessage, VLOG +#include // for basic_json, json_ref, iter_impl, operator<< +#include // IWYU pragma: keep +#include // for cast, object::cast +#include // for object, none, dict, isinstance, list, str, value_error, generic_item +#include // IWYU pragma: keep +#include // for cast_from_pyobject #include // for optional, nullopt #include // for basic_ostream, operator<< #include // for regex_search, regex #include // for runtime_error #include // for pair +// IWYU pragma: no_include namespace py = pybind11; using namespace py::literals; diff --git a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp index 554edacca9..b60cd17bf1 100644 --- a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp +++ b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp @@ -25,7 +25,7 @@ #include #include -#include // for atomic +#include // for atomic #include // for getenv #include #include diff --git a/python/morpheus/morpheus/_lib/stages/module.cpp b/python/morpheus/morpheus/_lib/stages/module.cpp index f026d81fa3..266455177e 100644 --- a/python/morpheus/morpheus/_lib/stages/module.cpp +++ b/python/morpheus/morpheus/_lib/stages/module.cpp @@ -31,7 +31,6 @@ #include "morpheus/stages/preprocess_nlp.hpp" // for PreprocessNLPStage, PreprocessNLPStageInterfaceProxy #include "morpheus/stages/serialize.hpp" // for SerializeStage, SerializeStageInterfaceProxy #include "morpheus/stages/write_to_file.hpp" // for WriteToFileStage, WriteToFileStageInterfaceProxy -#include "morpheus/utilities/cudf_util.hpp" // for CudfHelper #include "morpheus/utilities/http_server.hpp" // for DefaultMaxPayloadSize #include "morpheus/version.hpp" // for morpheus_VERSION_MAJOR, morpheus_VERSION_MINOR, morp... diff --git a/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp b/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp index 893a8f3a2d..1f1d2c0620 100644 --- a/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp +++ b/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp @@ -32,7 +32,6 @@ #include "py_llm_lambda_node.hpp" #include "morpheus/messages/control.hpp" // IWYU pragma: keep -#include "morpheus/utilities/cudf_util.hpp" #include "morpheus/version.hpp" #include // for Object, ObjectProperties From 64ab1a72b96233805eca1ec6f2c9508400b1f618 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 13:39:30 -0700 Subject: [PATCH 222/347] lint fixes --- examples/cpu_only/run.py | 2 +- .../morpheus/benchmarks/benchmark_conf_generator.py | 1 - .../production/morpheus/dfp/utils/config_generator.py | 1 - .../morpheus/dfp_integrated_training_batch_pipeline.py | 9 +-------- .../dfp_integrated_training_streaming_pipeline.py | 9 +-------- .../notebooks/dfp_azure_integrated_training.ipynb | 3 +-- .../morpheus/notebooks/dfp_duo_integrated_training.ipynb | 1 - examples/sid_visualization/run.py | 3 +-- .../test_dfp_rolling_window_stage.py | 2 -- 9 files changed, 5 insertions(+), 26 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index d87545c9c2..58727d1e32 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -124,7 +124,7 @@ def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") - pipeline.run() known_gpu_packages = ['cudf', 'cuml', 'cupy', 'tensorrt', 'torch'] - known_gpu_packages_loaded = [pkg in sys.modules.keys() for pkg in known_gpu_packages] + known_gpu_packages_loaded = [pkg in sys.modules for pkg in known_gpu_packages] if any(known_gpu_packages_loaded): for (i, pkg) in enumerate(known_gpu_packages): diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py b/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py index 734e3e17ed..f455b3eea6 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py @@ -99,7 +99,6 @@ def _create_config(self): config = generate_ae_config(source=(self._pipe_conf.get('source')), userid_column_name=(self._pipe_conf.get('userid_column_name')), timestamp_column_name=(self._pipe_conf.get('timestamp_column_name')), - use_cpp=(self._pipe_conf.get('use_cpp')), pipeline_batch_size=(self._pipe_conf.get('pipeline_batch_size')), edge_buffer_size=(self._pipe_conf.get('edge_buffer_size')), num_threads=(self._pipe_conf.get('num_threads'))) diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py b/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py index 3b6d6a86c9..6e961de179 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/utils/config_generator.py @@ -172,7 +172,6 @@ def generate_ae_config(source: str, timestamp_column_name: str, pipeline_batch_size: int = 0, edge_buffer_size: int = 0, - use_cpp: bool = False, num_threads: int = len(os.sched_getaffinity(0))): config = Config() config.num_threads = num_threads diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_batch_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_batch_pipeline.py index 5cd551055d..fe3d99f187 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_batch_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_batch_pipeline.py @@ -74,12 +74,6 @@ default="60d", help="The training duration to run starting from start_time", ) -@click.option( - "--use_cpp", - type=click.BOOL, - default=True, - help=("Indicates what type of logs are going to be used in the workload."), -) @click.option( "--cache_dir", type=str, @@ -135,7 +129,6 @@ def run_pipeline(source: str, sample_rate_s: int, tracking_uri, silence_monitors, - use_cpp, mlflow_experiment_name_template, mlflow_model_name_template, **kwargs): @@ -167,7 +160,7 @@ def run_pipeline(source: str, # Default timestamp column -- override with ControlMessage timestamp_column_name = "timestamp" - config: Config = generate_ae_config(source, userid_column_name, timestamp_column_name, use_cpp=use_cpp) + config: Config = generate_ae_config(source, userid_column_name, timestamp_column_name) # Construct the data frame Schema used to normalize incoming data schema_builder = SchemaBuilder(config, source) diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_streaming_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_streaming_pipeline.py index 55ebcf71a9..93df5c3e53 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_streaming_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_streaming_pipeline.py @@ -74,12 +74,6 @@ default="60d", help="The training duration to run starting from start_time", ) -@click.option( - "--use_cpp", - type=click.BOOL, - default=True, - help=("Indicates what type of logs are going to be used in the workload."), -) @click.option( "--cache_dir", type=str, @@ -147,7 +141,6 @@ def run_pipeline(source: str, sample_rate_s: int, tracking_uri, silence_monitors, - use_cpp, mlflow_experiment_name_template, mlflow_model_name_template, **kwargs): @@ -180,7 +173,7 @@ def run_pipeline(source: str, # Default timestamp column -- override with ControlMessage timestamp_column_name = "timestamp" - config: Config = generate_ae_config(source, userid_column_name, timestamp_column_name, use_cpp=use_cpp) + config: Config = generate_ae_config(source, userid_column_name, timestamp_column_name) # Construct the data frame Schema used to normalize incoming data schema_builder = SchemaBuilder(config, source) diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb index 75c14999a6..1ea01f0a87 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb @@ -187,8 +187,7 @@ "config: Config = generate_ae_config(\n", " source,\n", " userid_column_name=\"username\",\n", - " timestamp_column_name=\"timestamp\",\n", - " use_cpp=True,\n", + " timestamp_column_name=\"timestamp\"\n", ")\n", "\n", "# Construct the dataframe Schema which is used to normalize incoming azure logs\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb index db57a85a1e..64b2b92ef5 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb @@ -190,7 +190,6 @@ " source,\n", " userid_column_name=\"username\",\n", " timestamp_column_name=\"timestamp\",\n", - " use_cpp=True,\n", ")\n", "\n", "# Construct the dataframe Schema which is used to normalize incoming duo logs\n", diff --git a/examples/sid_visualization/run.py b/examples/sid_visualization/run.py index 17c85cbfbc..b2736972b3 100644 --- a/examples/sid_visualization/run.py +++ b/examples/sid_visualization/run.py @@ -119,7 +119,6 @@ def _generate_frames(self): @click.command() @click.option("--debug/--no-debug", default=False) -@click.option('--use_cpp', default=True) @click.option( "--num_threads", default=len(os.sched_getaffinity(0)), @@ -148,7 +147,7 @@ def _generate_frames(self): help="The name of the model that is deployed on Tritonserver.", ) @click.option("--triton_server_url", default="localhost:8001", required=True, help="Tritonserver url.") -def run_pipeline(debug, use_cpp, num_threads, input_file, max_batch_size, model_name, triton_server_url): +def run_pipeline(debug, num_threads, input_file, max_batch_size, model_name, triton_server_url): if debug: configure_logging(log_level=logging.DEBUG) diff --git a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py index 23c42ea727..15ad17c3cb 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py @@ -157,8 +157,6 @@ def test_build_window(config: Config, train_df: pd.DataFrame): from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage - from morpheus.messages import ControlMessage - stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') # Create an overlap From acbd057b88878d405124ef83c3a9c628109ff53a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 15:16:59 -0700 Subject: [PATCH 223/347] Disable the too-many-ancestors check --- python/morpheus/morpheus/stages/postprocess/validation_stage.py | 2 +- tests/pipeline/test_pipeline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/stages/postprocess/validation_stage.py b/python/morpheus/morpheus/stages/postprocess/validation_stage.py index 99da57b36d..e39c814136 100644 --- a/python/morpheus/morpheus/stages/postprocess/validation_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/validation_stage.py @@ -30,7 +30,7 @@ @register_stage("validate") -class ValidationStage(CompareDataFrameStage): +class ValidationStage(CompareDataFrameStage): # pylint: disable=too-many-ancestors """ Validate pipeline output for testing. diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index aded507af6..56aa234ffd 100755 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -38,7 +38,7 @@ from morpheus.utils.type_aliases import DataFrameType -class SourceTestStage(InMemorySourceStage): +class SourceTestStage(InMemorySourceStage): # pylint: disable=too-many-ancestors def __init__(self, config, From db57b0f3ca264ddf40251a3b3c2c886fa3257f11 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 17 Sep 2024 15:53:25 -0700 Subject: [PATCH 224/347] disably cyclic check for control message --- python/morpheus/morpheus/messages/__init__.py | 2 ++ python/morpheus/morpheus/messages/control_message.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/python/morpheus/morpheus/messages/__init__.py b/python/morpheus/morpheus/messages/__init__.py index c0b3256c9c..c6cb27c15c 100644 --- a/python/morpheus/morpheus/messages/__init__.py +++ b/python/morpheus/morpheus/messages/__init__.py @@ -31,10 +31,12 @@ from morpheus.messages.message_base import MessageBase from morpheus.messages.message_meta import MessageMeta from morpheus.messages.message_meta import UserMessageMeta +from morpheus.messages.control_message import ControlMessageType from morpheus.messages.control_message import ControlMessage __all__ = [ "ControlMessage", + "ControlMessageType", "DataLoaderRegistry", "InferenceMemory", "InferenceMemoryAE", diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index 1096815984..de625f0d03 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=cyclic-import + import dataclasses import logging import re From 07304074e627f9019f7e4f2978b3c4eca22c1adc Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 09:15:06 -0700 Subject: [PATCH 225/347] Use 'pandas' instead of 'Pandas' --- .../guides/6_digital_fingerprinting_reference.md | 10 +++++----- models/model-cards/dfp-model-card.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md b/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md index 19965504d7..66eeed523f 100644 --- a/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md +++ b/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md @@ -88,7 +88,7 @@ Defines a single column and type-cast. | Argument | Type | Description | | -------- | ---- | ----------- | | `name` | `str` | Name of the column | -| `dtype` | `str` or Python type | Any type string or Python class recognized by [Pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | +| `dtype` | `str` or Python type | Any type string or Python class recognized by [pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | #### Custom Column (`CustomColumn`) Subclass of `ColumnInfo`, defines a column to be computed by a user-defined function `process_column_fn`. @@ -96,7 +96,7 @@ Subclass of `ColumnInfo`, defines a column to be computed by a user-defined func | Argument | Type | Description | | -------- | ---- | ----------- | | `name` | `str` | Name of the column | -| `dtype` | `str` or Python type | Any type string or Python class recognized by [Pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | +| `dtype` | `str` or Python type | Any type string or Python class recognized by [pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | | `process_column_fn` | `function` | Function which receives the entire `DataFrame` as its only input, returning a new [`pandas.Series`](https://pandas.pydata.org/docs/reference/api/pandas.Series.html) object to be stored in column `name`. | | `input_column_types` | `dict[str, str]` | The input columns and the expected [`dtype` strings](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) that are needed for this Column to successfully process. Setting this as `None` will pass all columns. Specifying which columns are needed improves performance. | @@ -139,7 +139,7 @@ Subclass of `RenameColumn`, specific to casting UTC localized `datetime` values. | Argument | Type | Description | | -------- | ---- | ----------- | | `name` | `str` | Name of the destination column | -| `dtype` | `str` or Python type | Any type string or Python class recognized by [Pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | +| `dtype` | `str` or Python type | Any type string or Python class recognized by [pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | | `input_name` | `str` | Original column name | #### String-Join Column (`StringJoinColumn`) @@ -148,7 +148,7 @@ Subclass of `RenameColumn`, converts incoming `list` values to string by joining | Argument | Type | Description | | -------- | ---- | ----------- | | `name` | `str` | Name of the destination column | -| `dtype` | `str` or Python type | Any type string or Python class recognized by [Pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | +| `dtype` | `str` or Python type | Any type string or Python class recognized by [pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | | `input_name` | `str` | Original column name | | `sep` | `str` | Separator string to use for the join | @@ -158,7 +158,7 @@ Subclass of `ColumnInfo`, concatenates values from multiple columns into a new s | Argument | Type | Description | | -------- | ---- | ----------- | | `name` | `str` | Name of the destination column | -| `dtype` | `str` or Python type | Any type string or Python class recognized by [Pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | +| `dtype` | `str` or Python type | Any type string or Python class recognized by [pandas](https://pandas.pydata.org/docs/user_guide/basics.html#dtypes) | | `input_columns` | `List[str]` | List of columns to concatenate | | `sep` | `str` | Separator string | diff --git a/models/model-cards/dfp-model-card.md b/models/model-cards/dfp-model-card.md index 71c0eebc04..88b453d254 100644 --- a/models/model-cards/dfp-model-card.md +++ b/models/model-cards/dfp-model-card.md @@ -52,7 +52,7 @@ The model architecture consists of an Autoencoder, where the reconstruction loss * Reconstruction loss (per feature) **Output Parameters:** -* Pandas DataFrame +* pandas DataFrame ## Software Integration: **Runtime:** From 547db64e814dcce3f129bcaac0af0b5198bdee9b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 09:21:47 -0700 Subject: [PATCH 226/347] Fix? Can't find 'action.yml' error --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d7c748c181..4e277f1768 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -61,7 +61,7 @@ jobs: steps: - name: Get PR Info id: get-pr-info - uses: rapidsai/shared-action-workflows/get-pr-info@branch-23.08 + uses: nv-gha-runners/get-pr-info@branch-23.08 if: ${{ startsWith(github.ref_name, 'pull-request/') }} outputs: is_pr: ${{ startsWith(github.ref_name, 'pull-request/') }} From 2d780bb3252180daea4eedd5755ff0434fbc593d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 09:23:16 -0700 Subject: [PATCH 227/347] Fix? Can't find 'action.yml' error --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4e277f1768..5d7104a01d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -61,7 +61,7 @@ jobs: steps: - name: Get PR Info id: get-pr-info - uses: nv-gha-runners/get-pr-info@branch-23.08 + uses: nv-gha-runners/get-pr-info@main if: ${{ startsWith(github.ref_name, 'pull-request/') }} outputs: is_pr: ${{ startsWith(github.ref_name, 'pull-request/') }} From e22b01ade515ae8f14607bc792aa0a8e5efa2341 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 09:26:58 -0700 Subject: [PATCH 228/347] Fix? Can't find 'action.yml' error --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5d7104a01d..8b8340775f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -77,7 +77,7 @@ jobs: # Only run the CI pipeline if the PR does not have the skip-ci label and we are on a PR branch if: ${{ !fromJSON(needs.prepare.outputs.has_skip_ci_label) && fromJSON(needs.prepare.outputs.is_pr )}} secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 + uses: nv-gha-runners/.github/workflows/checks.yaml@main with: enable_check_generated_files: false From 66baad0202fbf175dc7cb11ea85365707eec685a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 09:31:19 -0700 Subject: [PATCH 229/347] Fix? Can't find 'action.yml' error --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8b8340775f..10359c073e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -77,7 +77,7 @@ jobs: # Only run the CI pipeline if the PR does not have the skip-ci label and we are on a PR branch if: ${{ !fromJSON(needs.prepare.outputs.has_skip_ci_label) && fromJSON(needs.prepare.outputs.is_pr )}} secrets: inherit - uses: nv-gha-runners/.github/workflows/checks.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false From 822c2fb1c0e06f40209a6860dce2be34781d2bec Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 09:33:03 -0700 Subject: [PATCH 230/347] Fix? Can't find 'action.yml' error --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 10359c073e..097d3c9e69 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -49,7 +49,7 @@ jobs: - checks - ci_pipe secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 prepare: # Executes the get-pr-info action to determine if the PR has the skip-ci label, if the action fails we assume the From ba385286d0426dd5512f3724988b5592da64e8b7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 10:54:16 -0700 Subject: [PATCH 231/347] Cleanups --- examples/cpu_only/run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index 58727d1e32..808f956802 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -92,7 +92,7 @@ def print_msg(msg: typing.Any) -> typing.Any: if isinstance(msg, MessageMeta): log_msg.append(f"- df type: {type(msg.df)}") - print(" ".join(log_msg)) + logger.debug(" ".join(log_msg)) return msg @@ -107,7 +107,7 @@ def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") - meta = msg.payload() with meta.mutable_dataframe() as df: - print(f"Received a ControlMessage with a dataframe of type {type(df)}") + logger.debug("Received a ControlMessage with a dataframe of type %s", type(df)) df[total_column_name] = df.select_dtypes(include="number").sum(axis=1) return msg @@ -123,7 +123,7 @@ def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") - pipeline.run() - known_gpu_packages = ['cudf', 'cuml', 'cupy', 'tensorrt', 'torch'] + known_gpu_packages = ['cudf', 'cuml', 'tensorrt', 'torch'] known_gpu_packages_loaded = [pkg in sys.modules for pkg in known_gpu_packages] if any(known_gpu_packages_loaded): @@ -135,7 +135,7 @@ def calculate_totals(msg: ControlMessage, *, total_column_name: str = "total") - else: logger.info(msg) else: - logger.infp("No GPU packages loaded") + logger.info("No GPU packages loaded") if __name__ == "__main__": From 90792c6d2e059146e48d74af384f4e02a3350539 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 11:09:48 -0700 Subject: [PATCH 232/347] Add cpu only support for testing the release container --- docker/run_container_release.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/run_container_release.sh b/docker/run_container_release.sh index 42575d394e..a97102380b 100755 --- a/docker/run_container_release.sh +++ b/docker/run_container_release.sh @@ -38,7 +38,15 @@ DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} popd &> /dev/null -DOCKER_ARGS="--runtime=nvidia --env WORKSPACE_VOLUME=${PWD} --net=host --gpus=all --cap-add=sys_nice ${DOCKER_EXTRA_ARGS}" +DOCKER_ARGS="--env WORKSPACE_VOLUME=${PWD} --net=host --cap-add=sys_nice ${DOCKER_EXTRA_ARGS}" + +if [[ -n "${CPU_ONLY}" ]]; then + echo -e "${b}Executing in CPU only mode${x}" + DOCKER_ARGS="${DOCKER_ARGS} --runtime=runc" +else + echo -e "${b}Executing in GPU mode${x}" + DOCKER_ARGS="${DOCKER_ARGS} --runtime=nvidia --gpus=all" +fi if [[ -n "${SSH_AUTH_SOCK}" ]]; then echo -e "${b}Setting up ssh-agent auth socket${x}" From e779d764768ecebc9c302d9ebe137c3e061c3a4c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 11:47:22 -0700 Subject: [PATCH 233/347] Fix type-o in variable name, remove warning log --- python/morpheus/morpheus/cli/commands.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/morpheus/morpheus/cli/commands.py b/python/morpheus/morpheus/cli/commands.py index 74e245a6ef..8cab85be1e 100644 --- a/python/morpheus/morpheus/cli/commands.py +++ b/python/morpheus/morpheus/cli/commands.py @@ -305,14 +305,11 @@ def run(ctx: click.Context, **kwargs): # Since the option isnt the same name as `should_use_cpp` anymore, manually set the value here. use_cpu_only = kwargs.pop("use_cpu_only") - use_cpu = kwargs.pop("use_cpp") + use_cpp = kwargs.pop("use_cpp") if use_cpu_only: - if use_cpu: - logger.warning("use_cpu_only is set to True, disabling C++ mode") - CppConfig.set_should_use_cpp(False) else: - CppConfig.set_should_use_cpp(use_cpu) + CppConfig.set_should_use_cpp(use_cpp) config = get_config_from_ctx(ctx) config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU From c50c7171dcba27de35001c4d232a129a382d8f6d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 12:35:04 -0700 Subject: [PATCH 234/347] Warn users regarding the use of the --use_cpp flag --- python/morpheus/morpheus/cli/commands.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/cli/commands.py b/python/morpheus/morpheus/cli/commands.py index 8cab85be1e..19b7f857b2 100644 --- a/python/morpheus/morpheus/cli/commands.py +++ b/python/morpheus/morpheus/cli/commands.py @@ -17,6 +17,7 @@ import logging import os import typing +import warnings import click @@ -302,8 +303,10 @@ def install(**kwargs): @prepare_command(parse_config=True) def run(ctx: click.Context, **kwargs): """Run subcommand, used for running a pipeline""" - # Since the option isnt the same name as `should_use_cpp` anymore, manually set the value here. + if ctx.get_parameter_source("use_cpp") is not click.core.ParameterSource.DEFAULT: + logger.warning("The --use_cpp flag is deprecated and will be removed in a future release") + # Since the option isnt the same name as `should_use_cpp` anymore, manually set the value here. use_cpu_only = kwargs.pop("use_cpu_only") use_cpp = kwargs.pop("use_cpp") if use_cpu_only: From c20a956d398b6b87428ec199841cd5925d080f93 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 13:18:19 -0700 Subject: [PATCH 235/347] Change default triton port to the one used by the C++ impl --- examples/developer_guide/2_1_real_world_phishing/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/developer_guide/2_1_real_world_phishing/run.py b/examples/developer_guide/2_1_real_world_phishing/run.py index b0907924aa..32e53042f7 100755 --- a/examples/developer_guide/2_1_real_world_phishing/run.py +++ b/examples/developer_guide/2_1_real_world_phishing/run.py @@ -75,7 +75,7 @@ default="phishing-bert-onnx", help="The name of the model that is deployed on Tritonserver.", ) -@click.option("--server_url", default='localhost:8001', help="Tritonserver url.") +@click.option("--server_url", default='localhost:8000', help="Tritonserver url.") @click.option( "--output_file", default=os.path.join(tempfile.gettempdir(), "detections.jsonlines"), From 3b8a0d4ecccffc962414d4a3629acbae2b8a84a2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 13:19:01 -0700 Subject: [PATCH 236/347] Support CPU only mode in WriteToRabbitMQStage, add --use_cpu_only flags to test scripts --- .../developer_guide/2_2_rabbitmq/read_simple.py | 5 ++++- .../developer_guide/2_2_rabbitmq/write_simple.py | 13 +++++++++---- .../2_2_rabbitmq/write_to_rabbitmq_stage.py | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/developer_guide/2_2_rabbitmq/read_simple.py b/examples/developer_guide/2_2_rabbitmq/read_simple.py index 2b26d2ba6a..dfcf8ef84f 100755 --- a/examples/developer_guide/2_2_rabbitmq/read_simple.py +++ b/examples/developer_guide/2_2_rabbitmq/read_simple.py @@ -22,6 +22,7 @@ from morpheus.common import FileTypes from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage @@ -33,11 +34,13 @@ is_flag=True, default=False, help="Use the function based version of the RabbitMQ source stage instead of the class") -def run_pipeline(use_source_function: bool): +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) +def run_pipeline(use_source_function: bool, use_cpu_only: bool): # Enable the Morpheus logger configure_logging(log_level=logging.DEBUG) config = Config() + config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU config.num_threads = len(os.sched_getaffinity(0)) # Create a linear pipeline object diff --git a/examples/developer_guide/2_2_rabbitmq/write_simple.py b/examples/developer_guide/2_2_rabbitmq/write_simple.py index 78fa2c3d26..2b2e4f0607 100755 --- a/examples/developer_guide/2_2_rabbitmq/write_simple.py +++ b/examples/developer_guide/2_2_rabbitmq/write_simple.py @@ -16,22 +16,27 @@ import logging import os +import click from write_to_rabbitmq_stage import WriteToRabbitMQStage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.pipeline import LinearPipeline from morpheus.stages.input.file_source_stage import FileSourceStage from morpheus.utils.logger import configure_logging -def run_pipeline(): +@click.command() +@click.option('--input_file', + type=click.Path(exists=True, readable=True), + default=os.path.join(os.environ['MORPHEUS_ROOT'], 'examples/data/email.jsonlines')) +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) +def run_pipeline(use_cpu_only: bool, input_file: str): # Enable the Morpheus logger configure_logging(log_level=logging.DEBUG) - root_dir = os.environ['MORPHEUS_ROOT'] - input_file = os.path.join(root_dir, 'examples/data/email.jsonlines') - config = Config() + config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU config.num_threads = len(os.sched_getaffinity(0)) # Create a linear pipeline object diff --git a/examples/developer_guide/2_2_rabbitmq/write_to_rabbitmq_stage.py b/examples/developer_guide/2_2_rabbitmq/write_to_rabbitmq_stage.py index 401d8b785e..fb5382eda6 100644 --- a/examples/developer_guide/2_2_rabbitmq/write_to_rabbitmq_stage.py +++ b/examples/developer_guide/2_2_rabbitmq/write_to_rabbitmq_stage.py @@ -22,6 +22,7 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages.message_meta import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @@ -29,7 +30,7 @@ @register_stage("to-rabbitmq") -class WriteToRabbitMQStage(PassThruTypeMixin, SinglePortStage): +class WriteToRabbitMQStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Source stage used to load messages from a RabbitMQ queue. From e6d19356424f0859d6728999300f01030538d543 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 13:33:24 -0700 Subject: [PATCH 237/347] write_simple.py now requires MORPHEUS_ROOT --- examples/developer_guide/2_2_rabbitmq/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/developer_guide/2_2_rabbitmq/README.md b/examples/developer_guide/2_2_rabbitmq/README.md index cadd6075a2..5b657b580f 100644 --- a/examples/developer_guide/2_2_rabbitmq/README.md +++ b/examples/developer_guide/2_2_rabbitmq/README.md @@ -54,6 +54,7 @@ If no exchange named 'logs' exists in RabbitMQ it will be created. By default th ## Launch the writer In a third terminal from the root of the Morpheus repo execute: ```bash +export MORPHEUS_ROOT=$(pwd) python examples/developer_guide/2_2_rabbitmq/write_simple.py ``` From 1fb03a4a955e999f594c5f4438c96bef42da38b4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 13:59:58 -0700 Subject: [PATCH 238/347] Update code snippets to match source code changes --- docs/source/developer_guide/guides/2_real_world_phishing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/developer_guide/guides/2_real_world_phishing.md b/docs/source/developer_guide/guides/2_real_world_phishing.md index a7776c8dc4..e8cb1cf74e 100644 --- a/docs/source/developer_guide/guides/2_real_world_phishing.md +++ b/docs/source/developer_guide/guides/2_real_world_phishing.md @@ -976,7 +976,7 @@ The code for our sink will be similar to other stages with a few changes. First, ```python @register_stage("to-rabbitmq") -class WriteToRabbitMQStage(PassThruTypeMixin, SinglePortStage): +class WriteToRabbitMQStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): ``` Our sink will function as a pass-through allowing the possibility of other sinks to be added to the pipeline. We could, hypothetically, have a pipeline where we emit the results to both RabbitMQ and a file. For this reason we will also be using the `PassThruTypeMixin`. @@ -1028,6 +1028,7 @@ import pika from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages.message_meta import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @@ -1035,7 +1036,7 @@ logger = logging.getLogger(__name__) @register_stage("to-rabbitmq") -class WriteToRabbitMQStage(PassThruTypeMixin, SinglePortStage): +class WriteToRabbitMQStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Source stage used to load messages from a RabbitMQ queue. From b6ce583d1291f258590fddfae07df9cf09ec303b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 14:00:37 -0700 Subject: [PATCH 239/347] Update python impls to match those from the 2_2 example --- .../src/rabbitmq_cpp_stage/rabbitmq_source_stage.py | 9 +++++++-- .../src/rabbitmq_cpp_stage/write_to_rabbitmq_stage.py | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py index e02b3dc6ab..c908baaaef 100755 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py @@ -26,15 +26,17 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages.message_meta import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) @register_stage("from-rabbitmq") -class RabbitMQSourceStage(PreallocatorMixin, SingleOutputSource): +class RabbitMQSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): """ Source stage used to load messages from a RabbitMQ queue. @@ -72,6 +74,9 @@ def __init__(self, self._poll_interval = pd.Timedelta(poll_interval) + # This will return either cudf.DataFrame or pandas.DataFrame depending on the execution mode + self._df_pkg = get_df_pkg(config.execution_mode) + @property def name(self) -> str: return "from-rabbitmq" @@ -119,7 +124,7 @@ def source_generator(self): if method_frame is not None: try: buffer = StringIO(body.decode("utf-8")) - df = cudf.io.read_json(buffer, orient='records', lines=True) + df = self._df_pkg.read_json(buffer, orient='records', lines=True) yield MessageMeta(df=df) except Exception as ex: logger.exception("Error occurred converting RabbitMQ message to Dataframe: %s", ex) diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/write_to_rabbitmq_stage.py b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/write_to_rabbitmq_stage.py index 401d8b785e..fb5382eda6 100644 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/write_to_rabbitmq_stage.py +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/write_to_rabbitmq_stage.py @@ -22,6 +22,7 @@ from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages.message_meta import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @@ -29,7 +30,7 @@ @register_stage("to-rabbitmq") -class WriteToRabbitMQStage(PassThruTypeMixin, SinglePortStage): +class WriteToRabbitMQStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Source stage used to load messages from a RabbitMQ queue. From 6fad250a20a76e4e557e31a6f6231fbb1963ec70 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 14:51:41 -0700 Subject: [PATCH 240/347] Convert to pandas prior to invoking the model --- .../morpheus/stages/inference/auto_encoder_inference_stage.py | 2 +- .../morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py b/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py index 81e6684a55..f731d77a36 100644 --- a/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/auto_encoder_inference_stage.py @@ -86,7 +86,7 @@ def process(self, batch: ControlMessage, callback: typing.Callable[[TensorMemory """ - data = batch.payload().get_data(batch.payload().df.columns.intersection(self._feature_columns)) + data = batch.payload().get_data(batch.payload().df.columns.intersection(self._feature_columns)).to_pandas() explain_cols = [x + "_z_loss" for x in self._feature_columns] + ["max_abs_z", "mean_abs_z"] explain_df = pd.DataFrame(np.empty((batch.tensors().count, (len(self._feature_columns) + 2)), dtype=object), diff --git a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py index 9f61dafe51..8cd18cfc87 100644 --- a/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py +++ b/python/morpheus/morpheus/stages/preprocess/preprocess_ae_stage.py @@ -78,7 +78,7 @@ def pre_process_batch(msg: ControlMessage, fea_len: int, feature_columns: typing morpheus.messages.ControlMessage """ - meta_df = msg.payload().get_data(msg.payload().df.columns.intersection(feature_columns)) + meta_df = msg.payload().get_data(msg.payload().df.columns.intersection(feature_columns)).to_pandas() autoencoder = msg.get_metadata("model") scores_mean = msg.get_metadata("train_scores_mean") From 877ae49f31a324f58ac2828a84632e5a62169655 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 14:53:01 -0700 Subject: [PATCH 241/347] Set columns file to be package relative, fix the probs type --- .../digital_fingerprinting/starter/run_cloudtrail_dfp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py b/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py index 0c4968b74b..ef4f00b7dc 100644 --- a/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py +++ b/examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py @@ -18,6 +18,8 @@ import click +from morpheus.cli.utils import MorpheusRelativePath +from morpheus.common import TypeId from morpheus.config import AEFeatureScalar from morpheus.config import Config from morpheus.config import ConfigAutoEncoder @@ -57,7 +59,7 @@ ) @click.option( "--columns_file", - type=click.Path(exists=True, readable=True), + type=MorpheusRelativePath(exists=True, readable=True), required=True, help="Feature columns file", ) @@ -134,7 +136,7 @@ def run_pipeline(num_threads, pipeline.add_stage(AutoEncoderInferenceStage(config)) # Add anomaly scores and z-scores to each message - pipeline.add_stage(AddScoresStage(config)) + pipeline.add_stage(AddScoresStage(config, probs_type=TypeId.FLOAT64)) # Add serialize stage pipeline.add_stage(SerializeStage(config)) From 23c34c99db8118724ad5eeae3aa4089f0d6beedc Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 14:56:01 -0700 Subject: [PATCH 242/347] Update help output, remove usage of --use_cpp=False, use package relative paths for the columns file --- .../digital_fingerprinting/starter/README.md | 121 ++++++++---------- 1 file changed, 56 insertions(+), 65 deletions(-) diff --git a/examples/digital_fingerprinting/starter/README.md b/examples/digital_fingerprinting/starter/README.md index 89c2e60a66..30e96f3011 100644 --- a/examples/digital_fingerprinting/starter/README.md +++ b/examples/digital_fingerprinting/starter/README.md @@ -31,69 +31,60 @@ DFP pipelines can be constructed and run using the Morpheus CLI command `morpheu Use `--help` to display information about the autoencoder pipeline command line options: ``` -morpheus run pipeline-ae --help +Usage: morpheus run pipeline-ae [OPTIONS] COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]... -Usage: morpheus run pipeline-ae [OPTIONS] COMMAND1 [ARGS]... [COMMAND2 - [ARGS]...]... + Configure and run the pipeline. To configure the pipeline, list the stages in the order that data should flow. The output of each stage will become the input for the next + stage. For example, to read, classify and write to a file, the following stages could be used - Configure and run the pipeline. To configure the pipeline, list the stages - in the order that data should flow. The output of each stage will become the - input for the next stage. For example, to read, classify and write to a - file, the following stages could be used + pipeline from-file --filename=my_dataset.json deserialize preprocess inf-triton --model_name=my_model --server_url=localhost:8001 filter --threshold=0.5 to-file + --filename=classifications.json - pipeline from-file --filename=my_dataset.json deserialize preprocess inf-triton --model_name=my_model - --server_url=localhost:8001 filter --threshold=0.5 to-file --filename=classifications.json - - Pipelines must follow a few rules: - 1. Data must originate in a source stage. Current options are `from-file` or `from-kafka` - 2. A `deserialize` stage must be placed between the source stages and the rest of the pipeline - 3. Only one inference stage can be used. Zero is also fine - 4. The following stages must come after an inference stage: `add-class`, `filter`, `gen-viz` + Pipelines must follow a few rules: 1. Data must originate in a source stage. Current options are `from-file` or `from-kafka` 2. A `deserialize` stage must be placed + between the source stages and the rest of the pipeline 3. Only one inference stage can be used. Zero is also fine 4. The following stages must come after an inference + stage: `add-class`, `filter`, `gen-viz` Options: - --columns_file FILE [default: ./morpheus/data/columns_ae_cloudtrail.txt] - --labels_file FILE Specifies a file to read labels from in - order to convert class IDs into labels. A - label file is a simple text file where each - line corresponds to a label. If unspecified, - only a single output label is created for - FIL - --userid_column_name TEXT Which column to use as the User ID. - [default: userIdentityaccountId; required] - --userid_filter TEXT Specifying this value will filter all - incoming data to only use rows with matching - User IDs. Which column is used for the User - ID is specified by `userid_column_name` - --feature_scaler TEXT Autoencoder feature scaler [default: - standard] - --use_generic_model BOOLEAN Whether to use a generic model when user does - not have minimum number of training rows - [default: False] - --viz_file FILE Save a visualization of the pipeline at the - specified location + --columns_file DATA FILE Specifies a file to read column features. [required] + --labels_file DATA FILE Specifies a file to read labels from in order to convert class IDs into labels. A label file is a simple text file where each line + corresponds to a label. + --userid_column_name TEXT Which column to use as the User ID. [default: userIdentityaccountId; required] + --userid_filter TEXT Specifying this value will filter all incoming data to only use rows with matching User IDs. Which column is used for the User ID is + specified by `userid_column_name` + --feature_scaler [NONE|STANDARD|GAUSSRANK] + Autoencoder feature scaler [default: STANDARD] + --use_generic_model Whether to use a generic model when user does not have minimum number of training rows + --viz_file FILE Save a visualization of the pipeline at the specified location + --viz_direction [BT|LR|RL|TB] Set the direction for the Graphviz pipeline diagram, ignored unless --viz_file is also specified. [default: LR] + --timestamp_column_name TEXT Which column to use as the timestamp. [default: timestamp; required] --help Show this message and exit. Commands: - add-class Add detected classifications to each message - add-scores Add probability scores to each message - buffer (Deprecated) Buffer results - delay (Deprecated) Delay results for a certain duration - filter Filter message by a classification threshold - from-azure Source stage is used to load Azure Active Directory messages. - from-cloudtrail Load messages from a CloudTrail directory - from-duo Source stage is used to load Duo Authentication messages. - gen-viz (Deprecated) Write out visualization data frames - inf-pytorch Perform inference with PyTorch - inf-triton Perform inference with Triton - monitor Display throughput numbers at a specific point in the - pipeline - preprocess Convert messages to tokens - serialize Include & exclude columns from messages - timeseries Perform time series anomaly detection and add prediction. - to-file Write all messages to a file - to-kafka Write all messages to a Kafka cluster - train-ae Deserialize source data from JSON - validate Validates pipeline output against an expected output + add-class Add detected classifications to each message. + add-scores Add probability scores to each message. + buffer (Deprecated) Buffer results. + delay (Deprecated) Delay results for a certain duration. + filter Filter message by a classification threshold. + from-arxiv Source stage that downloads PDFs from arxiv and converts them to dataframes. + from-azure Source stage is used to load Azure Active Directory messages. + from-cloudtrail Load messages from a CloudTrail directory. + from-databricks-deltalake Source stage used to load messages from a DeltaLake table. + from-duo Source stage is used to load Duo Authentication messages. + from-http Source stage that starts an HTTP server and listens for incoming requests on a specified endpoint. + from-http-client Source stage that polls a remote HTTP server for incoming data. + from-rss Load RSS feed items into a DataFrame. + inf-pytorch Perform inference with PyTorch. + monitor Display throughput numbers at a specific point in the pipeline. + preprocess Prepare Autoencoder input DataFrames for inference. + serialize Includes & excludes columns from messages. + timeseries Perform time series anomaly detection and add prediction. + to-elasticsearch This class writes the messages as documents to Elasticsearch. + to-file Write all messages to a file. + to-http Write all messages to an HTTP endpoint. + to-http-server Sink stage that starts an HTTP server and listens for incoming requests on a specified endpoint. + to-kafka Write all messages to a Kafka cluster. + train-ae Train an Autoencoder model on incoming data. + trigger Buffer data until the previous stage has completed. + validate Validate pipeline output for testing. ``` The commands above correspond to the Morpheus stages that can be used to construct your DFP pipeline. Options are available to configure pipeline and stages. The following table shows mapping between the main Morpheus CLI commands and underlying Morpheus Python stage classes: @@ -160,9 +151,9 @@ Run the following in your Morpheus container to start the CloudTrail DFP pipelin ``` morpheus --log_level=DEBUG \ - run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 --use_cpp=False \ + run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 \ pipeline-ae \ - --columns_file=morpheus/data/columns_ae_cloudtrail.txt \ + --columns_file=data/columns_ae_cloudtrail.txt \ --userid_column_name=userIdentitysessionContextsessionIssueruserName \ --userid_filter=user123 \ --feature_scaler=standard \ @@ -186,9 +177,9 @@ The following pipeline trains user models from downloaded training data and save on downloaded inference data. Inference results are written to `duo-detections.csv`. ``` morpheus --log_level=DEBUG \ - run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 --use_cpp=False \ + run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 \ pipeline-ae \ - --columns_file=morpheus/data/columns_ae_duo.txt \ + --columns_file=data/columns_ae_duo.txt \ --userid_column_name=username \ --feature_scaler=standard \ from-duo \ @@ -211,9 +202,9 @@ morpheus --log_level=DEBUG \ The following example shows how we can load pre-trained user models from the file (`models/dfp-models/duo_ae_user_models.pkl`) we created in the previous example. Pipeline then uses these models to run inference on validation data in `models/datasets/validation-data/duo`. Inference results are written to `duo-detections.csv`. ``` morpheus --log_level=DEBUG \ - run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 --use_cpp=False \ + run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 \ pipeline-ae \ - --columns_file=morpheus/data/columns_ae_duo.txt \ + --columns_file=data/columns_ae_duo.txt \ --userid_column_name=username \ --feature_scaler=standard \ from-duo \ @@ -260,9 +251,9 @@ morpheus --log_level=DEBUG \ The following example shows how we can load pre-trained user models from the file (`models/dfp-models/azure_ae_user_models.pkl`) we created in the previous example. Pipeline then uses these models to run inference on validation data in `models/datasets/validation-data/azure`. Inference results are written to `azure-detections.csv`. ``` morpheus --log_level=DEBUG \ - run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 --use_cpp=False \ + run --num_threads=1 --pipeline_batch_size=1024 --model_max_batch_size=1024 \ pipeline-ae \ - --columns_file=morpheus/data/columns_ae_azure.txt \ + --columns_file=data/columns_ae_azure.txt \ --userid_column_name=userPrincipalName \ --feature_scaler=standard \ from-azure \ @@ -287,7 +278,7 @@ run the example. Train user models from files in `models/datasets/training-data/dfp-cloudtrail-*.csv` and saves user models to file. Pipeline then uses these models to run inference on CloudTrail validation data in `models/datasets/validation-data/dfp-cloudtrail-*-input.csv`. Inference results are written to `cloudtrail-dfp-results.csv`. ``` python ./examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py \ - --columns_file=morpheus/data/columns_ae_cloudtrail.txt \ + --columns_file=data/columns_ae_cloudtrail.txt \ --input_glob=models/datasets/validation-data/dfp-cloudtrail-*-input.csv \ --train_data_glob=models/datasets/training-data/dfp-*.csv \ --models_output_filename=models/dfp-models/cloudtrail_ae_user_models.pkl \ @@ -297,7 +288,7 @@ python ./examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py \ Here we load pre-trained user models from the file (`models/dfp-models/cloudtrail_ae_user_models.pkl`) we created in the previous example. Pipeline then uses these models to run inference on validation data in `models/datasets/validation-data/dfp-cloudtrail-*-input.csv`. Inference results are written to `cloudtrail-dfp-results.csv`. ``` python ./examples/digital_fingerprinting/starter/run_cloudtrail_dfp.py \ - --columns_file=morpheus/data/columns_ae_cloudtrail.txt \ + --columns_file=data/columns_ae_cloudtrail.txt \ --input_glob=models/datasets/validation-data/dfp-cloudtrail-*-input.csv \ --pretrained_filename=models/dfp-models/cloudtrail_ae_user_models.pkl \ --output_file=./cloudtrail-dfp-results.csv From 2095ebd192c86f61dfacd25b6f622111b3add207 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 15:11:23 -0700 Subject: [PATCH 243/347] Change default triton port as we will only be using the C++ impl --- examples/llm/vdb_upload/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm/vdb_upload/run.py b/examples/llm/vdb_upload/run.py index f02ed5dfe0..b3099d845f 100644 --- a/examples/llm/vdb_upload/run.py +++ b/examples/llm/vdb_upload/run.py @@ -115,7 +115,7 @@ def run(): @click.option( "--triton_server_url", type=str, - default="localhost:8001", + default="localhost:8000", help="Triton server URL.", ) @click.option( From c87173dac50dc0d22a810eedf1e97b209df95a4d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 15:23:19 -0700 Subject: [PATCH 244/347] Fix inferring the dataframe type --- python/morpheus/morpheus/modules/output/write_to_vector_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/modules/output/write_to_vector_db.py b/python/morpheus/morpheus/modules/output/write_to_vector_db.py index a22acbcc4f..966ec0a12f 100644 --- a/python/morpheus/morpheus/modules/output/write_to_vector_db.py +++ b/python/morpheus/morpheus/modules/output/write_to_vector_db.py @@ -199,7 +199,7 @@ def on_data(msg: ControlMessage): (current_time - accum_stats.last_insert_time) >= write_time_interval): if accum_stats.data: - df_pkg = get_df_pkg_from_obj(accum_stats.data) + df_pkg = get_df_pkg_from_obj(accum_stats.data[0]) merged_df = df_pkg.concat(accum_stats.data) # pylint: disable=not-a-mapping From 60b0fb20eca31863416e73900346e6fc9f2a4652 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 15:38:28 -0700 Subject: [PATCH 245/347] Fix inferring the dataframe type --- python/morpheus/morpheus/modules/output/write_to_vector_db.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/modules/output/write_to_vector_db.py b/python/morpheus/morpheus/modules/output/write_to_vector_db.py index 966ec0a12f..2541cd788d 100644 --- a/python/morpheus/morpheus/modules/output/write_to_vector_db.py +++ b/python/morpheus/morpheus/modules/output/write_to_vector_db.py @@ -140,12 +140,11 @@ def on_completed(): final_df_references = [] if len(accumulator_dict): - df_pkg = get_df_pkg_from_obj(next(iter(accumulator_dict.values()))) - # Pushing remaining messages for key, accum_stats in accumulator_dict.items(): try: if accum_stats.data: + df_pkg = get_df_pkg_from_obj(accum_stats.data[0]) merged_df = df_pkg.concat(accum_stats.data) service.insert_dataframe(name=key, df=merged_df) final_df_references.append(accum_stats.data) From a22ccc997bf517db734d1b6f19553cd7f89cd8bf Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 16:44:25 -0700 Subject: [PATCH 246/347] Use the default port of the c++ triton impl --- examples/sid_visualization/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sid_visualization/run.py b/examples/sid_visualization/run.py index b2736972b3..6e58a92791 100644 --- a/examples/sid_visualization/run.py +++ b/examples/sid_visualization/run.py @@ -146,7 +146,7 @@ def _generate_frames(self): default="sid-minibert-onnx", help="The name of the model that is deployed on Tritonserver.", ) -@click.option("--triton_server_url", default="localhost:8001", required=True, help="Tritonserver url.") +@click.option("--triton_server_url", default="localhost:8000", required=True, help="Tritonserver url.") def run_pipeline(debug, num_threads, input_file, max_batch_size, model_name, triton_server_url): if debug: From d3eec63adeaf2b04e3d6976f1d88ac559836da91 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 16:45:20 -0700 Subject: [PATCH 247/347] Use triton http port as this is used by the C++ impl, fix path to vocab_has_file to be package relative --- examples/sid_visualization/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/sid_visualization/README.md b/examples/sid_visualization/README.md index 10aeb4cbee..c1d88b25b4 100644 --- a/examples/sid_visualization/README.md +++ b/examples/sid_visualization/README.md @@ -96,7 +96,7 @@ After the GUI has been launched, Morpheus now needs to be started. In the same s ```bash python examples/sid_visualization/run.py \ --debug \ - --triton_server_url=triton:8001 \ + --triton_server_url=triton:8000 \ --input_file=./examples/data/sid_visualization/group1-benign-2nodes.jsonlines \ --input_file=./examples/data/sid_visualization/group2-benign-50nodes.jsonlines \ --input_file=./examples/data/sid_visualization/group3-si-50nodes.jsonlines \ @@ -147,7 +147,7 @@ morpheus --log_level=DEBUG \ pipeline-nlp --model_seq_length=256 \ from-file --filename=${DEMO_DATASET} \ deserialize \ - preprocess --vocab_hash_file=morpheus/data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \ + preprocess --vocab_hash_file=data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \ inf-triton --model_name=sid-minibert-onnx --server_url=triton:8001 --force_convert_inputs=True \ monitor --description Inference\ Rate --unit=inf \ add-class \ From e224ab9e06d080d03703cc13a5bae298e6cb3151 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 16:48:32 -0700 Subject: [PATCH 248/347] Remove unused imports --- .../src/rabbitmq_cpp_stage/rabbitmq_source_stage.py | 2 -- python/morpheus/morpheus/cli/commands.py | 1 - 2 files changed, 3 deletions(-) diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py index c908baaaef..870bbd4554 100755 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py @@ -21,8 +21,6 @@ import pandas as pd import pika -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages.message_meta import MessageMeta diff --git a/python/morpheus/morpheus/cli/commands.py b/python/morpheus/morpheus/cli/commands.py index 19b7f857b2..6e6e3618a3 100644 --- a/python/morpheus/morpheus/cli/commands.py +++ b/python/morpheus/morpheus/cli/commands.py @@ -17,7 +17,6 @@ import logging import os import typing -import warnings import click From b47ef47fa78d0bae571c6e13fec6e9cad14a6248 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 18 Sep 2024 16:59:15 -0700 Subject: [PATCH 249/347] update read_simple and write_simple scripts --- .../2_2_rabbitmq/read_simple.py | 10 +++++++-- .../2_2_rabbitmq/write_simple.py | 10 +++++++-- .../4_rabbitmq_cpp_stage/src/read_simple.py | 9 ++++---- .../4_rabbitmq_cpp_stage/src/write_simple.py | 21 ++++++++++++++----- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/examples/developer_guide/2_2_rabbitmq/read_simple.py b/examples/developer_guide/2_2_rabbitmq/read_simple.py index dfcf8ef84f..c00e0728ed 100755 --- a/examples/developer_guide/2_2_rabbitmq/read_simple.py +++ b/examples/developer_guide/2_2_rabbitmq/read_simple.py @@ -35,13 +35,19 @@ default=False, help="Use the function based version of the RabbitMQ source stage instead of the class") @click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) -def run_pipeline(use_source_function: bool, use_cpu_only: bool): +@click.option( + "--num_threads", + default=len(os.sched_getaffinity(0)), + type=click.IntRange(min=1), + help="Number of internal pipeline threads to use", +) +def run_pipeline(use_source_function: bool, use_cpu_only: bool, num_threads: int): # Enable the Morpheus logger configure_logging(log_level=logging.DEBUG) config = Config() config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU - config.num_threads = len(os.sched_getaffinity(0)) + config.num_threads = num_threads # Create a linear pipeline object pipeline = LinearPipeline(config) diff --git a/examples/developer_guide/2_2_rabbitmq/write_simple.py b/examples/developer_guide/2_2_rabbitmq/write_simple.py index 2b2e4f0607..f2e6e76430 100755 --- a/examples/developer_guide/2_2_rabbitmq/write_simple.py +++ b/examples/developer_guide/2_2_rabbitmq/write_simple.py @@ -31,13 +31,19 @@ type=click.Path(exists=True, readable=True), default=os.path.join(os.environ['MORPHEUS_ROOT'], 'examples/data/email.jsonlines')) @click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) -def run_pipeline(use_cpu_only: bool, input_file: str): +@click.option( + "--num_threads", + default=len(os.sched_getaffinity(0)), + type=click.IntRange(min=1), + help="Number of internal pipeline threads to use", +) +def run_pipeline(use_cpu_only: bool, input_file: str, num_threads: int): # Enable the Morpheus logger configure_logging(log_level=logging.DEBUG) config = Config() config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU - config.num_threads = len(os.sched_getaffinity(0)) + config.num_threads = num_threads # Create a linear pipeline object pipeline = LinearPipeline(config) diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/src/read_simple.py b/examples/developer_guide/4_rabbitmq_cpp_stage/src/read_simple.py index b8271bb79a..66d5ffd76b 100755 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/src/read_simple.py +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/src/read_simple.py @@ -21,7 +21,7 @@ from morpheus.common import FileTypes from morpheus.config import Config -from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage @@ -29,20 +29,19 @@ @click.command() -@click.option('--use_cpp', default=True) +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) @click.option( "--num_threads", default=len(os.sched_getaffinity(0)), type=click.IntRange(min=1), help="Number of internal pipeline threads to use", ) -def run_pipeline(use_cpp, num_threads): +def run_pipeline(use_cpu_only: bool, num_threads: int): # Enable the Morpheus logger configure_logging(log_level=logging.DEBUG) - CppConfig.set_should_use_cpp(use_cpp) - config = Config() + config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU config.num_threads = num_threads # Create a linear pipeline object diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/src/write_simple.py b/examples/developer_guide/4_rabbitmq_cpp_stage/src/write_simple.py index b9cdf761e5..a4954d8ae1 100755 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/src/write_simple.py +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/src/write_simple.py @@ -16,23 +16,34 @@ import logging import os +import click from rabbitmq_cpp_stage.write_to_rabbitmq_stage import WriteToRabbitMQStage from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.pipeline import LinearPipeline from morpheus.stages.input.file_source_stage import FileSourceStage from morpheus.utils.logger import configure_logging -def run_pipeline(): +@click.command() +@click.option('--input_file', + type=click.Path(exists=True, readable=True), + default=os.path.join(os.environ['MORPHEUS_ROOT'], 'examples/data/email.jsonlines')) +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) +@click.option( + "--num_threads", + default=len(os.sched_getaffinity(0)), + type=click.IntRange(min=1), + help="Number of internal pipeline threads to use", +) +def run_pipeline(use_cpu_only: bool, input_file: str, num_threads: int): # Enable the Morpheus logger configure_logging(log_level=logging.DEBUG) - root_dir = os.environ['MORPHEUS_ROOT'] - input_file = os.path.join(root_dir, 'examples/data/email.jsonlines') - config = Config() - config.num_threads = len(os.sched_getaffinity(0)) + config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU + config.num_threads = num_threads # Create a linear pipeline object pipeline = LinearPipeline(config) From f70787b9a361ffe4aa16a2fd9a6c8373d5545219 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 19 Sep 2024 08:25:46 -0700 Subject: [PATCH 250/347] Mark the LLMEngineStage as supporting CPU & GPU execution modes --- .../morpheus_llm/stages/llm/llm_engine_stage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index f16442ac59..ff3aa57d76 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -15,14 +15,14 @@ import functools import logging import types -import typing import mrc from mrc.core import operators as ops from morpheus.config import Config -from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.messages import ControlMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus_llm.llm import LLMEngine @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -class LLMEngineStage(PassThruTypeMixin, SinglePortStage): +class LLMEngineStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Stage for executing an LLM engine within a Morpheus pipeline. @@ -52,7 +52,7 @@ def name(self) -> str: """Return the name of the stage""" return "llm-engine" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> tuple: """ Returns accepted input types for this stage. @@ -70,8 +70,8 @@ def supports_cpp_node(self): def _cast_control_message(self, message: ControlMessage, *, cpp_messages_lib: types.ModuleType) -> ControlMessage: """ - LLMEngineStage does not contain a Python implementation, however it is capable of running in Python/cpu-only - mode. This method is needed to cast the Python ControlMessage to a C++ ControlMessage. + LLMEngineStage does not contain a Python implementation, however it is capable of running in cpu-only mode. + This method is needed to cast the Python ControlMessage to a C++ ControlMessage. This is different than casting from the Python bindings for the C++ ControlMessage to a C++ ControlMessage. """ @@ -82,7 +82,7 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> node = _llm.LLMEngineStage(builder, self.unique_name, self._engine) node.launch_options.pe_count = 1 - if not CppConfig.get_should_use_cpp(): + if self._config.execution_mode == ExecutionMode.CPU: import morpheus._lib.messages as _messages cast_fn = functools.partial(self._cast_control_message, cpp_messages_lib=_messages) pre_node = builder.make_node(f"{self.unique_name}-pre-cast", ops.map(cast_fn)) From 51ab625a05bff5c03f0b74e64133cc1ae8607c74 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 19 Sep 2024 08:29:47 -0700 Subject: [PATCH 251/347] Add type hint --- python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index ff3aa57d76..7e61ae36d7 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -64,7 +64,7 @@ def accepted_types(self) -> tuple: """ return (ControlMessage, ) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: """Indicates whether this stage supports a C++ node.""" return True From 5bcd44dbc3bfd9ccaf8b263523c2898716ec38b2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 19 Sep 2024 08:44:00 -0700 Subject: [PATCH 252/347] Revert "Add type hint" This reverts commit 51ab625a05bff5c03f0b74e64133cc1ae8607c74. --- python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index 7e61ae36d7..ff3aa57d76 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -64,7 +64,7 @@ def accepted_types(self) -> tuple: """ return (ControlMessage, ) - def supports_cpp_node(self) -> bool: + def supports_cpp_node(self): """Indicates whether this stage supports a C++ node.""" return True From cf17c9f1cb1011cf449f72e27c5dd4839d7e060d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 19 Sep 2024 08:44:07 -0700 Subject: [PATCH 253/347] Revert "Mark the LLMEngineStage as supporting CPU & GPU execution modes" This reverts commit f70787b9a361ffe4aa16a2fd9a6c8373d5545219. --- .../morpheus_llm/stages/llm/llm_engine_stage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index ff3aa57d76..f16442ac59 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -15,14 +15,14 @@ import functools import logging import types +import typing import mrc from mrc.core import operators as ops from morpheus.config import Config -from morpheus.config import ExecutionMode +from morpheus.config import CppConfig from morpheus.messages import ControlMessage -from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus_llm.llm import LLMEngine @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -class LLMEngineStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): +class LLMEngineStage(PassThruTypeMixin, SinglePortStage): """ Stage for executing an LLM engine within a Morpheus pipeline. @@ -52,7 +52,7 @@ def name(self) -> str: """Return the name of the stage""" return "llm-engine" - def accepted_types(self) -> tuple: + def accepted_types(self) -> typing.Tuple: """ Returns accepted input types for this stage. @@ -70,8 +70,8 @@ def supports_cpp_node(self): def _cast_control_message(self, message: ControlMessage, *, cpp_messages_lib: types.ModuleType) -> ControlMessage: """ - LLMEngineStage does not contain a Python implementation, however it is capable of running in cpu-only mode. - This method is needed to cast the Python ControlMessage to a C++ ControlMessage. + LLMEngineStage does not contain a Python implementation, however it is capable of running in Python/cpu-only + mode. This method is needed to cast the Python ControlMessage to a C++ ControlMessage. This is different than casting from the Python bindings for the C++ ControlMessage to a C++ ControlMessage. """ @@ -82,7 +82,7 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> node = _llm.LLMEngineStage(builder, self.unique_name, self._engine) node.launch_options.pe_count = 1 - if self._config.execution_mode == ExecutionMode.CPU: + if not CppConfig.get_should_use_cpp(): import morpheus._lib.messages as _messages cast_fn = functools.partial(self._cast_control_message, cpp_messages_lib=_messages) pre_node = builder.make_node(f"{self.unique_name}-pre-cast", ops.map(cast_fn)) From 0539c06234c3730fc8917056ff9ba6c4d2d6b1b6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 19 Sep 2024 11:07:49 -0700 Subject: [PATCH 254/347] Revert "Revert "Mark the LLMEngineStage as supporting CPU & GPU execution modes"" This reverts commit cf17c9f1cb1011cf449f72e27c5dd4839d7e060d. --- .../morpheus_llm/stages/llm/llm_engine_stage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index f16442ac59..ff3aa57d76 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -15,14 +15,14 @@ import functools import logging import types -import typing import mrc from mrc.core import operators as ops from morpheus.config import Config -from morpheus.config import CppConfig +from morpheus.config import ExecutionMode from morpheus.messages import ControlMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus_llm.llm import LLMEngine @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -class LLMEngineStage(PassThruTypeMixin, SinglePortStage): +class LLMEngineStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ Stage for executing an LLM engine within a Morpheus pipeline. @@ -52,7 +52,7 @@ def name(self) -> str: """Return the name of the stage""" return "llm-engine" - def accepted_types(self) -> typing.Tuple: + def accepted_types(self) -> tuple: """ Returns accepted input types for this stage. @@ -70,8 +70,8 @@ def supports_cpp_node(self): def _cast_control_message(self, message: ControlMessage, *, cpp_messages_lib: types.ModuleType) -> ControlMessage: """ - LLMEngineStage does not contain a Python implementation, however it is capable of running in Python/cpu-only - mode. This method is needed to cast the Python ControlMessage to a C++ ControlMessage. + LLMEngineStage does not contain a Python implementation, however it is capable of running in cpu-only mode. + This method is needed to cast the Python ControlMessage to a C++ ControlMessage. This is different than casting from the Python bindings for the C++ ControlMessage to a C++ ControlMessage. """ @@ -82,7 +82,7 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> node = _llm.LLMEngineStage(builder, self.unique_name, self._engine) node.launch_options.pe_count = 1 - if not CppConfig.get_should_use_cpp(): + if self._config.execution_mode == ExecutionMode.CPU: import morpheus._lib.messages as _messages cast_fn = functools.partial(self._cast_control_message, cpp_messages_lib=_messages) pre_node = builder.make_node(f"{self.unique_name}-pre-cast", ops.map(cast_fn)) From 5a063c281ef2ed98f2d215912e72f3178b242836 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 19 Sep 2024 11:07:53 -0700 Subject: [PATCH 255/347] Revert "Revert "Add type hint"" This reverts commit 5bcd44dbc3bfd9ccaf8b263523c2898716ec38b2. --- python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index ff3aa57d76..7e61ae36d7 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -64,7 +64,7 @@ def accepted_types(self) -> tuple: """ return (ControlMessage, ) - def supports_cpp_node(self): + def supports_cpp_node(self) -> bool: """Indicates whether this stage supports a C++ node.""" return True From 11f58eaaf41372af0d0fd1b8e12ff5b7671c6a07 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 19 Sep 2024 12:12:19 -0700 Subject: [PATCH 256/347] WIP: not working --- examples/llm/completion/pipeline.py | 14 +++++++++----- examples/llm/completion/run.py | 1 + .../_lib/include/morpheus/messages/control.hpp | 6 +++++- python/morpheus/morpheus/_lib/messages/module.cpp | 9 +++++---- .../morpheus/_lib/src/messages/control.cpp | 11 +++++++++-- .../morpheus_llm/stages/llm/llm_engine_stage.py | 4 ++-- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/examples/llm/completion/pipeline.py b/examples/llm/completion/pipeline.py index 86b5df19d7..4087ce9ca6 100644 --- a/examples/llm/completion/pipeline.py +++ b/examples/llm/completion/pipeline.py @@ -15,9 +15,8 @@ import logging import time -import cudf - from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.io.deserializers import read_file_to_df from morpheus.pipeline.linear_pipeline import LinearPipeline @@ -26,6 +25,8 @@ from morpheus.stages.output.in_memory_sink_stage import InMemorySinkStage from morpheus.stages.preprocess.deserialize_stage import DeserializeStage from morpheus.utils.concat_df import concat_dataframes +from morpheus.utils.type_utils import exec_mode_to_df_type_str +from morpheus.utils.type_utils import get_df_class from morpheus_llm.llm import LLMEngine from morpheus_llm.llm.nodes.extracter_node import ExtracterNode from morpheus_llm.llm.nodes.llm_generate_node import LLMGenerateNode @@ -71,7 +72,8 @@ def _build_engine(llm_service: str): return engine -def pipeline(num_threads: int, +def pipeline(use_cpu_only: bool, + num_threads: int, pipeline_batch_size: int, model_max_batch_size: int, repeat_count: int, @@ -80,6 +82,7 @@ def pipeline(num_threads: int, shuffle: bool = False) -> float: config = Config() + config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU # Below properties are specified by the command line config.num_threads = num_threads @@ -89,9 +92,10 @@ def pipeline(num_threads: int, config.edge_buffer_size = 128 if input_file is not None: - source_df = read_file_to_df(input_file, df_type='cudf') + source_df = read_file_to_df(input_file, df_type=exec_mode_to_df_type_str(config.execution_mode)) else: - source_df = cudf.DataFrame({ + df_class = get_df_class(config.execution_mode) + source_df = df_class({ "country": [ "France", "Spain", diff --git a/examples/llm/completion/run.py b/examples/llm/completion/run.py index 611a5105db..4ba702c700 100644 --- a/examples/llm/completion/run.py +++ b/examples/llm/completion/run.py @@ -26,6 +26,7 @@ def run(): @run.command() +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) @click.option( "--num_threads", default=len(os.sched_getaffinity(0)), diff --git a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp index 259d6bd69f..2ca59cbbdd 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp @@ -21,6 +21,7 @@ #include "morpheus/messages/meta.hpp" // for MessageMeta #include "morpheus/utilities/json_types.hpp" // for json_t +#include // for PyHolder #include // for object, dict, list #include // IWYU pragma: keep @@ -169,6 +170,8 @@ class MORPHEUS_EXPORT ControlMessage */ void payload(const std::shared_ptr& payload); + void payload(pybind11::object payload); + /** * @brief Retrieves the tensor memory associated with the control message. * @@ -250,6 +253,7 @@ class MORPHEUS_EXPORT ControlMessage ControlMessageType to_task_type(const std::string& task_type, bool throw_on_error) const; ControlMessageType m_cm_type{ControlMessageType::NONE}; + std::unique_ptr m_py_payload{nullptr}; std::shared_ptr m_payload{nullptr}; std::shared_ptr m_tensors{nullptr}; @@ -308,7 +312,7 @@ struct MORPHEUS_EXPORT ControlMessageProxy * @brief Set the payload object given a Python instance of MessageMeta * @param meta */ - static void payload_from_python_meta(ControlMessage& self, const pybind11::object& meta); + static void payload_from_python_meta(ControlMessage& self, const pybind11::object& meta, bool no_cast = false); /** * @brief Sets a timestamp for a given key. diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index af559bde58..ef506acf2e 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -284,10 +284,11 @@ PYBIND11_MODULE(messages, _module) .def("list_metadata", &ControlMessageProxy::list_metadata) .def("payload", pybind11::overload_cast<>(&ControlMessage::payload)) .def("payload", pybind11::overload_cast&>(&ControlMessage::payload)) - .def( - "payload", - pybind11::overload_cast(&ControlMessageProxy::payload_from_python_meta), - py::arg("meta")) + .def("payload", + pybind11::overload_cast( + &ControlMessageProxy::payload_from_python_meta), + py::arg("meta"), + py::arg("no_cast") = false) .def("tensors", pybind11::overload_cast<>(&ControlMessage::tensors)) .def("tensors", pybind11::overload_cast&>(&ControlMessage::tensors)) .def("remove_task", &ControlMessage::remove_task, py::arg("task_type")) diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index d20334c35a..2d3b46c94d 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -415,9 +415,16 @@ void ControlMessageProxy::set_timestamp(ControlMessage& self, const std::string& } } -void ControlMessageProxy::payload_from_python_meta(ControlMessage& self, const pybind11::object& meta) +void ControlMessageProxy::payload_from_python_meta(ControlMessage& self, const pybind11::object& meta, bool no_cast) { - self.payload(MessageMetaInterfaceProxy::init_python_meta(meta)); + if (!no_cast) + { + self.payload(MessageMetaInterfaceProxy::init_python_meta(meta)); + } + else + { + // TODO + } } } // namespace morpheus diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index 7e61ae36d7..11d76b703c 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -58,7 +58,7 @@ def accepted_types(self) -> tuple: Returns ------- - typing.Tuple(`ControlMessage`, ) + tuple(`ControlMessage`, ) Accepted input types. """ @@ -75,7 +75,7 @@ def _cast_control_message(self, message: ControlMessage, *, cpp_messages_lib: ty This is different than casting from the Python bindings for the C++ ControlMessage to a C++ ControlMessage. """ - return cpp_messages_lib.ControlMessage(message) + return cpp_messages_lib.ControlMessage(message, no_cast=True) def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: import morpheus_llm._lib.llm as _llm From 3632e805e8e6a497c7bdefba99e721a16d03b126 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 09:32:06 -0700 Subject: [PATCH 257/347] Add work-around for issue #1891 --- examples/ransomware_detection/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/ransomware_detection/run.py b/examples/ransomware_detection/run.py index ec3ff4e015..dd9b18b559 100644 --- a/examples/ransomware_detection/run.py +++ b/examples/ransomware_detection/run.py @@ -193,7 +193,7 @@ def run_pipeline(debug, # Add a inference stage. # This stage sends inference requests to the Tritonserver and captures the response. - pipeline.add_stage( + inf_stage = pipeline.add_stage( TritonInferenceStage( config, model_name=model_name, @@ -201,6 +201,9 @@ def run_pipeline(debug, force_convert_inputs=True, )) + # Work-around for issue #1891 remove once resolved. + inf_stage._thread_count = 1 + # Add a monitor stage. # This stage logs the metrics (msg/sec) from the above stage. pipeline.add_stage(MonitorStage(config, description="Inference rate")) From 0a9d48c2c136ba044e6cfb6e6c8ef9ac7ee96efe Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 10:57:12 -0700 Subject: [PATCH 258/347] Expose thread_count as a constructor parameter to TritonInferenceStage and InferenceStage --- examples/ransomware_detection/run.py | 6 ++---- .../morpheus/morpheus/stages/inference/inference_stage.py | 8 +++++--- .../morpheus/stages/inference/triton_inference_stage.py | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/ransomware_detection/run.py b/examples/ransomware_detection/run.py index dd9b18b559..cd03bfa5d8 100644 --- a/examples/ransomware_detection/run.py +++ b/examples/ransomware_detection/run.py @@ -193,17 +193,15 @@ def run_pipeline(debug, # Add a inference stage. # This stage sends inference requests to the Tritonserver and captures the response. - inf_stage = pipeline.add_stage( + pipeline.add_stage( TritonInferenceStage( config, model_name=model_name, server_url=server_url, force_convert_inputs=True, + thread_count=1 # Work-around for issue #1891 remove once resolved. )) - # Work-around for issue #1891 remove once resolved. - inf_stage._thread_count = 1 - # Add a monitor stage. # This stage logs the metrics (msg/sec) from the above stage. pipeline.add_stage(MonitorStage(config, description="Inference rate")) diff --git a/python/morpheus/morpheus/stages/inference/inference_stage.py b/python/morpheus/morpheus/stages/inference/inference_stage.py index 6cc1763d33..4219064432 100644 --- a/python/morpheus/morpheus/stages/inference/inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/inference_stage.py @@ -148,10 +148,12 @@ class InferenceStage(ControlMessageStage): ---------- c : `morpheus.config.Config` Pipeline configuration instance. - + thread_count : int, optional + Number of threads to use for inference. If not provided, the `num_threads` attribute of the `Config` object + will be used. """ - def __init__(self, c: Config): + def __init__(self, c: Config, thread_count: int = None): super().__init__(c) # GPU only stage, assuming all messages are cuDF/CuPy based @@ -160,7 +162,7 @@ def __init__(self, c: Config): self._fea_length = c.feature_length - self._thread_count = c.num_threads + self._thread_count = thread_count or c.num_threads self._workers: typing.List[InferenceWorker] = [] self._inf_queue = ProducerConsumerQueue() diff --git a/python/morpheus/morpheus/stages/inference/triton_inference_stage.py b/python/morpheus/morpheus/stages/inference/triton_inference_stage.py index 2dc31925f7..62f0a51d8e 100644 --- a/python/morpheus/morpheus/stages/inference/triton_inference_stage.py +++ b/python/morpheus/morpheus/stages/inference/triton_inference_stage.py @@ -684,6 +684,9 @@ class TritonInferenceStage(InferenceStage): which will be inroduced as: inout_mapping={"mask": "input_mask", "output": "probs"} + thread_count : int, optional + Number of threads to use for inference. If not provided, the `num_threads` attribute of the `Config` object + will be used. """ _INFERENCE_WORKER_DEFAULT_INOUT_MAPPING = { @@ -710,8 +713,9 @@ def __init__(self, needs_logits: bool = None, inout_mapping: dict[str, str] = None, input_mapping: dict[str, str] = None, - output_mapping: dict[str, str] = None): - super().__init__(c) + output_mapping: dict[str, str] = None, + thread_count: int = None): + super().__init__(c, thread_count=thread_count) self._config = c From f8d2f810c24ef46b25a0e2dd3f0145b4fea7c809 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 11:10:11 -0700 Subject: [PATCH 259/347] Revert WIP changes --- .../_lib/include/morpheus/messages/control.hpp | 6 +----- python/morpheus/morpheus/_lib/messages/module.cpp | 9 ++++----- .../morpheus/morpheus/_lib/src/messages/control.cpp | 11 ++--------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp index 2ca59cbbdd..259d6bd69f 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp @@ -21,7 +21,6 @@ #include "morpheus/messages/meta.hpp" // for MessageMeta #include "morpheus/utilities/json_types.hpp" // for json_t -#include // for PyHolder #include // for object, dict, list #include // IWYU pragma: keep @@ -170,8 +169,6 @@ class MORPHEUS_EXPORT ControlMessage */ void payload(const std::shared_ptr& payload); - void payload(pybind11::object payload); - /** * @brief Retrieves the tensor memory associated with the control message. * @@ -253,7 +250,6 @@ class MORPHEUS_EXPORT ControlMessage ControlMessageType to_task_type(const std::string& task_type, bool throw_on_error) const; ControlMessageType m_cm_type{ControlMessageType::NONE}; - std::unique_ptr m_py_payload{nullptr}; std::shared_ptr m_payload{nullptr}; std::shared_ptr m_tensors{nullptr}; @@ -312,7 +308,7 @@ struct MORPHEUS_EXPORT ControlMessageProxy * @brief Set the payload object given a Python instance of MessageMeta * @param meta */ - static void payload_from_python_meta(ControlMessage& self, const pybind11::object& meta, bool no_cast = false); + static void payload_from_python_meta(ControlMessage& self, const pybind11::object& meta); /** * @brief Sets a timestamp for a given key. diff --git a/python/morpheus/morpheus/_lib/messages/module.cpp b/python/morpheus/morpheus/_lib/messages/module.cpp index ef506acf2e..af559bde58 100644 --- a/python/morpheus/morpheus/_lib/messages/module.cpp +++ b/python/morpheus/morpheus/_lib/messages/module.cpp @@ -284,11 +284,10 @@ PYBIND11_MODULE(messages, _module) .def("list_metadata", &ControlMessageProxy::list_metadata) .def("payload", pybind11::overload_cast<>(&ControlMessage::payload)) .def("payload", pybind11::overload_cast&>(&ControlMessage::payload)) - .def("payload", - pybind11::overload_cast( - &ControlMessageProxy::payload_from_python_meta), - py::arg("meta"), - py::arg("no_cast") = false) + .def( + "payload", + pybind11::overload_cast(&ControlMessageProxy::payload_from_python_meta), + py::arg("meta")) .def("tensors", pybind11::overload_cast<>(&ControlMessage::tensors)) .def("tensors", pybind11::overload_cast&>(&ControlMessage::tensors)) .def("remove_task", &ControlMessage::remove_task, py::arg("task_type")) diff --git a/python/morpheus/morpheus/_lib/src/messages/control.cpp b/python/morpheus/morpheus/_lib/src/messages/control.cpp index 2d3b46c94d..d20334c35a 100644 --- a/python/morpheus/morpheus/_lib/src/messages/control.cpp +++ b/python/morpheus/morpheus/_lib/src/messages/control.cpp @@ -415,16 +415,9 @@ void ControlMessageProxy::set_timestamp(ControlMessage& self, const std::string& } } -void ControlMessageProxy::payload_from_python_meta(ControlMessage& self, const pybind11::object& meta, bool no_cast) +void ControlMessageProxy::payload_from_python_meta(ControlMessage& self, const pybind11::object& meta) { - if (!no_cast) - { - self.payload(MessageMetaInterfaceProxy::init_python_meta(meta)); - } - else - { - // TODO - } + self.payload(MessageMetaInterfaceProxy::init_python_meta(meta)); } } // namespace morpheus From 203c56dfa4f5ed74ffe23cfd01427ae6a625e48a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 12:01:48 -0700 Subject: [PATCH 260/347] WIP - untested --- .../morpheus_llm/llm/nodes/extracter_node.py | 8 ++- .../llm/task_handlers/simple_task_handler.py | 4 +- .../stages/llm/llm_engine_stage.py | 56 ++++++++++++++++--- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/python/morpheus_llm/morpheus_llm/llm/nodes/extracter_node.py b/python/morpheus_llm/morpheus_llm/llm/nodes/extracter_node.py index 8027ad8178..710b0fe1e2 100644 --- a/python/morpheus_llm/morpheus_llm/llm/nodes/extracter_node.py +++ b/python/morpheus_llm/morpheus_llm/llm/nodes/extracter_node.py @@ -17,6 +17,7 @@ import numpy as np +from morpheus.messages import MessageMeta from morpheus_llm.llm import LLMContext from morpheus_llm.llm import LLMNodeBase @@ -59,7 +60,9 @@ async def execute(self, context: LLMContext) -> LLMContext: # pylint: disable=i # Get the keys from the task input_keys: list[str] = typing.cast(list[str], context.task()["input_keys"]) - with context.message().payload().mutable_dataframe() as df: + meta: MessageMeta = context.message().get_metadata("llm_message_meta") + + with meta.mutable_dataframe() as df: input_dict: list[dict] = df[input_keys].to_dict(orient="list") input_dict = _array_to_list(input_dict) @@ -95,7 +98,8 @@ def get_input_names(self) -> list[str]: async def execute(self, context: LLMContext) -> LLMContext: # pylint: disable=invalid-overridden-method # Get the data from the DataFrame - with context.message().payload().mutable_dataframe() as df: + meta: MessageMeta = context.message().get_metadata("llm_message_meta") + with meta.mutable_dataframe() as df: input_dict: list[dict] = df[self._input_names].to_dict(orient="list") input_dict = _array_to_list(input_dict) diff --git a/python/morpheus_llm/morpheus_llm/llm/task_handlers/simple_task_handler.py b/python/morpheus_llm/morpheus_llm/llm/task_handlers/simple_task_handler.py index c2461200ad..baf0db0310 100644 --- a/python/morpheus_llm/morpheus_llm/llm/task_handlers/simple_task_handler.py +++ b/python/morpheus_llm/morpheus_llm/llm/task_handlers/simple_task_handler.py @@ -15,6 +15,7 @@ import logging from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta from morpheus_llm.llm import LLMContext from morpheus_llm.llm import LLMTaskHandler @@ -48,7 +49,8 @@ async def try_handle(self, context: LLMContext) -> list[ControlMessage]: input_dict = context.get_inputs() - with context.message().payload().mutable_dataframe() as df: + meta: MessageMeta = context.message().get_metadata("llm_message_meta") + with meta.mutable_dataframe() as df: # Write the values to the dataframe for key, value in input_dict.items(): df[key] = value diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index 11d76b703c..ae9b0bd4d9 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -68,28 +68,66 @@ def supports_cpp_node(self) -> bool: """Indicates whether this stage supports a C++ node.""" return True - def _cast_control_message(self, message: ControlMessage, *, cpp_messages_lib: types.ModuleType) -> ControlMessage: + def _store_payload(self, message: ControlMessage) -> ControlMessage: + """ + Store the MessageMeta in the ControlMessage's metadata. + + In CPU-only allows the ControlMessage to hold an instance of a Python MessageMeta containing a pandas DataFrame. + """ + message.set_metadata("llm_message_meta", message.payload()) + return message + + def _cast_to_cpp_control_message(self, message: ControlMessage, *, + cpp_messages_lib: types.ModuleType) -> ControlMessage: """ LLMEngineStage does not contain a Python implementation, however it is capable of running in cpu-only mode. - This method is needed to cast the Python ControlMessage to a C++ ControlMessage. + This method is needed to create an instance of a C++ ControlMessage. This is different than casting from the Python bindings for the C++ ControlMessage to a C++ ControlMessage. """ - return cpp_messages_lib.ControlMessage(message, no_cast=True) + cm = cpp_messages_lib.ControlMessage() + metadata = message.get_metadata() + for (key, value) in metadata.items(): + cm.set_metadata(key, value) + + return cm + + def _restore_payload(self, message: ControlMessage) -> ControlMessage: + """ + Pop llm_message_meta from the metadata and set it as the payload. + + In CPU-only mode this has the effect of converting the C++ ControlMessage back to a Python ControlMessage. + """ + metadata = message.get_metadata() + message_meta = metadata.pop("llm_message_meta") + + out_message = ControlMessage() + out_message.payload(message_meta) + for (key, value) in metadata.items(): + out_message.set_metadata(key, value) + + return out_message def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: import morpheus_llm._lib.llm as _llm + + store_payload_node = builder.make_node(f"{self.unique_name}-store-payload", ops.map(self._store_payload)) + builder.make_edge(input_node, store_payload_node) + node = _llm.LLMEngineStage(builder, self.unique_name, self._engine) node.launch_options.pe_count = 1 if self._config.execution_mode == ExecutionMode.CPU: import morpheus._lib.messages as _messages - cast_fn = functools.partial(self._cast_control_message, cpp_messages_lib=_messages) - pre_node = builder.make_node(f"{self.unique_name}-pre-cast", ops.map(cast_fn)) - builder.make_edge(input_node, pre_node) + cast_to_cpp_fn = functools.partial(self._cast_to_cpp_control_message, cpp_messages_lib=_messages) + cast_to_cpp_node = builder.make_node(f"{self.unique_name}-pre-msg-cast", ops.map(cast_to_cpp_fn)) + builder.make_edge(store_payload_node, cast_to_cpp_node) + builder.make_edge(cast_to_cpp_node, node) - input_node = pre_node + else: + builder.make_edge(store_payload_node, node) - builder.make_edge(input_node, node) + restore_payload_node = builder.make_node(f"{self.unique_name}-restore-payload", ops.map(self._restore_payload)) + builder.make_edge(node, restore_payload_node) - return node + return restore_payload_node From 93597d9f57249b8aac3a70a369ecc2308c17f70d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 12:45:21 -0700 Subject: [PATCH 261/347] Fix casting of control messages --- .../stages/llm/llm_engine_stage.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index ae9b0bd4d9..a85a40b791 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -15,6 +15,8 @@ import functools import logging import types +import typing +from collections import deque import mrc from mrc.core import operators as ops @@ -77,7 +79,22 @@ def _store_payload(self, message: ControlMessage) -> ControlMessage: message.set_metadata("llm_message_meta", message.payload()) return message - def _cast_to_cpp_control_message(self, message: ControlMessage, *, + def _copy_tasks_and_metadata(self, + src: ControlMessage, + dst: ControlMessage, + metadata: dict[str, typing.Any] = None): + if metadata is None: + metadata = src.get_metadata() + + for (key, value) in metadata.items(): + dst.set_metadata(key, value) + + tasks = src.get_tasks() + for (task, task_value) in tasks.items(): + for tv in task_value: + dst.add_task(task, tv) + + def _cast_to_cpp_control_message(self, py_message: ControlMessage, *, cpp_messages_lib: types.ModuleType) -> ControlMessage: """ LLMEngineStage does not contain a Python implementation, however it is capable of running in cpu-only mode. @@ -85,12 +102,10 @@ def _cast_to_cpp_control_message(self, message: ControlMessage, *, This is different than casting from the Python bindings for the C++ ControlMessage to a C++ ControlMessage. """ - cm = cpp_messages_lib.ControlMessage() - metadata = message.get_metadata() - for (key, value) in metadata.items(): - cm.set_metadata(key, value) + cpp_message = cpp_messages_lib.ControlMessage() + self._copy_tasks_and_metadata(py_message, cpp_message) - return cm + return cpp_message def _restore_payload(self, message: ControlMessage) -> ControlMessage: """ @@ -103,8 +118,8 @@ def _restore_payload(self, message: ControlMessage) -> ControlMessage: out_message = ControlMessage() out_message.payload(message_meta) - for (key, value) in metadata.items(): - out_message.set_metadata(key, value) + + self._copy_tasks_and_metadata(message, out_message, metadata=metadata) return out_message From aec27d24145fef59853c995c642165b1ac1e0289 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 12:53:55 -0700 Subject: [PATCH 262/347] Add CPU only mode support to the simple agents pipeline --- examples/llm/agents/run.py | 1 + examples/llm/agents/simple_pipeline.py | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/llm/agents/run.py b/examples/llm/agents/run.py index b643926a2d..a97bce4b78 100644 --- a/examples/llm/agents/run.py +++ b/examples/llm/agents/run.py @@ -25,6 +25,7 @@ def run(): @run.command(help="Runs a simple finite pipeline with a single execution of a LangChain agent from a fixed input") +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) @click.option( "--num_threads", default=len(os.sched_getaffinity(0)), diff --git a/examples/llm/agents/simple_pipeline.py b/examples/llm/agents/simple_pipeline.py index 78bfc00039..7fd7e1fdcb 100644 --- a/examples/llm/agents/simple_pipeline.py +++ b/examples/llm/agents/simple_pipeline.py @@ -15,13 +15,13 @@ import logging import time -import cudf - from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.config import PipelineModes from morpheus.pipeline.linear_pipeline import LinearPipeline from morpheus.stages.input.in_memory_source_stage import InMemorySourceStage from morpheus.utils.concat_df import concat_dataframes +from morpheus.utils.type_utils import get_df_class from .common import build_common_pipeline @@ -29,6 +29,7 @@ def pipeline( + use_cpu_only: bool, num_threads: int, pipeline_batch_size, model_max_batch_size, @@ -36,6 +37,7 @@ def pipeline( repeat_count, ) -> float: config = Config() + config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU config.mode = PipelineModes.OTHER # Below properties are specified by the command line @@ -45,9 +47,9 @@ def pipeline( config.mode = PipelineModes.NLP config.edge_buffer_size = 128 + df_class = get_df_class(config.execution_mode) source_dfs = [ - cudf.DataFrame( - {"questions": ["Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?"]}) + df_class({"questions": ["Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?"]}) ] completion_task = {"task_type": "completion", "task_dict": {"input_keys": ["questions"], }} From 1d811c8f40cdf1f2e71df11fab934a03084b370a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 13:50:14 -0700 Subject: [PATCH 263/347] Remove unused import --- python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py index a85a40b791..289e447afa 100644 --- a/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py +++ b/python/morpheus_llm/morpheus_llm/stages/llm/llm_engine_stage.py @@ -16,7 +16,6 @@ import logging import types import typing -from collections import deque import mrc from mrc.core import operators as ops From 4646eb2f96eb2a7701f01240fa0ceb42035fa08e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 14:18:48 -0700 Subject: [PATCH 264/347] update tests to set the payload in the metadata --- tests/llm/nodes/test_extractor_node.py | 1 + tests/llm/nodes/test_manual_extractor_node.py | 1 + tests/llm/task_handlers/test_simple_task_handler.py | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/llm/nodes/test_extractor_node.py b/tests/llm/nodes/test_extractor_node.py index 355f16aeb5..38673c95fc 100644 --- a/tests/llm/nodes/test_extractor_node.py +++ b/tests/llm/nodes/test_extractor_node.py @@ -39,6 +39,7 @@ def test_execute(): df = cudf.DataFrame({"insects": insects.copy(), "mammals": mammals.copy(), "reptiles": reptiles.copy()}) message = ControlMessage() message.payload(MessageMeta(df)) + message.set_metadata("llm_message_meta", message.payload()) task_dict = {"input_keys": ["mammals", "reptiles"]} node = ExtracterNode() diff --git a/tests/llm/nodes/test_manual_extractor_node.py b/tests/llm/nodes/test_manual_extractor_node.py index 143636999d..7c3ef3542e 100644 --- a/tests/llm/nodes/test_manual_extractor_node.py +++ b/tests/llm/nodes/test_manual_extractor_node.py @@ -48,6 +48,7 @@ def test_execute(): df = cudf.DataFrame({"insects": insects.copy(), "mammals": mammals.copy(), "reptiles": reptiles.copy()}) message = ControlMessage() message.payload(MessageMeta(df)) + message.set_metadata("llm_message_meta", message.payload()) task_dict = {"input_keys": ["insects"]} node = ManualExtracterNode(["mammals", "reptiles"]) diff --git a/tests/llm/task_handlers/test_simple_task_handler.py b/tests/llm/task_handlers/test_simple_task_handler.py index 8439d2df3d..641f65b9b4 100644 --- a/tests/llm/task_handlers/test_simple_task_handler.py +++ b/tests/llm/task_handlers/test_simple_task_handler.py @@ -46,6 +46,7 @@ def test_try_handle(dataset_cudf: DatasetManager): message = ControlMessage() message.payload(MessageMeta(df)) + message.set_metadata("llm_message_meta", message.payload()) task_handler = SimpleTaskHandler(['reptiles']) From 6a23f7910e1bbe0e700c904ac5276f576a7fb871 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 23 Sep 2024 16:22:52 -0700 Subject: [PATCH 265/347] Fix conda path for missing llm packages --- .../morpheus_llm/morpheus_llm/llm/services/nemo_llm_service.py | 2 +- .../morpheus_llm/llm/services/openai_chat_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/morpheus_llm/morpheus_llm/llm/services/nemo_llm_service.py b/python/morpheus_llm/morpheus_llm/llm/services/nemo_llm_service.py index a3c2c2f6b5..ef80814929 100644 --- a/python/morpheus_llm/morpheus_llm/llm/services/nemo_llm_service.py +++ b/python/morpheus_llm/morpheus_llm/llm/services/nemo_llm_service.py @@ -27,7 +27,7 @@ IMPORT_ERROR_MESSAGE = ( "NemoLLM not found. Install it and other additional dependencies by running the following command:\n" "`conda env update --solver=libmamba -n morpheus " - "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/examples_cuda-121_arch-x86_64.yaml --prune`") try: import nemollm diff --git a/python/morpheus_llm/morpheus_llm/llm/services/openai_chat_service.py b/python/morpheus_llm/morpheus_llm/llm/services/openai_chat_service.py index a15f9c1a2d..d4eaac4503 100644 --- a/python/morpheus_llm/morpheus_llm/llm/services/openai_chat_service.py +++ b/python/morpheus_llm/morpheus_llm/llm/services/openai_chat_service.py @@ -32,7 +32,7 @@ IMPORT_ERROR_MESSAGE = ("OpenAIChatService & OpenAIChatClient require the openai package to be installed. " "Install it by running the following command:\n" "`conda env update --solver=libmamba -n morpheus " - "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/examples_cuda-121_arch-x86_64.yaml --prune`") try: import openai From a64de2cb707769dd78af3dc6208a59013e9543eb Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 25 Sep 2024 14:54:59 -0700 Subject: [PATCH 266/347] wip --- examples/abp_nvsmi_detection/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/abp_nvsmi_detection/README.md b/examples/abp_nvsmi_detection/README.md index eab83358e2..b29ad6bb84 100644 --- a/examples/abp_nvsmi_detection/README.md +++ b/examples/abp_nvsmi_detection/README.md @@ -61,7 +61,10 @@ In this example we will be using the `examples/data/nvsmi.jsonlines` dataset tha This example can be easily applied to datasets generated from your own NVIDIA GPU devices. If NetQ is not deployed in your environment, the `nvsmi_data_extract.py` script is provided which uses [pyNVML](https://pypi.org/project/nvidia-ml-py/) and [pandas](https://pandas.pydata.org/) to generate data similar to NetQ. `pyNVML` contains the Python bindings for NVIDIA Management Library (NVML), the same library used by `nvidia-smi`. -`pyNVML` and `pandas` come already installed on the Morpheus release and development Docker images. Otherwise, they will need to be installed before running the script. +pyNVML is not installed by default, use the following command to install it: +```bash +conda env update --solver=libmamba -n morpheus --file conda/environments/examples_cuda-121_arch-x86_64.yaml +``` Run the following to start generating your dataset: ``` From 36f51894d30501ebf877079b80f9d75d53cf695e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 25 Sep 2024 16:01:47 -0700 Subject: [PATCH 267/347] Replace manually invoking CudfHelper::load(), with re-inmplementing CudfHelper as a singleton --- .../morpheus/_lib/common/__init__.pyi | 3 -- .../morpheus/morpheus/_lib/common/module.cpp | 5 --- .../include/morpheus/utilities/cudf_util.hpp | 5 ++- .../morpheus/_lib/src/io/deserializers.cpp | 1 - .../morpheus/_lib/src/io/serializers.cpp | 1 - .../morpheus/_lib/src/utilities/cudf_util.cpp | 44 +++++++------------ .../_lib/tests/messages/test_dev_doc_ex3.cpp | 22 ++-------- .../_lib/tests/messages/test_messages.hpp | 17 +------ .../stages/test_triton_inference_stage.cpp | 15 +------ .../morpheus/_lib/tests/test_file_in_out.cpp | 16 +------ .../morpheus/_lib/tests/test_utils/common.cpp | 4 -- python/morpheus/morpheus/common/__init__.py | 2 - python/morpheus/morpheus/pipeline/pipeline.py | 5 --- tests/conftest.py | 7 --- 14 files changed, 28 insertions(+), 119 deletions(-) diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi index bba480d220..7e11e81ccd 100644 --- a/python/morpheus/morpheus/_lib/common/__init__.pyi +++ b/python/morpheus/morpheus/_lib/common/__init__.pyi @@ -18,7 +18,6 @@ __all__ = [ "Tensor", "TypeId", "determine_file_type", - "load_cudf_helper", "read_file_to_df", "typeid_is_fully_supported", "typeid_to_numpy_str", @@ -203,8 +202,6 @@ def determine_file_type(filename: os.PathLike) -> FileTypes: @typing.overload def determine_file_type(filename: str) -> FileTypes: pass -def load_cudf_helper() -> None: - pass def read_file_to_df(filename: str, file_type: FileTypes = FileTypes.Auto) -> object: pass def typeid_is_fully_supported(arg0: TypeId) -> bool: diff --git a/python/morpheus/morpheus/_lib/common/module.cpp b/python/morpheus/morpheus/_lib/common/module.cpp index 54f0cc850e..f9ba779f10 100644 --- a/python/morpheus/morpheus/_lib/common/module.cpp +++ b/python/morpheus/morpheus/_lib/common/module.cpp @@ -28,7 +28,6 @@ #include "morpheus/objects/filter_source.hpp" #include "morpheus/objects/tensor_object.hpp" // for TensorObject #include "morpheus/objects/wrapped_tensor.hpp" -#include "morpheus/utilities/cudf_util.hpp" #include "morpheus/utilities/http_server.hpp" #include "morpheus/version.hpp" @@ -143,10 +142,6 @@ PYBIND11_MODULE(common, _module) py::arg("filename"), py::arg("file_type") = FileTypes::Auto); - _module.def("load_cudf_helper", [] { - CudfHelper::load(); - }); - py::enum_( _module, "FilterSource", "Enum to indicate which source the FilterDetectionsStage should operate on.") .value("Auto", FilterSource::Auto) diff --git a/python/morpheus/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp b/python/morpheus/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp index 5eb6636919..7a87620b90 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp @@ -40,7 +40,7 @@ namespace morpheus { struct CudfHelper { public: - __attribute__((visibility("default"))) static void load(); + static void load(); /** * @brief Converts a C++ table to a Python DataTable object @@ -67,6 +67,9 @@ struct CudfHelper * @return TableInfoData */ static TableInfoData table_info_data_from_table(pybind11::object table); + + private: + CudfHelper(); }; /** @} */ // end of group diff --git a/python/morpheus/morpheus/_lib/src/io/deserializers.cpp b/python/morpheus/morpheus/_lib/src/io/deserializers.cpp index 8bd4ef5282..032cffd57b 100644 --- a/python/morpheus/morpheus/_lib/src/io/deserializers.cpp +++ b/python/morpheus/morpheus/_lib/src/io/deserializers.cpp @@ -98,7 +98,6 @@ cudf::io::table_with_metadata load_table_from_file(const std::string& filename, pybind11::object read_file_to_df(const std::string& filename, FileTypes file_type) { - CudfHelper::load(); auto table = load_table_from_file(filename, file_type); int index_col_count = prepare_df_index(table); diff --git a/python/morpheus/morpheus/_lib/src/io/serializers.cpp b/python/morpheus/morpheus/_lib/src/io/serializers.cpp index c0f5814305..54234f1592 100644 --- a/python/morpheus/morpheus/_lib/src/io/serializers.cpp +++ b/python/morpheus/morpheus/_lib/src/io/serializers.cpp @@ -252,7 +252,6 @@ void SerializersProxy::write_df_to_file(pybind11::object df, FileTypes file_type, const py::kwargs& kwargs) { - CudfHelper::load(); if (file_type == FileTypes::Auto) { file_type = determine_file_type(filename); // throws if it is unable to determine the type diff --git a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp index b60cd17bf1..f3f7a902c0 100644 --- a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp +++ b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp @@ -25,7 +25,7 @@ #include #include -#include // for atomic +#include // for atomic #include // for getenv #include #include @@ -38,49 +38,38 @@ */ #include "cudf_helpers_api.h" -namespace { -/* - * We want to prevent calling import_morpheus___lib__cudf_helpers() concurrently which is the purpose of the mutex. - * Although it is safe to call import_morpheus___lib__cudf_helpers() multiple times, we would still like to avoid it - * along with the call to std::getenv(), the atomic bool provides a cheap way to check if the library has been loaded. - */ -std::atomic g_cudf_helpers_loaded = false; -std::mutex g_cudf_helpers_load_mutex; -} // namespace - namespace morpheus { -void CudfHelper::load() +CudfHelper::CudfHelper() { - if (!g_cudf_helpers_loaded) + // Avoid loading cudf_helpers if we are in a sphinx build + if (std::getenv("MORPHEUS_IN_SPHINX_BUILD") == nullptr) { - // Avoid loading cudf_helpers if we are in a sphinx build - if (std::getenv("MORPHEUS_IN_SPHINX_BUILD") == nullptr) + if (import_morpheus___lib__cudf_helpers() != 0) { - std::lock_guard guard(g_cudf_helpers_load_mutex); - if (import_morpheus___lib__cudf_helpers() != 0) - { - pybind11::error_already_set ex; - - LOG(ERROR) << "Could not load cudf_helpers library: " << ex.what(); - throw ex; - } - else - { - g_cudf_helpers_loaded = true; - } + pybind11::error_already_set ex; + + LOG(ERROR) << "Could not load cudf_helpers library: " << ex.what(); + throw ex; } } } +void CudfHelper::load() +{ + static CudfHelper s; +} + pybind11::object proxy_table_from_table_with_metadata(cudf::io::table_with_metadata&& table, int index_col_count) { + CudfHelper::load(); return pybind11::reinterpret_steal( (PyObject*)make_table_from_table_with_metadata(std::move(table), index_col_count)); } morpheus::TableInfoData proxy_table_info_data_from_table(pybind11::object table) { + CudfHelper::load(); return make_table_info_data_from_table(table.ptr()); } @@ -91,6 +80,7 @@ pybind11::object CudfHelper::table_from_table_with_metadata(cudf::io::table_with pybind11::object CudfHelper::table_from_table_info(const TableInfoBase& table_info) { + CudfHelper::load(); // Get the table info data from the table_into auto table_info_data = table_info.get_data(); diff --git a/python/morpheus/morpheus/_lib/tests/messages/test_dev_doc_ex3.cpp b/python/morpheus/morpheus/_lib/tests/messages/test_dev_doc_ex3.cpp index 780ad48b37..94fd26aae3 100644 --- a/python/morpheus/morpheus/_lib/tests/messages/test_dev_doc_ex3.cpp +++ b/python/morpheus/morpheus/_lib/tests/messages/test_dev_doc_ex3.cpp @@ -17,10 +17,9 @@ #include "../test_utils/common.hpp" // IWYU pragma: associated -#include "morpheus/messages/control.hpp" // for ControlMessage -#include "morpheus/messages/meta.hpp" // for MessageMeta -#include "morpheus/objects/table_info.hpp" // for MutableTableInfo -#include "morpheus/utilities/cudf_util.hpp" // for CudfHelper +#include "morpheus/messages/control.hpp" // for ControlMessage +#include "morpheus/messages/meta.hpp" // for MessageMeta +#include "morpheus/objects/table_info.hpp" // for MutableTableInfo #include #include // for gil_scoped_release, gil_scoped_acquire @@ -34,20 +33,7 @@ using namespace morpheus; using namespace morpheus::test; class TestDevDocEx3 : public morpheus::test::TestWithPythonInterpreter -{ - protected: - void SetUp() override - { - morpheus::test::TestWithPythonInterpreter::SetUp(); - { - pybind11::gil_scoped_acquire gil; - - // Initially I ran into an issue bootstrapping cudf, I was able to work-around the issue, details in: - // https://github.com/rapidsai/cudf/issues/12862 - CudfHelper::load(); - } - } -}; +{}; TEST_F(TestDevDocEx3, TestPyObjFromMultiMesg) { diff --git a/python/morpheus/morpheus/_lib/tests/messages/test_messages.hpp b/python/morpheus/morpheus/_lib/tests/messages/test_messages.hpp index cf53f6ea2a..d1ca4a8dcb 100644 --- a/python/morpheus/morpheus/_lib/tests/messages/test_messages.hpp +++ b/python/morpheus/morpheus/_lib/tests/messages/test_messages.hpp @@ -19,26 +19,11 @@ #include "../test_utils/common.hpp" // IWYU pragma: associated -#include "morpheus/utilities/cudf_util.hpp" // for CudfHelper - #include namespace morpheus::test { class TestMessages : public morpheus::test::TestWithPythonInterpreter -{ - protected: - void SetUp() override - { - morpheus::test::TestWithPythonInterpreter::SetUp(); - { - pybind11::gil_scoped_acquire gil; - - // Initially I ran into an issue bootstrapping cudf, I was able to work-around the issue, details in: - // https://github.com/rapidsai/cudf/issues/12862 - CudfHelper::load(); - } - } -}; +{}; } // namespace morpheus::test diff --git a/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp b/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp index cbefd8355e..302889d37f 100644 --- a/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp +++ b/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp @@ -291,20 +291,7 @@ class ErrorProneTritonClient : public FakeTritonClient }; class TestTritonInferenceStage : public morpheus::test::TestWithPythonInterpreter -{ - protected: - void SetUp() override - { - morpheus::test::TestWithPythonInterpreter::SetUp(); - { - pybind11::gil_scoped_acquire gil; - - // Initially I ran into an issue bootstrapping cudf, I was able to work-around the issue, details in: - // https://github.com/rapidsai/cudf/issues/12862 - morpheus::CudfHelper::load(); - } - } -}; +{}; cudf::io::table_with_metadata create_test_table_with_metadata(uint32_t rows) { diff --git a/python/morpheus/morpheus/_lib/tests/test_file_in_out.cpp b/python/morpheus/morpheus/_lib/tests/test_file_in_out.cpp index 552e5bb8a7..55b5465ae3 100644 --- a/python/morpheus/morpheus/_lib/tests/test_file_in_out.cpp +++ b/python/morpheus/morpheus/_lib/tests/test_file_in_out.cpp @@ -20,7 +20,6 @@ #include "morpheus/io/deserializers.hpp" #include "morpheus/io/serializers.hpp" #include "morpheus/messages/meta.hpp" -#include "morpheus/utilities/cudf_util.hpp" #include #include @@ -48,20 +47,7 @@ std::string read_file(const std::filesystem::path& file_path) } class TestFileInOut : public morpheus::test::TestWithPythonInterpreter -{ - protected: - void SetUp() override - { - morpheus::test::TestWithPythonInterpreter::SetUp(); - { - pybind11::gil_scoped_acquire gil; - - // Initially I ran into an issue bootstrapping cudf, I was able to work-around the issue, details in: - // https://github.com/rapidsai/cudf/issues/12862 - CudfHelper::load(); - } - } -}; +{}; TEST_F(TestFileInOut, RoundTripCSV) { diff --git a/python/morpheus/morpheus/_lib/tests/test_utils/common.cpp b/python/morpheus/morpheus/_lib/tests/test_utils/common.cpp index 1c8eb86fa8..c58f708b6e 100644 --- a/python/morpheus/morpheus/_lib/tests/test_utils/common.cpp +++ b/python/morpheus/morpheus/_lib/tests/test_utils/common.cpp @@ -23,7 +23,6 @@ #include "morpheus/io/loaders/payload.hpp" #include "morpheus/io/loaders/rest.hpp" #include "morpheus/messages/meta.hpp" -#include "morpheus/utilities/cudf_util.hpp" #include "morpheus/utilities/string_util.hpp" #include // for PyStatus_Exception, PyConfig_Clear, PyConfig_InitPythonConfig @@ -81,9 +80,6 @@ void TestWithPythonInterpreter::SetUp() false); pybind11::gil_scoped_acquire gil; - - // Ensure that the cudf helpers are loaded so we can convert dataframes to MessageMeta - CudfHelper::load(); } void TestWithPythonInterpreter::TearDown() {} diff --git a/python/morpheus/morpheus/common/__init__.py b/python/morpheus/morpheus/common/__init__.py index 97f441c1fb..5935348143 100644 --- a/python/morpheus/morpheus/common/__init__.py +++ b/python/morpheus/morpheus/common/__init__.py @@ -23,7 +23,6 @@ from morpheus._lib.common import Tensor from morpheus._lib.common import TypeId from morpheus._lib.common import determine_file_type -from morpheus._lib.common import load_cudf_helper from morpheus._lib.common import read_file_to_df from morpheus._lib.common import typeid_is_fully_supported from morpheus._lib.common import typeid_to_numpy_str @@ -36,7 +35,6 @@ "FilterSource", "HttpEndpoint", "HttpServer", - "load_cudf_helper", "read_file_to_df", "Tensor", "typeid_is_fully_supported", diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index 0c8a66708a..5f5e41a2a8 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -29,7 +29,6 @@ from tqdm import tqdm import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import -from morpheus.common import load_cudf_helper from morpheus.config import Config from morpheus.config import ExecutionMode from morpheus.utils.type_utils import pretty_print_type_name @@ -296,10 +295,6 @@ def build(self): assert self._state == PipelineState.INITIALIZED, "Pipeline can only be built once!" assert len(self._sources) > 0, "Pipeline must have a source stage" - if (self._execution_mode == ExecutionMode.GPU): - # Load the cudf helper - load_cudf_helper() - self._pre_build() logger.info("====Registering Pipeline====") diff --git a/tests/conftest.py b/tests/conftest.py index e3666bc026..b047798e58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1094,13 +1094,6 @@ def mock_nemollm_fixture(): yield mock_nemollm -@pytest.fixture(name="load_cudf_helper", scope="session", autouse=True) -def load_cudf_helper_fixture(): - if os.environ.get("MORPHEUS_CPU_ONLY") is None: - from morpheus.common import load_cudf_helper - load_cudf_helper() - - @pytest.fixture(name="array_pkg") def array_pkg_fixture(execution_mode: "ExecutionMode") -> types.ModuleType: from morpheus.utils.type_utils import get_array_pkg From 36f441d122efa7f1db3dd7aa870ecc0df6741118 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 25 Sep 2024 16:03:20 -0700 Subject: [PATCH 268/347] remove unused import --- python/morpheus/morpheus/pipeline/pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/morpheus/morpheus/pipeline/pipeline.py b/python/morpheus/morpheus/pipeline/pipeline.py index 5f5e41a2a8..1298f6f020 100644 --- a/python/morpheus/morpheus/pipeline/pipeline.py +++ b/python/morpheus/morpheus/pipeline/pipeline.py @@ -30,7 +30,6 @@ import morpheus.pipeline as _pipeline # pylint: disable=cyclic-import from morpheus.config import Config -from morpheus.config import ExecutionMode from morpheus.utils.type_utils import pretty_print_type_name logger = logging.getLogger(__name__) From 8922dcf4b7c6142210f9d1ea4bf4c43f0eea9f4f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 25 Sep 2024 16:31:08 -0700 Subject: [PATCH 269/347] Mark --use_cpp flag as deprecated, disallow combining --use_cpp with --use_cpu_only --- python/morpheus/morpheus/cli/commands.py | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/python/morpheus/morpheus/cli/commands.py b/python/morpheus/morpheus/cli/commands.py index 461446feb6..78e5834e16 100644 --- a/python/morpheus/morpheus/cli/commands.py +++ b/python/morpheus/morpheus/cli/commands.py @@ -287,13 +287,14 @@ def install(**kwargs): @click.option('--use_cpp', default=True, type=bool, - help=("Whether or not to use C++ node and message types or to prefer python. " - "Only use as a last resort if bugs are encountered")) + help=("[Deprecated] Whether or not to use C++ node and message types or to prefer python. " + "Only use as a last resort if bugs are encountered. Cannot be used with --use_cpu_only")) @click.option('--use_cpu_only', default=False, type=bool, is_flag=True, - help=("Whether or not to run in CPU only mode, setting this to True will disable C++ mode.")) + help=("Whether or not to run in CPU only mode, setting this to True will disable C++ mode. " + "Cannot be used with --use_cpp")) @click.option('--manual_seed', default=None, type=click.IntRange(min=1), @@ -302,19 +303,26 @@ def install(**kwargs): @prepare_command(parse_config=True) def run(ctx: click.Context, **kwargs): """Run subcommand, used for running a pipeline""" - if ctx.get_parameter_source("use_cpp") is not click.core.ParameterSource.DEFAULT: - logger.warning("The --use_cpp flag is deprecated and will be removed in a future release") - # Since the option isnt the same name as `should_use_cpp` anymore, manually set the value here. + if (ctx.get_parameter_source("use_cpu_only") is not click.core.ParameterSource.DEFAULT + and ctx.get_parameter_source("use_cpp") is not click.core.ParameterSource.DEFAULT): + # If the user set explicit values for both use_cpu_only and use_cpp raise an error + raise click.UsageError("Cannot set both --use_cpp and --use_cpu_only. The --use_cpp flag is deprecated. " + "Use only --use_cpu_only.") + use_cpu_only = kwargs.pop("use_cpu_only") use_cpp = kwargs.pop("use_cpp") - if use_cpu_only: - CppConfig.set_should_use_cpp(False) + + # only check this value if the flag was explicitly set by the user + if ctx.get_parameter_source("use_cpp") is not click.core.ParameterSource.DEFAULT: + logger.warning("The --use_cpp flag is deprecated and will be removed in a future release") + + execution_mode = ExecutionMode.GPU if use_cpp else ExecutionMode.CPU else: - CppConfig.set_should_use_cpp(use_cpp) + execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU config = get_config_from_ctx(ctx) - config.execution_mode = ExecutionMode.CPU if use_cpu_only else ExecutionMode.GPU + config.execution_mode = execution_mode manual_seed_val = kwargs.pop("manual_seed", None) if manual_seed_val is not None: From 20b26db51f8ff005113463268ea6dfd67ccf7c60 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 25 Sep 2024 17:08:28 -0700 Subject: [PATCH 270/347] IWYU fixes --- python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp | 2 -- .../morpheus/_lib/tests/stages/test_triton_inference_stage.cpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp index f3f7a902c0..2e1c98a84d 100644 --- a/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp +++ b/python/morpheus/morpheus/_lib/src/utilities/cudf_util.cpp @@ -25,10 +25,8 @@ #include #include -#include // for atomic #include // for getenv #include -#include #include // Needed for logging #include // for move /* diff --git a/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp b/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp index 302889d37f..27f477511b 100644 --- a/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp +++ b/python/morpheus/morpheus/_lib/tests/stages/test_triton_inference_stage.cpp @@ -27,7 +27,6 @@ #include "morpheus/stages/inference_client_stage.hpp" // for TensorModelMapping, InferenceClientStage, IInferenceCl... #include "morpheus/stages/triton_inference.hpp" // for TritonInferenceClient, TritonInferInput, TritonInferRe... #include "morpheus/types.hpp" // for TensorMap -#include "morpheus/utilities/cudf_util.hpp" // for CudfHelper #include "morpheus/utilities/matx_util.hpp" // for MatxUtil #include // for cudaMemcpy, cudaMemcpyKind @@ -43,7 +42,6 @@ #include // for Error, InferOptions, InferenceServerHttpClient, InferR... #include // for Task #include // for TestScheduler -#include // for gil_scoped_acquire #include // for cuda_stream_per_thread #include // for device_buffer #include // for get_current_device_resource From c091271ee1cb438acc20d2475ab61228d53cba41 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 26 Sep 2024 15:06:45 -0700 Subject: [PATCH 271/347] Add explicit dependency for pynvml, this was already being pulled in as a dependency of cuml, but the dep is needed for the abp-nvsmi example --- conda/environments/examples_cuda-121_arch-x86_64.yaml | 1 + dependencies.yaml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/conda/environments/examples_cuda-121_arch-x86_64.yaml b/conda/environments/examples_cuda-121_arch-x86_64.yaml index 66b30db78e..53b128f437 100644 --- a/conda/environments/examples_cuda-121_arch-x86_64.yaml +++ b/conda/environments/examples_cuda-121_arch-x86_64.yaml @@ -42,6 +42,7 @@ dependencies: - pip - pluggy=1.3 - pydantic +- pynvml=11.4 - pypdf=3.17.4 - pypdfium2=4.30 - python-confluent-kafka>=1.9.2,<1.10.0a0 diff --git a/dependencies.yaml b/dependencies.yaml index e8f5525696..c8bec41e86 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -149,6 +149,7 @@ files: arch: [x86_64] includes: - cve-mitigation + - example-abp-nvsmi - example-dfp-prod - example-gnn - example-llms @@ -399,6 +400,12 @@ dependencies: - dgl==2.0.0 - dglgo + example-abp-nvsmi: + common: + - output_types: [conda] + packages: + - pynvml=11.4 + example-llms: common: - output_types: [conda] From e8e542f2d5250f847bf71fca79b93bb9a4505ef4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 26 Sep 2024 15:33:58 -0700 Subject: [PATCH 272/347] Document CPU-only example --- examples/cpu_only/README.md | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 examples/cpu_only/README.md diff --git a/examples/cpu_only/README.md b/examples/cpu_only/README.md new file mode 100644 index 0000000000..570cda8070 --- /dev/null +++ b/examples/cpu_only/README.md @@ -0,0 +1,72 @@ + + +# CPU Only Example Using Morpheus + +## Supported Environments +| Environment | Supported | Notes | +|-------------|-----------|-------| +| Conda | ✔ | | +| Morpheus Docker Container | ✔ | | +| Morpheus Release Container | ✔ | | +| Dev Container | ✔ | | + +## CPU Only Pipeline +This example demonstrates a simple Morpheus pipeline which is able to opperate on a host without access GPU. + +> **Note**: A more complex example of a pipeline that can exexute without a GPU is also available at `examples/llm/completion/README.md` + +From the root of the Morpheus repo, run: +```bash +python examples/cpu_only/run.py --help +``` + +Output: +``` +Usage: run.py [OPTIONS] + +Options: + --use_cpu_only Whether or not to run in CPU only mode, + setting this to True will disable C++ mode. + --log_level [CRITICAL|FATAL|ERROR|WARN|WARNING|INFO|DEBUG] + Specify the logging level to use. [default: + DEBUG] + --in_file PATH Input file [required] + --out_file FILE Output file [required] + --help Show this message and exit. +``` + +To launch the configured Morpheus pipeline with the sample data that is provided in `examples/data`, run the following: + +```bash +python examples/cpu_only/run.py --use_cpu_only --in_file=examples/data/email.jsonlines --out_file=.tmp/out.jsonlines +``` + +### CLI Example + +From the root of the Morpheus repo, run: +```bash +morpheus --log_level INFO \ + run --use_cpu_only \ + pipeline-other \ + from-file --filename=examples/data/email.jsonlines \ + monitor --description "source" \ + deserialize \ + monitor --description "deserialize" \ + serialize \ + to-file --filename=.tmp/out.jsonlines --overwrite +``` From 966fb99200470a508a20a2cc78ca7817168de8cb Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 26 Sep 2024 15:44:00 -0700 Subject: [PATCH 273/347] Document the --use_cpu_only flag. --- examples/developer_guide/2_2_rabbitmq/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/developer_guide/2_2_rabbitmq/README.md b/examples/developer_guide/2_2_rabbitmq/README.md index 5b657b580f..5407a9810b 100644 --- a/examples/developer_guide/2_2_rabbitmq/README.md +++ b/examples/developer_guide/2_2_rabbitmq/README.md @@ -62,6 +62,7 @@ This will read JSON data from the `examples/data/email.jsonlines` file and publi The `write_simple.py` script will exit as soon as the message is written to the queue. The `read_simple.py` script will continue reading from the queue until explicitly shut down with a control-C. +> **Note**: Both the `read_simple.py` and `write_simple.py` scipts will launch independent Morpheus pipelines, both of which can optionally execute in CPU-only mode by setting the `--use_cpu_only` flag. ## Alternate Morpheus CLI usage In the above examples we defined the pipeline using the Python API in the `read_simple.py` and `write_simple.py` scripts. Alternately, we could have defined the same pipelines using the Morpheus CLI tool. From 9d7c343d4bdc7001b3cde6d5b6850c7fda52e3f3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 26 Sep 2024 15:53:26 -0700 Subject: [PATCH 274/347] Remove out of date documentation regarting the --use_cpp and --num_threads flags --- examples/developer_guide/4_rabbitmq_cpp_stage/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/README.md b/examples/developer_guide/4_rabbitmq_cpp_stage/README.md index 1fba854fde..05deed426a 100644 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/README.md +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/README.md @@ -18,8 +18,6 @@ limitations under the License. # Example RabbitMQ stages This example builds upon the `examples/developer_guide/2_2_rabbitmq` example adding a C++ implementation for the `RabbitMQSourceStage` along with adding package install scripts. -This example adds two flags to the `read_simple.py` script. A `--use_cpp` flag which defaults to `True` and a `--num_threads` flag which defaults to the number of cores on the system as returned by `len(os.sched_getaffinity(0))`. - ## Supported Environments | Environment | Supported | Notes | |-------------|-----------|-------| From a6b75d7da05f37c8a3ae9cb2c832a4ab922d8506 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 26 Sep 2024 16:01:16 -0700 Subject: [PATCH 275/347] Remove explicit usage of CppConfig --- examples/cpu_only/run.py | 2 -- examples/llm/cli.py | 8 -------- 2 files changed, 10 deletions(-) diff --git a/examples/cpu_only/run.py b/examples/cpu_only/run.py index 808f956802..f0a50a47e0 100644 --- a/examples/cpu_only/run.py +++ b/examples/cpu_only/run.py @@ -73,8 +73,6 @@ def run_pipeline(log_level: int, use_cpu_only: bool, in_file: pathlib.Path, out_ else: execution_mode = ExecutionMode.GPU - CppConfig.set_should_use_cpp(not use_cpu_only) - config = Config() config.execution_mode = execution_mode diff --git a/examples/llm/cli.py b/examples/llm/cli.py index cf81557bfd..cb57624ada 100644 --- a/examples/llm/cli.py +++ b/examples/llm/cli.py @@ -35,23 +35,15 @@ type=click.Choice(get_log_levels(), case_sensitive=False), callback=parse_log_level, help="Specify the logging level to use.") -@click.option('--use_cpp', - default=True, - type=bool, - help=("Whether or not to use C++ node and message types or to prefer python. " - "Only use as a last resort if bugs are encountered")) @click.version_option() @click.pass_context def cli(ctx: click.Context, log_level: int, use_cpp: bool): """Main entrypoint for the LLM Examples""" - from morpheus.config import CppConfig from morpheus.utils.logger import configure_logging ctx_dict = ctx.ensure_object(dict) - CppConfig.set_should_use_cpp(use_cpp) - # Configure the logging configure_logging(log_level=log_level) From e1c8796a253dd63c3c8f576c9e1bc6602ed70d02 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 11:07:30 -0700 Subject: [PATCH 276/347] Remove use_cpp arg --- examples/llm/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm/cli.py b/examples/llm/cli.py index cb57624ada..d849113560 100644 --- a/examples/llm/cli.py +++ b/examples/llm/cli.py @@ -37,7 +37,7 @@ help="Specify the logging level to use.") @click.version_option() @click.pass_context -def cli(ctx: click.Context, log_level: int, use_cpp: bool): +def cli(ctx: click.Context, log_level: int): """Main entrypoint for the LLM Examples""" from morpheus.utils.logger import configure_logging From b7348b1a076dc404b9e1b4af31ec0d9811e87156 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 11:08:19 -0700 Subject: [PATCH 277/347] Default to CPP mode --- python/morpheus/morpheus/modules/filter_detections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/modules/filter_detections.py b/python/morpheus/morpheus/modules/filter_detections.py index 94ef301862..c0793a6092 100644 --- a/python/morpheus/morpheus/modules/filter_detections.py +++ b/python/morpheus/morpheus/modules/filter_detections.py @@ -81,7 +81,7 @@ def filter_detections(builder: mrc.Builder): field_name = config.get("field_name", "probs") threshold = config.get("threshold", 0.5) filter_source = config.get("filter_source", "AUTO") - use_cpp = config.get("use_cpp", False) + use_cpp = config.get("use_cpp", True) filter_source_dict = {"AUTO": FilterSource.Auto, "DATAFRAME": FilterSource.DATAFRAME, "TENSOR": FilterSource.TENSOR} From f1879ded05b8748172dccf426ea7542a3135017d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 11:56:44 -0700 Subject: [PATCH 278/347] Remove setting cppconfig in the grafana pipeline --- examples/digital_fingerprinting/production/grafana/run.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/digital_fingerprinting/production/grafana/run.py b/examples/digital_fingerprinting/production/grafana/run.py index 2bb7ade0e4..c64e156a4d 100644 --- a/examples/digital_fingerprinting/production/grafana/run.py +++ b/examples/digital_fingerprinting/production/grafana/run.py @@ -45,7 +45,6 @@ from morpheus.common import FilterSource from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.stages.output.write_to_file_stage import WriteToFileStage @@ -242,9 +241,6 @@ def run_pipeline(train_users, logger.info("Tracking URI: %s", mlflow.get_tracking_uri()) config = Config() - - CppConfig.set_should_use_cpp(False) - config.num_threads = len(os.sched_getaffinity(0)) config.ae = ConfigAutoEncoder() From b19469d3df14a5e566b46b5a760a2b677cb298ae Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 12:18:20 -0700 Subject: [PATCH 279/347] Remove setting cppconfig in the viz pipelines --- .../visualization/dfp_viz_azure_pipeline.py | 4 ---- .../visualization/dfp_viz_duo_pipeline.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py index afa9706ff7..eb8e863754 100644 --- a/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py +++ b/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py @@ -41,7 +41,6 @@ from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.utils.column_info import ColumnInfo @@ -180,9 +179,6 @@ def run_pipeline(train_users, logger.info("Tracking URI: %s", mlflow.get_tracking_uri()) config = Config() - - CppConfig.set_should_use_cpp(False) - config.num_threads = len(os.sched_getaffinity(0)) config.ae = ConfigAutoEncoder() diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py index 28f36995c1..325db0f540 100644 --- a/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py +++ b/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py @@ -41,7 +41,6 @@ from morpheus.common import FileTypes from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.pipeline import LinearPipeline from morpheus.stages.general.monitor_stage import MonitorStage from morpheus.utils.column_info import BoolColumn @@ -183,9 +182,6 @@ def run_pipeline(train_users, logger.info("Tracking URI: %s", mlflow.get_tracking_uri()) config = Config() - - CppConfig.set_should_use_cpp(False) - config.num_threads = len(os.sched_getaffinity(0)) config.ae = ConfigAutoEncoder() From 38968de9afae724fb1847adb16acbbea187e4470 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 12:24:41 -0700 Subject: [PATCH 280/347] Remove setting CppConfig --- .../production/morpheus/notebooks/dfp_azure_training.ipynb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb index acb759341b..0e471d9361 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb @@ -62,7 +62,6 @@ "from morpheus.cli.utils import parse_log_level\n", "from morpheus.config import Config\n", "from morpheus.config import ConfigAutoEncoder\n", - "from morpheus.config import CppConfig\n", "from morpheus.pipeline import LinearPipeline\n", "from morpheus.utils.column_info import ColumnInfo\n", "from morpheus.utils.column_info import DataFrameInputSchema\n", @@ -191,9 +190,6 @@ "configure_logging(log_level=logging.DEBUG)\n", "\n", "config = Config()\n", - "\n", - "CppConfig.set_should_use_cpp(False)\n", - "\n", "config.num_threads = len(os.sched_getaffinity(0))\n", "\n", "config.ae = ConfigAutoEncoder()\n", From 20e355e956c579ffb93be10f3164cf722ed7c4ae Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 13:15:51 -0700 Subject: [PATCH 281/347] Remove uneeded pip install, use conda with the libmamba solver, remove use_cpp --- .../production/morpheus/benchmarks/README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md index b002e7fa47..2b2a41d43e 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md @@ -38,14 +38,9 @@ In the `/workspace` directory of the container, run the following to compile Mor ./scripts/compile.sh ``` -Now install Morpheus: -```bash -pip install -e /workspace -``` - Install additional required dependencies: ```bash -mamba env update \ +conda env update --solver=libmamba \ -n ${CONDA_DEFAULT_ENV} \ --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` @@ -87,8 +82,7 @@ Morpheus pipeline configurations for each workflow are managed using [pipelines_ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, ... ``` From ad5f92a415838625d80c34495c788744f0b777bd Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 13:17:48 -0700 Subject: [PATCH 282/347] Remove use_cpp --- .../benchmarks/resource/pipelines_conf.json | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json b/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json index a15edde34f..6049014497 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json @@ -9,8 +9,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": true + "source": "azure" }, "test_dfp_modules_azure_payload_lti_e2e": { "message_path": "../control_messages/azure_payload_lti.json", @@ -21,8 +20,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": true + "source": "azure" }, "test_dfp_modules_azure_payload_lti_s3_e2e": { "message_path": "../control_messages/azure_payload_lti_s3.json", @@ -33,8 +31,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": true + "source": "azure" }, "test_dfp_modules_azure_payload_training_e2e": { "message_path": "../control_messages/azure_payload_training.json", @@ -45,8 +42,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": true + "source": "azure" }, "test_dfp_modules_azure_streaming_inference_e2e": { "message_path": "../control_messages/azure_streaming_inference.json", @@ -57,8 +53,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": true + "source": "azure" }, "test_dfp_modules_azure_streaming_lti_e2e": { "message_path": "../control_messages/azure_streaming_lti.json", @@ -69,8 +64,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": true + "source": "azure" }, "test_dfp_modules_azure_streaming_training_e2e": { "message_path": "../control_messages/azure_streaming_training.json", @@ -81,8 +75,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": true + "source": "azure" }, "test_dfp_modules_duo_payload_inference_e2e": { "message_path": "../control_messages/duo_payload_inference.json", @@ -93,8 +86,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_payload_lti_e2e": { "message_path": "../control_messages/duo_payload_lti.json", @@ -105,8 +97,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_payload_only_load_e2e": { "message_path": "../control_messages/duo_payload_only_load.json", @@ -117,8 +108,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_payload_training_e2e": { "message_path": "../control_messages/duo_payload_training.json", @@ -129,8 +119,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_streaming_inference_e2e": { "message_path": "../control_messages/duo_streaming_inference.json", @@ -141,8 +130,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_streaming_lti_e2e": { "message_path": "../control_messages/duo_streaming_lti.json", @@ -153,8 +141,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_streaming_only_load_e2e": { "message_path": "../control_messages/duo_streaming_only_load.json", @@ -165,8 +152,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_streaming_payload_e2e": { "message_path": "../control_messages/duo_streaming_payload.json", @@ -177,8 +163,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_modules_duo_streaming_training_e2e": { "message_path": "../control_messages/duo_streaming_training.json", @@ -189,8 +174,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": true + "source": "duo" }, "test_dfp_stages_azure_training_e2e": { "glob_path": "../../../../data/dfp/azure-training-data/*.json", @@ -201,8 +185,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": false + "source": "azure" }, "test_dfp_stages_azure_inference_e2e": { "glob_path": "../../../../data/dfp/azure-inference-data/*.json", @@ -213,8 +196,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "azure", - "use_cpp": false + "source": "azure" }, "test_dfp_stages_duo_training_e2e": { "glob_path": "../../../../data/dfp/duo-training-data/*.json", @@ -225,8 +207,7 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": false + "source": "duo" }, "test_dfp_stages_duo_inference_e2e": { "glob_path": "../../../../data/dfp/duo-inference-data/*.json", @@ -237,7 +218,6 @@ "duration": "60d", "userid_column_name": "username", "timestamp_column_name": "timestamp", - "source": "duo", - "use_cpp": false + "source": "duo" } } From 309da4aec41a989aa2a8a207563abb730203f500 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 13:47:24 -0700 Subject: [PATCH 283/347] Replace mamba with conda using the libmamba solver, remove --use_cpp flag from CLI example --- examples/gnn_fraud_detection_pipeline/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/gnn_fraud_detection_pipeline/README.md b/examples/gnn_fraud_detection_pipeline/README.md index 8bb1ab1570..97ae02e2fc 100644 --- a/examples/gnn_fraud_detection_pipeline/README.md +++ b/examples/gnn_fraud_detection_pipeline/README.md @@ -30,7 +30,7 @@ All environments require additional Conda packages which can be installed with e Prior to running the GNN fraud detection pipeline, additional requirements must be installed in to your Conda environment. A supplemental requirements file has been provided in this example directory. ```bash -mamba env update \ +conda env update --solver=libmamba \ -n ${CONDA_DEFAULT_ENV} \ --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` @@ -117,7 +117,7 @@ From the root of the Morpheus repo, run: PYTHONPATH="examples" \ morpheus --log_level INFO \ --plugin "gnn_fraud_detection_pipeline" \ - run --use_cpp False --pipeline_batch_size 1024 --model_max_batch_size 32 --edge_buffer_size 4 \ + run --pipeline_batch_size 1024 --model_max_batch_size 32 --edge_buffer_size 4 \ pipeline-other --model_fea_length 70 --label=probs \ from-file --filename examples/gnn_fraud_detection_pipeline/validation.csv --filter_null False \ deserialize \ From 71d37e036dc0f600d68451dd209093bd0245c355 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 14:25:23 -0700 Subject: [PATCH 284/347] Cleanup help string --- examples/llm/agents/run.py | 2 +- examples/llm/completion/run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm/agents/run.py b/examples/llm/agents/run.py index a97bce4b78..60d85eac84 100644 --- a/examples/llm/agents/run.py +++ b/examples/llm/agents/run.py @@ -25,7 +25,7 @@ def run(): @run.command(help="Runs a simple finite pipeline with a single execution of a LangChain agent from a fixed input") -@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help="Run in CPU only mode") @click.option( "--num_threads", default=len(os.sched_getaffinity(0)), diff --git a/examples/llm/completion/run.py b/examples/llm/completion/run.py index 4ba702c700..ed2e8a6c3d 100644 --- a/examples/llm/completion/run.py +++ b/examples/llm/completion/run.py @@ -26,7 +26,7 @@ def run(): @run.command() -@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help=("Whether or not to run in CPU only mode")) +@click.option('--use_cpu_only', default=False, type=bool, is_flag=True, help="Run in CPU only mode") @click.option( "--num_threads", default=len(os.sched_getaffinity(0)), From 0a8ad97eff7d15c65eaf5003b04f81f68751feff Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 14:26:40 -0700 Subject: [PATCH 285/347] Document the --use_cpu_only flag, use conda rather than mamba --- examples/llm/agents/README.md | 6 +++++- examples/llm/completion/README.md | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/llm/agents/README.md b/examples/llm/agents/README.md index d8c2944b26..97b8e913b4 100644 --- a/examples/llm/agents/README.md +++ b/examples/llm/agents/README.md @@ -104,7 +104,7 @@ export SERPAPI_API_KEY="" Install the required dependencies. ```bash -mamba env update \ +conda env update --solver=libmamba \ -n ${CONDA_DEFAULT_ENV} \ --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` @@ -131,6 +131,10 @@ python examples/llm/main.py agents simple [OPTIONS] ``` ### Options: +- `--use_cpu_only` + - **Description**: Run in CPU only mode + - **Default**: `False` + - `--num_threads INTEGER RANGE` - **Description**: Number of internal pipeline threads to use. - **Default**: `12` diff --git a/examples/llm/completion/README.md b/examples/llm/completion/README.md index c619546c47..2ae2f37b6b 100644 --- a/examples/llm/completion/README.md +++ b/examples/llm/completion/README.md @@ -78,7 +78,7 @@ Before running the pipeline, ensure that the `NGC_API_KEY` environment variable Install the required dependencies. ```bash -mamba env update \ +conda env update --solver=libmamba \ -n ${CONDA_DEFAULT_ENV} \ --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` @@ -114,6 +114,9 @@ python examples/llm/main.py completion [OPTIONS] COMMAND [ARGS]... - `pipeline` ##### Options: +- `--use_cpu_only` + - **Description**: Run in CPU only mode + - **Default**: `False` - `--num_threads INTEGER RANGE` - **Description**: Number of internal pipeline threads to use. From c342c29e927878bb7184d8c90af02b17a4b912ba Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 14:33:49 -0700 Subject: [PATCH 286/347] Remove --use_cpp flag --- examples/ransomware_detection/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/ransomware_detection/README.md b/examples/ransomware_detection/README.md index 0619af26ec..4b15a30b71 100644 --- a/examples/ransomware_detection/README.md +++ b/examples/ransomware_detection/README.md @@ -88,7 +88,6 @@ Usage: run.py [OPTIONS] Options: --debug BOOLEAN - --use_cpp BOOLEAN --num_threads INTEGER RANGE Number of internal pipeline threads to use [x>=1] --n_dask_workers INTEGER RANGE Number of dask workers [x>=2] From bcf7eb86af7d183f65983bbabb11a9cd38d7b136 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 14:36:59 -0700 Subject: [PATCH 287/347] Remove --use_cpp flag, fix documented number of threads --- examples/root_cause_analysis/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/root_cause_analysis/README.md b/examples/root_cause_analysis/README.md index 5d038fa959..943c00fad2 100644 --- a/examples/root_cause_analysis/README.md +++ b/examples/root_cause_analysis/README.md @@ -105,8 +105,8 @@ From the Morpheus repo root directory, run: ```bash export MORPHEUS_ROOT=$(pwd) morpheus --log_level=DEBUG \ -`# Run a pipeline with 5 threads and a model batch size of 32 (Must match Triton config)` \ -run --num_threads=8 --edge_buffer_size=4 --use_cpp=True --pipeline_batch_size=1024 --model_max_batch_size=32 \ +`# Run a pipeline with 8 threads and a model batch size of 32 (Must match Triton config)` \ +run --num_threads=8 --edge_buffer_size=4 --pipeline_batch_size=1024 --model_max_batch_size=32 \ `# Specify a NLP pipeline with 128 sequence length (Must match Triton config)` \ pipeline-nlp --model_seq_length=128 --label=not_root_cause --label=is_root_cause \ `# 1st Stage: Read from file` \ From 67c6b2b316d89be133e901b2fce49f4420b49d7a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 15:05:13 -0700 Subject: [PATCH 288/347] Remove out of date documentation about additional packages being needed, conda_docs.yml no longer exists, and the tools needed to perform the build all exist in the development yaml --- docs/README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/README.md b/docs/README.md index 469303430e..b0a3162a6a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,18 +17,10 @@ # Building Documentation -Additional packages required for building the documentation are defined in `./conda_docs.yml`. - -## Install Additional Dependencies -From the root of the Morpheus repo: -```bash -conda env update --solver=libmamba -n morpheus --file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune -``` - ## Build Morpheus and Documentation ``` CMAKE_CONFIGURE_EXTRA_ARGS="-DMORPHEUS_BUILD_DOCS=ON" ./scripts/compile.sh --target morpheus_docs ``` Outputs to `build/docs/html` - + If the documentation build is unsuccessful, refer to the **Out of Date Build Cache** section in [Troubleshooting](./source/extra_info/troubleshooting.md) to troubleshoot. From 75ae25aaa93af303dc66288823392324e199b623 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 15:10:53 -0700 Subject: [PATCH 289/347] Remove --use_cpp flag --- docs/source/cloud_deployment_guide.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/cloud_deployment_guide.md b/docs/source/cloud_deployment_guide.md index 1dac95c9ae..61608f8b0d 100644 --- a/docs/source/cloud_deployment_guide.md +++ b/docs/source/cloud_deployment_guide.md @@ -438,7 +438,6 @@ helm install --set ngc.apiKey="$API_KEY" \ --edge_buffer_size=4 \ --pipeline_batch_size=1024 \ --model_max_batch_size=1024 \ - --use_cpp=False \ pipeline-ae \ --columns_file=data/columns_ae_cloudtrail.txt \ --userid_filter=user123 \ From 16bad6f4269f30602b4da2aa4095130a7562d1a3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 15:25:29 -0700 Subject: [PATCH 290/347] Remove usage --use_cpp, also remove usage of the --num_threads flag which dates back to a time when it defaulted to 1 and most pipelines were Python based --- docs/source/cloud_deployment_guide.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/docs/source/cloud_deployment_guide.md b/docs/source/cloud_deployment_guide.md index 61608f8b0d..060e85a452 100644 --- a/docs/source/cloud_deployment_guide.md +++ b/docs/source/cloud_deployment_guide.md @@ -434,7 +434,6 @@ Inference and training based on a user ID (`user123`). The model is trained once ```bash helm install --set ngc.apiKey="$API_KEY" \ --set sdk.args="morpheus --log_level=DEBUG run \ - --num_threads=2 \ --edge_buffer_size=4 \ --pipeline_batch_size=1024 \ --model_max_batch_size=1024 \ @@ -479,11 +478,9 @@ Pipeline example to read data from a file, run inference using a `phishing-bert- ```bash helm install --set ngc.apiKey="$API_KEY" \ --set sdk.args="morpheus --log_level=DEBUG run \ - --num_threads=2 \ --edge_buffer_size=4 \ --pipeline_batch_size=1024 \ --model_max_batch_size=32 \ - --use_cpp=True \ pipeline-nlp \ --model_seq_length=128 \ --labels_file=data/labels_phishing.txt \ @@ -509,11 +506,9 @@ Pipeline example to read messages from an input Kafka topic, run inference using ```bash helm install --set ngc.apiKey="$API_KEY" \ --set sdk.args="morpheus --log_level=DEBUG run \ - --num_threads=2 \ --edge_buffer_size=4 \ --pipeline_batch_size=1024 \ --model_max_batch_size=32 \ - --use_cpp=True \ pipeline-nlp \ --model_seq_length=128 \ --labels_file=data/labels_phishing.txt \ @@ -556,9 +551,7 @@ Pipeline example to read data from a file, run inference using a `sid-minibert-o ```bash helm install --set ngc.apiKey="$API_KEY" \ --set sdk.args="morpheus --log_level=DEBUG run \ - --num_threads=3 \ --edge_buffer_size=4 \ - --use_cpp=True \ --pipeline_batch_size=1024 \ --model_max_batch_size=32 \ pipeline-nlp \ @@ -585,9 +578,7 @@ Pipeline example to read messages from an input Kafka topic, run inference using ```bash helm install --set ngc.apiKey="$API_KEY" \ --set sdk.args="morpheus --log_level=DEBUG run \ - --num_threads=3 \ --edge_buffer_size=4 \ - --use_cpp=True \ --pipeline_batch_size=1024 \ --model_max_batch_size=32 \ pipeline-nlp \ @@ -630,11 +621,9 @@ Pipeline example to read data from a file, run inference using an `abp-nvsmi-xgb ```bash helm install --set ngc.apiKey="$API_KEY" \ --set sdk.args="morpheus --log_level=DEBUG run \ - --num_threads=3 \ --edge_buffer_size=4 \ --pipeline_batch_size=1024 \ --model_max_batch_size=64 \ - --use_cpp=True \ pipeline-fil --columns_file=data/columns_fil.txt \ from-file --filename=./examples/data/nvsmi.jsonlines \ monitor --description 'FromFile Rate' --smoothing=0.001 \ @@ -656,10 +645,8 @@ Pipeline example to read messages from an input Kafka topic, run inference using ```bash helm install --set ngc.apiKey="$API_KEY" \ --set sdk.args="morpheus --log_level=DEBUG run \ - --num_threads=3 \ --pipeline_batch_size=1024 \ --model_max_batch_size=64 \ - --use_cpp=True \ pipeline-fil --columns_file=data/columns_fil.txt \ from-kafka --input_topic --bootstrap_servers broker:9092 \ monitor --description 'FromKafka Rate' --smoothing=0.001 \ From f13c545682940a0b4c99285bd5e1c0f04c14fa69 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 15:52:36 -0700 Subject: [PATCH 291/347] Specify filter source in the filter stage (fixes an error) Fix classname for ControlMessage Fix triton start command Remove --num_threads flag Set url port to the http port --- docs/source/basics/building_a_pipeline.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/basics/building_a_pipeline.md b/docs/source/basics/building_a_pipeline.md index 65fadb0cf6..06985d5ef6 100644 --- a/docs/source/basics/building_a_pipeline.md +++ b/docs/source/basics/building_a_pipeline.md @@ -107,7 +107,7 @@ morpheus --log_level=DEBUG run pipeline-other \ Then the following error displays: ``` -RuntimeError: The to-file stage cannot handle input of . Accepted input types: (,) +RuntimeError: The to-file stage cannot handle input of . Accepted input types: (,) ``` This indicates that the ``to-file`` stage cannot accept the input type of `morpheus.messages.ControlMessage`. This is because the ``to-file`` stage has no idea how to write that class to a file; it only knows how to write instances of `morpheus.messages.message_meta.MessageMeta`. To ensure you have a valid pipeline, examine the `Accepted input types: (,)` portion of the message. This indicates you need a stage that converts from the output type of the `deserialize` stage, `ControlMessage`, to `MessageMeta`, which is exactly what the `serialize` stage does. @@ -207,7 +207,7 @@ This example shows an NLP Pipeline which uses several stages available in Morphe #### Launching Triton Run the following to launch Triton and load the `sid-minibert` model: ```bash -docker run --rm -ti --gpus=all -p8000:8000 -p8001:8001 -p8002:8002 nvcr.io/nvidia/morpheus/morpheus-tritonserver-models:24.10 --model-repository=/models/triton-model-repo --exit-on-error=false --model-control-mode=explicit --load-model sid-minibert-onnx +docker run --rm -ti --gpus=all -p8000:8000 -p8001:8001 -p8002:8002 nvcr.io/nvidia/morpheus/morpheus-tritonserver-models:24.10 tritonserver --model-repository=/models/triton-model-repo --exit-on-error=false --model-control-mode=explicit --load-model sid-minibert-onnx ``` #### Launching Kafka @@ -216,15 +216,15 @@ Follow steps 1-8 in [Quick Launch Kafka Cluster](../developer_guide/contributing ![../img/nlp_kitchen_sink.png](../img/nlp_kitchen_sink.png) ```bash -morpheus --log_level=INFO run --num_threads=8 --pipeline_batch_size=1024 --model_max_batch_size=32 \ +morpheus --log_level=INFO run --pipeline_batch_size=1024 --model_max_batch_size=32 \ pipeline-nlp --viz_file=.tmp/nlp_kitchen_sink.png \ from-file --filename examples/data/pcap_dump.jsonlines \ deserialize \ preprocess \ - inf-triton --model_name=sid-minibert-onnx --server_url=localhost:8001 \ + inf-triton --model_name=sid-minibert-onnx --server_url=localhost:8000 \ monitor --description "Inference Rate" --smoothing=0.001 --unit "inf" \ add-class \ - filter --threshold=0.8 \ + filter --filter_source=TENSOR --threshold=0.8 \ serialize --include 'timestamp' --exclude '^_ts_' \ to-kafka --bootstrap_servers localhost:9092 --output_topic "inference_output" \ monitor --description "ToKafka Rate" --smoothing=0.001 --unit "msg" From 6240b4199b246d3d2a5087f8a489d5ba86cc18b9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 15:55:37 -0700 Subject: [PATCH 292/347] Update cli flags --- docs/source/basics/overview.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/source/basics/overview.rst b/docs/source/basics/overview.rst index ca1f8b6981..4b00a59062 100644 --- a/docs/source/basics/overview.rst +++ b/docs/source/basics/overview.rst @@ -39,16 +39,22 @@ run: $ morpheus run --help Usage: morpheus run [OPTIONS] COMMAND [ARGS]... + Run subcommand, used for running a pipeline + Options: - --num_threads INTEGER RANGE Number of internal pipeline threads to use [default: 12; x>=1] + --num_threads INTEGER RANGE Number of internal pipeline threads to use [default: 64; x>=1] --pipeline_batch_size INTEGER RANGE Internal batch size for the pipeline. Can be much larger than the model batch size. Also used for Kafka consumers [default: 256; x>=1] --model_max_batch_size INTEGER RANGE Max batch size to use for the model [default: 8; x>=1] --edge_buffer_size INTEGER RANGE - The size of buffered channels to use between nodes in a pipeline. Larger values reduce backpressure at the cost of memory. Smaller values will push - messages through the pipeline quicker. Must be greater than 1 and a power of 2 (i.e. 2, 4, 8, 16, etc.) [default: 128; x>=2] - --use_cpp BOOLEAN Whether or not to use C++ node and message types or to prefer python. Only use as a last resort if bugs are encountered [default: True] + The size of buffered channels to use between nodes in a pipeline. Larger values reduce backpressure at the cost of memory. Smaller + values will push messages through the pipeline quicker. Must be greater than 1 and a power of 2 (i.e. 2, 4, 8, 16, etc.) [default: + 128; x>=2] + --use_cpp BOOLEAN [Deprecated] Whether or not to use C++ node and message types or to prefer python. Only use as a last resort if bugs are encountered. + Cannot be used with --use_cpu_only [default: True] + --use_cpu_only Whether or not to run in CPU only mode, setting this to True will disable C++ mode. Cannot be used with --use_cpp + --manual_seed INTEGER RANGE Manually seed the random number generators used by Morpheus, useful for testing. [x>=1] --help Show this message and exit. Commands: @@ -57,6 +63,7 @@ run: pipeline-nlp Run the inference pipeline with a NLP model pipeline-other Run a custom inference pipeline without a specific model type + Currently, Morpheus pipeline can be operated in four different modes. * ``pipeline-ae`` From f692ff6838a29acc0efc4666cbedabf9d2e0d0b3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 16:19:04 -0700 Subject: [PATCH 293/347] Specify stage name --- .../developer_guide/1_simple_python_stage/pass_thru_deco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py b/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py index 9755f63765..cd71e83b63 100644 --- a/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py +++ b/examples/developer_guide/1_simple_python_stage/pass_thru_deco.py @@ -19,7 +19,7 @@ from morpheus.pipeline.stage_decorator import stage -@stage(execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) +@stage(name="pass-thru", execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) def pass_thru_stage(message: typing.Any) -> typing.Any: # Return the message for the next stage return message From 8ab412afeedf79836bcfcd21a0f7fb9959515e4f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 16:19:23 -0700 Subject: [PATCH 294/347] Document execution modes --- .../guides/1_simple_python_stage.md | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/source/developer_guide/guides/1_simple_python_stage.md b/docs/source/developer_guide/guides/1_simple_python_stage.md index 0ed1a08d59..c6cb7cf11c 100644 --- a/docs/source/developer_guide/guides/1_simple_python_stage.md +++ b/docs/source/developer_guide/guides/1_simple_python_stage.md @@ -52,6 +52,20 @@ def pass_thru_stage(message: typing.Any) -> typing.Any: return message ``` +By default, Morpheus stages are assumed to require a GPU. However since this stage doesn't perform any specific GPU operations. We can indicate that the stage does not require a GPU by passing a tuple of supported execution modes to the decorator as follows: +```python +import typing + +from morpheus.config import ExecutionMode +from morpheus.pipeline.stage_decorator import stage + + +@stage(name="pass-thru", execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) +def pass_thru_stage(message: typing.Any) -> typing.Any: + # Return the message for the next stage + return message +``` + We can then add our stage to a pipeline as follows: ```python config = Config() @@ -60,7 +74,7 @@ pipeline = LinearPipeline(config) pipeline.add_stage(pass_thru_stage(config)) ``` -It is possible to provide additional keyword arguments to the function. Consider the following example: +It is also possible to provide additional keyword arguments to the function. Consider the following example: ```python @stage def multiplier(message: MessageMeta, *, column: str, value: int | float = 2.0) -> MessageMeta: @@ -80,6 +94,8 @@ The class based approach to defining a stage offers a bit more flexibility, spec Stages in Morpheus define what types of data they accept, and the type of data that they emit. In this example we are emitting messages of the same type that is received, this is actually quite common and Morpheus provides a mixin class, `PassThruTypeMixin`, to simplify this. +Similar to the function based stage, the class based stage will be not require a GPU, and we will indicate that it is able to be used in both GPU and CPU execution modes by utilizing the `GpuAndCpuMixin`. + Optionally, stages can be registered as a command with the Morpheus CLI using the `register_stage` decorator. This allows for pipelines to be constructed from both pre-built stages and custom user stages via the command line. Any constructor arguments will be introspected using [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/) and exposed as command line flags. Similarly, the class's docstrings will be exposed in the help string of the stage on the command line. We start our class definition with a few basic imports: @@ -91,12 +107,13 @@ import mrc from mrc.core import operators as ops from morpheus.cli.register_stage import register_stage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @register_stage("pass-thru") -class PassThruStage(PassThruTypeMixin, SinglePortStage): +class PassThruStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): ``` There are four methods that need to be defined in our new subclass to implement the stage interface: `name`, `accepted_types`, `compute_schema`, `supports_cpp_node`, and `_build_single`. In practice, it is often necessary to define at least one more method which will perform the actual work of the stage; by convention, this method is typically named `on_data`, which we will define in our examples. @@ -171,12 +188,13 @@ import mrc from mrc.core import operators as ops from morpheus.cli.register_stage import register_stage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage @register_stage("pass-thru") -class PassThruStage(PassThruTypeMixin, SinglePortStage): +class PassThruStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): """ A Simple Pass Through Stage """ @@ -191,12 +209,11 @@ class PassThruStage(PassThruTypeMixin, SinglePortStage): def supports_cpp_node(self) -> bool: return False - def on_data(self, message: typing.Any): + def on_data(self, message: typing.Any) -> typing.Any: # Return the message for the next stage return message - def _build_single(self, builder: mrc.Builder, - input_node: mrc.SegmentObject) -> mrc.SegmentObject: + def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: node = builder.make_node(self.unique_name, ops.map(self.on_data)) builder.make_edge(input_node, node) From d4d18953dacee6a7677fff8332e4fccddcab0744 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 30 Sep 2024 16:37:55 -0700 Subject: [PATCH 295/347] Update to document GPU execution, use the http triton server port --- docs/source/developer_guide/guides/2_real_world_phishing.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/developer_guide/guides/2_real_world_phishing.md b/docs/source/developer_guide/guides/2_real_world_phishing.md index 0d27f0de98..22e4d8b48d 100644 --- a/docs/source/developer_guide/guides/2_real_world_phishing.md +++ b/docs/source/developer_guide/guides/2_real_world_phishing.md @@ -99,7 +99,7 @@ def __init__(self, config: Config): Refer to the [Stage Constructors](#stage-constructors) section for more details. -Since the purpose of this stage is specifically tied to pre-processing text data for an NLP pipeline, when we register the stage, we will explicitly limit the stage to NLP pipelines: +Since the purpose of this stage is specifically tied to pre-processing text data for an NLP pipeline, when we register the stage, we will explicitly limit the stage to NLP pipelines. In addition to this since the pipeline our stage is operating in is a GPU pipeline, we will not be utilizing the `GpuAndCpuMixin` mixin from the previous example.: ```python @register_stage("recipient-features", modes=[PipelineModes.NLP]) class RecipientFeaturesStage(PassThruTypeMixin, SinglePortStage): @@ -540,7 +540,7 @@ MORPHEUS_ROOT = os.environ['MORPHEUS_ROOT'] default="phishing-bert-onnx", help="The name of the model that is deployed on Tritonserver.", ) -@click.option("--server_url", default='localhost:8001', help="Tritonserver url.") +@click.option("--server_url", default='localhost:8000', help="Tritonserver url.") @click.option( "--output_file", default=os.path.join(tempfile.gettempdir(), "detections.jsonlines"), @@ -630,7 +630,7 @@ morpheus --log_level=debug --plugin examples/developer_guide/2_1_real_world_phis recipient-features \ deserialize \ preprocess --vocab_hash_file=data/bert-base-uncased-hash.txt --truncation=true --do_lower_case=true --add_special_tokens=false \ - inf-triton --model_name=phishing-bert-onnx --server_url=localhost:8001 --force_convert_inputs=true \ + inf-triton --model_name=phishing-bert-onnx --server_url=localhost:8000 --force_convert_inputs=true \ monitor --description="Inference Rate" --smoothing=0.001 --unit=inf \ add-scores --label=is_phishing \ serialize \ From 2ba0c96cdd5486bcec177504999641e755ef75c6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 08:35:13 -0700 Subject: [PATCH 296/347] wip --- .../guides/2_real_world_phishing.md | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/docs/source/developer_guide/guides/2_real_world_phishing.md b/docs/source/developer_guide/guides/2_real_world_phishing.md index 22e4d8b48d..43783b3e8e 100644 --- a/docs/source/developer_guide/guides/2_real_world_phishing.md +++ b/docs/source/developer_guide/guides/2_real_world_phishing.md @@ -748,6 +748,12 @@ In this example, we will create a source that reads messages from a [RabbitMQ](h The `PreallocatorMixin` when added to a stage class, typically a source stage, indicates that the stage emits newly constructed DataFrames either directly or contained in a `MessageMeta` instance into the pipeline. Adding this mixin allows any columns needed by other stages to be inserted into the DataFrame. +Similar to the pass through stage, this new source stage should be able to operate in both GPU and CPU execution modes, as such we will be using the `GpuAndCpuMixin` mixin. One thing to note is that the DataFrame payload of a `MessageMeta` object is always a `cudf.DataFrame` when running in GPU mode and a `pandas.DataFrame` when running in CPU mode. When supporting both GPU and CPU execution modes, care must be taken to avoid directly importing `cudf` (or any other package requiring a GPU) when running in CPU mode on a system without a GPU and would therefore result in an error. Stages are able to examine the execution mode with the `morpheus.config.Config.execution_mode` attribute. The `morpheus.utils.type_utils.get_df_pkg` helper method is used to import the appropriate DataFrame package based on the execution mode in the constructor: +```python + # This will return either cudf.DataFrame or pandas.DataFrame depending on the execution mode + self._df_pkg = get_df_pkg(config.execution_mode) +``` + The `compute_schema` method allows us to define our output type of `MessageMeta`, we do so by calling the `set_type` method of the `output_schema` attribute of the `StageSchema` object passed into the method. Of note here is that it is perfectly valid for a stage to determine its output type based upon configuration arguments passed into the constructor. However the stage must document a single output type per output port. If a stage emitted multiple output types, then the types must share a common base class which would serve as the stage's output type. ```python def compute_schema(self, schema: StageSchema): @@ -771,7 +777,7 @@ def source_generator(self, subscription: mrc.Subscription) -> collections.abc.It if method_frame is not None: try: buffer = StringIO(body.decode("utf-8")) - df = cudf.io.read_json(buffer, orient='records', lines=True) + df = self._df_pkg.read_json(buffer, orient='records', lines=True) yield MessageMeta(df=df) except Exception as ex: logger.exception("Error occurred converting RabbitMQ message to Dataframe: %s", ex) @@ -799,20 +805,20 @@ import mrc import pandas as pd import pika -import cudf - from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages.message_meta import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema +from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) @register_stage("from-rabbitmq") -class RabbitMQSourceStage(PreallocatorMixin, SingleOutputSource): +class RabbitMQSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): """ Source stage used to load messages from a RabbitMQ queue. @@ -854,6 +860,9 @@ class RabbitMQSourceStage(PreallocatorMixin, SingleOutputSource): self._poll_interval = pd.Timedelta(poll_interval) + # This will return either cudf.DataFrame or pandas.DataFrame depending on the execution mode + self._df_pkg = get_df_pkg(config.execution_mode) + @property def name(self) -> str: return "from-rabbitmq" @@ -874,7 +883,7 @@ class RabbitMQSourceStage(PreallocatorMixin, SingleOutputSource): if method_frame is not None: try: buffer = StringIO(body.decode("utf-8")) - df = cudf.io.read_json(buffer, orient='records', lines=True) + df = self._df_pkg.read_json(buffer, orient='records', lines=True) yield MessageMeta(df=df) except Exception as ex: logger.exception("Error occurred converting RabbitMQ message to Dataframe: %s", ex) @@ -889,7 +898,7 @@ class RabbitMQSourceStage(PreallocatorMixin, SingleOutputSource): ``` ### Function Based Approach -Similar to the `stage` decorator used in previous examples Morpheus provides a `source` decorator which wraps a generator function to be used as a source stage. In the class based approach we explicitly added the `PreallocatorMixin`, when using the `source` decorator the return type annotation will be inspected and a stage will be created with the `PreallocatorMixin` if the return type is a `DataFrame` type or a message which contains a `DataFrame` (`MessageMeta` and `ControlMessage`). +Similar to the `stage` decorator used in previous examples Morpheus provides a `source` decorator which wraps a generator function to be used as a source stage. In the class based approach we explicitly added the `PreallocatorMixin`, when using the `source` decorator the return type annotation will be inspected and a stage will be created with the `PreallocatorMixin` if the return type is a `DataFrame` type or a message which contains a `DataFrame` (`MessageMeta` and `ControlMessage`). We will also indicate which execution modes are supported by the stage by setting the `execution_modes` argument to the decorator. The code for the function will first perform the same setup as was used in the class constructor, then entering a nearly identical loop as that in the `source_generator` method. @@ -903,15 +912,15 @@ import mrc import pandas as pd import pika -import cudf - +from morpheus.config import ExecutionMode from morpheus.messages.message_meta import MessageMeta from morpheus.pipeline.stage_decorator import source +from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) -@source(name="from-rabbitmq") +@source(name="from-rabbitmq", execution_modes=(ExecutionMode.GPU, ExecutionMode.CPU)) def rabbitmq_source(subscription: mrc.Subscription, host: str, exchange: str, @@ -950,13 +959,15 @@ def rabbitmq_source(subscription: mrc.Subscription, poll_interval = pd.Timedelta(poll_interval) + df_pkg = get_df_pkg() + try: while subscription.is_subscribed(): (method_frame, _, body) = channel.basic_get(queue_name) if method_frame is not None: try: buffer = StringIO(body.decode("utf-8")) - df = cudf.io.read_json(buffer, orient='records', lines=True) + df = df_pkg.read_json(buffer, orient='records', lines=True) yield MessageMeta(df=df) except Exception as ex: logger.exception("Error occurred converting RabbitMQ message to Dataframe: %s", ex) @@ -995,16 +1006,21 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> return node ``` -Similar to our previous examples, most of the actual business logic of the stage is contained in the `on_data` method. In this case, we grab a reference to the [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/) attached to the incoming message. We then serialize to an [`io.StringIO`](https://docs.python.org/3.10/library/io.html?highlight=stringio#io.StringIO) buffer, which is then sent to RabbitMQ. +Similar to our previous examples, most of the actual business logic of the stage is contained in the `on_data` method. In this case, we grab a reference to the DataFrane attached to the incoming message. We then serialize to an [`io.StringIO`](https://docs.python.org/3.10/library/io.html?highlight=stringio#io.StringIO) buffer, which is then sent to RabbitMQ. + +> **Note**: This stage supports both GPU and CPU execution modes. When running in GPU mode, the payload of a `MessageMeta` object is always a [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/). When running in CPU mode, the payload is always a [pandas](https://pandas.pydata.org/) [DataFrane](https://pandas.pydata.org/docs/reference/frame.html). In many cases the two will be API compatible without requiring any changes to the code. In some cases however, the API may differ slightly and there is a need to know the pyaload type, care must be taken not to directly import `cudf` or any other package requiring a GPU when running in CPU mode on a system without a GPU. Morpheus provides some helper methods to assist with this, such as `morpheus.utils.type_utils.is_cudf_type` and `morpheus.utils.type_utils.get_df_pkg_from_obj`. ```python -def on_data(self, message: MessageMeta): - df = message.df - buffer = StringIO() - df.to_json(buffer, orient='records', lines=True) - body = buffer.getvalue().strip() - self._channel.basic_publish(exchange=self._exchange, routing_key=self._routing_key, body=body) - return message + def on_data(self, message: MessageMeta) -> MessageMeta: + df = message.df + + buffer = StringIO() + df.to_json(buffer, orient='records', lines=True) + body = buffer.getvalue().strip() + + self._channel.basic_publish(exchange=self._exchange, routing_key=self._routing_key, body=body) + + return message ``` The two new methods introduced in this example are the `on_error` and `on_complete` methods. For both methods, we want to make sure the [connection](https://pika.readthedocs.io/en/stable/modules/connection.html) object is properly closed. From a60a9b982fc5c73bc85145c4afe09dcf326b7e2c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 09:20:10 -0700 Subject: [PATCH 297/347] WIP --- .../developer_guide/guides/1_simple_python_stage.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/developer_guide/guides/1_simple_python_stage.md b/docs/source/developer_guide/guides/1_simple_python_stage.md index c6cb7cf11c..27586de578 100644 --- a/docs/source/developer_guide/guides/1_simple_python_stage.md +++ b/docs/source/developer_guide/guides/1_simple_python_stage.md @@ -29,7 +29,7 @@ To start, we will implement a single stage that could be included in a pipeline. ### Stand-alone Function -The stand-alone function approach is the simplest way to define a stage. The function should accept a single argument, which will be the input message, and return a single value, which will be the output message. The function should be decorated with the `morpheus.pipeline.stage_decorator.stage` decorator. +The stand-alone function approach is the simplest way to define a stage. The function should accept a single argument, which will be the input message, and return a single value, which will be the output message. The function should be decorated with the {py:func}`~morpheus.pipeline.stage_decorator.stage` decorator. ```python import typing @@ -90,13 +90,13 @@ pipe.add_stage(multiplier(config, column='probs', value=5)) ### Stage Class -The class based approach to defining a stage offers a bit more flexibility, specifically the ability to validate constructor arguments, and perform any needed setup prior to being invoked in a pipeline. Defining this stage requires us to specify the stage type. Morpheus stages which contain a single input and a single output typically inherit from `SinglePortStage`. Stages that act as sources of data, in that they do not take an input from a prior stage but rather produce data from a source such as a file, Kafka service, or other external sources, will need to inherit from the `SingleOutputSource` base class. +The class based approach to defining a stage offers a bit more flexibility, specifically the ability to validate constructor arguments, and perform any needed setup prior to being invoked in a pipeline. Defining this stage requires us to specify the stage type. Morpheus stages which contain a single input and a single output typically inherit from {py:class}`~morpheus.pipeline.single_port_stage.SinglePortStage`. Stages that act as sources of data, in that they do not take an input from a prior stage but rather produce data from a source such as a file, Kafka service, or other external sources, will need to inherit from the {py:class}`~morpheus.pipeline.single_output_source.SingleOutputSource` base class. -Stages in Morpheus define what types of data they accept, and the type of data that they emit. In this example we are emitting messages of the same type that is received, this is actually quite common and Morpheus provides a mixin class, `PassThruTypeMixin`, to simplify this. +Stages in Morpheus define what types of data they accept, and the type of data that they emit. In this example we are emitting messages of the same type that is received, this is actually quite common and Morpheus provides a mixin class, {py:class}`~morpheus.pipeline.pass_thru_type_mixin.PassThruTypeMixin`, to simplify this. -Similar to the function based stage, the class based stage will be not require a GPU, and we will indicate that it is able to be used in both GPU and CPU execution modes by utilizing the `GpuAndCpuMixin`. +Similar to the function based stage, the class based stage will be not require a GPU, and we will indicate that it is able to be used in both GPU and CPU execution modes by utilizing the {py:class}`~morpheus.pipeline.execution_mode_mixins.GpuAndCpuMixin`. -Optionally, stages can be registered as a command with the Morpheus CLI using the `register_stage` decorator. This allows for pipelines to be constructed from both pre-built stages and custom user stages via the command line. Any constructor arguments will be introspected using [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/) and exposed as command line flags. Similarly, the class's docstrings will be exposed in the help string of the stage on the command line. +Optionally, stages can be registered as a command with the Morpheus CLI using the {py:func}`~morpheus.cli.register_stage.register_stage` decorator. This allows for pipelines to be constructed from both pre-built stages and custom user stages via the command line. Any constructor arguments will be introspected using [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/) and exposed as command line flags. Similarly, the class's docstrings will be exposed in the help string of the stage on the command line. We start our class definition with a few basic imports: @@ -125,7 +125,7 @@ There are four methods that need to be defined in our new subclass to implement return "pass-thru" ``` -The `accepted_types` method returns a tuple of message classes that this stage is able to accept as input. Morpheus uses this to validate that the parent of this stage emits a message that this stage can accept. Since our stage is a pass through, we will declare that we can accept any incoming message type. Note that production stages will often declare only a single Morpheus message class such as `MessageMeta` or `ControlMessage` (refer to the message classes defined in `morpheus.messages` for a complete list). +The `accepted_types` method returns a tuple of message classes that this stage is able to accept as input. Morpheus uses this to validate that the parent of this stage emits a message that this stage can accept. Since our stage is a pass through, we will declare that we can accept any incoming message type. Note that production stages will often declare only a single Morpheus message class such as {py:class}`~morpheus.messages.MessageMeta` or {py:class}`~morpheus.messages.ControlMessage` (refer to the message classes defined in {py:mod}`~morpheus.messages` for a complete list). ```python def accepted_types(self) -> tuple: return (typing.Any,) From 2d70d3dae1b2e1af0950257d308451a95567aa70 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 10:01:55 -0700 Subject: [PATCH 298/347] wip --- .../guides/2_real_world_phishing.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/developer_guide/guides/2_real_world_phishing.md b/docs/source/developer_guide/guides/2_real_world_phishing.md index 43783b3e8e..bea1e0dfc3 100644 --- a/docs/source/developer_guide/guides/2_real_world_phishing.md +++ b/docs/source/developer_guide/guides/2_real_world_phishing.md @@ -29,7 +29,7 @@ For this task, we'll need to define a new stage, which we will call our `Recipie 1. Count the number of recipients in the email's metadata. 1. Emit a Morpheus `MessageMeta` object that will contain the record content along with the augmented metadata. -For this stage, the code will be similar to the previous example with a few notable changes. We will be working with the `MessageMeta` class. This is a Morpheus message containing a [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/). Since we will expect our new stage to operate on `MessageMeta` types, our new `accepted_types` method is defined as: +For this stage, the code will be similar to the previous example with a few notable changes. We will be working with the {py:class}`~morpheus.messages.MessageMeta` class. This is a Morpheus message containing a [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/). Since we will expect our new stage to operate on `MessageMeta` types, our new `accepted_types` method is defined as: ```python def accepted_types(self) -> tuple: @@ -105,7 +105,7 @@ Since the purpose of this stage is specifically tied to pre-processing text data class RecipientFeaturesStage(PassThruTypeMixin, SinglePortStage): ``` -Our `_build_single` method remains unchanged from the previous example; even though we are modifying the incoming messages, our input and output types remain the same and we continue to make use of the `PassThruTypeMixin`. +Our `_build_single` method remains unchanged from the previous example; even though we are modifying the incoming messages, our input and output types remain the same and we continue to make use of the {py:class}`~morpheus.pipeline.pass_thru_type_mixin.PassThruTypeMixin`. ### The Completed Preprocessing Stage @@ -639,7 +639,7 @@ morpheus --log_level=debug --plugin examples/developer_guide/2_1_real_world_phis ## Stage Constructors -In our `RecipientFeaturesStage` example we added a constructor to our stage, however we didn't go into much detail on the implementation. Every stage constructor must receive an instance of a `morpheus.config.Config` object as its first argument and is then free to define additional stage-specific arguments after that. The Morpheus configuration object will contain configuration parameters needed by multiple stages in the pipeline, and the constructor in each Morpheus stage is free to inspect these. In contrast, parameters specific to a single stage are typically defined as constructor arguments. It is a best practice to perform any necessary validation checks in the constructor, and raising an exception in the case of mis-configuration. This allows us to fail early rather than after the pipeline has started. +In our `RecipientFeaturesStage` example we added a constructor to our stage, however we didn't go into much detail on the implementation. Every stage constructor must receive an instance of a {py:class}`~morpheus.config.Config` object as its first argument and is then free to define additional stage-specific arguments after that. The Morpheus configuration object will contain configuration parameters needed by multiple stages in the pipeline, and the constructor in each Morpheus stage is free to inspect these. In contrast, parameters specific to a single stage are typically defined as constructor arguments. It is a best practice to perform any necessary validation checks in the constructor, and raising an exception in the case of mis-configuration. This allows us to fail early rather than after the pipeline has started. In our `RecipientFeaturesStage` example, we hard-coded the Bert separator token. Let's instead refactor the code to receive that as a constructor argument. This new constructor argument is documented following the [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/format.html#parameters) formatting style allowing it to be documented properly for both API and CLI users. Let's also take the opportunity to verify that the pipeline mode is set to `morpheus.config.PipelineModes.NLP`. @@ -742,13 +742,13 @@ Options: ### Class Based Approach -Creating a new source stage is similar to defining any other stage with a few differences. First, we will be subclassing `SingleOutputSource` including the `PreallocatorMixin`. Second, the required methods are the `name` property, `_build_source`, `compute_schema` and `supports_cpp_node` methods. +Creating a new source stage is similar to defining any other stage with a few differences. First, we will be subclassing {py:class}`~morpheus.pipeline.single_output_source.SingleOutputSource` and including the `PreallocatorMixin`. Second, the required methods are the `name` property, `_build_source`, `compute_schema` and `supports_cpp_node` methods. In this example, we will create a source that reads messages from a [RabbitMQ](https://www.rabbitmq.com/) queue using the [pika](https://pika.readthedocs.io/en/stable/#) client for Python. For simplicity, we will assume that authentication is not required for our RabbitMQ exchange and that the body of the RabbitMQ messages will be JSON formatted. Both authentication and support for other formats could be easily added later. The `PreallocatorMixin` when added to a stage class, typically a source stage, indicates that the stage emits newly constructed DataFrames either directly or contained in a `MessageMeta` instance into the pipeline. Adding this mixin allows any columns needed by other stages to be inserted into the DataFrame. -Similar to the pass through stage, this new source stage should be able to operate in both GPU and CPU execution modes, as such we will be using the `GpuAndCpuMixin` mixin. One thing to note is that the DataFrame payload of a `MessageMeta` object is always a `cudf.DataFrame` when running in GPU mode and a `pandas.DataFrame` when running in CPU mode. When supporting both GPU and CPU execution modes, care must be taken to avoid directly importing `cudf` (or any other package requiring a GPU) when running in CPU mode on a system without a GPU and would therefore result in an error. Stages are able to examine the execution mode with the `morpheus.config.Config.execution_mode` attribute. The `morpheus.utils.type_utils.get_df_pkg` helper method is used to import the appropriate DataFrame package based on the execution mode in the constructor: +Similar to the pass through stage, this new source stage should be able to operate in both GPU and CPU execution modes, as such we will be using the `GpuAndCpuMixin` mixin. One thing to note is that the DataFrame payload of a `MessageMeta` object is always a `cudf.DataFrame` when running in GPU mode and a `pandas.DataFrame` when running in CPU mode. When supporting both GPU and CPU execution modes, care must be taken to avoid directly importing `cudf` (or any other package requiring a GPU) when running in CPU mode on a system without a GPU and would therefore result in an error. Stages are able to examine the execution mode with the `morpheus.config.Config.execution_mode` attribute. The {py:func}`~morpheus.utils.type_utils.get_df_pkg` helper method is used to import the appropriate DataFrame package based on the execution mode in the constructor: ```python # This will return either cudf.DataFrame or pandas.DataFrame depending on the execution mode self._df_pkg = get_df_pkg(config.execution_mode) @@ -898,7 +898,7 @@ class RabbitMQSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource) ``` ### Function Based Approach -Similar to the `stage` decorator used in previous examples Morpheus provides a `source` decorator which wraps a generator function to be used as a source stage. In the class based approach we explicitly added the `PreallocatorMixin`, when using the `source` decorator the return type annotation will be inspected and a stage will be created with the `PreallocatorMixin` if the return type is a `DataFrame` type or a message which contains a `DataFrame` (`MessageMeta` and `ControlMessage`). We will also indicate which execution modes are supported by the stage by setting the `execution_modes` argument to the decorator. +Similar to the `stage` decorator used in previous examples Morpheus provides a {py:func}`~morpheus.pipeline.stage_decorator.source` decorator which wraps a generator function to be used as a source stage. In the class based approach we explicitly added the `PreallocatorMixin`, when using the `source` decorator the return type annotation will be inspected and a stage will be created with the `PreallocatorMixin` if the return type is a `DataFrame` type or a message which contains a `DataFrame` (`MessageMeta` and `ControlMessage`). We will also indicate which execution modes are supported by the stage by setting the `execution_modes` argument to the decorator. The code for the function will first perform the same setup as was used in the class constructor, then entering a nearly identical loop as that in the `source_generator` method. @@ -1008,7 +1008,7 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> Similar to our previous examples, most of the actual business logic of the stage is contained in the `on_data` method. In this case, we grab a reference to the DataFrane attached to the incoming message. We then serialize to an [`io.StringIO`](https://docs.python.org/3.10/library/io.html?highlight=stringio#io.StringIO) buffer, which is then sent to RabbitMQ. -> **Note**: This stage supports both GPU and CPU execution modes. When running in GPU mode, the payload of a `MessageMeta` object is always a [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/). When running in CPU mode, the payload is always a [pandas](https://pandas.pydata.org/) [DataFrane](https://pandas.pydata.org/docs/reference/frame.html). In many cases the two will be API compatible without requiring any changes to the code. In some cases however, the API may differ slightly and there is a need to know the pyaload type, care must be taken not to directly import `cudf` or any other package requiring a GPU when running in CPU mode on a system without a GPU. Morpheus provides some helper methods to assist with this, such as `morpheus.utils.type_utils.is_cudf_type` and `morpheus.utils.type_utils.get_df_pkg_from_obj`. +> **Note**: This stage supports both GPU and CPU execution modes. When running in GPU mode, the payload of a `MessageMeta` object is always a [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/). When running in CPU mode, the payload is always a [pandas](https://pandas.pydata.org/) [DataFrane](https://pandas.pydata.org/docs/reference/frame.html). In many cases the two will be API compatible without requiring any changes to the code. In some cases however, the API may differ slightly and there is a need to know the pyaload type, care must be taken not to directly import `cudf` or any other package requiring a GPU when running in CPU mode on a system without a GPU. Morpheus provides some helper methods to assist with this in the {py:mod}`~morpheus.utils.type_utils` module, such as {py:func}`~morpheus.utils.type_utils.is_cudf_type` and {py:func}`~morpheus.utils.type_utils.get_df_pkg_from_obj`. ```python def on_data(self, message: MessageMeta) -> MessageMeta: From 4fddd4835544c050640c556117827647a0702c81 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 10:52:54 -0700 Subject: [PATCH 299/347] Fix spelling errors --- .../guides/2_real_world_phishing.md | 2 +- .../guides/3_simple_cpp_stage.md | 23 ++++++++----------- examples/cpu_only/README.md | 4 ++-- .../developer_guide/2_2_rabbitmq/README.md | 2 +- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/docs/source/developer_guide/guides/2_real_world_phishing.md b/docs/source/developer_guide/guides/2_real_world_phishing.md index bea1e0dfc3..b1ae038f1a 100644 --- a/docs/source/developer_guide/guides/2_real_world_phishing.md +++ b/docs/source/developer_guide/guides/2_real_world_phishing.md @@ -1008,7 +1008,7 @@ def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> Similar to our previous examples, most of the actual business logic of the stage is contained in the `on_data` method. In this case, we grab a reference to the DataFrane attached to the incoming message. We then serialize to an [`io.StringIO`](https://docs.python.org/3.10/library/io.html?highlight=stringio#io.StringIO) buffer, which is then sent to RabbitMQ. -> **Note**: This stage supports both GPU and CPU execution modes. When running in GPU mode, the payload of a `MessageMeta` object is always a [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/). When running in CPU mode, the payload is always a [pandas](https://pandas.pydata.org/) [DataFrane](https://pandas.pydata.org/docs/reference/frame.html). In many cases the two will be API compatible without requiring any changes to the code. In some cases however, the API may differ slightly and there is a need to know the pyaload type, care must be taken not to directly import `cudf` or any other package requiring a GPU when running in CPU mode on a system without a GPU. Morpheus provides some helper methods to assist with this in the {py:mod}`~morpheus.utils.type_utils` module, such as {py:func}`~morpheus.utils.type_utils.is_cudf_type` and {py:func}`~morpheus.utils.type_utils.get_df_pkg_from_obj`. +> **Note**: This stage supports both GPU and CPU execution modes. When running in GPU mode, the payload of a `MessageMeta` object is always a [cuDF](https://docs.rapids.ai/api/cudf/stable/) [DataFrame](https://docs.rapids.ai/api/cudf/stable/user_guide/api_docs/dataframe/). When running in CPU mode, the payload is always a [pandas](https://pandas.pydata.org/) [DataFrane](https://pandas.pydata.org/docs/reference/frame.html). In many cases the two will be API compatible without requiring any changes to the code. In some cases however, the API may differ slightly and there is a need to know the payload type, care must be taken not to directly import `cudf` or any other package requiring a GPU when running in CPU mode on a system without a GPU. Morpheus provides some helper methods to assist with this in the {py:mod}`~morpheus.utils.type_utils` module, such as {py:func}`~morpheus.utils.type_utils.is_cudf_type` and {py:func}`~morpheus.utils.type_utils.get_df_pkg_from_obj`. ```python def on_data(self, message: MessageMeta) -> MessageMeta: diff --git a/docs/source/developer_guide/guides/3_simple_cpp_stage.md b/docs/source/developer_guide/guides/3_simple_cpp_stage.md index f21317475d..676a2df8a6 100644 --- a/docs/source/developer_guide/guides/3_simple_cpp_stage.md +++ b/docs/source/developer_guide/guides/3_simple_cpp_stage.md @@ -34,18 +34,13 @@ pip install ./ ## Overview Morpheus offers the choice of writing pipeline stages in either Python or C++. For many use cases, a Python stage is perfectly fine. However, in the event that a Python stage becomes a bottleneck for the pipeline, then writing a C++ implementation for the stage becomes advantageous. The C++ implementations of Morpheus stages and messages utilize the [pybind11](https://pybind11.readthedocs.io/en/stable/index.html) library to provide Python bindings. -We have been defining our stages in Python up to this point, the option of defining a C++ implementation is only available to stages implemented as classes. Many of the stages included with Morpheus have both a Python and a C++ implementation, and Morpheus will use the C++ implementations by default. You can explicitly disable the use of C++ stage implementations by calling `morpheus.config.CppConfig.set_should_use_cpp(False)`: +We have been defining our stages in Python up to this point, the option of defining a C++ implementation is only available to stages implemented as classes. Many of the stages included with Morpheus have both a Python and a C++ implementation, and Morpheus will use the C++ implementations by default when running in the GPU execution mode. When running in the CPU execution mode, Morpheus will always use the Python implementation. -```python -from morpheus.config import CppConfig -CppConfig.set_should_use_cpp(False) -``` +If a stage does not have a C++ implementation, Morpheus will fall back to the Python implementation without any additional configuration. Morpheus stages which only contain a C++ implementation, still require a Python class to register the stage, and provide the stage's configuration. -If a stage does not have a C++ implementation, Morpheus will fall back to the Python implementation without any additional configuration and operate in a hybrid execution mode. +In addition to C++ accelerated stage implementations, Morpheus also provides a C++ implementation for message primitives. When using the GPU GPU execution mode (the default), constructing one of the Python message classes defined under {py:mod}`~morpheus.messages` will return a Python object with bindings to the underlying C++ implementation. -In addition to C++ accelerated stage implementations, Morpheus also provides a C++ implementation for message primitives. When C++ execution is enabled, constructing one of the Python message classes defined under `morpheus.messages` will return a Python object with bindings to the underlying C++ implementation. - -Since we are defining our stages in Python, it becomes the responsibility of the Python stage to build a C++ accelerated node. This happens in the `_build_source` and `_build_single` methods. Ultimately it is the decision of a Python stage to build a Python node or a C++ node. It is perfectly acceptable to build a Python node when `morpheus.config.CppConfig.get_should_use_cpp()` is configured to `True`. It is not acceptable, however, to build a C++ node when `morpheus.config.CppConfig.get_should_use_cpp() == False`. The reason is the C++ implementations of Morpheus' messages can be consumed by Python and C++ stage implementations alike. However when `morpheus.config.CppConfig.get_should_use_cpp() == False`, the Python implementations of each message type will be used which cannot be consumed by the C++ implementations of stages. +Since we are defining our stages in Python, it becomes the responsibility of the Python stage to build a C++ accelerated node. This happens in the `_build_source` and `_build_single` methods. The Python stage should call `self._build_cpp_node()` to determine if a C++ node should be built, and ultimately it is the decision of a Python stage to build a Python node or a C++ node. It is perfectly acceptable to build a Python node when `self._build_cpp_node()` is returns `True`. It is not acceptable, however, to build a C++ node when `self._build_cpp_node()` returns `False`. The reason is the C++ implementations of Morpheus messages can be consumed by Python and C++ stage implementations alike. However the Python implementations of Morpheus messages cannot be consumed by the C++ implementations of stages. Python stages which have a C++ implementation must advertise this functionality by returning a value of `True` from the `supports_cpp_node` method: @@ -84,7 +79,7 @@ Both the `PythonSource` and `PythonNode` classes are defined in the `pymrc/node. As in our Python guide, we will start with a simple pass through stage which can be used as a starting point for future development of other stages. Note that by convention, C++ classes in Morpheus have the same name as their corresponding Python classes and are located under a directory named `_lib`. We will be following that convention. To start, we will create a `_lib` directory and a new empty `__init__.py` file. -While our Python implementation accepts messages of any type (in the form of Python objects), on the C++ side we don't have that flexibility since our node is subject to C++ static typing rules. In practice, this isn't a limitation as we usually know which specific message types we need to work with. For this example we will be working with the `ControlMessage` as our input and output type, it is also a common base type for many other Morpheus message classes. This means that at build time our Python stage implementation is able to build a C++ node when the incoming type is `ControlMessage`, while falling back to the existing Python implementation otherwise. +While our Python implementation accepts messages of any type (in the form of Python objects), on the C++ side we don't have that flexibility since our node is subject to C++ static typing rules. In practice, this isn't a limitation as we usually know which specific message types we need to work with. For this example we will be working with the `ControlMessage` as our input and output type. This means that at build time our Python stage implementation is able to build a C++ node when the incoming type is `ControlMessage`, while falling back to the existing Python implementation otherwise. To start with, we have our Morpheus and MRC-specific includes: @@ -371,7 +366,7 @@ def compute_schema(self, schema: StageSchema): ``` > **Note**: We are still using the `PassThruTypeMixin` to handle the requirements of setting the output type. -As mentioned in the previous section, our `_build_single` method needs to be updated to build a C++ node when the input type is `ControlMessage` and when `morpheus.config.CppConfig.get_should_use_cpp()` is `True` using the `self._build_cpp_node()` method. The `_build_cpp_node()` method compares both `morpheus.config.CppConfig.get_should_use_cpp()` and `supports_cpp_node()` and returns `True` only when both methods return `True`. +As mentioned in the previous section, our `_build_single` method needs to be updated to build a C++ node when the input type is `ControlMessage` and when `self._build_cpp_node()` returns `True`. ```python def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: @@ -398,13 +393,14 @@ from mrc.core import operators as ops from morpheus.cli.register_stage import register_stage from morpheus.config import Config from morpheus.messages import ControlMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema @register_stage("pass-thru") -class PassThruStage(PassThruTypeMixin, SinglePortStage): +class PassThruStage(PassThruTypeMixin, GpuAndCpuMixin, SinglePortStage): def __init__(self, config: Config): super().__init__(config) @@ -438,11 +434,10 @@ class PassThruStage(PassThruTypeMixin, SinglePortStage): builder.make_edge(input_node, node) return node - ``` ## Testing the Stage -To test the updated stage we will build a simple pipeline using the Morpheus command line tool. In order to illustrate the stage building a C++ node only when the input type is a `ControlMessage` we will insert the `pass-thru` stage in twice in the pipeline. In the first instance the input type will be `MessageMeta` and the stage will fallback to using a Python node, and in the second instance the input type will be a `ControlMessage` and the stage will build a C++ node. +To test the updated stage we will build a simple pipeline using the Morpheus command line tool. In order to illustrate the stage building a C++ node only when the input type is a `ControlMessage` we will insert the `pass-thru` stage in twice in the pipeline. In the first instance the input type will be `MessageMeta` and the stage will fallback to using a Python node, and in the second instance the input type will be a `ControlMessage` and the stage will build a C++ node. ```bash PYTHONPATH="examples/developer_guide/3_simple_cpp_stage/src" \ diff --git a/examples/cpu_only/README.md b/examples/cpu_only/README.md index 570cda8070..feac382a3f 100644 --- a/examples/cpu_only/README.md +++ b/examples/cpu_only/README.md @@ -26,9 +26,9 @@ limitations under the License. | Dev Container | ✔ | | ## CPU Only Pipeline -This example demonstrates a simple Morpheus pipeline which is able to opperate on a host without access GPU. +This example demonstrates a simple Morpheus pipeline which is able to operate on a host without access GPU. -> **Note**: A more complex example of a pipeline that can exexute without a GPU is also available at `examples/llm/completion/README.md` +> **Note**: A more complex example of a pipeline that can execute without a GPU is also available at `examples/llm/completion/README.md` From the root of the Morpheus repo, run: ```bash diff --git a/examples/developer_guide/2_2_rabbitmq/README.md b/examples/developer_guide/2_2_rabbitmq/README.md index 5407a9810b..db9465a31e 100644 --- a/examples/developer_guide/2_2_rabbitmq/README.md +++ b/examples/developer_guide/2_2_rabbitmq/README.md @@ -62,7 +62,7 @@ This will read JSON data from the `examples/data/email.jsonlines` file and publi The `write_simple.py` script will exit as soon as the message is written to the queue. The `read_simple.py` script will continue reading from the queue until explicitly shut down with a control-C. -> **Note**: Both the `read_simple.py` and `write_simple.py` scipts will launch independent Morpheus pipelines, both of which can optionally execute in CPU-only mode by setting the `--use_cpu_only` flag. +> **Note**: Both the `read_simple.py` and `write_simple.py` scripts will launch independent Morpheus pipelines, both of which can optionally execute in CPU-only mode by setting the `--use_cpu_only` flag. ## Alternate Morpheus CLI usage In the above examples we defined the pipeline using the Python API in the `read_simple.py` and `write_simple.py` scripts. Alternately, we could have defined the same pipelines using the Morpheus CLI tool. From b05bb1d7b6b5a37e9c1fded2fbc76aff96c33626 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 11:20:13 -0700 Subject: [PATCH 300/347] Assume pandas in the Pythin impl --- .../src/rabbitmq_cpp_stage/rabbitmq_source_stage.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py index 752ee0fb01..a408ca0b49 100755 --- a/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py +++ b/examples/developer_guide/4_rabbitmq_cpp_stage/src/rabbitmq_cpp_stage/rabbitmq_source_stage.py @@ -28,7 +28,6 @@ from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema -from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) @@ -72,9 +71,6 @@ def __init__(self, self._poll_interval = pd.Timedelta(poll_interval) - # This will return either cudf.DataFrame or pandas.DataFrame depending on the execution mode - self._df_pkg = get_df_pkg(config.execution_mode) - @property def name(self) -> str: return "from-rabbitmq" @@ -122,7 +118,7 @@ def source_generator(self, subscription: mrc.Subscription): if method_frame is not None: try: buffer = StringIO(body.decode("utf-8")) - df = self._df_pkg.read_json(buffer, orient='records', lines=True) + df = pd.read_json(buffer, orient='records', lines=True) yield MessageMeta(df=df) except Exception as ex: logger.exception("Error occurred converting RabbitMQ message to Dataframe: %s", ex) From 47251fbd9f0252d1dd2e4465209e41bba76ed542 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 11:31:23 -0700 Subject: [PATCH 301/347] WIP --- docs/source/developer_guide/guides/4_source_cpp_stage.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/source/developer_guide/guides/4_source_cpp_stage.md b/docs/source/developer_guide/guides/4_source_cpp_stage.md index 49beda09c2..02049e595b 100644 --- a/docs/source/developer_guide/guides/4_source_cpp_stage.md +++ b/docs/source/developer_guide/guides/4_source_cpp_stage.md @@ -36,6 +36,8 @@ For this example, we are going to add a C++ implementation for the `RabbitMQSour For communicating with [RabbitMQ](https://www.rabbitmq.com/) we will be using the [SimpleAmqpClient](https://github.com/alanxz/SimpleAmqpClient) library, and [libcudf](https://docs.rapids.ai/api/libcudf/stable/index.html) for constructing the `DataFrame`. +> **Note**: Since the C++ implementation will only be used when the execution mode is set to GPU. It is safe to assume the C++ implementation will always interact with cuDF DataFrames, and the Python implementation will always interact with pandas DataFrames. + ## Header Definition Our includes: @@ -477,7 +479,8 @@ PYBIND11_MODULE(rabbitmq_cpp_stage, m) ## Python Changes -Previously, our stage connected to the RabbitMQ server in the constructor. This is no longer advantageous to us when C++ execution is enabled. Instead, we will record our constructor arguments and move the connection code to a new `connect` method. Our new constructor and `connect` methods are updated to: +Previously, our stage connected to the RabbitMQ server in the constructor. This is no longer advantageous to us when C++ execution is enabled. Instead, we will record our constructor arguments and move the connection code to a new `connect` method. Since this stage's C++ implementation will always be used when running in GPU mode, we can assume the Python implementation will always interact with pandas DataFrames. +Our new constructor and `connect` methods are updated to: ```python def __init__(self, @@ -513,7 +516,7 @@ def connect(self): self._channel.queue_bind(exchange=self._exchange, queue=self._queue_name) ``` -Lastly, our `_build_source` method needs to be updated to build a C++ node when `morpheus.config.CppConfig.get_should_use_cpp()` is configured to `True` by using the `self._build_cpp_node()` method. +Lastly, our `_build_source` method needs to be updated to build a C++ node when `self._build_cpp_node()` returns `True`. ```python def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: From 13ae779cda627a67c4f18dbbbcb86696b2cd4913 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 11:43:00 -0700 Subject: [PATCH 302/347] WIP --- .../developer_guide/guides/5_digital_fingerprinting.md | 7 +++++-- examples/gnn_fraud_detection_pipeline/README.md | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/source/developer_guide/guides/5_digital_fingerprinting.md b/docs/source/developer_guide/guides/5_digital_fingerprinting.md index e64b8a91d4..4ad65fa6d2 100644 --- a/docs/source/developer_guide/guides/5_digital_fingerprinting.md +++ b/docs/source/developer_guide/guides/5_digital_fingerprinting.md @@ -186,11 +186,14 @@ docker compose build > This is most likely due to using an older version of the `docker-compose` command, instead re-run the build with `docker compose`. Refer to [Migrate to Compose V2](https://docs.docker.com/compose/migrate/) for more information. #### Downloading the example datasets -First, we will need to install `s3fs` and then run the `examples/digital_fingerprinting/fetch_example_data.py` script. This will download the example data into the `examples/data/dfp` dir. +First, we will need to install additional requirements in to the Conda environment. Then run the `examples/digital_fingerprinting/fetch_example_data.py` script. This will download the example data into the `examples/data/dfp` dir. From the Morpheus repo, run: ```bash -pip install s3fs +conda env update --solver=libmamba \ + -n ${CONDA_DEFAULT_ENV} \ + --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml + python examples/digital_fingerprinting/fetch_example_data.py all ``` diff --git a/examples/gnn_fraud_detection_pipeline/README.md b/examples/gnn_fraud_detection_pipeline/README.md index 97ae02e2fc..a09c0e3e9b 100644 --- a/examples/gnn_fraud_detection_pipeline/README.md +++ b/examples/gnn_fraud_detection_pipeline/README.md @@ -27,7 +27,7 @@ All environments require additional Conda packages which can be installed with e ## Requirements -Prior to running the GNN fraud detection pipeline, additional requirements must be installed in to your Conda environment. A supplemental requirements file has been provided in this example directory. +Prior to running the GNN fraud detection pipeline, additional requirements must be installed in to your Conda environment. ```bash conda env update --solver=libmamba \ From 77a87d218a510785eab15b36b13bfedb88554c99 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 11:47:42 -0700 Subject: [PATCH 303/347] Remove CppConfig --- .../guides/6_digital_fingerprinting_reference.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md b/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md index fd96515309..9fab800a82 100644 --- a/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md +++ b/docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md @@ -33,13 +33,10 @@ import os from morpheus.config import Config from morpheus.config import ConfigAutoEncoder -from morpheus.config import CppConfig from morpheus.cli.utils import get_package_relative_file from morpheus.utils.file_utils import load_labels_file ``` ```python -CppConfig.set_should_use_cpp(False) - config = Config() config.num_threads = len(os.sched_getaffinity(0)) config.ae = ConfigAutoEncoder() From 7078dc793e43aed9b7bb42cd117990d32279d38d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 12:01:56 -0700 Subject: [PATCH 304/347] Fix imports of ControlMessage and MessageMeta Remove code snippet that illustrated importing MultiMessage since it has been removed. --- .../guides/9_control_messages.md | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/docs/source/developer_guide/guides/9_control_messages.md b/docs/source/developer_guide/guides/9_control_messages.md index 2be63e2081..f852f2cc86 100644 --- a/docs/source/developer_guide/guides/9_control_messages.md +++ b/docs/source/developer_guide/guides/9_control_messages.md @@ -32,13 +32,13 @@ Control Messages are straightforward objects that contain `tasks`, `metadata`, a Control Messages can handle tasks such as `training`, `inference`, and a catchall category `other`. Tasks can be added, checked for existence, or removed from the Control Message using methods like `add_task`, `has_task`, and `remove_task`. ```python -import morpheus._lib.messages as messages +from morpheus.messages import ControlMessage task_data = { "....": "...." } -msg = messages.ControlMessage() +msg = ControlMessage() msg.add_task("training", task_data) if msg.has_task("training"): task = msg.remove_task("training") @@ -49,9 +49,9 @@ if msg.has_task("training"): Metadata is a set of key-value pairs that offer supplementary information about the Control Message and must be JSON serializable. You can set, check, and retrieve metadata values using the `set_metadata`, `has_metadata`, and `get_metadata` methods, respectively. ```python -import morpheus._lib.messages as messages +from morpheus.messages import ControlMessage -msg = messages.ControlMessage() +msg = ControlMessage() msg.set_metadata("description", "This is a sample control message.") if msg.has_metadata("description"): description = msg.get_metadata("description") @@ -63,12 +63,13 @@ The payload of a Control Message is a Morpheus `MessageMeta` object that can car ```python import cudf -import morpheus._lib.messages as messages +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta data = cudf.DataFrame() # some data -msg_meta = messages.MessageMeta(data) -msg = messages.ControlMessage() +msg_meta = MessageMeta(data) +msg = ControlMessage() msg.payload(msg_meta) @@ -82,25 +83,18 @@ msg_meta == retrieved_payload # True **The `MultiMessage` type was deprecated in 24.06 and has been completely removed in version 24.10.** When upgrading to 24.10, all uses of `MultiMessage` need to be converted to `ControlMessage`. Each `MultiMessage` functionality has a corresponding equivalent in `ControlMessage`, as illustrated below. -```python -import cudf -from morpheus.messages import MultiMessage, ControlMessage - -data = cudf.DataFrame() -msg_meta = MessageMeta(data) -``` | **Functionality** | **MultiMessage** | **ControlMessage** | | -------------------------------------------------------------- | ------------------------------------------ | ------------------------------------------------------------------- | | Initialization | `multi_msg = MultiMessage(msg_meta)` | `control_msg = ControlMessage()`
`control_msg.payload(msg_meta)` | -| Get `cudf.DataFrame` | `multi_msg.get_meta()` | `control_msg.payload().get_data()` | -| Get columns from `cudf.DataFrame` | `multi_msg.get_meta(col_name)` | `control_msg.payload().get_data(col_name)` | -| Set columns values to `cudf.DataFrame` | `multi_msg.set_meta(col_name, value)` | `control_msg.payload().set_data(col_name, value)` | -| Get sliced `cudf.DataFrame` for given start and stop positions | `multi_msg.get_slice(start, stop)` | `control_msg.payload().get_slice(start, stop)` | -| Copy the `cudf.DataFrame` for given ranges of rows | `multi_msg.copy_ranges(ranges)` | `control_msg.payload().copy_ranges(ranges)` | +| Get `DataFrame` | `multi_msg.get_meta()` | `control_msg.payload().get_data()` | +| Get columns from `DataFrame` | `multi_msg.get_meta(col_name)` | `control_msg.payload().get_data(col_name)` | +| Set columns values to `DataFrame` | `multi_msg.set_meta(col_name, value)` | `control_msg.payload().set_data(col_name, value)` | +| Get sliced `DataFrame` for given start and stop positions | `multi_msg.get_slice(start, stop)` | `control_msg.payload().get_slice(start, stop)` | +| Copy the `DataFrame` for given ranges of rows | `multi_msg.copy_ranges(ranges)` | `control_msg.payload().copy_ranges(ranges)` | | | **MultiTensorMessage** | **ControlMessage** | -| Get the inference tensor `cupy.ndarray` | `multi_tensor_msg.tensor()` | `control_msg.tensors()` | +| Get the inference tensor `ndarray` | `multi_tensor_msg.tensor()` | `control_msg.tensors()` | | Get a specific inference tensor | `multi_tensor_msg.get_tensor(tensor_name)` | `control_msg.tensors().get_tensor(tensor_name)` | -Note that the `get_slice()` and `copy_ranges()` functions in `ControlMessage` return the `MessageMeta` after slicing, whereas these functions in `MultiMessage` return a new `MultiMessage` instance. +Note that in the `ControlMessage` column the `get_slice()` and `copy_ranges()` methods are being called on the `MessageMeta` payload and thus return a `MessageMeta` after slicing, whereas these functions in `MultiMessage` return a new `MultiMessage` instance. From eaa5198c463adeaef04c12018414f606127e40cc Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 12:05:42 -0700 Subject: [PATCH 305/347] Remove usage of --use_cpp flag --- .../guides/10_modular_pipeline_digital_fingerprinting.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md b/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md index e48b8c6df2..74ddb500cb 100644 --- a/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md +++ b/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md @@ -539,7 +539,6 @@ To run the DFP pipelines with the example datasets within the container, run the ```bash python dfp_integrated_training_batch_pipeline.py \ --log_level DEBUG \ - --use_cpp=true \ --source duo \ --start_time "2022-08-01" \ --duration "60d" \ @@ -551,7 +550,6 @@ To run the DFP pipelines with the example datasets within the container, run the ```bash python dfp_integrated_training_batch_pipeline.py \ --log_level DEBUG \ - --use_cpp=true \ --source duo \ --start_time "2022-08-30" \ --input_file "./control_messages/duo_payload_inference.json" @@ -561,7 +559,6 @@ To run the DFP pipelines with the example datasets within the container, run the ```bash python dfp_integrated_training_batch_pipeline.py \ --log_level DEBUG \ - --use_cpp=true \ --source duo \ --start_time "2022-08-01" \ --duration "60d" \ @@ -573,7 +570,6 @@ To run the DFP pipelines with the example datasets within the container, run the ```bash python dfp_integrated_training_batch_pipeline.py \ --log_level DEBUG \ - --use_cpp=true \ --source azure \ --start_time "2022-08-01" \ --duration "60d" \ @@ -585,7 +581,6 @@ To run the DFP pipelines with the example datasets within the container, run the ```bash python dfp_integrated_training_batch_pipeline.py \ --log_level DEBUG \ - --use_cpp=true \ --source azure \ --start_time "2022-08-30" \ --input_file "./control_messages/azure_payload_inference.json" @@ -595,7 +590,6 @@ To run the DFP pipelines with the example datasets within the container, run the ```bash python dfp_integrated_training_batch_pipeline.py \ --log_level DEBUG \ - --use_cpp=true \ --source azure \ --start_time "2022-08-01" \ --duration "60d" \ From 687827f2f0710e08ae940d97e1fdefb4da43cd80 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 14:22:52 -0700 Subject: [PATCH 306/347] Remove references to using mamba, just document conda with libmamba, fix numbering of steps, remove redundant pip installs, remove section about installing cuML, update sections about tests --- docs/source/developer_guide/contributing.md | 60 +++++++++++---------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 22bca9fdd0..573f183e2e 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -153,14 +153,12 @@ This workflow utilizes a Docker container to set up most dependencies ensuring a If a Conda environment on the host machine is preferred over Docker, it is relatively easy to install the necessary dependencies (In reality, the Docker workflow creates a Conda environment inside the container). -Note: These instructions assume the user is using `mamba` instead of `conda` since its improved solver speed is very helpful when working with a large number of dependencies. If you are not familiar with `mamba` you can install it with `conda install -n base -c conda-forge mamba` (Make sure to only install into the base environment). `mamba` is a drop in replacement for `conda` and all Conda commands are compatible between the two. - #### Prerequisites - Volta architecture GPU or better - [CUDA 12.1](https://developer.nvidia.com/cuda-12-1-0-download-archive) -- `conda` and `mamba` - - If `conda` and `mamba` are not installed, we recommend using the MiniForge install guide which is located [here](https://github.com/conda-forge/miniforge). This will install both `conda` and `mamba` and set the channel default to use `conda-forge`. +- `conda` + - If `conda` is not installed, we recommend using the [MiniForge install guide](https://github.com/conda-forge/miniforge). This will install `conda` and set the channel default to use `conda-forge`. 1. Set up environment variables and clone the repo: ```bash @@ -168,13 +166,10 @@ Note: These instructions assume the user is using `mamba` instead of `conda` sin git clone https://github.com/nv-morpheus/Morpheus.git $MORPHEUS_ROOT cd $MORPHEUS_ROOT ``` - -2. Ensure all submodules are checked out: - -```bash -git submodule update --init --recursive -``` - +1. Ensure all submodules are checked out: + ```bash + git submodule update --init --recursive + ``` 1. Create the Morpheus Conda environment ```bash conda env create --solver=libmamba -n morpheus --file conda/environments/dev_cuda-121_arch-x86_64.yaml @@ -182,18 +177,18 @@ git submodule update --init --recursive ``` This creates a new environment named `morpheus`, and activates that environment. -1. Build Morpheus + + > **Note**: The `dev_cuda-121_arch-x86_64.yaml` Conda environment file specifies all of the dependences required to build Morpheus and run Morpheus. However many of the examples, and optional packages such as `morpheus_llm` require additional dependencies. Alternately the following command can be used to create the Conda environment: ```bash - ./scripts/compile.sh + conda env create --solver=libmamba -n morpheus --file conda/environments/all_cuda-121_arch-x86_64.yaml + conda activate morpheus ``` - This script will run both CMake Configure with default options and CMake build. -1. Install Morpheus +1. Build Morpheus ```bash - pip install -e ${MORPHEUS_ROOT}/python/morpheus - pip install -e ${MORPHEUS_ROOT}/python/morpheus_llm + ./scripts/compile.sh ``` - Once Morpheus has been built, it can be installed into the current virtual environment. -1. Test the build (Note: some tests will be skipped)\ + This script will build and install Morpheus into the conda environment. +1. Test the build (Note: some tests will be skipped) Some of the tests will rely on external data sets. ```bash MORPHEUS_ROOT=${PWD} @@ -212,15 +207,26 @@ git submodule update --init --recursive npm install -g camouflage-server@0.15 ``` - Run all tests: - ```bash - pytest --run_slow - ``` -1. Optional: Install cuML - - Many users may wish to install cuML. Due to the complex dependency structure and versioning requirements, we need to specify exact versions of each package. The command to accomplish this is: + - Run end-to-end (aka slow) tests: ```bash - mamba install -c rapidsai -c nvidia -c conda-forge cuml=23.06 + pytest --run_slow ``` +1. Optional: Run Kafka and Milvus tests + - Download Kafka: + ```bash + python ./ci/scripts/download_kafka.py + ``` + + - Run all tests (this will skip over tests that require optional dependencies which are not installed): + ```bash + pytest --run_slow --run_kafka --run_milvus + ``` + + - Run all tests including those that require optional dependencies: + ```bash + pytest --fail_missing --run_slow --run_kafka --run_milvus + ``` + 1. Run Morpheus ```bash morpheus run pipeline-nlp ... @@ -400,7 +406,7 @@ Third-party code included in the source tree (that is not pulled in as an extern Ex: ``` /** - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) , NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); From 023e3653d815e506b79c295f58bc5a76a6d7c45e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 14:32:59 -0700 Subject: [PATCH 307/347] wip --- docs/source/developer_guide/contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 573f183e2e..7564fa6c29 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -178,7 +178,7 @@ If a Conda environment on the host machine is preferred over Docker, it is relat This creates a new environment named `morpheus`, and activates that environment. - > **Note**: The `dev_cuda-121_arch-x86_64.yaml` Conda environment file specifies all of the dependences required to build Morpheus and run Morpheus. However many of the examples, and optional packages such as `morpheus_llm` require additional dependencies. Alternately the following command can be used to create the Conda environment: + > **Note**: The `dev_cuda-121_arch-x86_64.yaml` Conda environment file specifies all of the dependencies required to build Morpheus and run Morpheus. However many of the examples, and optional packages such as `morpheus_llm` require additional dependencies. Alternately the following command can be used to create the Conda environment: ```bash conda env create --solver=libmamba -n morpheus --file conda/environments/all_cuda-121_arch-x86_64.yaml conda activate morpheus @@ -187,7 +187,7 @@ If a Conda environment on the host machine is preferred over Docker, it is relat ```bash ./scripts/compile.sh ``` - This script will build and install Morpheus into the conda environment. + This script will build and install Morpheus into the Conda environment. 1. Test the build (Note: some tests will be skipped) Some of the tests will rely on external data sets. ```bash From e0512f83ac40c2b4c061a68c218f63b1850b3310 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 16:01:21 -0700 Subject: [PATCH 308/347] Add additional troubleshooting sections covering removing in-place build artifacts as well as running CI locally --- docs/source/developer_guide/contributing.md | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 7564fa6c29..c8a7a614ea 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -377,6 +377,35 @@ Due to the large number of dependencies, it's common to run into build issues. T - Message indicating `git apply ...` failed - Many of the dependencies require small patches to make them work. These patches must be applied once and only once. If this error displays, try deleting the offending package from the `build/_deps/` directory or from `.cache/cpm/`. - If all else fails, delete the entire `build/` directory and `.cache/` directory. + - Older build artifacts when performing an in-place build. + - When built with `MORPHEUS_PYTHON_INPLACE_BUILD=ON` compiled libraries will be deployed in-place in the source tree, and older build artifacts exist in the source tree. Remove these with: + ```bash + find ./python -name "*.so" -delete + find ./examples -name "*.so" -delete + ``` + - Issues building documentation + - Intermediate documentation build artifacts can cause errors for Sphinx. To remove these, run: + ```bash + rm -rf build/docs/ docs/source/_modules docs/source/_lib + ``` + - CI Issues + - To run CI locally, the `ci/scripts/run_ci_local.sh` script can be used. For example to run a local CI build: + ```bash + ci/scripts/run_ci_local.sh build + ``` + - Build artifacts resulting from a local CI run can be found in the `.tmp/local_ci_tmp/` directory. + - To troubleshoot a particular CI stage it can be helpful to run: + ```bash + ci/scripts/run_ci_local.sh bash + ``` + + This will open a bash shell inside the CI container with all of the environment variables typically set during a CI run. From here you can run the commands that would typically be run by one of the CI scripts in `ci/scripts/github`. + + To run a CI stage requiring a GPU (ex: `test`), set the `USE_GPU` environment variable to `1`: + ```bash + USE_GPU=1 ci/scripts/run_ci_local.sh bash + ``` + ## Licensing Morpheus is licensed under the Apache v2.0 license. All new source files including CMake and other build scripts should contain the Apache v2.0 license header. Any edits to existing source code should update the date range of the copyright to the current year. The format for the license header is: From 1f85cb2a4961e0053419eb06a55666df91cbca89 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 1 Oct 2024 16:22:30 -0700 Subject: [PATCH 309/347] Refer to the troubleshooting guide --- docs/source/developer_guide/contributing.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index c8a7a614ea..8be29e3c5b 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -406,6 +406,7 @@ Due to the large number of dependencies, it's common to run into build issues. T USE_GPU=1 ci/scripts/run_ci_local.sh bash ``` +Refer to the [troubleshooting guide](../extra_info/troubleshooting.md) for more information on common issues and how to resolve them. ## Licensing Morpheus is licensed under the Apache v2.0 license. All new source files including CMake and other build scripts should contain the Apache v2.0 license header. Any edits to existing source code should update the date range of the copyright to the current year. The format for the license header is: From c13826b1d9d0fa31cf688ff3ac6e2eeb13dcf66e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 4 Oct 2024 08:40:08 -0700 Subject: [PATCH 310/347] Fix merge error --- .../test_dfp_rolling_window_stage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py index be73c4bc08..06d142f91c 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py @@ -88,7 +88,7 @@ def test_get_user_cache_miss(config: Config): def test_build_window_no_new(config: Config, control_message: ControlMessage): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -99,7 +99,7 @@ def test_build_window_no_new(config: Config, control_message: ControlMessage): def test_build_window_not_enough_data(config: Config, control_message: ControlMessage): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -109,7 +109,7 @@ def test_build_window_not_enough_data(config: Config, control_message: ControlMe def test_build_window_min_increment(config: Config, control_message: ControlMessage): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -119,7 +119,7 @@ def test_build_window_min_increment(config: Config, control_message: ControlMess def test_build_window_invalid(config: Config, control_message: ControlMessage, train_df: pd.DataFrame): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -134,7 +134,7 @@ def test_build_window_invalid(config: Config, control_message: ControlMessage, t def test_build_window_overlap(config: Config, control_message: ControlMessage, train_df: pd.DataFrame): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -155,7 +155,7 @@ def test_build_window(config: Config, control_message: ControlMessage, dataset_pandas: DatasetManager, train_df: pd.DataFrame): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') From f71e3b2e5d19ff6a57b29322e87d684b14e0a9fb Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 4 Oct 2024 16:07:21 -0700 Subject: [PATCH 311/347] Document each of the conda environments --- docs/source/developer_guide/contributing.md | 42 ++++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 8be29e3c5b..59c2a56fa5 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -151,7 +151,35 @@ This workflow utilizes a Docker container to set up most dependencies ensuring a ### Build in a Conda Environment -If a Conda environment on the host machine is preferred over Docker, it is relatively easy to install the necessary dependencies (In reality, the Docker workflow creates a Conda environment inside the container). +If a [Conda](https://docs.conda.io/projects/conda/en/latest/) environment on the host machine is preferred over Docker, it is relatively easy to install the necessary dependencies (In reality, the Docker workflow creates a Conda environment inside the container). + +#### Conda Environment YAML Files +Morpheus provides multiple Conda environment files to support different workflows. Morpheus utilizes [rapids-dependency-file-generator](https://pypi.org/project/rapids-dependency-file-generator/) to manage these multiple environment files. All of Morpheus' Conda and [pip](https://pip.pypa.io/en/stable/) dependencies along with the different environments are defined in the `dependencies.yaml` file. + +The following are the available Conda environment files, all are located in the `conda/environments` directory, with the following naming convention: `__arch-.yaml`. +| Environment | File | Description | +| --- | --- | --- | +| `all` | `all_cuda-121_arch-x86_64.yaml` | All dependencies required to build, run and test Morpheus, along with all of the examples. This is a superset of the `dev`, `runtime` and `examples` environments. | +| `dev` | `dev_cuda-121_arch-x86_64.yaml` | Dependencies required to build, run and test Morpheus. This is a super-set of the `runtime` environment. | +| `examples` | `examples_cuda-121_arch-x86_64.yaml` | Dependencies required to run all examples. This is a super-set of the `runtime` environment. | +| `model-utils` | `model-utils_cuda-121_arch-x86_64.yaml` | Dependencies required to train models independent of Morpheus. | +| `runtime` | `runtime_cuda-121_arch-x86_64.yaml` | Minimal set of dependencies strictly required to run Morpheus. | + + +##### Updating Morpheus Dependencies +Changes to Morpheus dependencies can be made in the `dependencies.yaml` file, then run `rapids-dependency-file-generator` to update the individual environment files in the `conda/environments` directory . + +Install `rapids-dependency-file-generator` into the base Conda environment: +```bash +conda run -n base --live-stream pip install rapids-dependency-file-generator +``` + +To run: +```bash +conda run -n base --live-stream rapids-dependency-file-generator +``` + +When ready commit the changes to the `dependencies.yaml` file and the updated environment files. #### Prerequisites @@ -170,19 +198,21 @@ If a Conda environment on the host machine is preferred over Docker, it is relat ```bash git submodule update --init --recursive ``` -1. Create the Morpheus Conda environment +1. Create the Morpheus Conda environment using either the `dev` or `all` environment file. Refer to the [Conda Environment YAML Files](#conda-environment-yaml-files) section for more information. ```bash conda env create --solver=libmamba -n morpheus --file conda/environments/dev_cuda-121_arch-x86_64.yaml - conda activate morpheus ``` + or + ```bash + conda env create --solver=libmamba -n morpheus --file conda/environments/all_cuda-121_arch-x86_64.yaml - This creates a new environment named `morpheus`, and activates that environment. + ``` - > **Note**: The `dev_cuda-121_arch-x86_64.yaml` Conda environment file specifies all of the dependencies required to build Morpheus and run Morpheus. However many of the examples, and optional packages such as `morpheus_llm` require additional dependencies. Alternately the following command can be used to create the Conda environment: + This creates a new environment named `morpheus`. Activate the environment with: ```bash - conda env create --solver=libmamba -n morpheus --file conda/environments/all_cuda-121_arch-x86_64.yaml conda activate morpheus ``` + 1. Build Morpheus ```bash ./scripts/compile.sh From fdd115940f41e9d3404e264b1f7e10adcd2ba499 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 4 Oct 2024 16:10:29 -0700 Subject: [PATCH 312/347] superset not super-set --- docs/source/developer_guide/contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 59c2a56fa5..03e1c23bdf 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -160,8 +160,8 @@ The following are the available Conda environment files, all are located in the | Environment | File | Description | | --- | --- | --- | | `all` | `all_cuda-121_arch-x86_64.yaml` | All dependencies required to build, run and test Morpheus, along with all of the examples. This is a superset of the `dev`, `runtime` and `examples` environments. | -| `dev` | `dev_cuda-121_arch-x86_64.yaml` | Dependencies required to build, run and test Morpheus. This is a super-set of the `runtime` environment. | -| `examples` | `examples_cuda-121_arch-x86_64.yaml` | Dependencies required to run all examples. This is a super-set of the `runtime` environment. | +| `dev` | `dev_cuda-121_arch-x86_64.yaml` | Dependencies required to build, run and test Morpheus. This is a superset of the `runtime` environment. | +| `examples` | `examples_cuda-121_arch-x86_64.yaml` | Dependencies required to run all examples. This is a superset of the `runtime` environment. | | `model-utils` | `model-utils_cuda-121_arch-x86_64.yaml` | Dependencies required to train models independent of Morpheus. | | `runtime` | `runtime_cuda-121_arch-x86_64.yaml` | Minimal set of dependencies strictly required to run Morpheus. | From 79fbc889d34d1e98acd802b5d54dd5f0b0ba3776 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 4 Oct 2024 16:10:48 -0700 Subject: [PATCH 313/347] Add 'superset' to our dictionary --- ci/vale/styles/config/vocabularies/morpheus/accept.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/vale/styles/config/vocabularies/morpheus/accept.txt b/ci/vale/styles/config/vocabularies/morpheus/accept.txt index 285a85c7d8..f31ace2e17 100644 --- a/ci/vale/styles/config/vocabularies/morpheus/accept.txt +++ b/ci/vale/styles/config/vocabularies/morpheus/accept.txt @@ -71,6 +71,7 @@ pytest [Ss]ubcard(s?) [Ss]ubgraph(s?) [Ss]ubword(s?) +[Ss]uperset(s?) [Tt]imestamp(s?) [Tt]okenization [Tt]okenizer(s?) From e6a46a6a78166bb76def4948b8ca4a5ff28fdce4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 7 Oct 2024 11:13:09 -0700 Subject: [PATCH 314/347] Consolidate common code in the docker run scripts --- docker/run_container.sh | 57 +++++++++++++++++++++++++++++++++ docker/run_container_dev.sh | 42 +++--------------------- docker/run_container_release.sh | 43 +++---------------------- 3 files changed, 66 insertions(+), 76 deletions(-) create mode 100755 docker/run_container.sh diff --git a/docker/run_container.sh b/docker/run_container.sh new file mode 100755 index 0000000000..192f2eab5f --- /dev/null +++ b/docker/run_container.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Color variables +b="\033[0;36m" +g="\033[0;32m" +r="\033[0;31m" +e="\033[0;90m" +y="\033[0;33m" +x="\033[0m" + +_UNDEF_VAR_ERROR_MSG="Use the dev/release scripts to set these automatically" + +DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:?"Must set \$DOCKER_IMAGE_NAME. ${_UNDEF_VAR_ERROR_MSG}"} +DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:?"Must set \$DOCKER_IMAGE_TAG. ${_UNDEF_VAR_ERROR_MSG}"} + +# DOCKER_ARGS are set by the dev/release scripts +# DOCKER_EXTRA_ARGS are optionally set by the user +DOCKER_ARGS=${DOCKER_ARGS:?"Must set \$DOCKER_ARGS. ${_UNDEF_VAR_ERROR_MSG}"} +DOCKER_ARGS="${DOCKER_ARGS} --net=host --cap-add=sys_nice ${DOCKER_EXTRA_ARGS}" +DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} + +if [[ -n "${CPU_ONLY}" ]]; then + echo -e "${b}Executing in CPU only mode${x}" + DOCKER_ARGS="${DOCKER_ARGS} --runtime=runc" +else + echo -e "${b}Executing in GPU mode${x}" + DOCKER_ARGS="${DOCKER_ARGS} --runtime=nvidia --gpus=all" +fi + +if [[ -n "${SSH_AUTH_SOCK}" ]]; then + echo -e "${b}Setting up ssh-agent auth socket${x}" + DOCKER_ARGS="${DOCKER_ARGS} -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent:ro -e SSH_AUTH_SOCK=/ssh-agent" +fi + +echo -e "${g}Launching ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}...${x}" + +# Enable command logging to show what is being executed +set -x +docker run ${DOCA_EXTRA_ARGS} --rm -ti ${DOCKER_ARGS} ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}" + +{ EXIT_CODE=$?; set +x; } 2>/dev/null + +exit $EXIT_CODE diff --git a/docker/run_container_dev.sh b/docker/run_container_dev.sh index fd05aa7031..cce4402b7c 100755 --- a/docker/run_container_dev.sh +++ b/docker/run_container_dev.sh @@ -14,46 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# set -x - -# Color variables -b="\033[0;36m" -g="\033[0;32m" -r="\033[0;31m" -e="\033[0;90m" -y="\033[0;33m" -x="\033[0m" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"morpheus"} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"dev-$(date +'%y%m%d')"} -DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} - -DOCKER_ARGS=" --env WORKSPACE_VOLUME=${PWD} -v $PWD:/workspace --net=host --cap-add=sys_nice" - -if [[ -n "${CPU_ONLY}" ]]; then - echo -e "${b}Executing in CPU only mode${x}" - DOCKER_ARGS="${DOCKER_ARGS} --runtime=runc" -else - echo -e "${b}Executing in GPU mode${x}" - DOCKER_ARGS="${DOCKER_ARGS} --runtime=nvidia --gpus=all" -fi - -if [[ -n "${SSH_AUTH_SOCK}" ]]; then - echo -e "${b}Setting up ssh-agent auth socket${x}" - DOCKER_ARGS="${DOCKER_ARGS} -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent:ro -e SSH_AUTH_SOCK=/ssh-agent" -fi - -echo -e "${g}Launching ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}...${x}" - -set -x -docker run \ - -v /dev/hugepages:/dev/hugepages \ - --privileged \ - --rm \ - -ti \ - ${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS} \ - ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}" -{ EXIT_CODE=$?; set +x; } 2>/dev/null +DOCKER_ARGS=" --env WORKSPACE_VOLUME=${PWD} -v $PWD:/workspace -v /dev/hugepages:/dev/hugepages --privileged" -exit $EXIT_CODE +# Call the general run script +${SCRIPT_DIR}/run_container.sh diff --git a/docker/run_container_release.sh b/docker/run_container_release.sh index 5882b4115e..ac559b01fa 100755 --- a/docker/run_container_release.sh +++ b/docker/run_container_release.sh @@ -16,56 +16,23 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -# Color variables -b="\033[0;36m" -g="\033[0;32m" -r="\033[0;31m" -e="\033[0;90m" -y="\033[0;33m" -x="\033[0m" - # Change to the script file to ensure we are in the correct repo (in case were in a submodule) pushd ${SCRIPT_DIR} &> /dev/null MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} -MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON} -MORPHEUS_BUILD_MORPHEUS_DFP=${MORPHEUS_BUILD_MORPHEUS_DFP:-ON} DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/morpheus"} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"$(git describe --tags --abbrev=0)-runtime"} -# This variable is used for passing extra arguments to the docker run command. Do not use DOCKER_ARGS for this purpose. -DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} - popd &> /dev/null -DOCKER_ARGS="--env WORKSPACE_VOLUME=${PWD} --net=host --cap-add=sys_nice ${DOCKER_EXTRA_ARGS}" - -if [[ -n "${CPU_ONLY}" ]]; then - echo -e "${b}Executing in CPU only mode${x}" - DOCKER_ARGS="${DOCKER_ARGS} --runtime=runc" -else - echo -e "${b}Executing in GPU mode${x}" - DOCKER_ARGS="${DOCKER_ARGS} --runtime=nvidia --gpus=all" -fi - -if [[ -n "${SSH_AUTH_SOCK}" ]]; then - echo -e "${b}Setting up ssh-agent auth socket${x}" - DOCKER_ARGS="${DOCKER_ARGS} -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent:ro -e SSH_AUTH_SOCK=/ssh-agent" -fi - -# DPDK requires hugepage and privileged container -DOCA_EXTRA_ARGS="" +# DPDK (and thus DOCA) requires hugepage and privileged container +DOCKER_ARGS="" if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then - echo -e "${b}Enabling DOCA Support. Mounting /dev/hugepages and running in privileged mode${x}" + echo -e "Enabling DOCA Support. Mounting /dev/hugepages and running in privileged mode" DOCKER_ARGS="${DOCKER_ARGS} -v /dev/hugepages:/dev/hugepages --privileged" fi - -echo -e "${g}Launching ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}...${x}" - -# Enable command logging to show what is being executed -set -x -docker run ${DOCA_EXTRA_ARGS} --rm -ti ${DOCKER_ARGS} ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}" -set +x +# Call the general run script +${SCRIPT_DIR}/run_container.sh From 5d26520c754f1a4056401f6e1fd269412283e16d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 7 Oct 2024 11:25:11 -0700 Subject: [PATCH 315/347] Cleanup handling of DOCKER_ARGS, remove unused WORKSPACE_VOLUME env var --- docker/run_container.sh | 2 +- docker/run_container_dev.sh | 6 +++--- docker/run_container_release.sh | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/run_container.sh b/docker/run_container.sh index 192f2eab5f..7d368556ef 100755 --- a/docker/run_container.sh +++ b/docker/run_container.sh @@ -29,7 +29,7 @@ DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:?"Must set \$DOCKER_IMAGE_TAG. ${_UNDEF_VAR_ # DOCKER_ARGS are set by the dev/release scripts # DOCKER_EXTRA_ARGS are optionally set by the user -DOCKER_ARGS=${DOCKER_ARGS:?"Must set \$DOCKER_ARGS. ${_UNDEF_VAR_ERROR_MSG}"} +DOCKER_ARGS=${DOCKER_ARGS:-""} DOCKER_ARGS="${DOCKER_ARGS} --net=host --cap-add=sys_nice ${DOCKER_EXTRA_ARGS}" DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} diff --git a/docker/run_container_dev.sh b/docker/run_container_dev.sh index cce4402b7c..0caa949c80 100755 --- a/docker/run_container_dev.sh +++ b/docker/run_container_dev.sh @@ -16,10 +16,10 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"morpheus"} -DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"dev-$(date +'%y%m%d')"} +export DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"morpheus"} +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"dev-$(date +'%y%m%d')"} -DOCKER_ARGS=" --env WORKSPACE_VOLUME=${PWD} -v $PWD:/workspace -v /dev/hugepages:/dev/hugepages --privileged" +export DOCKER_ARGS="-v $PWD:/workspace -v /dev/hugepages:/dev/hugepages --privileged" # Call the general run script ${SCRIPT_DIR}/run_container.sh diff --git a/docker/run_container_release.sh b/docker/run_container_release.sh index ac559b01fa..5ea4e3fd74 100755 --- a/docker/run_container_release.sh +++ b/docker/run_container_release.sh @@ -21,13 +21,13 @@ pushd ${SCRIPT_DIR} &> /dev/null MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} -DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/morpheus"} -DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"$(git describe --tags --abbrev=0)-runtime"} +export DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/morpheus"} +export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"$(git describe --tags --abbrev=0)-runtime"} popd &> /dev/null # DPDK (and thus DOCA) requires hugepage and privileged container -DOCKER_ARGS="" +export DOCKER_ARGS="" if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then echo -e "Enabling DOCA Support. Mounting /dev/hugepages and running in privileged mode" From 43c253403deec490b74c8fdeef1e361ae929bcc8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 10 Oct 2024 11:22:22 -0700 Subject: [PATCH 316/347] Switch to using a forkserver, docs state that using fork is problematic in a multithreaded env --- python/morpheus/morpheus/utils/shared_process_pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/utils/shared_process_pool.py b/python/morpheus/morpheus/utils/shared_process_pool.py index 1abfe06d30..520ce7ec89 100644 --- a/python/morpheus/morpheus/utils/shared_process_pool.py +++ b/python/morpheus/morpheus/utils/shared_process_pool.py @@ -136,7 +136,7 @@ def _initialize(self): self._total_max_workers = math.floor(max(1, len(os.sched_getaffinity(0)) * cpu_usage)) self._processes = [] - self._context = mp.get_context("fork") + self._context = mp.get_context("forkserver") self._manager = self._context.Manager() self._task_queues = self._manager.dict() self._stage_semaphores = self._manager.dict() From dd54ce1aef867eadb3f4711fdd8486823d695a98 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 10 Oct 2024 11:33:11 -0700 Subject: [PATCH 317/347] Support CPU & GPU execution mode in MultiProcessingStage --- .../morpheus/stages/general/multi_processing_stage.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/morpheus/morpheus/stages/general/multi_processing_stage.py b/python/morpheus/morpheus/stages/general/multi_processing_stage.py index f96939841a..2beb14eefd 100644 --- a/python/morpheus/morpheus/stages/general/multi_processing_stage.py +++ b/python/morpheus/morpheus/stages/general/multi_processing_stage.py @@ -22,6 +22,7 @@ import mrc.core.operators as ops from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.pipeline.stage_schema import StageSchema from morpheus.utils.shared_process_pool import SharedProcessPool @@ -180,6 +181,12 @@ def __init__(self, def name(self) -> str: return self._name + def supported_execution_modes(self) -> tuple[ExecutionMode]: + """ + Returns a tuple of supported execution modes of this stage. + """ + return (ExecutionMode.GPU, ExecutionMode.CPU) + def _on_data(self, data: InputT) -> OutputT: task = self._shared_process_pool.submit_task(self.name, self._process_fn, data) result = task.result() From 55a5b972ccb1b913af662a3f431bd4aab385d579 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 10 Oct 2024 11:48:07 -0700 Subject: [PATCH 318/347] Test CPU and GPU execution modes --- tests/test_multi_processing_stage.py | 35 ++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/test_multi_processing_stage.py b/tests/test_multi_processing_stage.py index d83e9b8d3d..abf3a25dda 100644 --- a/tests/test_multi_processing_stage.py +++ b/tests/test_multi_processing_stage.py @@ -26,6 +26,7 @@ from _utils import assert_results from _utils.dataset_manager import DatasetManager from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.messages import ControlMessage from morpheus.messages import MessageMeta from morpheus.pipeline import LinearPipeline @@ -49,6 +50,7 @@ def _process_df(df: pd.DataFrame, column: str, value: str) -> pd.DataFrame: return df +@pytest.mark.gpu_and_cpu_mode def test_create_stage_type_deduction(config: Config, dataset_pandas: DatasetManager): # Test create() with normal function @@ -105,26 +107,39 @@ def __init__(self, self._add_column_name = add_column_name self._shared_process_pool.set_usage(self.name, self._process_pool_usage) + self._execution_mode = c.execution_mode @property def name(self) -> str: return "derived-multi-processing-stage" + def supported_execution_modes(self) -> tuple[ExecutionMode]: + """ + Returns a tuple of supported execution modes of this stage. + """ + return (ExecutionMode.GPU, ExecutionMode.CPU) + def _on_data(self, data: ControlMessage) -> ControlMessage: input_df = data.payload().copy_dataframe() - pdf = input_df.to_pandas() + if self._execution_mode == ExecutionMode.GPU: + input_df = input_df.to_pandas() + partial_process_fn = partial(_process_df, column=self._add_column_name, value="Hello") - task = self._shared_process_pool.submit_task(self.name, partial_process_fn, pdf) + task = self._shared_process_pool.submit_task(self.name, partial_process_fn, input_df) + + df = task.result() + if self._execution_mode == ExecutionMode.GPU: + df = cudf.DataFrame.from_pandas(df) - df = cudf.DataFrame.from_pandas(task.result()) meta = MessageMeta(df) data.payload(meta) return data +@pytest.mark.gpu_and_cpu_mode def test_derived_stage_type_deduction(config: Config): mp_stage = DerivedMultiProcessingStage(c=config, process_pool_usage=0.1, add_column_name="new_column") @@ -137,12 +152,11 @@ def test_derived_stage_type_deduction(config: Config): def pandas_dataframe_generator(dataset_pandas: DatasetManager, count: int) -> Generator[pd.DataFrame, None, None]: - - df = dataset_pandas["csv_sample.csv"] for _ in range(count): - yield df + yield dataset_pandas["csv_sample.csv"] +@pytest.mark.gpu_and_cpu_mode def test_created_stage_pipe(config: Config, dataset_pandas: DatasetManager): config.num_threads = os.cpu_count() @@ -171,6 +185,7 @@ def test_created_stage_pipe(config: Config, dataset_pandas: DatasetManager): assert df.equals(expected_df) +@pytest.mark.gpu_and_cpu_mode def test_derived_stage_pipe(config: Config, dataset_pandas: DatasetManager): config.num_threads = os.cpu_count() @@ -181,7 +196,7 @@ def test_derived_stage_pipe(config: Config, dataset_pandas: DatasetManager): expected_df[add_column_name] = "Hello" pipe = LinearPipeline(config) - pipe.set_source(InMemorySourceStage(config, [cudf.DataFrame(input_df)])) + pipe.set_source(InMemorySourceStage(config, [input_df])) pipe.add_stage(DeserializeStage(config, ensure_sliceable_index=True)) pipe.add_stage(DerivedMultiProcessingStage(c=config, process_pool_usage=0.1, add_column_name=add_column_name)) pipe.add_stage(SerializeStage(config)) @@ -192,6 +207,7 @@ def test_derived_stage_pipe(config: Config, dataset_pandas: DatasetManager): assert_results(comp_stage.get_results()) +@pytest.mark.gpu_and_cpu_mode def test_multiple_stages_pipe(config: Config, dataset_pandas: DatasetManager): config.num_threads = os.cpu_count() @@ -206,9 +222,8 @@ def test_multiple_stages_pipe(config: Config, dataset_pandas: DatasetManager): partial_fn = partial(_process_df, column="new_column_1", value="new_value") - @stage - def pdf_to_control_message_stage(pdf: pd.DataFrame) -> ControlMessage: - df = cudf.DataFrame.from_pandas(pdf) + @stage(execution_modes=(ExecutionMode.CPU, ExecutionMode.GPU)) + def pdf_to_control_message_stage(df: pd.DataFrame) -> ControlMessage: meta = MessageMeta(df) msg = ControlMessage() msg.payload(meta) From 5e50dcde13b5806f4bb5a2b5288d1923985a6c7b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 10 Oct 2024 12:13:23 -0700 Subject: [PATCH 319/347] IWYU fixes --- .../morpheus/_lib/include/morpheus/messages/control.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp index 259d6bd69f..c10bdc4a78 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/messages/control.hpp @@ -24,13 +24,16 @@ #include // for object, dict, list #include // IWYU pragma: keep -#include // for system_clock, time_point +// for system_clock, time_point +#include // IWYU pragma: keep #include // for map #include // for shared_ptr #include // for optional #include // for string #include // for vector +// IWYU pragma: no_include + namespace morpheus { enum class MORPHEUS_EXPORT ControlMessageType From dd46f7198397057fa828bfa3fb70be73fb380b84 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 10 Oct 2024 14:14:13 -0700 Subject: [PATCH 320/347] Remove unused include --- python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp b/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp index f388689df7..51323b8ef6 100644 --- a/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp +++ b/python/morpheus_llm/morpheus_llm/_lib/llm/module.cpp @@ -33,7 +33,6 @@ #include "morpheus/messages/control.hpp" // IWYU pragma: keep #include "morpheus/pybind11/json.hpp" // IWYU pragma: keep -#include "morpheus/utilities/cudf_util.hpp" #include "morpheus/utilities/json_types.hpp" #include "morpheus/version.hpp" From 986dbc4241daff70aaf43cd3ddf373bf29a9eb2f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 14 Oct 2024 10:35:08 -0700 Subject: [PATCH 321/347] Specify a cuda build of pytorch, install pytorch via conda rather than pip, remove commented out pin of libwevp --- conda/environments/all_cuda-125_arch-x86_64.yaml | 3 +-- conda/environments/dev_cuda-125_arch-x86_64.yaml | 3 +-- conda/environments/examples_cuda-125_arch-x86_64.yaml | 3 +-- conda/environments/runtime_cuda-125_arch-x86_64.yaml | 3 +-- dependencies.yaml | 4 +--- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 0281cf6b68..c08b83ec70 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -98,6 +98,7 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 +- pytorch=2.4.0=cuda120* - rapidjson=1.1.0 - rapids-dask-dependency=24.10 - rdma-core>=48 @@ -125,7 +126,6 @@ dependencies: - websockets - yapf=0.40.1 - pip: - - --extra-index-url https://download.pytorch.org/whl/cu124 - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - databricks-cli < 0.100 @@ -140,5 +140,4 @@ dependencies: - nemollm==0.3.5 - pymilvus==2.3.6 - pytest-kafka==0.6.0 - - torch==2.4.0+cu124 name: all_cuda-125_arch-x86_64 diff --git a/conda/environments/dev_cuda-125_arch-x86_64.yaml b/conda/environments/dev_cuda-125_arch-x86_64.yaml index af599fb7de..dcee6f758b 100644 --- a/conda/environments/dev_cuda-125_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-125_arch-x86_64.yaml @@ -82,6 +82,7 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 +- pytorch=2.4.0=cuda120* - rapidjson=1.1.0 - rapids-dask-dependency=24.10 - rdma-core>=48 @@ -105,11 +106,9 @@ dependencies: - websockets - yapf=0.40.1 - pip: - - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 - pymilvus==2.3.6 - pytest-kafka==0.6.0 - - torch==2.4.0+cu124 name: dev_cuda-125_arch-x86_64 diff --git a/conda/environments/examples_cuda-125_arch-x86_64.yaml b/conda/environments/examples_cuda-125_arch-x86_64.yaml index ffcae28e4a..8b9c483cbd 100644 --- a/conda/environments/examples_cuda-125_arch-x86_64.yaml +++ b/conda/environments/examples_cuda-125_arch-x86_64.yaml @@ -48,6 +48,7 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 +- pytorch=2.4.0=cuda120* - rapids-dask-dependency=24.10 - requests - requests-cache=1.1 @@ -63,7 +64,6 @@ dependencies: - watchdog=3.0 - websockets - pip: - - --extra-index-url https://download.pytorch.org/whl/cu124 - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - databricks-cli < 0.100 @@ -77,5 +77,4 @@ dependencies: - milvus==2.3.5 - nemollm==0.3.5 - pymilvus==2.3.6 - - torch==2.4.0+cu124 name: examples_cuda-125_arch-x86_64 diff --git a/conda/environments/runtime_cuda-125_arch-x86_64.yaml b/conda/environments/runtime_cuda-125_arch-x86_64.yaml index 2551739061..d41f0164ec 100644 --- a/conda/environments/runtime_cuda-125_arch-x86_64.yaml +++ b/conda/environments/runtime_cuda-125_arch-x86_64.yaml @@ -36,6 +36,7 @@ dependencies: - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz - python=3.10 +- pytorch=2.4.0=cuda120* - rapids-dask-dependency=24.10 - requests - requests-cache=1.1 @@ -47,10 +48,8 @@ dependencies: - watchdog=3.0 - websockets - pip: - - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 - pymilvus==2.3.6 - - torch==2.4.0+cu124 name: runtime_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 5c2eb2a5b2..2f45a1a5d0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -339,7 +339,6 @@ dependencies: - feedparser=6.0 - grpcio - grpcio-status - # - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 ## - mlflow #>=2.10.0,<3 - mrc=24.10 - networkx=2.8.8 @@ -347,6 +346,7 @@ dependencies: - pydantic - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz + - pytorch=2.4.0=cuda120* - pluggy=1.3 - rapids-dask-dependency=24.10 # provides dask and distributed - requests @@ -360,12 +360,10 @@ dependencies: - websockets - pip - pip: - - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 # update to match pymilvus when available - pymilvus==2.3.6 - - torch==2.4.0+cu124 test_python_morpheus: common: From 27cfba0df6750c3d84516becafbf84d0843787d4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 14 Oct 2024 11:04:01 -0700 Subject: [PATCH 322/347] Fix the config fixture to always run in gpu mode --- tests/morpheus_dfp/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/morpheus_dfp/conftest.py b/tests/morpheus_dfp/conftest.py index 44992d0e06..74e2036472 100644 --- a/tests/morpheus_dfp/conftest.py +++ b/tests/morpheus_dfp/conftest.py @@ -61,12 +61,11 @@ def ae_feature_cols_fixture(): @pytest.fixture(name="config") -def config_fixture(config_no_cpp: Config, ae_feature_cols: typing.List[str]): +def config_fixture(config: Config, ae_feature_cols: typing.List[str]): """ The digital_fingerprinting production example utilizes the Auto Encoder config, and requires C++ execution disabled. """ from morpheus.config import ConfigAutoEncoder - config = config_no_cpp config.ae = ConfigAutoEncoder() config.ae.feature_columns = ae_feature_cols yield config From 14e6cdcdab5b6899b57b88f853473558aa92c572 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 14 Oct 2024 13:33:56 -0700 Subject: [PATCH 323/347] Revert "Specify a cuda build of pytorch, install pytorch via conda rather than pip, remove commented out pin of libwevp" This reverts commit 986dbc4241daff70aaf43cd3ddf373bf29a9eb2f. --- conda/environments/all_cuda-125_arch-x86_64.yaml | 3 ++- conda/environments/dev_cuda-125_arch-x86_64.yaml | 3 ++- conda/environments/examples_cuda-125_arch-x86_64.yaml | 3 ++- conda/environments/runtime_cuda-125_arch-x86_64.yaml | 3 ++- dependencies.yaml | 4 +++- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index c08b83ec70..0281cf6b68 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -98,7 +98,6 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 -- pytorch=2.4.0=cuda120* - rapidjson=1.1.0 - rapids-dask-dependency=24.10 - rdma-core>=48 @@ -126,6 +125,7 @@ dependencies: - websockets - yapf=0.40.1 - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - databricks-cli < 0.100 @@ -140,4 +140,5 @@ dependencies: - nemollm==0.3.5 - pymilvus==2.3.6 - pytest-kafka==0.6.0 + - torch==2.4.0+cu124 name: all_cuda-125_arch-x86_64 diff --git a/conda/environments/dev_cuda-125_arch-x86_64.yaml b/conda/environments/dev_cuda-125_arch-x86_64.yaml index dcee6f758b..af599fb7de 100644 --- a/conda/environments/dev_cuda-125_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-125_arch-x86_64.yaml @@ -82,7 +82,6 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 -- pytorch=2.4.0=cuda120* - rapidjson=1.1.0 - rapids-dask-dependency=24.10 - rdma-core>=48 @@ -106,9 +105,11 @@ dependencies: - websockets - yapf=0.40.1 - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 - pymilvus==2.3.6 - pytest-kafka==0.6.0 + - torch==2.4.0+cu124 name: dev_cuda-125_arch-x86_64 diff --git a/conda/environments/examples_cuda-125_arch-x86_64.yaml b/conda/environments/examples_cuda-125_arch-x86_64.yaml index 8b9c483cbd..ffcae28e4a 100644 --- a/conda/environments/examples_cuda-125_arch-x86_64.yaml +++ b/conda/environments/examples_cuda-125_arch-x86_64.yaml @@ -48,7 +48,6 @@ dependencies: - python-docx==1.1.0 - python-graphviz - python=3.10 -- pytorch=2.4.0=cuda120* - rapids-dask-dependency=24.10 - requests - requests-cache=1.1 @@ -64,6 +63,7 @@ dependencies: - watchdog=3.0 - websockets - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - databricks-cli < 0.100 @@ -77,4 +77,5 @@ dependencies: - milvus==2.3.5 - nemollm==0.3.5 - pymilvus==2.3.6 + - torch==2.4.0+cu124 name: examples_cuda-125_arch-x86_64 diff --git a/conda/environments/runtime_cuda-125_arch-x86_64.yaml b/conda/environments/runtime_cuda-125_arch-x86_64.yaml index d41f0164ec..2551739061 100644 --- a/conda/environments/runtime_cuda-125_arch-x86_64.yaml +++ b/conda/environments/runtime_cuda-125_arch-x86_64.yaml @@ -36,7 +36,6 @@ dependencies: - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz - python=3.10 -- pytorch=2.4.0=cuda120* - rapids-dask-dependency=24.10 - requests - requests-cache=1.1 @@ -48,8 +47,10 @@ dependencies: - watchdog=3.0 - websockets - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 - pymilvus==2.3.6 + - torch==2.4.0+cu124 name: runtime_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 68e6ffac22..95809bb0ee 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -358,6 +358,7 @@ dependencies: - feedparser=6.0 - grpcio - grpcio-status + # - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 ## - mlflow #>=2.10.0,<3 - mrc=24.10 - networkx=2.8.8 @@ -365,7 +366,6 @@ dependencies: - pydantic - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz - - pytorch=2.4.0=cuda120* - pluggy=1.3 - rapids-dask-dependency=24.10 # provides dask and distributed - requests @@ -379,10 +379,12 @@ dependencies: - websockets - pip - pip: + - --extra-index-url https://download.pytorch.org/whl/cu124 - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 # update to match pymilvus when available - pymilvus==2.3.6 + - torch==2.4.0+cu124 test_python_morpheus: common: From 7500d37684d1c746e24ebf5e4c80fb42e59e4325 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 09:13:09 -0700 Subject: [PATCH 324/347] Fix merge error --- examples/ransomware_detection/stages/create_features.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/ransomware_detection/stages/create_features.py b/examples/ransomware_detection/stages/create_features.py index 4c70c8c119..92f0b187b8 100644 --- a/examples/ransomware_detection/stages/create_features.py +++ b/examples/ransomware_detection/stages/create_features.py @@ -17,6 +17,7 @@ from mrc.core import operators as ops from dask.distributed import Client +import cudf from morpheus.cli.register_stage import register_stage from morpheus.config import Config From ac2eae642e7602335125f289f955c8d89d9d0577 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 09:15:53 -0700 Subject: [PATCH 325/347] Move SupportedTypes to a class property of MonitorController --- python/morpheus/morpheus/controllers/monitor_controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/controllers/monitor_controller.py b/python/morpheus/morpheus/controllers/monitor_controller.py index 74e5b829ba..940d079097 100644 --- a/python/morpheus/morpheus/controllers/monitor_controller.py +++ b/python/morpheus/morpheus/controllers/monitor_controller.py @@ -28,8 +28,6 @@ logger = logging.getLogger(__name__) -SupportedTypes = typing.Union[DataFrameType, MessageMeta, ControlMessage, list] - class MonitorController: """ @@ -59,6 +57,7 @@ class MonitorController: Custom implementation of tqdm if required. """ + SupportedTypes = typing.Union[DataFrameType, MessageMeta, ControlMessage, list] controller_count: int = 0 def __init__(self, From d243108bf72fc1b30f97924b6069a0fed2a9114e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 09:18:39 -0700 Subject: [PATCH 326/347] Rename DataFrameTypeStr to DataFrameModule --- .../morpheus/controllers/rss_controller.py | 4 ++-- python/morpheus/morpheus/io/deserializers.py | 6 +++--- python/morpheus/morpheus/io/utils.py | 4 ++-- python/morpheus/morpheus/parsers/zeek.py | 4 ++-- .../morpheus/morpheus/utils/type_aliases.py | 2 +- python/morpheus/morpheus/utils/type_utils.py | 20 +++++++++---------- tests/_utils/dataset_manager.py | 14 ++++++------- tests/morpheus/io/test_io_utils.py | 4 ++-- tests/morpheus/utils/test_type_utils.py | 12 +++++------ tests/test_conftest.py | 4 ++-- 10 files changed, 37 insertions(+), 37 deletions(-) diff --git a/python/morpheus/morpheus/controllers/rss_controller.py b/python/morpheus/morpheus/controllers/rss_controller.py index fb1e85c81f..d63e992588 100644 --- a/python/morpheus/morpheus/controllers/rss_controller.py +++ b/python/morpheus/morpheus/controllers/rss_controller.py @@ -29,7 +29,7 @@ from morpheus.messages import MessageMeta from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_utils import get_df_class logger = logging.getLogger(__name__) @@ -107,7 +107,7 @@ def __init__(self, stop_after: int = 0, interval_secs: float = 600, should_stop_fn: Callable[[], bool] = None, - df_type: DataFrameTypeStr = "cudf"): + df_type: DataFrameModule = "cudf"): if IMPORT_EXCEPTION is not None: raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index 0e93f11ae5..b41c083c02 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -26,7 +26,7 @@ from morpheus.io.utils import filter_null_data from morpheus.io.utils import get_json_reader from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_utils import df_type_str_to_pkg @@ -34,7 +34,7 @@ def _read_file_to_df_py(*, file_name: typing.Union[str, io.IOBase], file_type: FileTypes, parser_kwargs: dict, - df_type: DataFrameTypeStr) -> DataFrameType: + df_type: DataFrameModule) -> DataFrameType: if (parser_kwargs is None): parser_kwargs = {} @@ -93,7 +93,7 @@ def read_file_to_df(file_name: typing.Union[str, io.IOBase], parser_kwargs: dict = None, filter_nulls: bool = True, filter_null_columns: list[str] | str = 'data', - df_type: DataFrameTypeStr = "pandas") -> DataFrameType: + df_type: DataFrameModule = "pandas") -> DataFrameType: """ Reads a file into a dataframe and performs any of the necessary cleanup. diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index c3cb5d61fb..db0661d7fc 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -22,7 +22,7 @@ from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_aliases import SeriesType from morpheus.utils.type_utils import df_type_str_to_exec_mode from morpheus.utils.type_utils import is_cudf_type @@ -141,7 +141,7 @@ def truncate_string_cols_by_bytes(df: DataFrameType, @typing.overload -def get_json_reader(df_type_str: DataFrameTypeStr) -> typing.Callable[..., DataFrameType]: +def get_json_reader(df_type_str: DataFrameModule) -> typing.Callable[..., DataFrameType]: ... diff --git a/python/morpheus/morpheus/parsers/zeek.py b/python/morpheus/morpheus/parsers/zeek.py index ac9a4a24b8..3f3bf61e74 100644 --- a/python/morpheus/morpheus/parsers/zeek.py +++ b/python/morpheus/morpheus/parsers/zeek.py @@ -13,7 +13,7 @@ # limitations under the License. from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_utils import get_df_pkg TYPE_DICT = { @@ -38,7 +38,7 @@ } -def parse(filepath: str, df_type: DataFrameTypeStr = "cudf") -> DataFrameType: +def parse(filepath: str, df_type: DataFrameModule = "cudf") -> DataFrameType: """ Parse Zeek log file and return cuDF dataframe. Uses header comments to get column names/types and configure parser. diff --git a/python/morpheus/morpheus/utils/type_aliases.py b/python/morpheus/morpheus/utils/type_aliases.py index b6df387c1d..5e977918c7 100644 --- a/python/morpheus/morpheus/utils/type_aliases.py +++ b/python/morpheus/morpheus/utils/type_aliases.py @@ -22,7 +22,7 @@ import cudf -DataFrameTypeStr = typing.Literal["cudf", "pandas"] +DataFrameModule = typing.Literal["cudf", "pandas"] DataFrameType = typing.Union["pandas.DataFrame", "cudf.DataFrame"] SeriesType = typing.Union["pandas.Series", "cudf.Series"] diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index f1892074d3..2cbb5fac75 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -23,7 +23,7 @@ from morpheus.config import CppConfig from morpheus.config import ExecutionMode from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule # pylint: disable=invalid-name T_co = typing.TypeVar("T_co", covariant=True) @@ -172,7 +172,7 @@ def get_full_qualname(klass: type) -> str: return module + '.' + klass.__qualname__ -def df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr) -> ExecutionMode: +def df_type_str_to_exec_mode(df_type_str: DataFrameModule) -> ExecutionMode: """ Return the appropriate execution mode based on the DataFrame type string. """ @@ -181,11 +181,11 @@ def df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr) -> ExecutionMode: if df_type_str == "pandas": return ExecutionMode.CPU - valid_values = ", ".join(typing.get_args(DataFrameTypeStr)) + valid_values = ", ".join(typing.get_args(DataFrameModule)) raise ValueError(f"Invalid DataFrame type string: {df_type_str}, valid values are: {valid_values}") -def exec_mode_to_df_type_str(execution_mode: ExecutionMode) -> DataFrameTypeStr: +def exec_mode_to_df_type_str(execution_mode: ExecutionMode) -> DataFrameModule: if execution_mode == ExecutionMode.GPU: return "cudf" @@ -198,7 +198,7 @@ def cpp_mode_to_exec_mode() -> ExecutionMode: return ExecutionMode.CPU -def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: +def df_type_str_to_pkg(df_type_str: DataFrameModule) -> types.ModuleType: """ Return the appropriate DataFrame package based on the DataFrame type string. """ @@ -208,12 +208,12 @@ def df_type_str_to_pkg(df_type_str: DataFrameTypeStr) -> types.ModuleType: if df_type_str == "pandas": return pd - valid_values = ", ".join(typing.get_args(DataFrameTypeStr)) + valid_values = ", ".join(typing.get_args(DataFrameModule)) raise ValueError(f"Invalid DataFrame type string: {df_type_str}, valid values are: {valid_values}") @typing.overload -def get_df_pkg(selector: DataFrameTypeStr = None) -> types.ModuleType: +def get_df_pkg(selector: DataFrameModule = None) -> types.ModuleType: ... @@ -222,7 +222,7 @@ def get_df_pkg(selector: ExecutionMode = None) -> types.ModuleType: ... -def get_df_pkg(selector: ExecutionMode | DataFrameTypeStr = None) -> types.ModuleType: +def get_df_pkg(selector: ExecutionMode | DataFrameModule = None) -> types.ModuleType: """ Return the appropriate DataFrame package based on the execution mode. """ @@ -241,7 +241,7 @@ def get_df_pkg(selector: ExecutionMode | DataFrameTypeStr = None) -> types.Modul @typing.overload -def get_df_class(selector: DataFrameTypeStr = None) -> type[DataFrameType]: +def get_df_class(selector: DataFrameModule = None) -> type[DataFrameType]: ... @@ -250,7 +250,7 @@ def get_df_class(selector: ExecutionMode = None) -> type[DataFrameType]: ... -def get_df_class(selector: ExecutionMode | DataFrameTypeStr = None) -> type[DataFrameType]: +def get_df_class(selector: ExecutionMode | DataFrameModule = None) -> type[DataFrameType]: """ Return the appropriate DataFrame class based on the execution mode. """ diff --git a/tests/_utils/dataset_manager.py b/tests/_utils/dataset_manager.py index 404462a3ce..802e491de5 100644 --- a/tests/_utils/dataset_manager.py +++ b/tests/_utils/dataset_manager.py @@ -30,7 +30,7 @@ from morpheus.io.deserializers import read_file_to_df from morpheus.utils import compare_df from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_aliases import SeriesType @@ -44,15 +44,15 @@ class DatasetManager: Type of DataFrame to return unless otherwise explicitly specified. """ - __df_cache: dict[tuple[DataFrameTypeStr, str], DataFrameType] = {} + __df_cache: dict[tuple[DataFrameModule, str], DataFrameType] = {} # Values in `__instances` are instances of `DatasetLoader` - __instances: dict[DataFrameTypeStr, "DatasetManager"] = {} + __instances: dict[DataFrameModule, "DatasetManager"] = {} # Explicitly using __new__ instead of of an __init__ to implement this as a singleton for each dataframe type. # Initialization is also being performed here instead of an __init__ method as an __init__ method would be re-run # the __init__ on the singleton instance for each cache hit. - def __new__(cls, df_type: DataFrameTypeStr): + def __new__(cls, df_type: DataFrameModule): """Returns the singleton instance of `DatasetManager` for the specified `df_type`.""" try: return cls.__instances[df_type] @@ -63,7 +63,7 @@ def __new__(cls, df_type: DataFrameTypeStr): return instance @staticmethod - def get_alt_df_type(df_type: DataFrameTypeStr) -> DataFrameTypeStr: + def get_alt_df_type(df_type: DataFrameModule) -> DataFrameModule: """Returns the other possible df type.""" return 'cudf' if df_type == 'pandas' else 'pandas' @@ -73,7 +73,7 @@ def clear(self): def get_df(self, file_path: str, - df_type: DataFrameTypeStr = None, + df_type: DataFrameModule = None, no_cache: bool = False, **reader_kwargs) -> DataFrameType: """ @@ -125,7 +125,7 @@ def get_df(self, return df.copy(deep=True) - def __getitem__(self, item: str | tuple[str] | tuple[str, DataFrameTypeStr]) -> DataFrameType: + def __getitem__(self, item: str | tuple[str] | tuple[str, DataFrameModule]) -> DataFrameType: """Implements `__getitem__` to allow for fetching DataFrames using the `[]` operator.""" if not isinstance(item, tuple): item = (item, ) diff --git a/tests/morpheus/io/test_io_utils.py b/tests/morpheus/io/test_io_utils.py index 84655c2055..4becc3e24b 100755 --- a/tests/morpheus/io/test_io_utils.py +++ b/tests/morpheus/io/test_io_utils.py @@ -26,7 +26,7 @@ from morpheus.config import ExecutionMode from morpheus.io import utils as io_utils from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule MULTI_BYTE_STRINGS = ["ñäμɛ", "Moρφέας", "taç"] @@ -141,7 +141,7 @@ def test_truncate_string_cols_by_bytes(dataset: DatasetManager, @pytest.mark.parametrize("mode, expected", [(ExecutionMode.GPU, cudf.read_json), (ExecutionMode.CPU, pd.read_json), ("cudf", cudf.read_json), ("pandas", pd.read_json)]) -def test_get_json_reader(mode: typing.Union[ExecutionMode, DataFrameTypeStr], expected: Callable[..., DataFrameType]): +def test_get_json_reader(mode: typing.Union[ExecutionMode, DataFrameModule], expected: Callable[..., DataFrameType]): reader = io_utils.get_json_reader(mode) if hasattr(reader, "func"): # Unwrap partial diff --git a/tests/morpheus/utils/test_type_utils.py b/tests/morpheus/utils/test_type_utils.py index 1a01e2df67..ab06f39fcb 100644 --- a/tests/morpheus/utils/test_type_utils.py +++ b/tests/morpheus/utils/test_type_utils.py @@ -23,7 +23,7 @@ import cudf from morpheus.config import ExecutionMode -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_utils import df_type_str_to_exec_mode from morpheus.utils.type_utils import df_type_str_to_pkg from morpheus.utils.type_utils import exec_mode_to_df_type_str @@ -35,13 +35,13 @@ @pytest.mark.parametrize("mode, expected", [(ExecutionMode.GPU, cudf.DataFrame), (ExecutionMode.CPU, pd.DataFrame), ("cudf", cudf.DataFrame), ("pandas", pd.DataFrame)]) -def test_get_df_class(mode: typing.Union[ExecutionMode, DataFrameTypeStr], expected: types.ModuleType): +def test_get_df_class(mode: typing.Union[ExecutionMode, DataFrameModule], expected: types.ModuleType): assert get_df_class(mode) is expected @pytest.mark.parametrize("mode, expected", [(ExecutionMode.GPU, cudf), (ExecutionMode.CPU, pd), ("cudf", cudf), ("pandas", pd)]) -def test_get_df_pkg(mode: typing.Union[ExecutionMode, DataFrameTypeStr], expected: types.ModuleType): +def test_get_df_pkg(mode: typing.Union[ExecutionMode, DataFrameModule], expected: types.ModuleType): assert get_df_pkg(mode) is expected @@ -79,7 +79,7 @@ def test_is_cudf_type(obj: typing.Any, expected: bool): @pytest.mark.parametrize("df_type_str, expected", [("cudf", cudf), ("pandas", pd)], ids=["cudf", "pandas"]) -def test_df_type_str_to_pkg(df_type_str: DataFrameTypeStr, expected: types.ModuleType): +def test_df_type_str_to_pkg(df_type_str: DataFrameModule, expected: types.ModuleType): assert df_type_str_to_pkg(df_type_str) is expected @@ -91,7 +91,7 @@ def test_df_type_str_to_pkg_invalid(invalid_type_str: typing.Any): @pytest.mark.parametrize("df_type_str, expected", [("cudf", ExecutionMode.GPU), ("pandas", ExecutionMode.CPU)], ids=["cudf", "pandas"]) -def test_df_type_str_to_exec_mode(df_type_str: DataFrameTypeStr, expected: ExecutionMode): +def test_df_type_str_to_exec_mode(df_type_str: DataFrameModule, expected: ExecutionMode): assert df_type_str_to_exec_mode(df_type_str) == expected @@ -103,5 +103,5 @@ def test_df_type_str_to_exec_mode_invalid(invalid_type_str: typing.Any): @pytest.mark.parametrize("exec_mode, expected", [(ExecutionMode.GPU, "cudf"), (ExecutionMode.CPU, "pandas")], ids=["GPU", "CPU"]) -def test_exec_mode_to_df_type_str(exec_mode: ExecutionMode, expected: DataFrameTypeStr): +def test_exec_mode_to_df_type_str(exec_mode: ExecutionMode, expected: DataFrameModule): assert exec_mode_to_df_type_str(exec_mode) == expected diff --git a/tests/test_conftest.py b/tests/test_conftest.py index 9752803496..42fdd2dd2c 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -24,7 +24,7 @@ from morpheus.config import Config from morpheus.config import CppConfig from morpheus.config import ExecutionMode -from morpheus.utils.type_aliases import DataFrameTypeStr +from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_utils import exec_mode_to_df_type_str @@ -258,7 +258,7 @@ def test_df_type_no_marks(df_type, df_type_from_marker): assert df_type == df_type_from_marker -def test_df_type_matches_execution_mode(df_type: DataFrameTypeStr, execution_mode: ExecutionMode): +def test_df_type_matches_execution_mode(df_type: DataFrameModule, execution_mode: ExecutionMode): assert df_type == exec_mode_to_df_type_str(execution_mode) From f7d4bf3735024525db9a8523bdfa3970aa4ba8b1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 09:28:18 -0700 Subject: [PATCH 327/347] Fix overloads for get_json_reader --- python/morpheus/morpheus/io/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index db0661d7fc..0a5d77131f 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -141,16 +141,21 @@ def truncate_string_cols_by_bytes(df: DataFrameType, @typing.overload -def get_json_reader(df_type_str: DataFrameModule) -> typing.Callable[..., DataFrameType]: +def get_json_reader(selector: DataFrameModule) -> typing.Callable[..., DataFrameType]: ... +@typing.overload +def get_json_reader(selector: ExecutionMode) -> typing.Callable[..., DataFrameType]: + ... -def get_json_reader(execution_mode: ExecutionMode) -> typing.Callable[..., DataFrameType]: +def get_json_reader(selector: DataFrameModule| ExecutionMode) -> typing.Callable[..., DataFrameType]: """ Return the appropriate JSON reader based on the execution mode. """ - if not isinstance(execution_mode, ExecutionMode): - execution_mode = df_type_str_to_exec_mode(execution_mode) + if not isinstance(selector, ExecutionMode): + execution_mode = df_type_str_to_exec_mode(selector) + else: + execution_mode = selector if (execution_mode == ExecutionMode.GPU): import cudf From 1b89232de8982244c222dc9c4c74048822883b56 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 11:45:46 -0700 Subject: [PATCH 328/347] Add property methods to the mixin to wrap common type_utils uses --- .../pipeline/execution_mode_mixins.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/morpheus/morpheus/pipeline/execution_mode_mixins.py b/python/morpheus/morpheus/pipeline/execution_mode_mixins.py index 6bef674d8d..0b740e2d99 100644 --- a/python/morpheus/morpheus/pipeline/execution_mode_mixins.py +++ b/python/morpheus/morpheus/pipeline/execution_mode_mixins.py @@ -17,9 +17,13 @@ based upon configuration or runtime conditions. """ +import types from abc import ABC from morpheus.config import ExecutionMode +from morpheus.utils import type_utils +from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType class CpuOnlyMixin(ABC): @@ -44,3 +48,24 @@ def supported_execution_modes(self) -> tuple[ExecutionMode]: Returns a tuple of supported execution modes of this stage. """ return (ExecutionMode.GPU, ExecutionMode.CPU) + + @property + def df_type_str(self) -> DataFrameModule: + """ + Returns the DataFrame module that should be used for the given execution mode. + """ + return type_utils.exec_mode_to_df_type_str(self._config.execution_mode) + + @property + def df_pkg(self) -> types.ModuleType: + """ + Returns the DataFrame package that should be used for the given execution mode. + """ + return type_utils.get_df_pkg(self._config.execution_mode) + + @property + def df_class(self) -> type[DataFrameType]: + """ + Returns the DataFrame class that should be used for the given execution mode. + """ + return type_utils.get_df_class(self._config.execution_mode) From 93625629a8248c814f74116ad193444743ca5cfa Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 11:46:28 -0700 Subject: [PATCH 329/347] Formatting fixes --- examples/ransomware_detection/stages/create_features.py | 1 + python/morpheus/morpheus/controllers/rss_controller.py | 2 +- python/morpheus/morpheus/io/deserializers.py | 2 +- python/morpheus/morpheus/io/utils.py | 6 ++++-- python/morpheus/morpheus/parsers/zeek.py | 2 +- python/morpheus/morpheus/utils/type_utils.py | 2 +- tests/_utils/dataset_manager.py | 2 +- tests/morpheus/io/test_io_utils.py | 2 +- 8 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/ransomware_detection/stages/create_features.py b/examples/ransomware_detection/stages/create_features.py index 92f0b187b8..3ca214caad 100644 --- a/examples/ransomware_detection/stages/create_features.py +++ b/examples/ransomware_detection/stages/create_features.py @@ -17,6 +17,7 @@ from mrc.core import operators as ops from dask.distributed import Client + import cudf from morpheus.cli.register_stage import register_stage diff --git a/python/morpheus/morpheus/controllers/rss_controller.py b/python/morpheus/morpheus/controllers/rss_controller.py index d63e992588..6334f4f23c 100644 --- a/python/morpheus/morpheus/controllers/rss_controller.py +++ b/python/morpheus/morpheus/controllers/rss_controller.py @@ -28,8 +28,8 @@ import requests_cache from morpheus.messages import MessageMeta -from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_utils import get_df_class logger = logging.getLogger(__name__) diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index b41c083c02..518b62d93f 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -25,8 +25,8 @@ from morpheus.config import CppConfig from morpheus.io.utils import filter_null_data from morpheus.io.utils import get_json_reader -from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_utils import df_type_str_to_pkg diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index 0a5d77131f..9875c30506 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -21,8 +21,8 @@ import pandas as pd from morpheus.config import ExecutionMode -from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import SeriesType from morpheus.utils.type_utils import df_type_str_to_exec_mode from morpheus.utils.type_utils import is_cudf_type @@ -144,11 +144,13 @@ def truncate_string_cols_by_bytes(df: DataFrameType, def get_json_reader(selector: DataFrameModule) -> typing.Callable[..., DataFrameType]: ... + @typing.overload def get_json_reader(selector: ExecutionMode) -> typing.Callable[..., DataFrameType]: ... -def get_json_reader(selector: DataFrameModule| ExecutionMode) -> typing.Callable[..., DataFrameType]: + +def get_json_reader(selector: DataFrameModule | ExecutionMode) -> typing.Callable[..., DataFrameType]: """ Return the appropriate JSON reader based on the execution mode. """ diff --git a/python/morpheus/morpheus/parsers/zeek.py b/python/morpheus/morpheus/parsers/zeek.py index 3f3bf61e74..8d28a1bc24 100644 --- a/python/morpheus/morpheus/parsers/zeek.py +++ b/python/morpheus/morpheus/parsers/zeek.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_utils import get_df_pkg TYPE_DICT = { diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 2cbb5fac75..7dd629b687 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -22,8 +22,8 @@ from morpheus.config import CppConfig from morpheus.config import ExecutionMode -from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType # pylint: disable=invalid-name T_co = typing.TypeVar("T_co", covariant=True) diff --git a/tests/_utils/dataset_manager.py b/tests/_utils/dataset_manager.py index 802e491de5..40202b4025 100644 --- a/tests/_utils/dataset_manager.py +++ b/tests/_utils/dataset_manager.py @@ -29,8 +29,8 @@ from _utils import assert_results from morpheus.io.deserializers import read_file_to_df from morpheus.utils import compare_df -from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import SeriesType diff --git a/tests/morpheus/io/test_io_utils.py b/tests/morpheus/io/test_io_utils.py index 4becc3e24b..3c3e241ce8 100755 --- a/tests/morpheus/io/test_io_utils.py +++ b/tests/morpheus/io/test_io_utils.py @@ -25,8 +25,8 @@ from _utils.dataset_manager import DatasetManager from morpheus.config import ExecutionMode from morpheus.io import utils as io_utils -from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_aliases import DataFrameModule +from morpheus.utils.type_aliases import DataFrameType MULTI_BYTE_STRINGS = ["ñäμɛ", "Moρφέας", "taç"] From fb90067573020474d570b785b30f32ab76e6e732 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 12:09:03 -0700 Subject: [PATCH 330/347] make the get_df_pkg and get_df_class wrappers methods not properties as these come at a cost --- python/morpheus/morpheus/pipeline/execution_mode_mixins.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/morpheus/morpheus/pipeline/execution_mode_mixins.py b/python/morpheus/morpheus/pipeline/execution_mode_mixins.py index 0b740e2d99..e5483501fa 100644 --- a/python/morpheus/morpheus/pipeline/execution_mode_mixins.py +++ b/python/morpheus/morpheus/pipeline/execution_mode_mixins.py @@ -56,15 +56,13 @@ def df_type_str(self) -> DataFrameModule: """ return type_utils.exec_mode_to_df_type_str(self._config.execution_mode) - @property - def df_pkg(self) -> types.ModuleType: + def get_df_pkg(self) -> types.ModuleType: """ Returns the DataFrame package that should be used for the given execution mode. """ return type_utils.get_df_pkg(self._config.execution_mode) - @property - def df_class(self) -> type[DataFrameType]: + def get_df_class(self) -> type[DataFrameType]: """ Returns the DataFrame class that should be used for the given execution mode. """ From 075b0fa7d6325cff01ddff7fc1780024b0043f66 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 12:10:06 -0700 Subject: [PATCH 331/347] Update stages to use GpuAndCpuMixin helpers --- .../morpheus/stages/input/autoencoder_source_stage.py | 3 +-- python/morpheus/morpheus/stages/input/file_source_stage.py | 5 +---- python/morpheus/morpheus/stages/input/rss_source_stage.py | 6 +++--- .../morpheus/stages/output/http_server_sink_stage.py | 3 +-- .../stages/postprocess/generate_viz_frames_stage.py | 6 +++--- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py index 97f3afb1aa..7174650893 100644 --- a/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py +++ b/python/morpheus/morpheus/stages/input/autoencoder_source_stage.py @@ -29,7 +29,6 @@ from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema from morpheus.utils.directory_watcher import DirectoryWatcher -from morpheus.utils.type_utils import get_df_class class AutoencoderSourceStage(PreallocatorMixin, GpuAndCpuMixin, SingleOutputSource): @@ -99,7 +98,7 @@ def __init__(self, # is good for interleaving source stages. self._repeat_count = repeat - self._df_class = get_df_class(c.execution_mode) + self._df_class = self.get_df_class() self._watcher = DirectoryWatcher(input_glob=input_glob, watch_directory=watch_directory, diff --git a/python/morpheus/morpheus/stages/input/file_source_stage.py b/python/morpheus/morpheus/stages/input/file_source_stage.py index 2292501c86..398c1b126e 100644 --- a/python/morpheus/morpheus/stages/input/file_source_stage.py +++ b/python/morpheus/morpheus/stages/input/file_source_stage.py @@ -29,7 +29,6 @@ from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema -from morpheus.utils.type_utils import exec_mode_to_df_type_str logger = logging.getLogger(__name__) @@ -99,8 +98,6 @@ def __init__(self, self._iterative = iterative self._repeat_count = repeat - self._df_type = exec_mode_to_df_type_str(c.execution_mode) - @property def name(self) -> str: """Return the name of the stage""" @@ -142,7 +139,7 @@ def _generate_frames(self, subscription: mrc.Subscription) -> typing.Iterable[Me filter_nulls=self._filter_null, filter_null_columns=self._filter_null_columns, parser_kwargs=self._parser_kwargs, - df_type=self._df_type, + df_type=self.df_type_str, ) for i in range(self._repeat_count): diff --git a/python/morpheus/morpheus/stages/input/rss_source_stage.py b/python/morpheus/morpheus/stages/input/rss_source_stage.py index 0e76d7dd5f..a5dc473189 100644 --- a/python/morpheus/morpheus/stages/input/rss_source_stage.py +++ b/python/morpheus/morpheus/stages/input/rss_source_stage.py @@ -20,16 +20,16 @@ from morpheus.config import Config from morpheus.controllers.rss_controller import RSSController from morpheus.messages import MessageMeta +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.preallocator_mixin import PreallocatorMixin from morpheus.pipeline.single_output_source import SingleOutputSource from morpheus.pipeline.stage_schema import StageSchema -from morpheus.utils.type_utils import exec_mode_to_df_type_str logger = logging.getLogger(__name__) @register_stage("from-rss") -class RSSSourceStage(PreallocatorMixin, SingleOutputSource): +class RSSSourceStage(GpuAndCpuMixin, PreallocatorMixin, SingleOutputSource): """ Load RSS feed items into a DataFrame. @@ -84,7 +84,7 @@ def __init__(self, stop_after=stop_after, interval_secs=interval_secs, should_stop_fn=self.is_stop_requested, - df_type=exec_mode_to_df_type_str(c.execution_mode)) + df_type=self.df_type_str) @property def name(self) -> str: diff --git a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py index f268ad1866..448285f018 100644 --- a/python/morpheus/morpheus/stages/output/http_server_sink_stage.py +++ b/python/morpheus/morpheus/stages/output/http_server_sink_stage.py @@ -35,7 +35,6 @@ from morpheus.utils.http_utils import HttpParseResponse from morpheus.utils.http_utils import MimeTypes from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_utils import get_df_pkg logger = logging.getLogger(__name__) @@ -115,7 +114,7 @@ def __init__(self, self._df_serializer_fn = df_serializer_fn or self._default_df_serializer - self._df_pkg = get_df_pkg(config.execution_mode) + self._df_pkg = self.get_df_pkg() # FiberQueue doesn't have a way to check the size, nor does it have a way to check if it's empty without # attempting to perform a read. We'll keep track of the size ourselves. diff --git a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py index be1934111d..25f02d4ea0 100644 --- a/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py +++ b/python/morpheus/morpheus/stages/postprocess/generate_viz_frames_stage.py @@ -31,18 +31,18 @@ from morpheus.config import Config from morpheus.config import PipelineModes from morpheus.messages import ControlMessage +from morpheus.pipeline.execution_mode_mixins import GpuAndCpuMixin from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin from morpheus.pipeline.single_port_stage import SinglePortStage from morpheus.utils.producer_consumer_queue import AsyncIOProducerConsumerQueue from morpheus.utils.producer_consumer_queue import Closed from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_utils import get_df_class logger = logging.getLogger(__name__) @register_stage("gen-viz", modes=[PipelineModes.NLP], command_args={"deprecated": True}) -class GenerateVizFramesStage(PassThruTypeMixin, SinglePortStage): +class GenerateVizFramesStage(GpuAndCpuMixin, PassThruTypeMixin, SinglePortStage): """ Write out visualization DataFrames. @@ -81,7 +81,7 @@ def __init__(self, self._server_task: asyncio.Task = None self._server_close_event: asyncio.Event = None - self._df_class: type[DataFrameType] = get_df_class(c.execution_mode) + self._df_class: type[DataFrameType] = self.get_df_class() @property def name(self) -> str: From 319595e35ac121f94a4642369d02641658cb9480 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 12:43:18 -0700 Subject: [PATCH 332/347] Add helper methods for getting a read_csv and read_parquet method --- python/morpheus/morpheus/io/deserializers.py | 45 ++++++------- python/morpheus/morpheus/io/utils.py | 69 +++++++++++++++++--- 2 files changed, 82 insertions(+), 32 deletions(-) diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index 518b62d93f..c67ea91872 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -24,12 +24,25 @@ from morpheus.common import read_file_to_df as read_file_to_df_cpp from morpheus.config import CppConfig from morpheus.io.utils import filter_null_data +from morpheus.io.utils import get_csv_reader from morpheus.io.utils import get_json_reader +from morpheus.io.utils import get_parquet_reader from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_utils import df_type_str_to_pkg +def get_reader(file_type: FileTypes, df_type: DataFrameModule) -> typing.Callable[..., DataFrameType]: + if (file_type == FileTypes.CSV): + return get_csv_reader(df_type) + elif (file_type == FileTypes.JSON): + return get_json_reader(df_type) + elif (file_type == FileTypes.PARQUET): + return get_parquet_reader(df_type) + + raise ValueError(f"Unsupported file type: {file_type}") + + def _read_file_to_df_py(*, file_name: typing.Union[str, io.IOBase], file_type: FileTypes, @@ -60,30 +73,14 @@ def _read_file_to_df_py(*, # Update with any args set by the user. User values overwrite defaults kwargs.update(parser_kwargs) - - df = None - if (mode == FileTypes.JSON): - reader = get_json_reader(df_type) - df = reader(file_name, **kwargs) - - else: - df_class = df_type_str_to_pkg(df_type) - - if (mode == FileTypes.CSV): - df: DataFrameType = df_class.read_csv(file_name, **kwargs) - - if (len(df.columns) > 1 and df.columns[0] == "Unnamed: 0" and df.iloc[:, 0].dtype == np.dtype(int)): - df.set_index("Unnamed: 0", drop=True, inplace=True) - df.index.name = "" - df.sort_index(inplace=True) - - elif (mode == FileTypes.PARQUET): - df = df_class.read_parquet(file_name, **kwargs) - - else: - assert False, f"Unsupported file type mode: {mode}" - - assert df is not None + reader = get_reader(mode, df_type) + + df: DataFrameType = reader(file_name, **kwargs) + if (mode == FileTypes.CSV): + if (len(df.columns) > 1 and df.columns[0] == "Unnamed: 0" and df.iloc[:, 0].dtype == np.dtype(int)): + df.set_index("Unnamed: 0", drop=True, inplace=True) + df.index.name = "" + df.sort_index(inplace=True) return df diff --git a/python/morpheus/morpheus/io/utils.py b/python/morpheus/morpheus/io/utils.py index 9875c30506..58e5413f91 100644 --- a/python/morpheus/morpheus/io/utils.py +++ b/python/morpheus/morpheus/io/utils.py @@ -140,6 +140,47 @@ def truncate_string_cols_by_bytes(df: DataFrameType, return performed_truncation +def _selector_to_exec_mode(selector: DataFrameModule | ExecutionMode) -> ExecutionMode: + if not isinstance(selector, ExecutionMode): + execution_mode = df_type_str_to_exec_mode(selector) + else: + execution_mode = selector + + return execution_mode + + +def _get_df_method(selector: DataFrameModule | ExecutionMode, method_name: str) -> typing.Callable[..., DataFrameType]: + """ + Return the appropriate DataFrame method based on the execution mode. + """ + execution_mode = _selector_to_exec_mode(selector) + + if (execution_mode == ExecutionMode.GPU): + import cudf + method = getattr(cudf, method_name) + else: + method = getattr(pd, method_name) + + return method + + +@typing.overload +def get_csv_reader(selector: DataFrameModule) -> typing.Callable[..., DataFrameType]: + ... + + +@typing.overload +def get_csv_reader(selector: ExecutionMode) -> typing.Callable[..., DataFrameType]: + ... + + +def get_csv_reader(selector: DataFrameModule | ExecutionMode) -> typing.Callable[..., DataFrameType]: + """ + Return the appropriate CSV reader based on the execution mode. + """ + return _get_df_method(selector, 'read_csv') + + @typing.overload def get_json_reader(selector: DataFrameModule) -> typing.Callable[..., DataFrameType]: ... @@ -154,15 +195,27 @@ def get_json_reader(selector: DataFrameModule | ExecutionMode) -> typing.Callabl """ Return the appropriate JSON reader based on the execution mode. """ - if not isinstance(selector, ExecutionMode): - execution_mode = df_type_str_to_exec_mode(selector) - else: - execution_mode = selector + execution_mode = _selector_to_exec_mode(selector) + reader = _get_df_method(execution_mode, 'read_json') if (execution_mode == ExecutionMode.GPU): - import cudf - reader = functools.partial(cudf.read_json, engine='cudf') - else: - reader = pd.read_json + reader = functools.partial(reader, engine='cudf') return reader + + +@typing.overload +def get_parquet_reader(selector: DataFrameModule) -> typing.Callable[..., DataFrameType]: + ... + + +@typing.overload +def get_parquet_reader(selector: ExecutionMode) -> typing.Callable[..., DataFrameType]: + ... + + +def get_parquet_reader(selector: DataFrameModule | ExecutionMode) -> typing.Callable[..., DataFrameType]: + """ + Return the appropriate Parquet reader based on the execution mode. + """ + return _get_df_method(selector, 'read_parquet') From ce8ea07ad116ce79c9d7fd8e158239d2cbab1d32 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 12:48:27 -0700 Subject: [PATCH 333/347] Use get_csv_reader --- python/morpheus/morpheus/parsers/zeek.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/morpheus/morpheus/parsers/zeek.py b/python/morpheus/morpheus/parsers/zeek.py index 8d28a1bc24..37cf555cc2 100644 --- a/python/morpheus/morpheus/parsers/zeek.py +++ b/python/morpheus/morpheus/parsers/zeek.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from morpheus.utils.io.utils import get_csv_reader from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_aliases import DataFrameType from morpheus.utils.type_utils import get_df_pkg @@ -55,15 +56,15 @@ def parse(filepath: str, df_type: DataFrameModule = "cudf") -> DataFrameType: DataFrameType Parsed Zeek log dataframe """ - df_pkg = get_df_pkg(df_type) - header_gdf = df_pkg.read_csv(filepath, names=["line"], nrows=8) + csv_reader = get_csv_reader(df_type) + header_gdf = csv_reader(filepath, names=["line"], nrows=8) lines_gdf = header_gdf["line"].str.split() column_names = lines_gdf.iloc[6][1:] column_types = lines_gdf.iloc[7][1:] column_dtypes = list(map(lambda x: TYPE_DICT.get(x, "str"), column_types)) - log_gdf = df_pkg.read_csv( + log_gdf = csv_reader( filepath, delimiter="\t", dtype=column_dtypes, From 7be0402bd824399c39780454c090a4a9866689c4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 12:52:14 -0700 Subject: [PATCH 334/347] Fix imports --- python/morpheus/morpheus/parsers/zeek.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/parsers/zeek.py b/python/morpheus/morpheus/parsers/zeek.py index 37cf555cc2..bc8d5683b7 100644 --- a/python/morpheus/morpheus/parsers/zeek.py +++ b/python/morpheus/morpheus/parsers/zeek.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from morpheus.utils.io.utils import get_csv_reader +from morpheus.io.utils import get_csv_reader from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_utils import get_df_pkg TYPE_DICT = { "bool": "bool", From fa67db6abba8d3a9398bb8ae33ea186bd29e4a61 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:17:16 -0700 Subject: [PATCH 335/347] Rase a ValueError instead of a KeyError to match behavior of the C++ impl --- python/morpheus/morpheus/messages/control_message.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/morpheus/morpheus/messages/control_message.py b/python/morpheus/morpheus/messages/control_message.py index de625f0d03..8c958572e8 100644 --- a/python/morpheus/morpheus/messages/control_message.py +++ b/python/morpheus/morpheus/messages/control_message.py @@ -162,9 +162,9 @@ def set_timestamp(self, key: str, timestamp: datetime): def get_timestamp(self, key: str, fail_if_nonexist: bool = False) -> datetime | None: try: return self._timestamps[key] - except KeyError: + except KeyError as e: if fail_if_nonexist: - raise + raise ValueError("Timestamp for the specified key does not exist.") from e return None def filter_timestamp(self, regex_filter: str) -> dict[str, datetime]: From 485764c7ceaf2de1352ec16ae3495ecc0fb894b1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:18:27 -0700 Subject: [PATCH 336/347] Remove inconsistent tensor reshaping issue #1955 --- python/morpheus/morpheus/messages/memory/tensor_memory.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/morpheus/morpheus/messages/memory/tensor_memory.py b/python/morpheus/morpheus/messages/memory/tensor_memory.py index ce193cd5da..33c6d4f2bd 100644 --- a/python/morpheus/morpheus/messages/memory/tensor_memory.py +++ b/python/morpheus/morpheus/messages/memory/tensor_memory.py @@ -176,7 +176,5 @@ def set_tensor(self, name: str, tensor: NDArrayType): ValueError If the number of rows in `tensor` does not match `count` """ - # Ensure that we have 2D array here (`ensure_2d` inserts the wrong axis) - reshaped_tensor = tensor if tensor.ndim == 2 else np.reshape(tensor, (tensor.shape[0], -1)) - self._check_tensor(reshaped_tensor) - self._tensors[name] = reshaped_tensor + self._check_tensor(tensor) + self._tensors[name] = tensor From d903feba8d6cac34c44737a42e93983b1777ccda Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:19:12 -0700 Subject: [PATCH 337/347] Update ControlMemory tests to test the Python and C++ impls --- .../morpheus/messages/test_control_message.py | 127 ++++++++++-------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/tests/morpheus/messages/test_control_message.py b/tests/morpheus/messages/test_control_message.py index 85f2aa344f..8ba17ab527 100644 --- a/tests/morpheus/messages/test_control_message.py +++ b/tests/morpheus/messages/test_control_message.py @@ -18,24 +18,25 @@ import io import sys -import cupy as cp import pytest from _utils.dataset_manager import DatasetManager from morpheus import messages +from morpheus.config import Config from morpheus.messages import TensorMemory +from morpheus.utils.type_utils import get_array_pkg # pylint: disable=unsupported-membership-test # pylint: disable=unsubscriptable-object -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_control_message_init(): messages.ControlMessage() # noqa: F841 messages.ControlMessage({"test": "test"}) # noqa: F841 -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_control_message_tasks(): message = messages.ControlMessage() assert len(message.get_tasks()) == 0 @@ -70,12 +71,6 @@ def test_control_message_tasks(): assert message.get_tasks()["type_a"][0]["key_x"] == "value_x" assert message.get_tasks()["type_a"][1]["key_y"] == "value_y" - # Ensure the underlying tasks cannot are not modified - message = messages.ControlMessage() - tasks = message.get_tasks() - tasks["type_a"] = [{"key_x", "value_x"}] # pylint: disable=unsupported-assignment-operation - assert len(message.get_tasks()) == 0 - message = messages.ControlMessage() message.add_task("type_a", {"key_x": "value_x"}) message.add_task("type_a", {"key_y": "value_y"}) @@ -86,7 +81,7 @@ def test_control_message_tasks(): assert message.get_tasks()["type_a"][1]["key_y"] == "value_y" -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_control_message_metadata(): message = messages.ControlMessage() @@ -108,11 +103,8 @@ def test_control_message_metadata(): assert message.get_metadata()["key_y"] == "value_yy" - message.get_metadata()["not_mutable"] = 5 # pylint: disable=unsupported-assignment-operation - - assert "not_mutable" not in message.get_metadata() - +@pytest.mark.gpu_and_cpu_mode def test_set_and_get_metadata(): message = messages.ControlMessage() @@ -132,6 +124,7 @@ def test_set_and_get_metadata(): assert all_metadata["another_key"] == "another_value" +@pytest.mark.gpu_and_cpu_mode def test_list_metadata(): message = messages.ControlMessage() @@ -146,6 +139,7 @@ def test_list_metadata(): assert set(keys) == {"key1", "key2", "key3"} +@pytest.mark.gpu_and_cpu_mode def test_get_metadata_default_value(): message = messages.ControlMessage() @@ -159,7 +153,7 @@ def test_get_metadata_default_value(): assert message.get_metadata("non_existing_key", "default_value") == "default_value" -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_control_message_get(): raw_control_message = messages.ControlMessage({ "test": "test_rcm", "tasks": [{ @@ -183,7 +177,7 @@ def test_control_message_get(): assert (control_message.has_task("load")) -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_control_message_set(): raw_control_message = messages.ControlMessage() control_message = messages.ControlMessage() @@ -204,6 +198,7 @@ def test_control_message_set(): assert (control_message.has_task("load")) +@pytest.mark.gpu_and_cpu_mode def test_control_message_set_and_get_payload(dataset: DatasetManager): df = dataset["test_dataframe.jsonlines"] @@ -217,7 +212,7 @@ def test_control_message_set_and_get_payload(dataset: DatasetManager): DatasetManager.assert_df_equal(payload.df, payload2.df) -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_set_and_get_timestamp_single(): # Create a ControlMessage instance msg = messages.ControlMessage() @@ -234,7 +229,7 @@ def test_set_and_get_timestamp_single(): assert result == timestamp, "The retrieved timestamp should match the one that was set." -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_filter_timestamp(): # Create a ControlMessage instance msg = messages.ControlMessage() @@ -255,7 +250,7 @@ def test_filter_timestamp(): assert result[f"{group}::key2"] == timestamp2, "The timestamp for key2 should match." -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_modetest_tensor_manipulation_after_retrieval def test_get_timestamp_fail_if_nonexist(): # Create a ControlMessage instance msg = messages.ControlMessage() @@ -269,10 +264,15 @@ def test_get_timestamp_fail_if_nonexist(): assert str(exc_info.value) == "Timestamp for the specified key does not exist." -# Test setting and getting tensors with cupy arrays -@pytest.mark.usefixtures("config_only_cpp") -def test_tensors_setting_and_getting(): - data = {"input_ids": cp.array([1, 2, 3]), "input_mask": cp.array([1, 1, 1]), "segment_ids": cp.array([0, 0, 1])} +@pytest.mark.gpu_and_cpu_mode +def test_tensors_setting_and_getting(config: Config): + # Test setting and getting tensors with cupy/numpy arrays + array_pkg = get_array_pkg(config.execution_mode) + data = { + "input_ids": array_pkg.array([1, 2, 3]), + "input_mask": array_pkg.array([1, 1, 1]), + "segment_ids": array_pkg.array([0, 0, 1]) + } message = messages.ControlMessage() tensor_memory = TensorMemory(count=data["input_ids"].shape[0]) tensor_memory.set_tensors(data) @@ -283,14 +283,17 @@ def test_tensors_setting_and_getting(): assert retrieved_tensors.count == data["input_ids"].shape[0], "Tensor count mismatch." for key, val in data.items(): - assert cp.allclose(retrieved_tensors.get_tensor(key), val), f"Mismatch in tensor data for {key}." + assert array_pkg.allclose(retrieved_tensors.get_tensor(key), val), f"Mismatch in tensor data for {key}." -# Test retrieving tensor names and checking specific tensor existence -@pytest.mark.usefixtures("config_only_cpp") -def test_tensor_names_and_existence(): +@pytest.mark.gpu_and_cpu_mode +def test_tensor_names_and_existence(config: Config): + # Test retrieving tensor names and checking specific tensor existence + array_pkg = get_array_pkg(config.execution_mode) tokenized_data = { - "input_ids": cp.array([1, 2, 3]), "input_mask": cp.array([1, 1, 1]), "segment_ids": cp.array([0, 0, 1]) + "input_ids": array_pkg.array([1, 2, 3]), + "input_mask": array_pkg.array([1, 1, 1]), + "segment_ids": array_pkg.array([0, 0, 1]) } message = messages.ControlMessage() tensor_memory = TensorMemory(count=tokenized_data["input_ids"].shape[0], tensors=tokenized_data) @@ -303,11 +306,14 @@ def test_tensor_names_and_existence(): assert retrieved_tensors.has_tensor(key), f"Tensor {key} should exist." -# Test manipulating tensors after retrieval -@pytest.mark.usefixtures("config_only_cpp") -def test_tensor_manipulation_after_retrieval(): +@pytest.mark.gpu_and_cpu_mode +def test_tensor_manipulation_after_retrieval(config: Config): + # Test manipulating tensors after retrieval + array_pkg = get_array_pkg(config.execution_mode) tokenized_data = { - "input_ids": cp.array([1, 2, 3]), "input_mask": cp.array([1, 1, 1]), "segment_ids": cp.array([0, 0, 1]) + "input_ids": array_pkg.array([1, 2, 3]), + "input_mask": array_pkg.array([1, 1, 1]), + "segment_ids": array_pkg.array([0, 0, 1]) } message = messages.ControlMessage() tensor_memory = TensorMemory(count=3, tensors=tokenized_data) @@ -315,17 +321,20 @@ def test_tensor_manipulation_after_retrieval(): message.tensors(tensor_memory) retrieved_tensors = message.tensors() - new_tensor = cp.array([4, 5, 6]) + new_tensor = array_pkg.array([4, 5, 6]) retrieved_tensors.set_tensor("new_tensor", new_tensor) - assert cp.allclose(retrieved_tensors.get_tensor("new_tensor"), new_tensor), "New tensor data mismatch." + assert array_pkg.allclose(retrieved_tensors.get_tensor("new_tensor"), new_tensor), "New tensor data mismatch." -# Assuming there's functionality to update all tensors at once -@pytest.mark.usefixtures("config_only_cpp") -def test_tensor_update(): +@pytest.mark.gpu_and_cpu_mode +def test_tensor_update(config: Config): + # Assuming there's functionality to update all tensors at once + array_pkg = get_array_pkg(config.execution_mode) tokenized_data = { - "input_ids": cp.array([1, 2, 3]), "input_mask": cp.array([1, 1, 1]), "segment_ids": cp.array([0, 0, 1]) + "input_ids": array_pkg.array([1, 2, 3]), + "input_mask": array_pkg.array([1, 1, 1]), + "segment_ids": array_pkg.array([0, 0, 1]) } message = messages.ControlMessage() tensor_memory = TensorMemory(count=3, tensors=tokenized_data) @@ -334,7 +343,9 @@ def test_tensor_update(): # Update tensors with new data new_tensors = { - "input_ids": cp.array([4, 5, 6]), "input_mask": cp.array([1, 0, 1]), "segment_ids": cp.array([1, 1, 0]) + "input_ids": array_pkg.array([4, 5, 6]), + "input_mask": array_pkg.array([1, 0, 1]), + "segment_ids": array_pkg.array([1, 1, 0]) } tensor_memory.set_tensors(new_tensors) @@ -342,13 +353,14 @@ def test_tensor_update(): updated_tensors = message.tensors() for key, val in new_tensors.items(): - assert cp.allclose(updated_tensors.get_tensor(key), val), f"Mismatch in updated tensor data for {key}." + assert array_pkg.allclose(updated_tensors.get_tensor(key), val), f"Mismatch in updated tensor data for {key}." -@pytest.mark.usefixtures("config_only_cpp") -def test_update_individual_tensor(): - initial_data = {"input_ids": cp.array([1, 2, 3]), "input_mask": cp.array([1, 1, 1])} - update_data = {"input_ids": cp.array([4, 5, 6])} +@pytest.mark.gpu_and_cpu_mode +def test_update_individual_tensor(config: Config): + array_pkg = get_array_pkg(config.execution_mode) + initial_data = {"input_ids": array_pkg.array([1, 2, 3]), "input_mask": array_pkg.array([1, 1, 1])} + update_data = {"input_ids": array_pkg.array([4, 5, 6])} message = messages.ControlMessage() tensor_memory = TensorMemory(count=3, tensors=initial_data) message.tensors(tensor_memory) @@ -358,14 +370,14 @@ def test_update_individual_tensor(): retrieved_tensors = message.tensors() # Check updated tensor - assert cp.allclose(retrieved_tensors.get_tensor("input_ids"), + assert array_pkg.allclose(retrieved_tensors.get_tensor("input_ids"), update_data["input_ids"]), "Input IDs update mismatch." # Ensure other tensor remains unchanged - assert cp.allclose(retrieved_tensors.get_tensor("input_mask"), + assert array_pkg.allclose(retrieved_tensors.get_tensor("input_mask"), initial_data["input_mask"]), "Input mask should remain unchanged after updating input_ids." -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_and_cpu_mode def test_behavior_with_empty_tensors(): message = messages.ControlMessage() tensor_memory = TensorMemory(count=0) @@ -376,25 +388,26 @@ def test_behavior_with_empty_tensors(): assert len(retrieved_tensors.tensor_names) == 0, "There should be no tensor names for empty tensor memory." -@pytest.mark.usefixtures("config_only_cpp") -def test_consistency_after_multiple_operations(): - initial_data = {"input_ids": cp.array([1, 2, 3]), "input_mask": cp.array([1, 1, 1])} +@pytest.mark.gpu_and_cpu_mode +def test_consistency_after_multiple_operations(config: Config): + array_pkg = get_array_pkg(config.execution_mode) + initial_data = {"input_ids": array_pkg.array([1, 2, 3]), "input_mask": array_pkg.array([1, 1, 1])} message = messages.ControlMessage() tensor_memory = TensorMemory(count=3, tensors=initial_data) message.tensors(tensor_memory) # Update a tensor - tensor_memory.set_tensor("input_ids", cp.array([4, 5, 6])) + tensor_memory.set_tensor("input_ids", array_pkg.array([4, 5, 6])) # Remove another tensor # Add a new tensor - new_tensor = {"new_tensor": cp.array([7, 8, 9])} + new_tensor = {"new_tensor": array_pkg.array([7, 8, 9])} tensor_memory.set_tensor("new_tensor", new_tensor["new_tensor"]) retrieved_tensors = message.tensors() assert retrieved_tensors.count == 3, "Tensor count mismatch after multiple operations." - assert cp.allclose(retrieved_tensors.get_tensor("input_ids"), - cp.array([4, 5, 6])), "Mismatch in input_ids after update." - assert cp.allclose(retrieved_tensors.get_tensor("new_tensor"), + assert array_pkg.allclose(retrieved_tensors.get_tensor("input_ids"), + array_pkg.array([4, 5, 6])), "Mismatch in input_ids after update." + assert array_pkg.allclose(retrieved_tensors.get_tensor("new_tensor"), new_tensor["new_tensor"]), "New tensor data mismatch." @@ -428,7 +441,7 @@ def fixture_pyobject(request): return request.param() -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_mode def test_metadata_holds_non_serializable_python_obj(py_object): message = messages.ControlMessage() @@ -452,7 +465,7 @@ def test_metadata_holds_non_serializable_python_obj(py_object): assert obj is metadata_dict_with_obj["nested_obj"] -@pytest.mark.usefixtures("config_only_cpp") +@pytest.mark.gpu_mode def test_tasks_hold_non_serializable_python_obj(py_object): message = messages.ControlMessage() From ef466ae383bafda0c044dccdae4bca6c6a950535 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:19:48 -0700 Subject: [PATCH 338/347] Test should be run in both cpu and gpu modes --- tests/morpheus/stages/test_file_source_stage_pipe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/morpheus/stages/test_file_source_stage_pipe.py b/tests/morpheus/stages/test_file_source_stage_pipe.py index 59f9c76d63..33e9ad8af6 100755 --- a/tests/morpheus/stages/test_file_source_stage_pipe.py +++ b/tests/morpheus/stages/test_file_source_stage_pipe.py @@ -32,6 +32,7 @@ @pytest.mark.slow +@pytest.mark.gpu_and_cpu_mode @pytest.mark.parametrize("input_file", [ os.path.join(TEST_DIRS.tests_data_dir, "filter_probs.csv"), From 357a6e21f60a5dbc603d2eb6004c6afc2ac99dd3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:20:07 -0700 Subject: [PATCH 339/347] Remove unused config_only_cpp and config_no_cpp fixtures --- tests/conftest.py | 33 --------------------------------- tests/test_conftest.py | 9 ++------- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6869e8c2c3..4132045924 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -193,39 +193,6 @@ def pytest_runtest_teardown(item, nextitem): reset_logging(logger_name=None) # Reset the root logger as well -@pytest.fixture(scope="function") -def config_only_cpp(): - """ - Use this fixture in unittest style tests to indicate a lack of support for C++. Use via - `@pytest.mark.usefixtures("config_only_cpp")` - """ - - from morpheus.config import Config - from morpheus.config import CppConfig - - CppConfig.set_should_use_cpp(True) - - yield Config() - - -@pytest.fixture(scope="function") -def config_no_cpp(): - """ - Use this fixture in unittest style tests to indicate support for C++. Use via - `@pytest.mark.usefixtures("config_no_cpp")` - """ - - from morpheus.config import Config - from morpheus.config import CppConfig - from morpheus.config import ExecutionMode - - CppConfig.set_should_use_cpp(False) - config = Config() - config.execution_mode = ExecutionMode.CPU - - yield config - - @pytest.fixture(scope="function") def df_type(request: pytest.FixtureRequest): diff --git a/tests/test_conftest.py b/tests/test_conftest.py index 42fdd2dd2c..5856152771 100644 --- a/tests/test_conftest.py +++ b/tests/test_conftest.py @@ -191,13 +191,8 @@ def test_fixture_neither(execution_mode: ExecutionMode): # === Config Fixture === -@pytest.mark.usefixtures("config_no_cpp") -def test_config_fixture_no_cpp(): - assert not CppConfig.get_should_use_cpp() - - -@pytest.mark.usefixtures("config_only_cpp") -def test_config_fixture_only_cpp(): +@pytest.mark.usefixtures("config") +def test_config_fixture(): assert CppConfig.get_should_use_cpp() From 1998922921242c073fde8d78439ed957f4d1b71e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:28:59 -0700 Subject: [PATCH 340/347] lint fixes --- python/morpheus/morpheus/io/deserializers.py | 7 ++++--- python/morpheus/morpheus/messages/memory/tensor_memory.py | 2 -- tests/morpheus/messages/test_control_message.py | 8 ++++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index c67ea91872..34703ac50e 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -29,15 +29,16 @@ from morpheus.io.utils import get_parquet_reader from morpheus.utils.type_aliases import DataFrameModule from morpheus.utils.type_aliases import DataFrameType -from morpheus.utils.type_utils import df_type_str_to_pkg def get_reader(file_type: FileTypes, df_type: DataFrameModule) -> typing.Callable[..., DataFrameType]: if (file_type == FileTypes.CSV): return get_csv_reader(df_type) - elif (file_type == FileTypes.JSON): + + if (file_type == FileTypes.JSON): return get_json_reader(df_type) - elif (file_type == FileTypes.PARQUET): + + if (file_type == FileTypes.PARQUET): return get_parquet_reader(df_type) raise ValueError(f"Unsupported file type: {file_type}") diff --git a/python/morpheus/morpheus/messages/memory/tensor_memory.py b/python/morpheus/morpheus/messages/memory/tensor_memory.py index 33c6d4f2bd..2e3164585e 100644 --- a/python/morpheus/morpheus/messages/memory/tensor_memory.py +++ b/python/morpheus/morpheus/messages/memory/tensor_memory.py @@ -16,8 +16,6 @@ import dataclasses import typing -import numpy as np - import morpheus._lib.messages as _messages from morpheus.messages.message_base import MessageData from morpheus.utils.type_aliases import NDArrayType diff --git a/tests/morpheus/messages/test_control_message.py b/tests/morpheus/messages/test_control_message.py index 8ba17ab527..b9ba42d079 100644 --- a/tests/morpheus/messages/test_control_message.py +++ b/tests/morpheus/messages/test_control_message.py @@ -371,10 +371,10 @@ def test_update_individual_tensor(config: Config): # Check updated tensor assert array_pkg.allclose(retrieved_tensors.get_tensor("input_ids"), - update_data["input_ids"]), "Input IDs update mismatch." + update_data["input_ids"]), "Input IDs update mismatch." # Ensure other tensor remains unchanged assert array_pkg.allclose(retrieved_tensors.get_tensor("input_mask"), - initial_data["input_mask"]), "Input mask should remain unchanged after updating input_ids." + initial_data["input_mask"]), "input_mask should be unchanged after updating input_ids." @pytest.mark.gpu_and_cpu_mode @@ -406,9 +406,9 @@ def test_consistency_after_multiple_operations(config: Config): retrieved_tensors = message.tensors() assert retrieved_tensors.count == 3, "Tensor count mismatch after multiple operations." assert array_pkg.allclose(retrieved_tensors.get_tensor("input_ids"), - array_pkg.array([4, 5, 6])), "Mismatch in input_ids after update." + array_pkg.array([4, 5, 6])), "Mismatch in input_ids after update." assert array_pkg.allclose(retrieved_tensors.get_tensor("new_tensor"), - new_tensor["new_tensor"]), "New tensor data mismatch." + new_tensor["new_tensor"]), "New tensor data mismatch." class NonSerializablePyClass(): From a0f5e5ba43d23912660fcb39a960d907bdf31598 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:36:37 -0700 Subject: [PATCH 341/347] Fix cpu-mode json tests --- tests/morpheus/stages/test_file_source_stage_pipe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/morpheus/stages/test_file_source_stage_pipe.py b/tests/morpheus/stages/test_file_source_stage_pipe.py index 33e9ad8af6..0f5c1fdb2e 100755 --- a/tests/morpheus/stages/test_file_source_stage_pipe.py +++ b/tests/morpheus/stages/test_file_source_stage_pipe.py @@ -25,6 +25,7 @@ from morpheus.common import FileTypes from morpheus.common import determine_file_type from morpheus.config import Config +from morpheus.config import ExecutionMode from morpheus.io.deserializers import read_file_to_df from morpheus.pipeline import LinearPipeline from morpheus.stages.input.file_source_stage import FileSourceStage @@ -47,7 +48,7 @@ def test_file_source_stage_pipe(config: Config, input_file: str, filter_null: bo parser_kwargs = {} if determine_file_type(input_file) == FileTypes.JSON: # kwarg specific to pandas.read_json - parser_kwargs['convert_dates'] = False + parser_kwargs['convert_dates'] = config.execution_mode == ExecutionMode.CPU expected_df = read_file_to_df(file_name=input_file, filter_nulls=filter_null, From 80e817aa717cdc9b2027dc15af771534c7e84b52 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 16:56:11 -0700 Subject: [PATCH 342/347] Fix misleading runtime erro messafe, remove unused inferring gpu mode based on the use_cudf marker --- tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4132045924..3dca6bc243 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -209,7 +209,7 @@ def df_type(request: pytest.FixtureRequest): use_cudf = request.node.get_closest_marker("use_cudf") is not None if (use_pandas and use_cudf): - raise RuntimeError(f"Both markers (gpu_mode and cpu_mode) were added to function {request.node.nodeid}. " + raise RuntimeError(f"Both markers (use_pandas and use_cudf) were added to function {request.node.nodeid}. " "Remove markers to support both.") # This will default to "cudf" or follow use_pandas @@ -235,7 +235,6 @@ def _get_execution_mode(request: pytest.FixtureRequest) -> "ExecutionMode": # if both are undefined, infer based on the df_type if (not gpu_mode and not cpu_mode): - gpu_mode = request.node.get_closest_marker("use_cudf") is not None cpu_mode = request.node.get_closest_marker("use_pandas") is not None # This will default to True or follow gpu_mode From 32fb3b83fa0a7bfdcdb64c96aee1a3d08f06e9b9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 17 Oct 2024 12:29:07 -0700 Subject: [PATCH 343/347] Ignore code.visualstudio.com, these urls are being forbidden when run from inside CI --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index aa59786e26..d743df18e1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -199,6 +199,7 @@ r'^http://$', r'^https://$', r'https://(platform\.)?openai.com', + r'https://code.visualstudio.com' ] # Add any paths that contain templates here, relative to this directory. From 341ffe0f4274b19175cd396ddfb0c5bfd2491fed Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 18 Oct 2024 10:59:48 -0700 Subject: [PATCH 344/347] Improved doc strings --- python/morpheus/morpheus/io/deserializers.py | 2 +- .../morpheus/morpheus/utils/type_aliases.py | 12 +- python/morpheus/morpheus/utils/type_utils.py | 131 +++++++++++++++++- 3 files changed, 139 insertions(+), 6 deletions(-) diff --git a/python/morpheus/morpheus/io/deserializers.py b/python/morpheus/morpheus/io/deserializers.py index 34703ac50e..15867664f7 100644 --- a/python/morpheus/morpheus/io/deserializers.py +++ b/python/morpheus/morpheus/io/deserializers.py @@ -107,7 +107,7 @@ def read_file_to_df(file_name: typing.Union[str, io.IOBase], Whether to filter null rows after loading, by default True. filter_null_columns : list[str]|str, default = 'data' Column or columns to filter null values from. Ignored when `filter_null` is False. - df_type : typing.Literal[, optional + df_type : typing.Literal["cudf", "pandas"], optional What type of parser to use. Options are 'cudf' and 'pandas', by default "pandas". Returns diff --git a/python/morpheus/morpheus/utils/type_aliases.py b/python/morpheus/morpheus/utils/type_aliases.py index 5e977918c7..0028c076fe 100644 --- a/python/morpheus/morpheus/utils/type_aliases.py +++ b/python/morpheus/morpheus/utils/type_aliases.py @@ -23,8 +23,18 @@ import cudf DataFrameModule = typing.Literal["cudf", "pandas"] +"""Valid DataFrame modules.""" + DataFrameType = typing.Union["pandas.DataFrame", "cudf.DataFrame"] +"""Alias for pandas and cuDF DataFrame types.""" + SeriesType = typing.Union["pandas.Series", "cudf.Series"] +"""Alias for pandas and cuDF Series types.""" NDArrayType = typing.Union["numpy.ndarray", "cupy.ndarray"] -TensorMapType = dict[str, NDArrayType] +"""Alias for NumPy and CuPy ndarray types.""" + +# Intentionally using `typing.Dict` instead of `dict` to avoid a Sphinx build error. +# https://github.com/nv-morpheus/Morpheus/issues/1956 +TensorMapType = typing.Dict[str, NDArrayType] +"""Alias for a dictionary of tensor names to tensors represented as either a NumPy or CuPy ndarray.""" diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 7dd629b687..09bf5e8c96 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -175,6 +175,16 @@ def get_full_qualname(klass: type) -> str: def df_type_str_to_exec_mode(df_type_str: DataFrameModule) -> ExecutionMode: """ Return the appropriate execution mode based on the DataFrame type string. + + Parameters + ---------- + df_type_str : `morpheus.utils.type_aliases.DataFrameModule` + The DataFrame type string. + + Returns + ------- + `morpheus.config.ExecutionMode` + The associated execution mode based on the DataFrame type string. """ if df_type_str == "cudf": return ExecutionMode.GPU @@ -186,6 +196,19 @@ def df_type_str_to_exec_mode(df_type_str: DataFrameModule) -> ExecutionMode: def exec_mode_to_df_type_str(execution_mode: ExecutionMode) -> DataFrameModule: + """ + Return the appropriate DataFrame type string based on the execution mode. + + Parameters + ---------- + execution_mode : `morpheus.config.ExecutionMode` + The execution mode. + + Returns + ------- + `morpheus.utils.type_aliases.DataFrameModule` + The associated DataFrame type string based on the execution mode. + """ if execution_mode == ExecutionMode.GPU: return "cudf" @@ -193,6 +216,14 @@ def exec_mode_to_df_type_str(execution_mode: ExecutionMode) -> DataFrameModule: def cpp_mode_to_exec_mode() -> ExecutionMode: + """ + Return the execution mode based on the configuration of the global `morpheus.config.CppConfig` singleton. + + Returns + ------- + `morpheus.config.ExecutionMode` + The execution mode. + """ if CppConfig.get_should_use_cpp(): return ExecutionMode.GPU return ExecutionMode.CPU @@ -200,7 +231,17 @@ def cpp_mode_to_exec_mode() -> ExecutionMode: def df_type_str_to_pkg(df_type_str: DataFrameModule) -> types.ModuleType: """ - Return the appropriate DataFrame package based on the DataFrame type string. + Import and return the appropriate DataFrame package based on the DataFrame type string. + + Parameters + ---------- + df_type_str : `morpheus.utils.type_aliases.DataFrameModule` + The DataFrame type string. + + Returns + ------- + types.ModuleType + The associated DataFrame package based on the DataFrame type string. """ if df_type_str == "cudf": import cudf @@ -224,7 +265,28 @@ def get_df_pkg(selector: ExecutionMode = None) -> types.ModuleType: def get_df_pkg(selector: ExecutionMode | DataFrameModule = None) -> types.ModuleType: """ - Return the appropriate DataFrame package based on the execution mode. + Return the appropriate DataFrame package based on `selector` which can be either an `ExecutionMode` instance, a + DataFrame type string, or `None`. + + When `None` the execution mode is determined by the global `morpheus.config.CppConfig` singleton. + + This method is best used within code which needs to operate in both CPU and GPU modes, where simply importing `cudf` + would cause an import error if the user is not using a GPU. + Example usage:: + + from morpheus.utils.type_utils import get_df_pkg + df_pkg = get_df_pkg() + ser = df_pkg.Series([1,2,3]) + + Parameters + ---------- + selector : `morpheus.utils.type_aliases.DataFrameModule` | `morpheus.config.ExecutionMode` | None, optional + The selector to determine the DataFrame package, by default None. + + Returns + ------- + types.ModuleType + The associated DataFrame package based on the selector. """ if selector is None: execution_mode = cpp_mode_to_exec_mode() @@ -252,7 +314,26 @@ def get_df_class(selector: ExecutionMode = None) -> type[DataFrameType]: def get_df_class(selector: ExecutionMode | DataFrameModule = None) -> type[DataFrameType]: """ - Return the appropriate DataFrame class based on the execution mode. + Return the appropriate DataFrame `selector` which can be either an `ExecutionMode` instance, a + DataFrame type string, or `None`. + + When `None` the execution mode is determined by the global `morpheus.config.CppConfig` singleton. + + This method is best used within code which needs to construct a DataFrame in both CPU and GPU modes. + Example usage:: + + from morpheus.utils.type_utils import get_df_class + df_class = get_df_class() + df = df_class({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}) + + Parameters + ---------- + selector : `morpheus.utils.type_aliases.DataFrameModule` | `morpheus.config.ExecutionMode` | None, optional + The selector to determine the DataFrame class, by default None. + + Returns + ------- + type[DataFrameType] """ df_pkg = get_df_pkg(selector) return df_pkg.DataFrame @@ -261,13 +342,33 @@ def get_df_class(selector: ExecutionMode | DataFrameModule = None) -> type[DataF def is_cudf_type(obj: typing.Any) -> bool: """ Check if a given object (DataFrame, Series, RangeIndex etc...) is a cuDF type. + + Parameters + ---------- + obj : typing.Any + The object to check. + + Returns + ------- + bool + `True` if the object is a cuDF type, `False` otherwise. """ return "cudf" in str(type(obj)) def get_df_pkg_from_obj(obj: typing.Any) -> types.ModuleType: """ - Return the appropriate DataFrame package based on the DataFrame object. + Return the appropriate DataFrame package based on a given object (DataFrame, Series, RangeIndex etc...). + + Parameters + ---------- + obj : typing.Any + The object to check. + + Returns + ------- + types.ModuleType + The associated DataFrame package based on the object. """ if is_cudf_type(obj): import cudf @@ -279,6 +380,16 @@ def get_df_pkg_from_obj(obj: typing.Any) -> types.ModuleType: def is_dataframe(obj: typing.Any) -> bool: """ Check if a given object is a pandas or cudf DataFrame. + + Parameters + ---------- + obj : typing.Any + The object to check. + + Returns + ------- + bool + `True` if the object is a DataFrame, `False` otherwise. """ df_pkg = get_df_pkg_from_obj(obj) return isinstance(obj, df_pkg.DataFrame) @@ -287,6 +398,18 @@ def is_dataframe(obj: typing.Any) -> bool: def get_array_pkg(execution_mode: ExecutionMode = None) -> types.ModuleType: """ Return the appropriate array package (CuPy for GPU, NumPy for CPU) based on the execution mode. + + When `None` the execution mode is determined by the global `morpheus.config.CppConfig` singleton. + + Parameters + ---------- + execution_mode : `morpheus.config.ExecutionMode`, optional + The execution mode, by default `None`. + + Returns + ------- + types.ModuleType + The associated array package based on the execution mode. """ if execution_mode is None: execution_mode = cpp_mode_to_exec_mode() From 368ea5deb23c13d46409327a4887d6dfb2087044 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 18 Oct 2024 14:49:19 -0700 Subject: [PATCH 345/347] Misc docstring improvements --- python/morpheus/morpheus/utils/type_utils.py | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/python/morpheus/morpheus/utils/type_utils.py b/python/morpheus/morpheus/utils/type_utils.py index 09bf5e8c96..95c870e30f 100644 --- a/python/morpheus/morpheus/utils/type_utils.py +++ b/python/morpheus/morpheus/utils/type_utils.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Utility functions for working with types.""" import inspect import types @@ -240,7 +241,7 @@ def df_type_str_to_pkg(df_type_str: DataFrameModule) -> types.ModuleType: Returns ------- - types.ModuleType + `types.ModuleType` The associated DataFrame package based on the DataFrame type string. """ if df_type_str == "cudf": @@ -285,7 +286,7 @@ def get_df_pkg(selector: ExecutionMode | DataFrameModule = None) -> types.Module Returns ------- - types.ModuleType + `types.ModuleType` The associated DataFrame package based on the selector. """ if selector is None: @@ -333,7 +334,7 @@ def get_df_class(selector: ExecutionMode | DataFrameModule = None) -> type[DataF Returns ------- - type[DataFrameType] + `type[DataFrameType]` """ df_pkg = get_df_pkg(selector) return df_pkg.DataFrame @@ -345,12 +346,12 @@ def is_cudf_type(obj: typing.Any) -> bool: Parameters ---------- - obj : typing.Any + obj : `typing.Any` The object to check. Returns ------- - bool + `bool` `True` if the object is a cuDF type, `False` otherwise. """ return "cudf" in str(type(obj)) @@ -362,12 +363,12 @@ def get_df_pkg_from_obj(obj: typing.Any) -> types.ModuleType: Parameters ---------- - obj : typing.Any + obj : `typing.Any` The object to check. Returns ------- - types.ModuleType + `types.ModuleType` The associated DataFrame package based on the object. """ if is_cudf_type(obj): @@ -383,12 +384,12 @@ def is_dataframe(obj: typing.Any) -> bool: Parameters ---------- - obj : typing.Any + obj : `typing.Any` The object to check. Returns ------- - bool + `bool` `True` if the object is a DataFrame, `False` otherwise. """ df_pkg = get_df_pkg_from_obj(obj) @@ -408,7 +409,7 @@ def get_array_pkg(execution_mode: ExecutionMode = None) -> types.ModuleType: Returns ------- - types.ModuleType + `types.ModuleType` The associated array package based on the execution mode. """ if execution_mode is None: From e907c67816fb9ce2069d168ffce8cb5ed095f041 Mon Sep 17 00:00:00 2001 From: David Gardner <96306125+dagardner-nv@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:27:47 -0700 Subject: [PATCH 346/347] Update docs/source/developer_guide/contributing.md Co-authored-by: Michael Demoret <42954918+mdemoret-nv@users.noreply.github.com> --- docs/source/developer_guide/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 98d662ba4c..75174f08f8 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -179,7 +179,7 @@ To run: conda run -n base --live-stream rapids-dependency-file-generator ``` -When ready commit the changes to the `dependencies.yaml` file and the updated environment files. +When ready, commit both the changes to the `dependencies.yaml` file and the updated environment files into the repo. #### Prerequisites From 94816a9f591971e19de5eadd1e9d0008fc5f781e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 21 Oct 2024 16:31:09 -0700 Subject: [PATCH 347/347] Expand instruction for running rapids-dependency-file-generator --- docs/source/developer_guide/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 98d662ba4c..dca96df595 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -174,7 +174,7 @@ Install `rapids-dependency-file-generator` into the base Conda environment: conda run -n base --live-stream pip install rapids-dependency-file-generator ``` -To run: +Then to generate update the individual environment files run: ```bash conda run -n base --live-stream rapids-dependency-file-generator ```