From f2db2b915b84b07f8542d490a4c95b9457cbb480 Mon Sep 17 00:00:00 2001 From: Anuradha Karuppiah Date: Wed, 18 Sep 2024 16:38:04 +0000 Subject: [PATCH] Update dfp imports in examples and tests Signed-off-by: Anuradha Karuppiah --- .../production/grafana/run.py | 22 ++++++++-------- .../benchmarks/benchmark_conf_generator.py | 9 ++++--- .../benchmarks/test_bench_e2e_dfp_pipeline.py | 26 +++++++++---------- .../notebooks/dfp_azure_inference.ipynb | 18 ++++++------- .../dfp_azure_integrated_training.ipynb | 12 ++++----- .../notebooks/dfp_azure_training.ipynb | 18 ++++++------- .../notebooks/dfp_duo_inference.ipynb | 18 ++++++------- .../dfp_duo_integrated_training.ipynb | 12 ++++----- .../morpheus/notebooks/dfp_duo_training.ipynb | 18 ++++++------- .../visualization/dfp_viz_azure_pipeline.py | 22 ++++++++-------- .../visualization/dfp_viz_duo_pipeline.py | 22 ++++++++-------- .../digital_fingerprinting/conftest.py | 3 ++- .../test_dfp_file_batcher_stage.py | 16 ++++++------ .../test_dfp_file_to_df.py | 8 +++--- .../test_dfp_inference_stage.py | 12 ++++----- .../test_dfp_mlflow_model_writer.py | 18 ++++++------- .../test_dfp_postprocessing_stage.py | 8 +++--- .../test_dfp_preprocessing_stage.py | 4 +-- .../test_dfp_rolling_window_stage.py | 21 +++++++-------- .../test_dfp_split_users_stage.py | 6 ++--- .../test_dfp_training.py | 13 +++++----- .../test_dfp_viz_postproc.py | 6 ++--- .../test_multifile_source.py | 8 +++--- .../test_write_to_s3_stage.py | 2 +- .../utils/test_config_generator.py | 6 ++--- 25 files changed, 164 insertions(+), 164 deletions(-) diff --git a/examples/digital_fingerprinting/production/grafana/run.py b/examples/digital_fingerprinting/production/grafana/run.py index f5768eab1c..014ca1d434 100644 --- a/examples/digital_fingerprinting/production/grafana/run.py +++ b/examples/digital_fingerprinting/production/grafana/run.py @@ -26,17 +26,6 @@ import logging_loki import mlflow import pandas as pd -from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage -from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage -from dfp.stages.dfp_inference_stage import DFPInferenceStage -from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage -from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage -from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage -from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage -from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage -from dfp.stages.dfp_training import DFPTraining -from dfp.stages.multi_file_source import MultiFileSource -from dfp.utils.regex_utils import iso_date_regex from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import get_package_relative_file @@ -61,6 +50,17 @@ from morpheus.utils.column_info import StringCatColumn from morpheus.utils.file_utils import date_extractor from morpheus.utils.logger import configure_logging +from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage +from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage +from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage +from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage +from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage +from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage +from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage +from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage +from morpheus_dfp.stages.dfp_training import DFPTraining +from morpheus_dfp.stages.multi_file_source import MultiFileSource +from morpheus_dfp.utils.regex_utils import iso_date_regex def _file_type_name_to_enum(file_type: str) -> FileTypes: diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py b/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py index 734e3e17ed..480893e8b8 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py @@ -25,10 +25,11 @@ import mlflow import pandas as pd -from dfp.utils.config_generator import ConfigGenerator -from dfp.utils.config_generator import generate_ae_config -from dfp.utils.dfp_arg_parser import DFPArgParser -from dfp.utils.schema_utils import SchemaBuilder + +from morpheus_dfp.utils.config_generator import ConfigGenerator +from morpheus_dfp.utils.config_generator import generate_ae_config +from morpheus_dfp.utils.dfp_arg_parser import DFPArgParser +from morpheus_dfp.utils.schema_utils import SchemaBuilder logger = logging.getLogger(f"morpheus.{__name__}") diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py b/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py index 9583821290..051e3b7f25 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py @@ -20,23 +20,11 @@ import typing import boto3 -import dfp.modules # noqa: F401 # pylint:disable=unused-import import pytest -from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage -from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage -from dfp.stages.dfp_inference_stage import DFPInferenceStage -from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage -from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage -from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage -from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage -from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage -from dfp.stages.dfp_training import DFPTraining -from dfp.stages.multi_file_source import MultiFileSource -from dfp.utils.regex_utils import iso_date_regex -from dfp.utils.schema_utils import Schema import morpheus.loaders # noqa: F401 # pylint:disable=unused-import import morpheus.modules # noqa: F401 # pylint:disable=unused-import +import morpheus_dfp.modules # noqa: F401 # pylint:disable=unused-import from benchmarks.benchmark_conf_generator import BenchmarkConfGenerator from benchmarks.benchmark_conf_generator import load_json from benchmarks.benchmark_conf_generator import set_mlflow_tracking_uri @@ -53,6 +41,18 @@ from morpheus.utils.column_info import DataFrameInputSchema from morpheus.utils.file_utils import date_extractor from morpheus.utils.logger import configure_logging +from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage +from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage +from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage +from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage +from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage +from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage +from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage +from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage +from morpheus_dfp.stages.dfp_training import DFPTraining +from morpheus_dfp.stages.multi_file_source import MultiFileSource +from morpheus_dfp.utils.regex_utils import iso_date_regex +from morpheus_dfp.utils.schema_utils import Schema logger = logging.getLogger(f"morpheus.{__name__}") diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb index 8e5413f71c..909523e8ff 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb @@ -47,15 +47,15 @@ "from datetime import datetime\n", "from functools import partial\n", "\n", - "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", - "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", - "from dfp.stages.dfp_inference_stage import DFPInferenceStage\n", - "from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n", - "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", - "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", - "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", - "from dfp.stages.multi_file_source import MultiFileSource\n", - "from dfp.utils.regex_utils import iso_date_regex\n", + "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", + "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", + "from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage\n", + "from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n", + "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", + "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", + "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", + "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n", + "from morpheus_dfp.utils.regex_utils import iso_date_regex\n", "\n", "from morpheus.common import FileTypes\n", "from morpheus.common import FilterSource\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb index 75c14999a6..0002a318b8 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb @@ -45,12 +45,12 @@ "\n", "# When segment modules are imported, they're added to the module registry.\n", "# To avoid flake8 warnings about unused code, the noqa flag is used during import.\n", - "import dfp.modules # noqa: F401\n", - "from dfp.utils.config_generator import ConfigGenerator\n", - "from dfp.utils.config_generator import generate_ae_config\n", - "from dfp.utils.dfp_arg_parser import DFPArgParser\n", - "from dfp.utils.schema_utils import Schema\n", - "from dfp.utils.schema_utils import SchemaBuilder\n", + "import morpheus_dfp.modules # noqa: F401\n", + "from morpheus_dfp.utils.config_generator import ConfigGenerator\n", + "from morpheus_dfp.utils.config_generator import generate_ae_config\n", + "from morpheus_dfp.utils.dfp_arg_parser import DFPArgParser\n", + "from morpheus_dfp.utils.schema_utils import Schema\n", + "from morpheus_dfp.utils.schema_utils import SchemaBuilder\n", "\n", "import morpheus.loaders # noqa: F401\n", "import morpheus.modules # noqa: F401\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb index fc49b736b6..5846857da2 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb @@ -45,15 +45,15 @@ "import typing\n", "from datetime import datetime\n", "\n", - "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", - "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", - "from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n", - "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", - "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", - "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", - "from dfp.stages.dfp_training import DFPTraining\n", - "from dfp.stages.multi_file_source import MultiFileSource\n", - "from dfp.utils.regex_utils import iso_date_regex\n", + "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", + "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", + "from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n", + "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", + "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", + "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", + "from morpheus_dfp.stages.dfp_training import DFPTraining\n", + "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n", + "from morpheus_dfp.utils.regex_utils import iso_date_regex\n", "\n", "from morpheus.common import FileTypes\n", "from morpheus.cli.utils import get_log_levels\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb index 8bb35d5f78..2f8a62d833 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb @@ -45,15 +45,15 @@ "import typing\n", "from datetime import datetime\n", "\n", - "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", - "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", - "from dfp.stages.dfp_inference_stage import DFPInferenceStage\n", - "from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n", - "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", - "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", - "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", - "from dfp.stages.multi_file_source import MultiFileSource\n", - "from dfp.utils.regex_utils import iso_date_regex\n", + "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", + "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", + "from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage\n", + "from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n", + "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", + "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", + "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", + "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n", + "from morpheus_dfp.utils.regex_utils import iso_date_regex\n", "\n", "from morpheus.common import FileTypes\n", "from morpheus.common import FilterSource\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb index db57a85a1e..086786e9a1 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb @@ -45,14 +45,14 @@ "\n", "# When segment modules are imported, they're added to the module registry.\n", "# To avoid flake8 warnings about unused code, the noqa flag is used during import.\n", - "import dfp.modules # noqa: F401\n", + "import morpheus_dfp.modules # noqa: F401\n", "from morpheus import modules # noqa: F401\n", "from morpheus import loaders # noqa: F401\n", - "from dfp.utils.config_generator import ConfigGenerator\n", - "from dfp.utils.config_generator import generate_ae_config\n", - "from dfp.utils.dfp_arg_parser import DFPArgParser\n", - "from dfp.utils.schema_utils import Schema\n", - "from dfp.utils.schema_utils import SchemaBuilder\n", + "from morpheus_dfp.utils.config_generator import ConfigGenerator\n", + "from morpheus_dfp.utils.config_generator import generate_ae_config\n", + "from morpheus_dfp.utils.dfp_arg_parser import DFPArgParser\n", + "from morpheus_dfp.utils.schema_utils import Schema\n", + "from morpheus_dfp.utils.schema_utils import SchemaBuilder\n", "\n", "import morpheus.loaders # noqa: F401\n", "import morpheus.modules # noqa: F401\n", diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb index 1b1837d3e5..867f39e727 100644 --- a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb +++ b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb @@ -45,15 +45,15 @@ "import typing\n", "from datetime import datetime\n", "\n", - "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", - "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", - "from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n", - "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", - "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", - "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", - "from dfp.stages.dfp_training import DFPTraining\n", - "from dfp.stages.multi_file_source import MultiFileSource\n", - "from dfp.utils.regex_utils import iso_date_regex\n", + "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n", + "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n", + "from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n", + "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n", + "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n", + "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n", + "from morpheus_dfp.stages.dfp_training import DFPTraining\n", + "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n", + "from morpheus_dfp.utils.regex_utils import iso_date_regex\n", "\n", "from morpheus.common import FileTypes\n", "from morpheus.cli.utils import get_log_levels\n", diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py index f67460a6bd..b3e99ccdd4 100644 --- a/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py +++ b/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py @@ -23,17 +23,6 @@ import click import mlflow import pandas as pd -from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage -from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage -from dfp.stages.dfp_inference_stage import DFPInferenceStage -from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage -from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage -from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage -from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage -from dfp.stages.dfp_training import DFPTraining -from dfp.stages.dfp_viz_postproc import DFPVizPostprocStage -from dfp.stages.multi_file_source import MultiFileSource -from dfp.utils.regex_utils import iso_date_regex from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import get_package_relative_file @@ -54,6 +43,17 @@ from morpheus.utils.column_info import StringCatColumn from morpheus.utils.file_utils import date_extractor from morpheus.utils.logger import configure_logging +from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage +from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage +from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage +from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage +from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage +from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage +from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage +from morpheus_dfp.stages.dfp_training import DFPTraining +from morpheus_dfp.stages.dfp_viz_postproc import DFPVizPostprocStage +from morpheus_dfp.stages.multi_file_source import MultiFileSource +from morpheus_dfp.utils.regex_utils import iso_date_regex @click.command() diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py index 19ab53c59d..984b8fc799 100644 --- a/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py +++ b/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py @@ -23,17 +23,6 @@ import click import mlflow import pandas as pd -from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage -from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage -from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage -from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage -from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage -from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage -from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage -from dfp.stages.dfp_training import DFPTraining -from dfp.stages.dfp_viz_postproc import DFPVizPostprocStage -from dfp.stages.multi_file_source import MultiFileSource -from dfp.utils.regex_utils import iso_date_regex from morpheus.cli.utils import get_log_levels from morpheus.cli.utils import get_package_relative_file @@ -55,6 +44,17 @@ from morpheus.utils.column_info import StringCatColumn from morpheus.utils.file_utils import date_extractor from morpheus.utils.logger import configure_logging +from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage +from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage +from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage +from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage +from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage +from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage +from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage +from morpheus_dfp.stages.dfp_training import DFPTraining +from morpheus_dfp.stages.dfp_viz_postproc import DFPVizPostprocStage +from morpheus_dfp.stages.multi_file_source import MultiFileSource +from morpheus_dfp.utils.regex_utils import iso_date_regex @click.command() diff --git a/tests/examples/digital_fingerprinting/conftest.py b/tests/examples/digital_fingerprinting/conftest.py index 25fb2b54c0..c441a24e6b 100644 --- a/tests/examples/digital_fingerprinting/conftest.py +++ b/tests/examples/digital_fingerprinting/conftest.py @@ -91,7 +91,8 @@ def dfp_prod_in_sys_path( @pytest.fixture(name="dfp_message_meta") def dfp_message_meta_fixture(config, dataset_pandas: DatasetManager): import pandas as pd - from dfp.messages.dfp_message_meta import DFPMessageMeta + + from morpheus_dfp.messages.dfp_message_meta import DFPMessageMeta user_id = 'test_user' df = dataset_pandas['filter_probs.csv'] diff --git a/tests/examples/digital_fingerprinting/test_dfp_file_batcher_stage.py b/tests/examples/digital_fingerprinting/test_dfp_file_batcher_stage.py index 6120867926..fd94312f97 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_file_batcher_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_file_batcher_stage.py @@ -48,7 +48,7 @@ def file_specs_fixture(test_data_dir: str): def test_constructor(config: Config): - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage def date_conversion_func(x): return x @@ -71,7 +71,7 @@ def date_conversion_func(x): def test_constructor_deprecated_args(config: Config): """Test that the deprecated sampling_rate_s arg is still supported""" - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage with pytest.deprecated_call(): stage = DFPFileBatcherStage(config, lambda x: x, sampling_rate_s=55) @@ -82,14 +82,14 @@ def test_constructor_deprecated_args(config: Config): def test_constructor_both_sample_args_error(config: Config): """Test that an error is raised if both sampling and sampling_rate_s are specified""" - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage with pytest.raises(AssertionError): DFPFileBatcherStage(config, lambda x: x, sampling=55, sampling_rate_s=20) def test_on_data(config: Config, date_conversion_func: typing.Callable, file_specs: typing.List[fsspec.core.OpenFile]): - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage stage = DFPFileBatcherStage(config, date_conversion_func) assert not stage.on_data([]) @@ -108,7 +108,7 @@ def test_on_data_two_batches(config: Config, file_specs: typing.List[fsspec.core.OpenFile], test_data_dir: str): # Test with a one-minute window which should split the data into two batches - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage stage = DFPFileBatcherStage(config, date_conversion_func, period='min') batches = stage.on_data(file_specs) assert len(batches) == 2 @@ -131,7 +131,7 @@ def test_on_data_start_time(config: Config, file_specs: typing.List[fsspec.core.OpenFile], test_data_dir: str): # Test with a start time that excludes some files - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage stage = DFPFileBatcherStage(config, date_conversion_func, period='min', @@ -151,7 +151,7 @@ def test_on_data_end_time(config: Config, file_specs: typing.List[fsspec.core.OpenFile], test_data_dir: str): # Test with a end time that excludes some files - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage stage = DFPFileBatcherStage(config, date_conversion_func, period='min', @@ -172,7 +172,7 @@ def test_on_data_start_time_end_time(config: Config, file_specs: typing.List[fsspec.core.OpenFile], test_data_dir: str): # Test with a start & end time that excludes some files - from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage + from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage stage = DFPFileBatcherStage(config, date_conversion_func, period='min', diff --git a/tests/examples/digital_fingerprinting/test_dfp_file_to_df.py b/tests/examples/digital_fingerprinting/test_dfp_file_to_df.py index 19fa6add61..557818d8c7 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_file_to_df.py +++ b/tests/examples/digital_fingerprinting/test_dfp_file_to_df.py @@ -77,7 +77,7 @@ def test_single_object_to_dataframe_timeout(): @pytest.mark.usefixtures("restore_environ") def test_constructor(config: Config): - from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage + from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage # The user may have this already set, ensure it is undefined os.environ.pop('MORPHEUS_FILE_DOWNLOAD_TYPE', None) @@ -118,7 +118,7 @@ def test_get_or_create_dataframe_from_batch_cache_miss(mock_proc_df: mock.MagicM tmp_path: str, single_file_obj: fsspec.core.OpenFile, dataset_pandas: DatasetManager): - from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage + from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage config.ae.timestamp_column_name = 'v1' mock_dask_cluster.return_value = mock_dask_cluster mock_dask_client.return_value = mock_dask_client @@ -195,7 +195,7 @@ def test_get_or_create_dataframe_from_batch_cache_hit(mock_obf_to_df: mock.Magic use_convert_to_dataframe: bool, tmp_path: str, dataset_pandas: DatasetManager): - from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage + from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage config.ae.timestamp_column_name = 'v1' mock_dask_cluster.return_value = mock_dask_cluster mock_dask_client.return_value = mock_dask_client @@ -253,7 +253,7 @@ def test_get_or_create_dataframe_from_batch_none_noop(mock_obf_to_df: mock.Magic dl_type: str, use_convert_to_dataframe: bool, tmp_path: str): - from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage + from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage mock_dask_cluster.return_value = mock_dask_cluster mock_dask_client.return_value = mock_dask_client diff --git a/tests/examples/digital_fingerprinting/test_dfp_inference_stage.py b/tests/examples/digital_fingerprinting/test_dfp_inference_stage.py index 1175d0a61e..722fadb993 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_inference_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_inference_stage.py @@ -30,20 +30,20 @@ @pytest.fixture(name="mock_mlflow_client", autouse=True) def mock_mlflow_client_fixture(): - with mock.patch("dfp.stages.dfp_inference_stage.MlflowClient") as mock_mlflow_client: + with mock.patch("morpheus_dfp.stages.dfp_inference_stage.MlflowClient") as mock_mlflow_client: mock_mlflow_client.return_value = mock_mlflow_client yield mock_mlflow_client @pytest.fixture(name="mock_model_manager", autouse=True) def mock_model_manager_fixture(): - with mock.patch("dfp.stages.dfp_inference_stage.ModelManager") as mock_model_manager: + with mock.patch("morpheus_dfp.stages.dfp_inference_stage.ModelManager") as mock_model_manager: mock_model_manager.return_value = mock_model_manager yield mock_model_manager def test_constructor(config: Config, mock_mlflow_client: mock.MagicMock, mock_model_manager: mock.MagicMock): - from dfp.stages.dfp_inference_stage import DFPInferenceStage + from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage stage = DFPInferenceStage(config, model_name_formatter="test_model_name-{user_id}-{user_md5}") @@ -58,7 +58,7 @@ def test_constructor(config: Config, mock_mlflow_client: mock.MagicMock, mock_mo def test_get_model(config: Config, mock_mlflow_client: mock.MagicMock, mock_model_manager: mock.MagicMock): - from dfp.stages.dfp_inference_stage import DFPInferenceStage + from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage mock_model_cache = mock.MagicMock() mock_model_manager.load_user_model.return_value = mock_model_cache @@ -80,7 +80,7 @@ def test_on_data( control_message: "ControlMessage", # noqa: F821 log_level: int, dataset_pandas: DatasetManager): - from dfp.stages.dfp_inference_stage import DFPInferenceStage + from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage set_log_level(log_level) @@ -116,7 +116,7 @@ def test_on_data_get_model_error( mock_model_manager: mock.MagicMock, control_message: "ControlMessage", # noqa: F821 raise_error: bool): - from dfp.stages.dfp_inference_stage import DFPInferenceStage + from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage # There are two error conditions that can occur in get_model can return None or raise an error if raise_error: diff --git a/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py b/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py index 49fc093ba4..b39e05a03d 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py +++ b/tests/examples/digital_fingerprinting/test_dfp_mlflow_model_writer.py @@ -110,7 +110,7 @@ def mock_mlflow(): def test_constructor(config: Config): - from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage + from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage stage = DFPMLFlowModelWriterStage(config, model_name_formatter="test_model_name-{user_id}-{user_md5}", @@ -131,7 +131,7 @@ def test_constructor(config: Config): ("test_model_name-{user_id}-{user_md5}", 'test_城安宮川', "test_model_name-test_城安宮川-c9acc3dec97777c8b6fd8ae70a744ea8") ]) def test_user_id_to_model(config: Config, model_name_formatter: str, user_id: str, expected_val: str): - from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage + from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage stage = DFPMLFlowModelWriterStage(config, model_name_formatter=model_name_formatter) assert stage._controller.user_id_to_model(user_id) == expected_val @@ -149,7 +149,7 @@ def test_user_id_to_model(config: Config, model_name_formatter: str, user_id: st 'test_城安宮川', "/test/expr/dfp-test_城安宮川-test_城安宮川-c9acc3dec97777c8b6fd8ae70a744ea8")]) def test_user_id_to_experiment(config: Config, experiment_name_formatter: str, user_id: str, expected_val: str): - from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage + from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage stage = DFPMLFlowModelWriterStage(config, model_name_formatter="dfp-{user_id}", @@ -178,7 +178,7 @@ def verify_apply_model_permissions(mock_requests: MockedRequests, def test_apply_model_permissions(config: Config, databricks_env: dict, mock_requests: MockedRequests): - from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage + from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage databricks_permissions = OrderedDict([('group1', 'CAN_READ'), ('group2', 'CAN_WRITE')]) stage = DFPMLFlowModelWriterStage(config, databricks_permissions=databricks_permissions, timeout=10) stage._controller._apply_model_permissions("test_experiment") @@ -206,7 +206,7 @@ def test_apply_model_permissions_no_perms_error(config: Config, else: os.environ.pop("DATABRICKS_TOKEN", None) - from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage + from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage stage = DFPMLFlowModelWriterStage(config) with pytest.raises(RuntimeError): stage._controller._apply_model_permissions("test_experiment") @@ -217,7 +217,7 @@ def test_apply_model_permissions_no_perms_error(config: Config, @pytest.mark.usefixtures("databricks_env") def test_apply_model_permissions_requests_error(config: Config, mock_requests: MockedRequests): - from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage + from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage mock_requests.get.side_effect = RuntimeError("test error") stage = DFPMLFlowModelWriterStage(config, timeout=10) @@ -238,9 +238,9 @@ def test_on_data( databricks_env: dict, databricks_permissions: dict, tracking_uri: str): - from dfp.messages.dfp_message_meta import DFPMessageMeta - from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage - from dfp.stages.dfp_mlflow_model_writer import conda_env + from morpheus_dfp.messages.dfp_message_meta import DFPMessageMeta + from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage + from morpheus_dfp.stages.dfp_mlflow_model_writer import conda_env should_apply_permissions = (databricks_permissions is not None and tracking_uri == "databricks") diff --git a/tests/examples/digital_fingerprinting/test_dfp_postprocessing_stage.py b/tests/examples/digital_fingerprinting/test_dfp_postprocessing_stage.py index b173c145dc..31176221a9 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_postprocessing_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_postprocessing_stage.py @@ -27,7 +27,7 @@ def test_constructor(config: Config): - from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage + from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage stage = DFPPostprocessingStage(config) assert isinstance(stage, SinglePortStage) assert stage._needed_columns['event_time'] == TypeId.STRING @@ -36,13 +36,13 @@ def test_constructor(config: Config): @pytest.mark.usefixtures("reset_loglevel") @pytest.mark.parametrize('use_on_data', [True, False]) @pytest.mark.parametrize('log_level', [logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]) -@mock.patch('dfp.stages.dfp_postprocessing_stage.datetime') +@mock.patch('morpheus_dfp.stages.dfp_postprocessing_stage.datetime') def test_process_events_on_data(mock_datetime: mock.MagicMock, config: Config, control_message: ControlMessage, use_on_data: bool, log_level: int): - from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage + from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage mock_dt_obj = mock.MagicMock() mock_dt_obj.strftime.return_value = '2021-01-01T00:00:00Z' @@ -68,7 +68,7 @@ def test_process_events_on_data(mock_datetime: mock.MagicMock, def test_on_data_none(config: Config): - from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage + from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage stage = DFPPostprocessingStage(config) assert stage.on_data(None) is None mock_payload = mock.MagicMock() diff --git a/tests/examples/digital_fingerprinting/test_dfp_preprocessing_stage.py b/tests/examples/digital_fingerprinting/test_dfp_preprocessing_stage.py index 538e20425e..49a2feea90 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_preprocessing_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_preprocessing_stage.py @@ -28,7 +28,7 @@ def test_constructor(config: Config): - from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage + from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage schema = DataFrameInputSchema() stage = DFPPreprocessingStage(config, input_schema=schema) @@ -43,7 +43,7 @@ def test_process_features( control_message: "ControlMessage", # noqa: F821 dataset_pandas: DatasetManager, log_level: int): - from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage + from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage set_log_level(log_level) diff --git a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py index 01504d7d47..b8f7e8cd18 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_rolling_window_stage.py @@ -41,7 +41,7 @@ def build_mock_user_cache(user_id: str = 'test_user', def test_constructor(config: Config): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') assert isinstance(stage, SinglePortStage) @@ -53,7 +53,7 @@ def test_constructor(config: Config): def test_get_user_cache_hit(config: Config): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -65,8 +65,8 @@ def test_get_user_cache_hit(config: Config): def test_get_user_cache_miss(config: Config): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage - from dfp.utils.cached_user_window import CachedUserWindow + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.utils.cached_user_window import CachedUserWindow config.ae.timestamp_column_name = 'test_timestamp_col' stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -85,7 +85,7 @@ def test_build_window_no_new( config: Config, dfp_message_meta: "DFPMessageMeta" # noqa: F821 ): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -99,7 +99,7 @@ def test_build_window_not_enough_data( config: Config, dfp_message_meta: "DFPMessageMeta" # noqa: F821 ): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -112,7 +112,7 @@ def test_build_window_min_increment( config: Config, dfp_message_meta: "DFPMessageMeta" # noqa: F821 ): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -125,7 +125,7 @@ def test_build_window_invalid( config: Config, dfp_message_meta: "DFPMessageMeta" # noqa: F821 ): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -144,7 +144,7 @@ def test_build_window_overlap( config: Config, dfp_message_meta: "DFPMessageMeta" # noqa: F821 ): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') @@ -165,9 +165,8 @@ def test_build_window( use_on_data: bool, dfp_message_meta: "DFPMessageMeta", # noqa: F821 dataset_pandas: DatasetManager): - from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage - from morpheus.messages import ControlMessage + from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage stage = DFPRollingWindowStage(config, min_history=5, min_increment=7, max_history=100, cache_dir='/test/path/cache') diff --git a/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py b/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py index 8189df73fe..8563fd7f9a 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py +++ b/tests/examples/digital_fingerprinting/test_dfp_split_users_stage.py @@ -27,7 +27,7 @@ def test_constructor(config: Config): - from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage + from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage stage = DFPSplitUsersStage(config, include_generic=False, include_individual=True) assert isinstance(stage, SinglePortStage) @@ -64,7 +64,7 @@ def test_extract_users(config: Config, include_individual: bool, skip_users: typing.List[str], only_users: typing.List[str]): - from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage + from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage config.ae.userid_column_name = "From" config.ae.fallback_username = "testy_testerson" @@ -120,7 +120,7 @@ def test_extract_users(config: Config, def test_extract_users_none_to_empty(config: Config): - from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage + from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage stage = DFPSplitUsersStage(config, include_generic=True, include_individual=True) assert not stage.extract_users(None) diff --git a/tests/examples/digital_fingerprinting/test_dfp_training.py b/tests/examples/digital_fingerprinting/test_dfp_training.py index 60cd545eab..e4683c1ea2 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_training.py +++ b/tests/examples/digital_fingerprinting/test_dfp_training.py @@ -25,7 +25,7 @@ def test_constructor(config: Config): - from dfp.stages.dfp_training import DFPTraining + from morpheus_dfp.stages.dfp_training import DFPTraining stage = DFPTraining(config, model_kwargs={'test': 'this'}, epochs=40, validation_size=0.5) assert isinstance(stage, SinglePortStage) @@ -36,24 +36,23 @@ def test_constructor(config: Config): @pytest.mark.parametrize('validation_size', [-1, -0.2, 1, 5]) def test_constructor_bad_validation_size(config: Config, validation_size: float): - from dfp.stages.dfp_training import DFPTraining + from morpheus_dfp.stages.dfp_training import DFPTraining with pytest.raises(ValueError): DFPTraining(config, validation_size=validation_size) @pytest.mark.parametrize('validation_size', [0., 0.2]) -@mock.patch('dfp.stages.dfp_training.AutoEncoder') -@mock.patch('dfp.stages.dfp_training.train_test_split') +@mock.patch('morpheus_dfp.stages.dfp_training.AutoEncoder') +@mock.patch('morpheus_dfp.stages.dfp_training.train_test_split') def test_on_data(mock_train_test_split: mock.MagicMock, mock_ae: mock.MagicMock, config: Config, dataset_pandas: DatasetManager, validation_size: float): - from dfp.messages.dfp_message_meta import DFPMessageMeta - from dfp.stages.dfp_training import DFPTraining - from morpheus.messages import ControlMessage + from morpheus_dfp.messages.dfp_message_meta import DFPMessageMeta + from morpheus_dfp.stages.dfp_training import DFPTraining mock_ae.return_value = mock_ae diff --git a/tests/examples/digital_fingerprinting/test_dfp_viz_postproc.py b/tests/examples/digital_fingerprinting/test_dfp_viz_postproc.py index 571f976712..ea15b2caee 100644 --- a/tests/examples/digital_fingerprinting/test_dfp_viz_postproc.py +++ b/tests/examples/digital_fingerprinting/test_dfp_viz_postproc.py @@ -54,7 +54,7 @@ def expected_df_fixture(config: Config, control_message: "ControlMessage"): # n def test_constructor(config: Config): - from dfp.stages.dfp_viz_postproc import DFPVizPostprocStage + from morpheus_dfp.stages.dfp_viz_postproc import DFPVizPostprocStage stage = DFPVizPostprocStage(config, period='M', output_dir='/fake/test/dir', output_prefix='test_prefix') assert isinstance(stage, SinglePortStage) @@ -72,7 +72,7 @@ def test_postprocess( control_message: "ControlMessage", # noqa: F821 expected_df: pd.DataFrame, dataset_pandas: DatasetManager): - from dfp.stages.dfp_viz_postproc import DFPVizPostprocStage + from morpheus_dfp.stages.dfp_viz_postproc import DFPVizPostprocStage # _postprocess doesn't write to disk, so the fake output_dir, shouldn't be an issue stage = DFPVizPostprocStage(config, period='min', output_dir='/fake/test/dir', output_prefix='test_prefix') @@ -88,7 +88,7 @@ def test_write_to_files( control_message: "ControlMessage", # noqa: F821 expected_df: pd.DataFrame, dataset_pandas: DatasetManager): - from dfp.stages.dfp_viz_postproc import DFPVizPostprocStage + from morpheus_dfp.stages.dfp_viz_postproc import DFPVizPostprocStage stage = DFPVizPostprocStage(config, period='min', output_dir=tmp_path, output_prefix='test_prefix_') assert stage._write_to_files(control_message) is control_message diff --git a/tests/examples/digital_fingerprinting/test_multifile_source.py b/tests/examples/digital_fingerprinting/test_multifile_source.py index c430e10c12..e0f982ce60 100644 --- a/tests/examples/digital_fingerprinting/test_multifile_source.py +++ b/tests/examples/digital_fingerprinting/test_multifile_source.py @@ -25,7 +25,7 @@ def test_constructor(config: Config): - from dfp.stages.multi_file_source import MultiFileSource + from morpheus_dfp.stages.multi_file_source import MultiFileSource batch_size = 1234 n_threads = 13 @@ -43,7 +43,7 @@ def test_constructor(config: Config): def test_generate_frames_fsspec(config: Config, tmp_path: str): - from dfp.stages.multi_file_source import MultiFileSource + from morpheus_dfp.stages.multi_file_source import MultiFileSource file_glob = os.path.join(TEST_DIRS.tests_data_dir, 'appshield', 'snapshot-1', '*.json') temp_glob = os.path.join(tmp_path, '*.json') # this won't match anything @@ -65,7 +65,7 @@ def test_generate_frames_fsspec(config: Config, tmp_path: str): @mock.patch('time.sleep') def test_polling_generate_frames_fsspec(amock_time: mock.MagicMock, config: Config, tmp_path: str): - from dfp.stages.multi_file_source import MultiFileSource + from morpheus_dfp.stages.multi_file_source import MultiFileSource file_glob = os.path.join(TEST_DIRS.tests_data_dir, 'appshield', 'snapshot-1', '*.json') temp_glob = os.path.join(tmp_path, '*.json') # this won't match anything @@ -88,7 +88,7 @@ def test_polling_generate_frames_fsspec(amock_time: mock.MagicMock, config: Conf def test_generate_frames_fsspec_no_files(config: Config, tmp_path: str): - from dfp.stages.multi_file_source import MultiFileSource + from morpheus_dfp.stages.multi_file_source import MultiFileSource assert os.listdir(tmp_path) == [] diff --git a/tests/examples/digital_fingerprinting/test_write_to_s3_stage.py b/tests/examples/digital_fingerprinting/test_write_to_s3_stage.py index 6c951ff175..beab5e3f2a 100644 --- a/tests/examples/digital_fingerprinting/test_write_to_s3_stage.py +++ b/tests/examples/digital_fingerprinting/test_write_to_s3_stage.py @@ -20,7 +20,7 @@ def test_constructor(config: Config): - from dfp.stages.write_to_s3_stage import WriteToS3Stage + from morpheus_dfp.stages.write_to_s3_stage import WriteToS3Stage mock_s3_writer = mock.MagicMock() stage = WriteToS3Stage(config, s3_writer=mock_s3_writer) diff --git a/tests/examples/digital_fingerprinting/utils/test_config_generator.py b/tests/examples/digital_fingerprinting/utils/test_config_generator.py index 40d4f37b67..eb4c850f94 100644 --- a/tests/examples/digital_fingerprinting/utils/test_config_generator.py +++ b/tests/examples/digital_fingerprinting/utils/test_config_generator.py @@ -23,7 +23,7 @@ @pytest.fixture(name="dfp_arg_parser") def dfp_arg_parser_fixture(): - from dfp.utils.dfp_arg_parser import DFPArgParser + from morpheus_dfp.utils.dfp_arg_parser import DFPArgParser dfp_arg_parser = DFPArgParser(skip_user=["unittest-skip-user"], only_user=["unittest-only-user"], start_time=datetime(1993, 4, 5, 6, 7, 8), @@ -43,13 +43,13 @@ def dfp_arg_parser_fixture(): @pytest.fixture(name="schema") def schema_fixture(config: Config): - from dfp.utils.schema_utils import SchemaBuilder + from morpheus_dfp.utils.schema_utils import SchemaBuilder schema_builder = SchemaBuilder(config, "duo") yield schema_builder.build_schema() def test_constructor(config: Config, dfp_arg_parser: "DFPArgParser", schema: "Schema"): # noqa: F821 - from dfp.utils.config_generator import ConfigGenerator + from morpheus_dfp.utils.config_generator import ConfigGenerator config_generator = ConfigGenerator(config=config, dfp_arg_parser=dfp_arg_parser, schema=schema, encoding="latin1")