Update dfp imports in examples and tests

Signed-off-by: Anuradha Karuppiah <[email protected]>
nv-morpheus · Sep 23, 2024 · f2db2b9 · f2db2b9
1 parent d093166
commit f2db2b9
Show file tree

Hide file tree

Showing 25 changed files with 164 additions and 164 deletions.
diff --git a/examples/digital_fingerprinting/production/grafana/run.py b/examples/digital_fingerprinting/production/grafana/run.py
@@ -26,17 +26,6 @@
 import logging_loki
 import mlflow
 import pandas as pd
-from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
-from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
-from dfp.stages.dfp_inference_stage import DFPInferenceStage
-from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
-from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage
-from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
-from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
-from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
-from dfp.stages.dfp_training import DFPTraining
-from dfp.stages.multi_file_source import MultiFileSource
-from dfp.utils.regex_utils import iso_date_regex
 
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import get_package_relative_file
@@ -61,6 +50,17 @@
 from morpheus.utils.column_info import StringCatColumn
 from morpheus.utils.file_utils import date_extractor
 from morpheus.utils.logger import configure_logging
+from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
+from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
+from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage
+from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
+from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage
+from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
+from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
+from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
+from morpheus_dfp.stages.dfp_training import DFPTraining
+from morpheus_dfp.stages.multi_file_source import MultiFileSource
+from morpheus_dfp.utils.regex_utils import iso_date_regex
 
 
 def _file_type_name_to_enum(file_type: str) -> FileTypes:

diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py b/examples/digital_fingerprinting/production/morpheus/benchmarks/benchmark_conf_generator.py
@@ -25,10 +25,11 @@
 
 import mlflow
 import pandas as pd
-from dfp.utils.config_generator import ConfigGenerator
-from dfp.utils.config_generator import generate_ae_config
-from dfp.utils.dfp_arg_parser import DFPArgParser
-from dfp.utils.schema_utils import SchemaBuilder
+
+from morpheus_dfp.utils.config_generator import ConfigGenerator
+from morpheus_dfp.utils.config_generator import generate_ae_config
+from morpheus_dfp.utils.dfp_arg_parser import DFPArgParser
+from morpheus_dfp.utils.schema_utils import SchemaBuilder
 
 logger = logging.getLogger(f"morpheus.{__name__}")
 

diff --git a/...ples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py b/...ples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py
@@ -20,23 +20,11 @@
 import typing
 
 import boto3
-import dfp.modules  # noqa: F401 # pylint:disable=unused-import
 import pytest
-from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
-from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
-from dfp.stages.dfp_inference_stage import DFPInferenceStage
-from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
-from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage
-from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
-from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
-from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
-from dfp.stages.dfp_training import DFPTraining
-from dfp.stages.multi_file_source import MultiFileSource
-from dfp.utils.regex_utils import iso_date_regex
-from dfp.utils.schema_utils import Schema
 
 import morpheus.loaders  # noqa: F401 # pylint:disable=unused-import
 import morpheus.modules  # noqa: F401 # pylint:disable=unused-import
+import morpheus_dfp.modules  # noqa: F401 # pylint:disable=unused-import
 from benchmarks.benchmark_conf_generator import BenchmarkConfGenerator
 from benchmarks.benchmark_conf_generator import load_json
 from benchmarks.benchmark_conf_generator import set_mlflow_tracking_uri
@@ -53,6 +41,18 @@
 from morpheus.utils.column_info import DataFrameInputSchema
 from morpheus.utils.file_utils import date_extractor
 from morpheus.utils.logger import configure_logging
+from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
+from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
+from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage
+from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
+from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage
+from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
+from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
+from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
+from morpheus_dfp.stages.dfp_training import DFPTraining
+from morpheus_dfp.stages.multi_file_source import MultiFileSource
+from morpheus_dfp.utils.regex_utils import iso_date_regex
+from morpheus_dfp.utils.schema_utils import Schema
 
 logger = logging.getLogger(f"morpheus.{__name__}")
 

diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_inference.ipynb
@@ -47,15 +47,15 @@
     "from datetime import datetime\n",
     "from functools import partial\n",
     "\n",
-    "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
-    "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
-    "from dfp.stages.dfp_inference_stage import DFPInferenceStage\n",
-    "from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n",
-    "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
-    "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
-    "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
-    "from dfp.stages.multi_file_source import MultiFileSource\n",
-    "from dfp.utils.regex_utils import iso_date_regex\n",
+    "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
+    "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
+    "from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage\n",
+    "from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n",
+    "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
+    "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
+    "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
+    "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n",
+    "from morpheus_dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
     "from morpheus.common import FileTypes\n",
     "from morpheus.common import FilterSource\n",

diff --git a/.../digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb b/.../digital_fingerprinting/production/morpheus/notebooks/dfp_azure_integrated_training.ipynb
@@ -45,12 +45,12 @@
     "\n",
     "# When segment modules are imported, they're added to the module registry.\n",
     "# To avoid flake8 warnings about unused code, the noqa flag is used during import.\n",
-    "import dfp.modules  # noqa: F401\n",
-    "from dfp.utils.config_generator import ConfigGenerator\n",
-    "from dfp.utils.config_generator import generate_ae_config\n",
-    "from dfp.utils.dfp_arg_parser import DFPArgParser\n",
-    "from dfp.utils.schema_utils import Schema\n",
-    "from dfp.utils.schema_utils import SchemaBuilder\n",
+    "import morpheus_dfp.modules  # noqa: F401\n",
+    "from morpheus_dfp.utils.config_generator import ConfigGenerator\n",
+    "from morpheus_dfp.utils.config_generator import generate_ae_config\n",
+    "from morpheus_dfp.utils.dfp_arg_parser import DFPArgParser\n",
+    "from morpheus_dfp.utils.schema_utils import Schema\n",
+    "from morpheus_dfp.utils.schema_utils import SchemaBuilder\n",
     "\n",
     "import morpheus.loaders  # noqa: F401\n",
     "import morpheus.modules  # noqa: F401\n",

diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_azure_training.ipynb
@@ -45,15 +45,15 @@
     "import typing\n",
     "from datetime import datetime\n",
     "\n",
-    "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
-    "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
-    "from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n",
-    "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
-    "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
-    "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
-    "from dfp.stages.dfp_training import DFPTraining\n",
-    "from dfp.stages.multi_file_source import MultiFileSource\n",
-    "from dfp.utils.regex_utils import iso_date_regex\n",
+    "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
+    "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
+    "from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n",
+    "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
+    "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
+    "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
+    "from morpheus_dfp.stages.dfp_training import DFPTraining\n",
+    "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n",
+    "from morpheus_dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
     "from morpheus.common import FileTypes\n",
     "from morpheus.cli.utils import get_log_levels\n",

diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_inference.ipynb
@@ -45,15 +45,15 @@
     "import typing\n",
     "from datetime import datetime\n",
     "\n",
-    "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
-    "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
-    "from dfp.stages.dfp_inference_stage import DFPInferenceStage\n",
-    "from dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n",
-    "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
-    "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
-    "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
-    "from dfp.stages.multi_file_source import MultiFileSource\n",
-    "from dfp.utils.regex_utils import iso_date_regex\n",
+    "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
+    "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
+    "from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage\n",
+    "from morpheus_dfp.stages.dfp_postprocessing_stage import DFPPostprocessingStage\n",
+    "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
+    "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
+    "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
+    "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n",
+    "from morpheus_dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
     "from morpheus.common import FileTypes\n",
     "from morpheus.common import FilterSource\n",

diff --git a/...es/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb b/...es/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_integrated_training.ipynb
@@ -45,14 +45,14 @@
     "\n",
     "# When segment modules are imported, they're added to the module registry.\n",
     "# To avoid flake8 warnings about unused code, the noqa flag is used during import.\n",
-    "import dfp.modules  # noqa: F401\n",
+    "import morpheus_dfp.modules  # noqa: F401\n",
     "from morpheus import modules  # noqa: F401\n",
     "from morpheus import loaders  # noqa: F401\n",
-    "from dfp.utils.config_generator import ConfigGenerator\n",
-    "from dfp.utils.config_generator import generate_ae_config\n",
-    "from dfp.utils.dfp_arg_parser import DFPArgParser\n",
-    "from dfp.utils.schema_utils import Schema\n",
-    "from dfp.utils.schema_utils import SchemaBuilder\n",
+    "from morpheus_dfp.utils.config_generator import ConfigGenerator\n",
+    "from morpheus_dfp.utils.config_generator import generate_ae_config\n",
+    "from morpheus_dfp.utils.dfp_arg_parser import DFPArgParser\n",
+    "from morpheus_dfp.utils.schema_utils import Schema\n",
+    "from morpheus_dfp.utils.schema_utils import SchemaBuilder\n",
     "\n",
     "import morpheus.loaders  # noqa: F401\n",
     "import morpheus.modules  # noqa: F401\n",

diff --git a/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb b/examples/digital_fingerprinting/production/morpheus/notebooks/dfp_duo_training.ipynb
@@ -45,15 +45,15 @@
     "import typing\n",
     "from datetime import datetime\n",
     "\n",
-    "from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
-    "from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
-    "from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n",
-    "from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
-    "from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
-    "from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
-    "from dfp.stages.dfp_training import DFPTraining\n",
-    "from dfp.stages.multi_file_source import MultiFileSource\n",
-    "from dfp.utils.regex_utils import iso_date_regex\n",
+    "from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage\n",
+    "from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage\n",
+    "from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage\n",
+    "from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage\n",
+    "from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage\n",
+    "from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage\n",
+    "from morpheus_dfp.stages.dfp_training import DFPTraining\n",
+    "from morpheus_dfp.stages.multi_file_source import MultiFileSource\n",
+    "from morpheus_dfp.utils.regex_utils import iso_date_regex\n",
     "\n",
     "from morpheus.common import FileTypes\n",
     "from morpheus.cli.utils import get_log_levels\n",

diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_azure_pipeline.py
@@ -23,17 +23,6 @@
 import click
 import mlflow
 import pandas as pd
-from dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
-from dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
-from dfp.stages.dfp_inference_stage import DFPInferenceStage
-from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
-from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
-from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
-from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
-from dfp.stages.dfp_training import DFPTraining
-from dfp.stages.dfp_viz_postproc import DFPVizPostprocStage
-from dfp.stages.multi_file_source import MultiFileSource
-from dfp.utils.regex_utils import iso_date_regex
 
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import get_package_relative_file
@@ -54,6 +43,17 @@
 from morpheus.utils.column_info import StringCatColumn
 from morpheus.utils.file_utils import date_extractor
 from morpheus.utils.logger import configure_logging
+from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
+from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
+from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage
+from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
+from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
+from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
+from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
+from morpheus_dfp.stages.dfp_training import DFPTraining
+from morpheus_dfp.stages.dfp_viz_postproc import DFPVizPostprocStage
+from morpheus_dfp.stages.multi_file_source import MultiFileSource
+from morpheus_dfp.utils.regex_utils import iso_date_regex
 
 
 @click.command()

diff --git a/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py b/examples/digital_fingerprinting/visualization/dfp_viz_duo_pipeline.py
@@ -23,17 +23,6 @@
 import click
 import mlflow
 import pandas as pd
-from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
-from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
-from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage
-from dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
-from dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
-from dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
-from dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
-from dfp.stages.dfp_training import DFPTraining
-from dfp.stages.dfp_viz_postproc import DFPVizPostprocStage
-from dfp.stages.multi_file_source import MultiFileSource
-from dfp.utils.regex_utils import iso_date_regex
 
 from morpheus.cli.utils import get_log_levels
 from morpheus.cli.utils import get_package_relative_file
@@ -55,6 +44,17 @@
 from morpheus.utils.column_info import StringCatColumn
 from morpheus.utils.file_utils import date_extractor
 from morpheus.utils.logger import configure_logging
+from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
+from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
+from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage
+from morpheus_dfp.stages.dfp_mlflow_model_writer import DFPMLFlowModelWriterStage
+from morpheus_dfp.stages.dfp_preprocessing_stage import DFPPreprocessingStage
+from morpheus_dfp.stages.dfp_rolling_window_stage import DFPRollingWindowStage
+from morpheus_dfp.stages.dfp_split_users_stage import DFPSplitUsersStage
+from morpheus_dfp.stages.dfp_training import DFPTraining
+from morpheus_dfp.stages.dfp_viz_postproc import DFPVizPostprocStage
+from morpheus_dfp.stages.multi_file_source import MultiFileSource
+from morpheus_dfp.utils.regex_utils import iso_date_regex
 
 
 @click.command()

diff --git a/tests/examples/digital_fingerprinting/conftest.py b/tests/examples/digital_fingerprinting/conftest.py
@@ -91,7 +91,8 @@ def dfp_prod_in_sys_path(
 @pytest.fixture(name="dfp_message_meta")
 def dfp_message_meta_fixture(config, dataset_pandas: DatasetManager):
     import pandas as pd
-    from dfp.messages.dfp_message_meta import DFPMessageMeta
+
+    from morpheus_dfp.messages.dfp_message_meta import DFPMessageMeta
 
     user_id = 'test_user'
     df = dataset_pandas['filter_probs.csv']