From 8235316abd9dbca88795230c9adfeac8d3934474 Mon Sep 17 00:00:00 2001 From: David Gardner <96306125+dagardner-nv@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:25:10 -0700 Subject: [PATCH] Fix `Unregistered type : mrc::pymrc::coro::BoostFibersMainPyAwaitable` error (#1869) * Ensure `BoostFibersMainPyAwaitable` is registered by importing `mrc.core.coro` * Work-around for undefined symbol link error with c-ares * Fix paths to data files in validation scripts * Fix path to C++ files for CI checks * Replace usage of `ie.` to avoid documentation CI error Closes #1868 ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - David Gardner (https://github.com/dagardner-nv) Approvers: - Anuradha Karuppiah (https://github.com/AnuradhaKaruppiah) URL: https://github.com/nv-morpheus/Morpheus/pull/1869 --- ci/scripts/common.sh | 2 +- conda/environments/all_cuda-121_arch-x86_64.yaml | 1 + conda/environments/dev_cuda-121_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + docs/source/developer_guide/contributing.md | 2 +- python/morpheus/morpheus/_lib/stages/__init__.pyi | 1 + python/morpheus/morpheus/_lib/stages/module.cpp | 3 +++ scripts/validation/val-run-pipeline.sh | 14 +++++++------- 8 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh index 258fe689c2..ebd4a2a39f 100644 --- a/ci/scripts/common.sh +++ b/ci/scripts/common.sh @@ -27,7 +27,7 @@ export PY_DIRS="${PY_ROOT} ci/scripts" export BASE_SHA=${CHANGE_TARGET:-${BASE_SHA:-$(${SCRIPT_DIR}/gitutils.py get_merge_target)}} export COMMIT_SHA=${GIT_COMMIT:-${COMMIT_SHA:-HEAD}} -export CPP_FILE_REGEX='^(\.\/)?(examples|morpheus|tests)\/.*\.(cc|cpp|h|hpp)$' +export CPP_FILE_REGEX='^(\.\/)?(examples|python|tests)\/.*\.(cc|cpp|h|hpp)$' export PYTHON_FILE_REGEX='^(\.\/)?(?!\.|build|external).*\.(py|pyx|pxd)$' # Use these options to skip any of the checks diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 003b1c017d..201b487b98 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -17,6 +17,7 @@ dependencies: - benchmark=1.8.3 - boto3 - breathe=4.35.0 +- c-ares=1.32 - ccache - clangdev=16 - click>=8 diff --git a/conda/environments/dev_cuda-121_arch-x86_64.yaml b/conda/environments/dev_cuda-121_arch-x86_64.yaml index 8416486552..2b00aaeab2 100644 --- a/conda/environments/dev_cuda-121_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-121_arch-x86_64.yaml @@ -14,6 +14,7 @@ dependencies: - beautifulsoup4=4.12 - benchmark=1.8.3 - breathe=4.35.0 +- c-ares=1.32 - ccache - clangdev=16 - click>=8 diff --git a/dependencies.yaml b/dependencies.yaml index 335abef0e7..e8f5525696 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -233,6 +233,7 @@ dependencies: # Non-Compiler Dependencies - automake=1.16.5 # Needed for DOCA build + - c-ares=1.32 # 1.33 causes an undefined symbol error - ccache - cmake=3.27 - cuda-cudart-dev=12.1 diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 052f58b7cf..38439e9d43 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -126,7 +126,7 @@ This workflow utilizes a Docker container to set up most dependencies ensuring a ```bash ./docker/run_container_dev.sh ``` - 1. The container tag follows the same rules as `build_container_dev.sh` and will default to the current `YYMMDD`. Specify the desired tag with `DOCKER_IMAGE_TAG`. i.e. `DOCKER_IMAGE_TAG=my_tag ./docker/run_container_dev.sh` + 1. The container tag follows the same rules as `build_container_dev.sh` and will default to the current `YYMMDD`. Specify the desired tag with `DOCKER_IMAGE_TAG`. For example, `DOCKER_IMAGE_TAG=my_tag ./docker/run_container_dev.sh` 2. This will automatically mount the current working directory to `/workspace`. 3. Some of the validation tests require launching the Morpheus models Docker container within the Morpheus container. To enable this you will need to grant the Morpheus container access to your host OS's Docker socket file with: ```bash diff --git a/python/morpheus/morpheus/_lib/stages/__init__.pyi b/python/morpheus/morpheus/_lib/stages/__init__.pyi index 12a274c601..86a91e9088 100644 --- a/python/morpheus/morpheus/_lib/stages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/stages/__init__.pyi @@ -10,6 +10,7 @@ import morpheus._lib.stages import typing from morpheus._lib.common import FilterSource import morpheus._lib.common +import mrc.core.coro import mrc.core.segment import os diff --git a/python/morpheus/morpheus/_lib/stages/module.cpp b/python/morpheus/morpheus/_lib/stages/module.cpp index dc6537fa7d..3709e67394 100644 --- a/python/morpheus/morpheus/_lib/stages/module.cpp +++ b/python/morpheus/morpheus/_lib/stages/module.cpp @@ -73,6 +73,9 @@ PYBIND11_MODULE(stages, _module) // Make sure to load mrc.core.segment to get ObjectProperties mrc::pymrc::import(_module, "mrc.core.segment"); + // Import the mrc coro module + mrc::pymrc::import(_module, "mrc.core.coro"); + mrc::pymrc::from_import(_module, "morpheus._lib.common", "FilterSource"); py::class_, diff --git a/scripts/validation/val-run-pipeline.sh b/scripts/validation/val-run-pipeline.sh index 65641a1370..0af859a6e0 100755 --- a/scripts/validation/val-run-pipeline.sh +++ b/scripts/validation/val-run-pipeline.sh @@ -41,7 +41,7 @@ function run_pipeline_sid_minibert(){ pipeline-nlp --model_seq_length=256 \ from-file --filename=${INPUT_FILE} \ deserialize \ - preprocess --vocab_hash_file=${MORPHEUS_ROOT}/morpheus/data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \ + preprocess --vocab_hash_file=${MORPHEUS_ROOT}/python/morpheus/morpheus/data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \ ${INFERENCE_STAGE} \ monitor --description "Inference Rate" --smoothing=0.001 --unit inf \ add-class --prefix="si_" \ @@ -62,7 +62,7 @@ function run_pipeline_sid_bert(){ pipeline-nlp --model_seq_length=256 \ from-file --filename=${INPUT_FILE} \ deserialize \ - preprocess --vocab_hash_file=${MORPHEUS_ROOT}/morpheus/data/bert-base-cased-hash.txt --truncation=True --do_lower_case=False --add_special_tokens=False \ + preprocess --vocab_hash_file=${MORPHEUS_ROOT}/python/morpheus/morpheus/data/bert-base-cased-hash.txt --truncation=True --do_lower_case=False --add_special_tokens=False \ ${INFERENCE_STAGE} \ monitor --description "Inference Rate" --smoothing=0.001 --unit inf \ add-class --prefix="si_" \ @@ -80,7 +80,7 @@ function run_pipeline_abp_nvsmi(){ VAL_OUTPUT=$5 morpheus --log_level=DEBUG run --num_threads=$(nproc) --pipeline_batch_size=1024 --model_max_batch_size=1024 --use_cpp=${USE_CPP} \ - pipeline-fil --columns_file=${MORPHEUS_ROOT}/morpheus/data/columns_fil.txt \ + pipeline-fil --columns_file=${MORPHEUS_ROOT}/python/morpheus/morpheus/data/columns_fil.txt \ from-file --filename=${INPUT_FILE} \ deserialize \ preprocess \ @@ -101,10 +101,10 @@ function run_pipeline_phishing_email(){ VAL_OUTPUT=$5 morpheus --log_level=DEBUG run --num_threads=$(nproc) --pipeline_batch_size=1024 --model_max_batch_size=32 --use_cpp=${USE_CPP} \ - pipeline-nlp --model_seq_length=128 --labels_file=${MORPHEUS_ROOT}/morpheus/data/labels_phishing.txt \ + pipeline-nlp --model_seq_length=128 --labels_file=${MORPHEUS_ROOT}/python/morpheus/morpheus/data/labels_phishing.txt \ from-file --filename=${INPUT_FILE} \ deserialize \ - preprocess --vocab_hash_file=${MORPHEUS_ROOT}/morpheus/data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \ + preprocess --vocab_hash_file=${MORPHEUS_ROOT}/python/morpheus/morpheus/data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \ ${INFERENCE_STAGE} \ monitor --description "Inference Rate" --smoothing=0.001 --unit inf \ add-class --label=is_phishing --threshold=0.7 \ @@ -122,7 +122,7 @@ function run_pipeline_hammah_user123(){ VAL_OUTPUT=$5 morpheus --log_level=DEBUG run --num_threads=$(nproc) --pipeline_batch_size=1024 --model_max_batch_size=1024 --use_cpp=${USE_CPP} \ - pipeline-ae --columns_file="${MORPHEUS_ROOT}/morpheus/data/columns_ae_cloudtrail.txt" --userid_filter="user123" --userid_column_name="userIdentitysessionContextsessionIssueruserName" --timestamp_column_name="event_dt" \ + pipeline-ae --columns_file="${MORPHEUS_ROOT}/python/morpheus/morpheus/data/columns_ae_cloudtrail.txt" --userid_filter="user123" --userid_column_name="userIdentitysessionContextsessionIssueruserName" --timestamp_column_name="event_dt" \ from-cloudtrail --input_glob="${MORPHEUS_ROOT}/models/datasets/validation-data/dfp-cloudtrail-*-input.csv" \ train-ae --train_data_glob="${MORPHEUS_ROOT}/models/datasets/training-data/dfp-cloudtrail-*.csv" --source_stage_class=morpheus.stages.input.cloud_trail_source_stage.CloudTrailSourceStage --seed 42 \ preprocess \ @@ -144,7 +144,7 @@ function run_pipeline_hammah_role-g(){ VAL_OUTPUT=$5 morpheus --log_level=DEBUG run --num_threads=$(nproc) --pipeline_batch_size=1024 --model_max_batch_size=1024 --use_cpp=${USE_CPP} \ - pipeline-ae --columns_file="${MORPHEUS_ROOT}/morpheus/data/columns_ae_cloudtrail.txt" --userid_filter="role-g" --userid_column_name="userIdentitysessionContextsessionIssueruserName" --timestamp_column_name="event_dt" \ + pipeline-ae --columns_file="${MORPHEUS_ROOT}/python/morpheus/morpheus/data/columns_ae_cloudtrail.txt" --userid_filter="role-g" --userid_column_name="userIdentitysessionContextsessionIssueruserName" --timestamp_column_name="event_dt" \ from-cloudtrail --input_glob="${MORPHEUS_ROOT}/models/datasets/validation-data/dfp-cloudtrail-*-input.csv" \ train-ae --train_data_glob="${MORPHEUS_ROOT}/models/datasets/training-data/dfp-cloudtrail-*.csv" --source_stage_class=morpheus.stages.input.cloud_trail_source_stage.CloudTrailSourceStage --seed 42 \ preprocess \