Skip to content

Commit

Permalink
Add tests to reproduce nv-morpheus#1639
Browse files Browse the repository at this point in the history
  • Loading branch information
@dagardner-nv
dagardner-nv committed Apr 18, 2024
1 parent ab8d0a7 commit af9bb17
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 44 deletions.
91 changes: 57 additions & 34 deletions tests/test_sid.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import os
from unittest import mock

Expand All @@ -25,6 +26,7 @@
from _utils import calc_error_val
from _utils import compare_class_to_scores
from _utils import mk_async_infer
from morpheus.config import Config
from morpheus.config import CppConfig
from morpheus.config import PipelineModes
from morpheus.pipeline import LinearPipeline
Expand All @@ -44,7 +46,13 @@
MODEL_MAX_BATCH_SIZE = 32


def _run_minibert_pipeline(config, tmp_path, model_name, truncated, data_col_name: str = "data"):
def _run_minibert_pipeline(*,
config: Config,
tmp_path: str,
model_name: str,
truncated: bool,
data_col_name: str = "data",
num_threads: int = 1):
"""
Runs just the Minibert Pipeline
"""
Expand All @@ -66,7 +74,7 @@ def _run_minibert_pipeline(config, tmp_path, model_name, truncated, data_col_nam
config.pipeline_batch_size = 1024
config.feature_length = FEATURE_LENGTH
config.edge_buffer_size = 128
config.num_threads = 1
config.num_threads = num_threads

val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'sid-validation-data.csv')
vocab_file_name = os.path.join(TEST_DIRS.data_dir, 'bert-base-uncased-hash.txt')
Expand Down Expand Up @@ -100,7 +108,8 @@ def _run_minibert_pipeline(config, tmp_path, model_name, truncated, data_col_nam
column=data_col_name))
pipe.add_stage(
TritonInferenceStage(config, model_name=model_name, server_url='localhost:8001', force_convert_inputs=True))
pipe.add_stage(MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf"))
pipe.add_stage(
MonitorStage(config, description="Inference Rate", smoothing=0.001, unit="inf", log_level=logging.INFO))
pipe.add_stage(AddClassificationsStage(config, threshold=0.5, prefix="si_"))
pipe.add_stage(AddScoresStage(config, prefix="score_"))
pipe.add_stage(
Expand All @@ -113,7 +122,13 @@ def _run_minibert_pipeline(config, tmp_path, model_name, truncated, data_col_nam
return calc_error_val(results_file_name)


def _run_minibert(config, tmp_path, model_name, truncated, data_col_name: str = "data"):
def _run_minibert(*,
config: Config,
tmp_path: str,
model_name: str,
truncated: bool,
data_col_name: str = "data",
num_threads: int = 1):
"""
Runs the minibert pipeline and mocks the Triton Python interface
"""
Expand Down Expand Up @@ -145,44 +160,52 @@ def _run_minibert(config, tmp_path, model_name, truncated, data_col_name: str =
async_infer = mk_async_infer(inf_results)
mock_triton_client.async_infer.side_effect = async_infer

return _run_minibert_pipeline(config, tmp_path, model_name, truncated, data_col_name)
return _run_minibert_pipeline(config=config,
tmp_path=tmp_path,
model_name=model_name,
truncated=truncated,
data_col_name=data_col_name,
num_threads=num_threads)


@pytest.mark.slow
@pytest.mark.use_cpp
@pytest.mark.usefixtures("launch_mock_triton")
def test_minibert_no_trunc(config, tmp_path):
@pytest.mark.parametrize("num_threads", [1, 4])
def test_minibert_no_trunc(config: Config, tmp_path: str, num_threads: int):

results = _run_minibert(config, tmp_path, "sid-minibert-onnx-no-trunc", False)
results = _run_minibert(config=config,
tmp_path=tmp_path,
model_name="sid-minibert-onnx-no-trunc",
truncated=False,
num_threads=num_threads)

# Not sure why these are different
if (CppConfig.get_should_use_cpp()):
assert results.diff_rows == 18
else:
assert results.diff_rows == 1333
# When threading is enabled, the results returned from the mocked Triton server won't match the expected results
if num_threads == 1:
# Not sure why these are different
if (CppConfig.get_should_use_cpp()):
assert results.diff_rows == 18
else:
assert results.diff_rows == 1333


@pytest.mark.slow
@pytest.mark.usefixtures("launch_mock_triton")
def test_minibert_truncated(config, tmp_path):

results = _run_minibert(config, tmp_path, 'sid-minibert-onnx', True)

# Not sure why these are different
if (CppConfig.get_should_use_cpp()):
assert results.diff_rows == 1204
else:
assert results.diff_rows == 1333


@pytest.mark.slow
@pytest.mark.usefixtures("launch_mock_triton")
def test_minibert_data_col_name(config, tmp_path):

results = _run_minibert(config, tmp_path, 'sid-minibert-onnx', True, "definitely_not_data")

# Not sure why these are different
if (CppConfig.get_should_use_cpp()):
assert results.diff_rows == 1204
else:
assert results.diff_rows == 1333
@pytest.mark.parametrize("data_col_name", ["data", "definitely_not_data"])
@pytest.mark.parametrize("num_threads", [1, 4])
def test_minibert_truncated(config: Config, tmp_path: str, data_col_name: str, num_threads: int):

results = _run_minibert(config=config,
tmp_path=tmp_path,
model_name='sid-minibert-onnx',
truncated=True,
data_col_name=data_col_name,
num_threads=num_threads)

# When threading is enabled, the results returned from the mocked Triton server won't match the expected results
if num_threads == 1:
# Not sure why these are different
if (CppConfig.get_should_use_cpp()):
assert results.diff_rows == 1204
else:
assert results.diff_rows == 1333
27 changes: 17 additions & 10 deletions tests/test_sid_kafka.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import os
import typing
from io import StringIO
Expand Down Expand Up @@ -52,13 +53,15 @@
@pytest.mark.kafka
@pytest.mark.slow
@pytest.mark.use_python
@pytest.mark.parametrize("num_threads", [1, 4])
@mock.patch('tritonclient.grpc.InferenceServerClient')
def test_minibert_no_cpp(mock_triton_client: mock.MagicMock,
dataset_pandas: DatasetManager,
config: Config,
kafka_bootstrap_servers: str,
kafka_topics: KafkaTopics,
kafka_consumer: "KafkaConsumer"):
kafka_consumer: "KafkaConsumer",
num_threads: int):
mock_metadata = {
"inputs": [{
"name": "input_ids", "datatype": "INT32", "shape": [-1, FEATURE_LENGTH]
Expand Down Expand Up @@ -101,7 +104,7 @@ def test_minibert_no_cpp(mock_triton_client: mock.MagicMock,
config.pipeline_batch_size = 1024
config.feature_length = FEATURE_LENGTH
config.edge_buffer_size = 128
config.num_threads = 1
config.num_threads = num_threads

val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'sid-validation-data.csv')
vocab_file_name = os.path.join(TEST_DIRS.data_dir, 'bert-base-uncased-hash.txt')
Expand Down Expand Up @@ -137,20 +140,23 @@ def test_minibert_no_cpp(mock_triton_client: mock.MagicMock,

assert len(output_df) == len(val_df)

results = compare_df(val_df, output_df, exclude_columns=[r'^ID$', r'^_ts_'], rel_tol=0.05)

assert results['diff_rows'] == 1333
# When threading is enabled, the results returned from the mocked Triton server won't match the expected results
if num_threads == 1:
results = compare_df(val_df, output_df, exclude_columns=[r'^ID$', r'^_ts_'], rel_tol=0.05)
assert results['diff_rows'] == 1333


@pytest.mark.kafka
@pytest.mark.slow
@pytest.mark.use_cpp
@pytest.mark.usefixtures("launch_mock_triton")
@pytest.mark.parametrize("num_threads", [1, 4])
def test_minibert_cpp(dataset_pandas: DatasetManager,
config: Config,
kafka_bootstrap_servers: str,
kafka_topics: KafkaTopics,
kafka_consumer: "KafkaConsumer"):
kafka_consumer: "KafkaConsumer",
num_threads: int):
config.mode = PipelineModes.NLP
config.class_labels = [
"address",
Expand All @@ -168,7 +174,7 @@ def test_minibert_cpp(dataset_pandas: DatasetManager,
config.pipeline_batch_size = 1024
config.feature_length = FEATURE_LENGTH
config.edge_buffer_size = 128
config.num_threads = 1
config.num_threads = num_threads

val_file_name = os.path.join(TEST_DIRS.validation_data_dir, 'sid-validation-data.csv')
vocab_file_name = os.path.join(TEST_DIRS.data_dir, 'bert-base-uncased-hash.txt')
Expand Down Expand Up @@ -207,6 +213,7 @@ def test_minibert_cpp(dataset_pandas: DatasetManager,

assert len(output_df) == len(val_df)

results = compare_df(val_df, output_df, exclude_columns=[r'^ID$', r'^_ts_'], rel_tol=0.05)

assert results['diff_rows'] == 1204
# When threading is enabled, the results returned from the mocked Triton server won't match the expected results
if num_threads == 1:
results = compare_df(val_df, output_df, exclude_columns=[r'^ID$', r'^_ts_'], rel_tol=0.05)
assert results['diff_rows'] == 1204

0 comments on commit af9bb17

Please sign in to comment.