diff --git a/examples/log_parsing/postprocessing.py b/examples/log_parsing/postprocessing.py index f19836d896..1e4d89689c 100644 --- a/examples/log_parsing/postprocessing.py +++ b/examples/log_parsing/postprocessing.py @@ -83,8 +83,12 @@ def compute_schema(self, schema: StageSchema): schema.output_schema.set_type(MessageMeta) def _postprocess(self, msg: ControlMessage): - infer_pdf = pd.DataFrame(msg.tensors().get_tensor('seq_ids').get()).astype(int) - infer_pdf.columns = ["doc", "start", "stop"] + with msg.payload().mutable_dataframe() as src_df: + src_index = src_df.index.to_pandas() + + seq_ids = msg.tensors().get_tensor('seq_ids').get() + infer_pdf = pd.DataFrame({"doc": src_index, "start": seq_ids[:, 1], "stop": seq_ids[:, 2]}) + infer_pdf["confidences"] = msg.tensors().get_tensor('confidences').tolist() infer_pdf["labels"] = msg.tensors().get_tensor('labels').tolist() infer_pdf["token_ids"] = msg.tensors().get_tensor('input_ids').tolist() diff --git a/python/morpheus/morpheus/_lib/include/morpheus/objects/table_info.hpp b/python/morpheus/morpheus/_lib/include/morpheus/objects/table_info.hpp index d4e719abd2..710158aec3 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/objects/table_info.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/objects/table_info.hpp @@ -70,7 +70,7 @@ struct MORPHEUS_EXPORT TableInfoBase std::vector get_column_names() const; /** - * @brief Get size of a index names in a data table + * @brief Get the number of indices in a data table * * @return cudf::size_type */ diff --git a/python/morpheus/morpheus/_lib/include/morpheus/stages/deserialize.hpp b/python/morpheus/morpheus/_lib/include/morpheus/stages/deserialize.hpp index 76d6051d13..6f4381b6f9 100644 --- a/python/morpheus/morpheus/_lib/include/morpheus/stages/deserialize.hpp +++ b/python/morpheus/morpheus/_lib/include/morpheus/stages/deserialize.hpp @@ -45,12 +45,6 @@ namespace morpheus { * @file */ -void make_output_message(std::shared_ptr& incoming_message, - TensorIndex start, - TensorIndex stop, - control_message_task_t* task, - std::shared_ptr& windowed_message); - /****** DeserializationStage********************************/ class MORPHEUS_EXPORT DeserializeStage : public mrc::pymrc::PythonNode, std::shared_ptr> diff --git a/tests/examples/log_parsing/test_postprocessing.py b/tests/examples/log_parsing/test_postprocessing.py index e6271d8a42..fe23c9736e 100644 --- a/tests/examples/log_parsing/test_postprocessing.py +++ b/tests/examples/log_parsing/test_postprocessing.py @@ -38,10 +38,11 @@ def fixture_model_config_file(): def build_post_proc_message(dataset_cudf: DatasetManager, log_test_data_dir: str): input_file = os.path.join(TEST_DIRS.validation_data_dir, 'log-parsing-validation-data-input.csv') - input_df = dataset_cudf[input_file] - meta = MessageMeta(input_df) # we have tensor data for the first five rows + input_df = dataset_cudf[input_file][:5] + meta = MessageMeta(input_df) + count = 5 tensors = {} for tensor_name in ['confidences', 'input_ids', 'labels']: