From f8f64d706e027abde79c991b4ac7b4f70cbaca7c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 5 Jul 2024 12:02:19 -0700 Subject: [PATCH 1/2] Add dataset for vdb_upload example --- examples/data/vdb_upload/doca_guides.jsonlines | 3 +++ examples/data/vdb_upload/nvidia_blogs.jsonlines | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 examples/data/vdb_upload/doca_guides.jsonlines create mode 100644 examples/data/vdb_upload/nvidia_blogs.jsonlines diff --git a/examples/data/vdb_upload/doca_guides.jsonlines b/examples/data/vdb_upload/doca_guides.jsonlines new file mode 100644 index 0000000000..9bb90be880 --- /dev/null +++ b/examples/data/vdb_upload/doca_guides.jsonlines @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ff28cce2fde3b3b625057d1c68d454b28026f0476dd8b5f8addf4137e7c00e +size 28661 diff --git a/examples/data/vdb_upload/nvidia_blogs.jsonlines b/examples/data/vdb_upload/nvidia_blogs.jsonlines new file mode 100644 index 0000000000..2efd5dd00c --- /dev/null +++ b/examples/data/vdb_upload/nvidia_blogs.jsonlines @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42c8fadcee174402e7fd50a060516a6f07a20f2ac0750210997442030c7982ad +size 10260 From 04014c73cf1982d35dbec7e723fef91dd40a00ad Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 5 Jul 2024 12:13:05 -0700 Subject: [PATCH 2/2] Update filesystem examples to use the new dataset --- examples/llm/vdb_upload/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm/vdb_upload/README.md b/examples/llm/vdb_upload/README.md index b8a3ef35e5..de6c18a81d 100644 --- a/examples/llm/vdb_upload/README.md +++ b/examples/llm/vdb_upload/README.md @@ -214,7 +214,7 @@ python examples/llm/main.py vdb_upload pipeline \ ```bash python examples/llm/main.py vdb_upload pipeline \ --source_type filesystem \ - --file_source "./morpheus/data/*" \ + --file_source="./examples/data/vdb_upload/*.jsonlines" \ --enable_monitors \ --embedding_model_name all-MiniLM-L6-v2 ``` @@ -224,7 +224,7 @@ python examples/llm/main.py vdb_upload pipeline \ ```bash python examples/llm/main.py vdb_upload pipeline \ --source_type rss --source_type filesystem \ - --file_source "./morpheus/data/*" \ + --file_source="./examples/data/vdb_upload/*.jsonlines" \ --interval_secs 600 \ --enable_cache \ --enable_monitors \