Fix merging of CLI args and Yaml configs in vdb_upload example

examples/llm/vdb_upload/langchain.py

@@ -20,15 +20,15 @@
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores.milvus import Milvus
 
-from examples.llm.vdb_upload.vdb_utils import build_rss_urls
+from examples.llm.vdb_upload.vdb_utils import DEFAULT_RSS_URLS
 from morpheus.utils.logging_timer import log_time
 
 logger = logging.getLogger(__name__)
 
 
 def chain(model_name, save_cache):
     with log_time(msg="Seeding with chain took {duration} ms. {rate_per_sec} docs/sec", log_fn=logger.debug) as log:
-        loader = RSSFeedLoader(urls=build_rss_urls())
+        loader = RSSFeedLoader(urls=DEFAULT_RSS_URLS.copy())
 
         documents = loader.load()
 

examples/llm/vdb_upload/run.py

@@ -16,8 +16,7 @@
 import os
 
 import click
-from vdb_upload.vdb_utils import build_cli_configs
-from vdb_upload.vdb_utils import build_final_config
+from vdb_upload.vdb_utils import build_config
 from vdb_upload.vdb_utils import is_valid_service
 
 logger = logging.getLogger(__name__)
@@ -144,7 +143,8 @@ def run():
     default="http://localhost:19530",
     help="URI for connecting to Vector Database server.",
 )
-def pipeline(**kwargs):
+@click.pass_context
+def pipeline(ctx: click.Context, **kwargs):
     """
     Configure and run the data processing pipeline based on the specified command-line options.
 
@@ -154,25 +154,28 @@ def pipeline(**kwargs):
 
     Parameters
     ----------
+    ctx: click.Context
+        Click context object.
     **kwargs : dict
         Keyword arguments containing command-line options.
 
     Returns
     -------
     The result of the internal pipeline function call.
     """
+
     vdb_config_path = kwargs.pop('vdb_config_path', None)
-    cli_source_conf, cli_embed_conf, cli_pipe_conf, cli_tok_conf, cli_vdb_conf = build_cli_configs(**kwargs)
-    final_config = build_final_config(vdb_config_path,
-                                      cli_source_conf,
-                                      cli_embed_conf,
-                                      cli_pipe_conf,
-                                      cli_tok_conf,
-                                      cli_vdb_conf)
 
+    # When a config file is provided, only merge the explicit flags set by the user
+    explicit_cli_args = {}
+    for (key, value) in kwargs.items():
+        if ctx.get_parameter_source(key) is not click.core.ParameterSource.DEFAULT:
+            explicit_cli_args[key] = value
+
+    config = build_config(vdb_conf_path=vdb_config_path, explicit_cli_args=explicit_cli_args, implicit_cli_args=kwargs)
     # Call the internal pipeline function with the final config dictionary
     from .pipeline import pipeline as _pipeline
-    return _pipeline(**final_config)
+    return _pipeline(**config)
 
 
 @run.command()

examples/llm/vdb_upload/vdb_config.yaml

@@ -31,7 +31,7 @@ vdb_pipeline:
 
   sources:
     - type: rss
-      name: "rss_cve"
+      name: "rss"
       config:
         batch_size: 128 # Number of rss feeds per batch
         cache_dir: "./.cache/http"
@@ -75,7 +75,6 @@ vdb_pipeline:
         output_batch_size: 2048 # Number of chunked documents per output batch
         request_timeout_sec: 2.0
         run_indefinitely: true
-        stop_after_rec: 0
         strip_markup: true
         web_scraper_config:
           chunk_overlap: 51