🔥 Removed generate_catalog_json task

dyvenia · Mar 5, 2024 · 64bce20 · 64bce20
1 parent 5cf158c
commit 64bce20
Showing 1 changed file with 7 additions and 18 deletions.
diff --git a/viadot/flows/transform_and_catalog.py b/viadot/flows/transform_and_catalog.py
@@ -191,8 +191,8 @@ def gen_flow(self) -> Flow:
         run_select = self.dbt_selects.get("run")
         run_select_safe = f"-s {run_select}" if run_select is not None else ""
 
-        run = ShellTask(
-            name="dbt_task_run",
+        run_model = ShellTask(
+            name="dbt_task_run_model",
             command=f"dbt run {run_select_safe} {dbt_target_option}",
             helper_script=f"cd {local_dbt_repo_path}",
             return_all=True,
@@ -202,24 +202,14 @@ def gen_flow(self) -> Flow:
         test_select = self.dbt_selects.get("test", run_select)
         test_select_safe = f"-s {test_select}" if test_select is not None else ""
 
-        test = ShellTask(
-            name="dbt_task_test",
+        run_tests = ShellTask(
+            name="dbt_task_run_tests",
             command=f"dbt test {test_select_safe} {dbt_target_option}",
             helper_script=f"cd {local_dbt_repo_path}",
             return_all=True,
             stream_output=True,
         ).bind(flow=self)
 
-        # Generate docs
-        # Produces `catalog.json`, `run-results.json`, and `manifest.json`
-
-        generate_catalog_json = custom_shell_task.bind(
-            name="dbt_task_docs_generate",
-            command=f"dbt docs generate {dbt_target_option} --no-compile",
-            helper_script=f"cd {self.dbt_project_path}",
-            flow=self,
-        )
-
         # Upload build metadata to Luma
         path_expanded = os.path.expandvars(self.metadata_dir_path)
         metadata_dir_path = Path(path_expanded)
@@ -237,8 +227,7 @@ def gen_flow(self) -> Flow:
 
         dbt_clean_up.set_upstream(clone, flow=self)
         pull_dbt_deps.set_upstream(dbt_clean_up, flow=self)
-        run.set_upstream(pull_dbt_deps, flow=self)
-        test.set_upstream(run, flow=self)
-        generate_catalog_json.set_upstream(test, flow=self)
-        upload_metadata_luma.set_upstream(generate_catalog_json, flow=self)
+        run_model.set_upstream(pull_dbt_deps, flow=self)
+        run_tests.set_upstream(run_model, flow=self)
+        upload_metadata_luma.set_upstream(run_tests, flow=self)
         _cleanup_repo.set_upstream(upload_metadata_luma, flow=self)