From 5cf158cc856056a7bb4ae26d72f4e0d827b861d1 Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Fri, 1 Mar 2024 15:46:36 +0000 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=9A=A7=20Added=20`no=20config`=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/tasks/luma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viadot/tasks/luma.py b/viadot/tasks/luma.py index 11eb91e45..3d0870657 100644 --- a/viadot/tasks/luma.py +++ b/viadot/tasks/luma.py @@ -41,7 +41,7 @@ def __init__( self.helper_script = dbt_project_path self.url = url self.metadata_dir_path = metadata_dir_path - self.command = f"luma dbt send-test-results --luma-url {url} --metadata-dir {metadata_dir_path}" + self.command = f"luma dbt send-test-results --luma-url {url} --metadata-dir {metadata_dir_path} --no config" self.return_all = True self.stream_output = True self.log_stderr = True From 64bce200d2c2db2f8d32a4c00f0aa4582362cbb5 Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Tue, 5 Mar 2024 10:37:15 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=94=A5=20Removed=20generate=5Fcatalog?= =?UTF-8?q?=5Fjson=20task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/transform_and_catalog.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/viadot/flows/transform_and_catalog.py b/viadot/flows/transform_and_catalog.py index 08ac6b895..6c739978e 100644 --- a/viadot/flows/transform_and_catalog.py +++ b/viadot/flows/transform_and_catalog.py @@ -191,8 +191,8 @@ def gen_flow(self) -> Flow: run_select = self.dbt_selects.get("run") run_select_safe = f"-s {run_select}" if run_select is not None else "" - run = ShellTask( - name="dbt_task_run", + run_model = ShellTask( + name="dbt_task_run_model", command=f"dbt run {run_select_safe} {dbt_target_option}", helper_script=f"cd {local_dbt_repo_path}", return_all=True, @@ -202,24 +202,14 @@ def gen_flow(self) -> Flow: test_select = self.dbt_selects.get("test", run_select) test_select_safe = f"-s {test_select}" if test_select is not None else "" - test = ShellTask( - name="dbt_task_test", + run_tests = ShellTask( + name="dbt_task_run_tests", command=f"dbt test {test_select_safe} {dbt_target_option}", helper_script=f"cd {local_dbt_repo_path}", return_all=True, stream_output=True, ).bind(flow=self) - # Generate docs - # Produces `catalog.json`, `run-results.json`, and `manifest.json` - - generate_catalog_json = custom_shell_task.bind( - name="dbt_task_docs_generate", - command=f"dbt docs generate {dbt_target_option} --no-compile", - helper_script=f"cd {self.dbt_project_path}", - flow=self, - ) - # Upload build metadata to Luma path_expanded = os.path.expandvars(self.metadata_dir_path) metadata_dir_path = Path(path_expanded) @@ -237,8 +227,7 @@ def gen_flow(self) -> Flow: dbt_clean_up.set_upstream(clone, flow=self) pull_dbt_deps.set_upstream(dbt_clean_up, flow=self) - run.set_upstream(pull_dbt_deps, flow=self) - test.set_upstream(run, flow=self) - generate_catalog_json.set_upstream(test, flow=self) - upload_metadata_luma.set_upstream(generate_catalog_json, flow=self) + run_model.set_upstream(pull_dbt_deps, flow=self) + run_tests.set_upstream(run_model, flow=self) + upload_metadata_luma.set_upstream(run_tests, flow=self) _cleanup_repo.set_upstream(upload_metadata_luma, flow=self) From c47c120223765f2814aa282bd02c3a56ea96471f Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Wed, 6 Mar 2024 15:11:28 +0000 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=90=9B=20Updated=20`--no-config`=20fl?= =?UTF-8?q?ag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/tasks/luma.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viadot/tasks/luma.py b/viadot/tasks/luma.py index 3d0870657..ebe709720 100644 --- a/viadot/tasks/luma.py +++ b/viadot/tasks/luma.py @@ -38,10 +38,10 @@ def __init__( secret=credentials_secret, vault_name=vault_name ) url = json.loads(credentials_str).get("url") - self.helper_script = dbt_project_path + self.helper_script = f"cd {dbt_project_path}" self.url = url self.metadata_dir_path = metadata_dir_path - self.command = f"luma dbt send-test-results --luma-url {url} --metadata-dir {metadata_dir_path} --no config" + self.command = f"luma dbt send-test-results --luma-url {url} --metadata-dir {metadata_dir_path} --no-config" self.return_all = True self.stream_output = True self.log_stderr = True From 0fe6897b0a37c7942de474a088dd011812db763e Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Tue, 9 Apr 2024 08:03:58 +0200 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=93=9D=20Added=20missing=20docstring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/tasks/luma.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/viadot/tasks/luma.py b/viadot/tasks/luma.py index ebe709720..bce06ef2d 100644 --- a/viadot/tasks/luma.py +++ b/viadot/tasks/luma.py @@ -12,7 +12,7 @@ def __init__( self, metadata_dir_path: str, url: str = None, - dbt_project_path: str = None, + dbt_project_path: str = "tmp_dbt_repo_dir", credentials_secret: str = None, vault_name: str = None, *args, @@ -26,6 +26,8 @@ def __init__( In the case of dbt, it's dbt project's `target` directory, which contains dbt artifacts (`sources.json`, `catalog.json`, `manifest.json`, and `run_results.json`). url (str, optional): The url of the Luma ingestion API. Defaults to None. + dbt_project_path (str, optional): The path to the dbt project (the directory containing + the `dbt_project.yml` file). Defaults to 'tmp_dbt_repo_dir'. credentials_secret (str, optional): The name of the Azure Key Vault secret containing Luma credentials. Defaults to None. vault_name (str, optional): The name of the vault from which to obtain the secrets. Defaults to None.