From b5fecfac79da76c3172b06e7b1896d5c0f2b0f7a Mon Sep 17 00:00:00 2001 From: himanshudube97 Date: Fri, 13 Dec 2024 14:34:47 +0530 Subject: [PATCH 1/5] code --- src/components/DBT/CreateOrgTaskForm.tsx | 3 ++- src/components/Flows/FlowCreate.tsx | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/components/DBT/CreateOrgTaskForm.tsx b/src/components/DBT/CreateOrgTaskForm.tsx index ca8e8301..222038cc 100644 --- a/src/components/DBT/CreateOrgTaskForm.tsx +++ b/src/components/DBT/CreateOrgTaskForm.tsx @@ -89,7 +89,8 @@ const CreateOrgTaskForm = ({ mutate, showForm, setShowForm }: CreateOrgTaskFormP .map((task: MasterTask) => { return { id: task.slug, label: task.slug }; }); - setMasterTasks(tasksDropDownRows); + console.log(tasksDropDownRows, 'taskdropdown'); + setMasterTasks([...tasksDropDownRows, { id: 'dbt-cloud', label: 'dbt-cloud' }]); } catch (err: any) { console.error(err); errorToast(err.message, [], globalContext); diff --git a/src/components/Flows/FlowCreate.tsx b/src/components/Flows/FlowCreate.tsx index f43387b7..2332d119 100644 --- a/src/components/Flows/FlowCreate.tsx +++ b/src/components/Flows/FlowCreate.tsx @@ -168,18 +168,30 @@ const FlowCreate = ({ setLoading(true); try { const data: any = await httpGet(session, `prefect/v1/flows/${flowId}`); - + console.log(data, 'data'); let tasksToApply = tasks.filter(ValidateDefaultTasksToApplyInPipeline); if (data.transformTasks.length === 0) { tasksToApply = []; } + //if "data.transformTasks" and "tasksToApply" are same then the alignment is simple else advanced. const ifTasksAligned = data.transformTasks.every( (task: { uuid: string; seq: number }, index: number) => task.uuid === tasksToApply[index].uuid ); + const dbtCloudRegx = /^dbtcloud(?:-.+)?/; + + const dbtCloudTasks = data.transformTasks.filter((item: any) => + dbtCloudRegx.test(item.slug) + ); + const dbtCliTasks = data.transformTasks.filter( + (item: any) => !dbtCloudRegx.test(item.slug) + ); + if (data.transformTasks.length > 0 && !ifTasksAligned) { + } + const arrangeTheDataAccToUUID = (transformTasks: any, alignment: string) => { const uuidOrder = data.transformTasks.reduce((acc: any, obj: any) => { acc[obj.uuid] = obj.seq; return acc; @@ -187,9 +199,8 @@ const FlowCreate = ({ tasksToApply = tasks .filter((obj) => uuidOrder.hasOwnProperty(obj.uuid)) .sort((a, b) => uuidOrder[a.uuid] - uuidOrder[b.uuid]); - setAlignment('advanced'); - } - + setAlignment(alignment); + }; const cronObject = convertCronToString(data.cron); reset({ @@ -421,6 +432,9 @@ const FlowCreate = ({ Advanced + + DBT Cloud + Date: Sat, 14 Dec 2024 03:26:44 +0530 Subject: [PATCH 2/5] sending aligment field in the request body to check if the task is a core task or a dbtcloud task --- src/components/Flows/FlowCreate.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/Flows/FlowCreate.tsx b/src/components/Flows/FlowCreate.tsx index 2332d119..01cf69fd 100644 --- a/src/components/Flows/FlowCreate.tsx +++ b/src/components/Flows/FlowCreate.tsx @@ -290,6 +290,7 @@ const FlowCreate = ({ const response = await httpPost(session, 'prefect/v1/flows/', { name: data.name, connections: selectedConns, + alignment: alignment, // this field will be used in backend to differentiate bw cli and cloud tasks. cron: cronExpression, transformTasks: data.tasks.map((task: TransformTask, index: number) => ({ uuid: task.uuid, From d3660db76eddea63998c48a2e92195ceb9bad282 Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Mon, 16 Dec 2024 23:30:44 +0530 Subject: [PATCH 3/5] orgtask creation from transform page --- src/components/DBT/CreateOrgTaskForm.tsx | 11 +++++++---- src/components/DBT/DBTTaskList.tsx | 10 +++++++++- src/config/constant.ts | 1 + 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/components/DBT/CreateOrgTaskForm.tsx b/src/components/DBT/CreateOrgTaskForm.tsx index 222038cc..96ced384 100644 --- a/src/components/DBT/CreateOrgTaskForm.tsx +++ b/src/components/DBT/CreateOrgTaskForm.tsx @@ -5,7 +5,7 @@ import ListItem from '@mui/material/ListItem'; import CustomDialog from '../Dialog/CustomDialog'; import { Controller, useFieldArray, useForm } from 'react-hook-form'; import Input from '../UI/Input/Input'; -import { TASK_GITPULL, TASK_DBTCLEAN, TASK_DOCSGENERATE } from '@/config/constant'; +import { TASK_GITPULL, TASK_DBTCLEAN, TASK_DBTCLOUD_JOB } from '@/config/constant'; import { useSession } from 'next-auth/react'; import { useContext, useEffect, useRef, useState } from 'react'; import { httpGet, httpPost } from '@/helpers/http'; @@ -90,7 +90,7 @@ const CreateOrgTaskForm = ({ mutate, showForm, setShowForm }: CreateOrgTaskFormP return { id: task.slug, label: task.slug }; }); console.log(tasksDropDownRows, 'taskdropdown'); - setMasterTasks([...tasksDropDownRows, { id: 'dbt-cloud', label: 'dbt-cloud' }]); + setMasterTasks([...tasksDropDownRows]); } catch (err: any) { console.error(err); errorToast(err.message, [], globalContext); @@ -156,6 +156,8 @@ const CreateOrgTaskForm = ({ mutate, showForm, setShowForm }: CreateOrgTaskFormP setLoading(false); }; + const isDbtCloudTask = () => selectedTask?.id === TASK_DBTCLOUD_JOB; + const FormContent = () => { return ( <> @@ -286,12 +288,13 @@ const CreateOrgTaskForm = ({ mutate, showForm, setShowForm }: CreateOrgTaskFormP - + + {!isDbtCloudTask() && ( - + )} ); }; diff --git a/src/components/DBT/DBTTaskList.tsx b/src/components/DBT/DBTTaskList.tsx index 8ca46546..15eba357 100644 --- a/src/components/DBT/DBTTaskList.tsx +++ b/src/components/DBT/DBTTaskList.tsx @@ -242,6 +242,14 @@ export const DBTTaskList = ({ const tempRows = tasks .filter((task: TransformTask) => task.slug != TASK_DOCSGENERATE) .map((task: TransformTask) => [ + + + {task.label} + + , diff --git a/src/config/constant.ts b/src/config/constant.ts index fc51988d..bc58638c 100644 --- a/src/config/constant.ts +++ b/src/config/constant.ts @@ -24,6 +24,7 @@ export const TASK_DBTCLEAN = 'dbt-clean'; export const TASK_DBTDEPS = 'dbt-deps'; export const TASK_GITPULL = 'git-pull'; export const TASK_DOCSGENERATE = 'dbt-docs-generate'; +export const TASK_DBTCLOUD_JOB = 'dbt-cloud-job'; // Demo account export const demoAccDestSchema = process.env.NEXT_PUBLIC_DEMO_ACCOUNT_DEST_SCHEMA; From b1c0ee3a521845651af030c698a339745c75265f Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Tue, 17 Dec 2024 00:24:44 +0530 Subject: [PATCH 4/5] pipeline creation/editing with dbt cloud tasks --- src/components/Flows/FlowCreate.tsx | 21 +++------------------ src/components/Flows/TaskSequence.tsx | 6 ++++-- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/components/Flows/FlowCreate.tsx b/src/components/Flows/FlowCreate.tsx index 01cf69fd..46ac6fb4 100644 --- a/src/components/Flows/FlowCreate.tsx +++ b/src/components/Flows/FlowCreate.tsx @@ -168,7 +168,6 @@ const FlowCreate = ({ setLoading(true); try { const data: any = await httpGet(session, `prefect/v1/flows/${flowId}`); - console.log(data, 'data'); let tasksToApply = tasks.filter(ValidateDefaultTasksToApplyInPipeline); if (data.transformTasks.length === 0) { @@ -180,18 +179,7 @@ const FlowCreate = ({ (task: { uuid: string; seq: number }, index: number) => task.uuid === tasksToApply[index].uuid ); - const dbtCloudRegx = /^dbtcloud(?:-.+)?/; - - const dbtCloudTasks = data.transformTasks.filter((item: any) => - dbtCloudRegx.test(item.slug) - ); - const dbtCliTasks = data.transformTasks.filter( - (item: any) => !dbtCloudRegx.test(item.slug) - ); - if (data.transformTasks.length > 0 && !ifTasksAligned) { - } - const arrangeTheDataAccToUUID = (transformTasks: any, alignment: string) => { const uuidOrder = data.transformTasks.reduce((acc: any, obj: any) => { acc[obj.uuid] = obj.seq; return acc; @@ -199,8 +187,9 @@ const FlowCreate = ({ tasksToApply = tasks .filter((obj) => uuidOrder.hasOwnProperty(obj.uuid)) .sort((a, b) => uuidOrder[a.uuid] - uuidOrder[b.uuid]); - setAlignment(alignment); - }; + setAlignment('advanced'); + } + const cronObject = convertCronToString(data.cron); reset({ @@ -290,7 +279,6 @@ const FlowCreate = ({ const response = await httpPost(session, 'prefect/v1/flows/', { name: data.name, connections: selectedConns, - alignment: alignment, // this field will be used in backend to differentiate bw cli and cloud tasks. cron: cronExpression, transformTasks: data.tasks.map((task: TransformTask, index: number) => ({ uuid: task.uuid, @@ -433,9 +421,6 @@ const FlowCreate = ({ Advanced - - DBT Cloud - {node.rowIndex + 1} - {node.data.command} + + {node.data.command || node.data?.slug.replace(/-/g, ' ')} + task.command} + getOptionLabel={(task: any) => task.command || task.slug?.replace(/-/g, ' ')} placeholder="Select" options={autocompleteOptions} onChange={handleSelect} From f055f4b76b53c6c2eded93e5f0742d6ff687c61d Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Tue, 17 Dec 2024 00:28:45 +0530 Subject: [PATCH 5/5] minor change --- src/components/Flows/TaskSequence.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/Flows/TaskSequence.tsx b/src/components/Flows/TaskSequence.tsx index 238ff4c4..fbd42264 100644 --- a/src/components/Flows/TaskSequence.tsx +++ b/src/components/Flows/TaskSequence.tsx @@ -100,7 +100,7 @@ export const TaskSequence = ({ field, options: initialOptions }: TaskSequencePro {node.rowIndex + 1} - {node.data.command || node.data?.slug.replace(/-/g, ' ')} + {node.data.command || node.data.slug.replace(/-/g, ' ')}