From f5f833f2deb2d22b26d7d995612983854f39b07a Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Mon, 16 Dec 2024 21:18:46 +0800 Subject: [PATCH 1/3] [BugFix][FlinkJar]Fix the issue where FlinkJar cannot use global variables (#4052) Co-authored-by: zackyoungh --- .../java/org/dinky/data/model/JarSubmitParam.java | 13 +++++++++++++ .../main/java/org/dinky/executor/Executor.java | 7 ++++++- .../main/java/org/dinky/explainer/Explainer.java | 15 --------------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/dinky-client/dinky-client-base/src/main/java/org/dinky/data/model/JarSubmitParam.java b/dinky-client/dinky-client-base/src/main/java/org/dinky/data/model/JarSubmitParam.java index dc4bb9b222..8da95a15d9 100644 --- a/dinky-client/dinky-client-base/src/main/java/org/dinky/data/model/JarSubmitParam.java +++ b/dinky-client/dinky-client-base/src/main/java/org/dinky/data/model/JarSubmitParam.java @@ -93,4 +93,17 @@ public String getArgs() { } return args; } + + public String getStatement() { + return StrUtil.format( + "EXECUTE JAR WITH (\n" + "'uri'='{}',\n" + + "'main-class'='{}',\n" + + "'args'='{}',\n" + + "'allowNonRestoredState'='{}'\n" + + ");", + getUri(), + getMainClass(), + getArgs(), + getAllowNonRestoredState()); + } } diff --git a/dinky-core/src/main/java/org/dinky/executor/Executor.java b/dinky-core/src/main/java/org/dinky/executor/Executor.java index b68bf736b3..1490eef394 100644 --- a/dinky-core/src/main/java/org/dinky/executor/Executor.java +++ b/dinky-core/src/main/java/org/dinky/executor/Executor.java @@ -25,6 +25,7 @@ import org.dinky.data.job.JobStatement; import org.dinky.data.job.JobStatementType; import org.dinky.data.job.SqlType; +import org.dinky.data.model.JarSubmitParam; import org.dinky.data.model.LineageRel; import org.dinky.data.result.SqlExplainResult; import org.dinky.explainer.print_table.PrintStatementExplainer; @@ -66,6 +67,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import cn.hutool.core.codec.Base64; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.util.ReflectUtil; import cn.hutool.core.util.URLUtil; @@ -214,7 +216,10 @@ public JobStatementPlan parseStatementIntoJobStatementPlan(String[] statements) if (operationType.equals(SqlType.SET) || operationType.equals(SqlType.RESET)) { jobStatementPlan.addJobStatement(statement, JobStatementType.SET, operationType); } else if (operationType.equals(SqlType.EXECUTE_JAR)) { - jobStatementPlan.addJobStatement(statement, JobStatementType.EXECUTE_JAR, operationType); + JarSubmitParam jarSubmitParam = JarSubmitParam.build(statement); + jarSubmitParam.setUri("base64@" + Base64.encode(pretreatStatement(jarSubmitParam.getArgs()))); + jobStatementPlan.addJobStatement( + jarSubmitParam.toString(), JobStatementType.EXECUTE_JAR, operationType); } else if (operationType.equals(SqlType.EXECUTE)) { jobStatementPlan.addJobStatement(statement, JobStatementType.PIPELINE, operationType); } else if (operationType.equals(SqlType.PRINT)) { diff --git a/dinky-core/src/main/java/org/dinky/explainer/Explainer.java b/dinky-core/src/main/java/org/dinky/explainer/Explainer.java index 0971e510c1..7b6e78583e 100644 --- a/dinky-core/src/main/java/org/dinky/explainer/Explainer.java +++ b/dinky-core/src/main/java/org/dinky/explainer/Explainer.java @@ -32,7 +32,6 @@ import org.dinky.explainer.mock.MockStatementExplainer; import org.dinky.function.data.model.UDF; import org.dinky.function.pool.UdfCodePool; -import org.dinky.function.util.UDFUtil; import org.dinky.job.JobConfig; import org.dinky.job.JobManager; import org.dinky.job.JobRunnerFactory; @@ -108,20 +107,6 @@ private void generateUDFStatement(JobStatementPlan jobStatementPlan) { } } - public List parseUDFFromStatements(String[] statements) { - List udfList = new ArrayList<>(); - for (String statement : statements) { - if (statement.isEmpty()) { - continue; - } - UDF udf = UDFUtil.toUDF(statement, jobManager.getDinkyClassLoader()); - if (Asserts.isNotNull(udf)) { - udfList.add(udf); - } - } - return udfList; - } - public ExplainResult explainSql(String statement) { log.info("Start explain FlinkSQL..."); JobStatementPlan jobStatementPlan; From 2b2f896b9b44166c1fc277695cb64a3652a9c526 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Tue, 17 Dec 2024 21:25:16 +0800 Subject: [PATCH 2/3] [FixBug] Fix pg batch insert error (#4059) Co-authored-by: Zzm0809 --- .../java/org/dinky/mapper/RoleMenuMapper.java | 6 - .../postgresql/V20241217.1.2.0__release.sql | 109 ++++++++++++++++++ .../main/resources/mapper/RoleMenuMapper.xml | 12 -- .../SqlTask/PushDolphin/index.tsx | 2 +- .../CenterTabContent/SqlTask/index.tsx | 29 +++-- 5 files changed, 129 insertions(+), 29 deletions(-) create mode 100644 dinky-admin/src/main/resources/db/migration/postgresql/V20241217.1.2.0__release.sql diff --git a/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java index c346ef9990..8e91c0ec59 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/RoleMenuMapper.java @@ -24,15 +24,9 @@ import org.apache.ibatis.annotations.Mapper; -import java.util.List; - /** MenuMapper */ @Mapper public interface RoleMenuMapper extends SuperMapper { int checkMenuExistRole(Integer menuId); - - int deleteRoleMenuByRoleId(Integer roleId); - - int batchRoleMenu(List roleMenuList); } diff --git a/dinky-admin/src/main/resources/db/migration/postgresql/V20241217.1.2.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20241217.1.2.0__release.sql new file mode 100644 index 0000000000..853b00ecb0 --- /dev/null +++ b/dinky-admin/src/main/resources/db/migration/postgresql/V20241217.1.2.0__release.sql @@ -0,0 +1,109 @@ + +-- Table structure for public.dinky_user_tenant +CREATE TABLE IF NOT EXISTS public.dinky_user_tenant_backup +( + id serial PRIMARY KEY, + user_id int NOT NULL, + tenant_id int NOT NULL, + tenant_admin_flag boolean DEFAULT false, + create_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX IF NOT EXISTS user_tenant_un_idx1 ON public.dinky_user_tenant_backup (user_id, tenant_id); + +COMMENT ON TABLE public.dinky_user_tenant_backup IS 'Relationship between users and tenants'; +COMMENT ON COLUMN public.dinky_user_tenant_backup.id IS 'id'; +COMMENT ON COLUMN public.dinky_user_tenant_backup.user_id IS 'user id'; +COMMENT ON COLUMN public.dinky_user_tenant_backup.tenant_id IS 'tenant id'; +COMMENT ON COLUMN public.dinky_user_tenant_backup.tenant_admin_flag IS 'tenant admin flag(0:false,1:true)'; + +CREATE OR REPLACE TRIGGER set_update_time_dinky_user_tenant + BEFORE UPDATE + ON public.dinky_user_tenant_backup + FOR EACH ROW +EXECUTE FUNCTION trigger_set_timestamp(); + +insert into public.dinky_user_tenant_backup(id, user_id, tenant_id, tenant_admin_flag, create_time, update_time) +select id, user_id, tenant_id, + case when tenant_admin_flag = 0 then false else true end as tenant_admin_flag, + create_time, update_time +from public.dinky_user_tenant; + +drop table if exists public.dinky_user_tenant; +alter table public.dinky_user_tenant_backup rename to dinky_user_tenant; + + + + + +DO $$ + DECLARE + table_seq_info text[][] := ARRAY[ + -- Enter the table name and the corresponding sequence name in order + ['dinky_alert_group', 'dinky_alert_group_id_seq'], + ['dinky_alert_history', 'dinky_alert_history_id_seq'], + ['dinky_alert_instance', 'dinky_alert_instance_id_seq'], + ['dinky_alert_rules', 'dinky_alert_rules_id_seq'], + ['dinky_alert_template', 'dinky_alert_template_id_seq'], + ['dinky_catalogue', 'dinky_catalogue_id_seq'], + ['dinky_cluster', 'dinky_cluster_id_seq'], + ['dinky_cluster_configuration', 'dinky_cluster_configuration_id_seq'], + ['dinky_dashboard', 'dinky_dashboard_id_seq'], + ['dinky_database', 'dinky_database_id_seq'], + ['dinky_flink_document', 'dinky_flink_document_id_seq'], + ['dinky_fragment', 'dinky_fragment_id_seq'], + ['dinky_git_project', 'dinky_git_project_id_seq'], + ['dinky_history', 'dinky_history_id_seq'], + ['dinky_job_history', 'dinky_job_history_id_seq'], + ['dinky_job_instance', 'dinky_job_instance_id_seq'], + ['dinky_metrics', 'dinky_metrics_id_seq'], + ['dinky_resources', 'dinky_resources_id_seq'], + ['dinky_role', 'dinky_role_id_seq'], + ['dinky_row_permissions', 'dinky_row_permissions_id_seq'], + ['dinky_savepoints', 'dinky_savepoints_id_seq'], + ['dinky_sys_config', 'dinky_sys_config_id_seq'], + ['dinky_sys_login_log', 'dinky_sys_login_log_id_seq'], + ['dinky_sys_menu', 'dinky_sys_menu_id_seq'], + ['dinky_sys_operate_log', 'dinky_sys_operate_log_id_seq'], + ['dinky_sys_role_menu', 'dinky_sys_role_menu_id_seq'], + ['dinky_sys_token', 'dinky_sys_token_id_seq'], + ['dinky_task', 'dinky_task_id_seq'], + ['dinky_task_version', 'dinky_task_version_id_seq'], + ['dinky_tenant', 'dinky_tenant_id_seq'], + ['dinky_udf_manage', 'dinky_udf_manage_id_seq'], + ['dinky_udf_template', 'dinky_udf_template_id_seq'], + ['dinky_user', 'dinky_user_id_seq'], + ['dinky_user_role', 'dinky_user_role_id_seq'], + ['dinky_user_tenant', 'dinky_user_tenant_id_seq'], + ['metadata_database', 'metadata_database_id_seq'], + ['metadata_function', 'metadata_function_id_seq'], + ['metadata_table', 'metadata_table_id_seq'] + ]; + i integer := 1; + BEGIN + -- Loop through each table-name and series-name combination + WHILE i <= array_length(table_seq_info, 1) LOOP + -- Obtain the current table name and sequence name + DECLARE + table_name text := table_seq_info[i][1]; + seq_name text := table_seq_info[i][2]; + max_id integer; + BEGIN + -- Step 1: Query the maximum value of ID in the current table (for each table) + EXECUTE format('SELECT MAX(id) FROM %I', table_name) INTO max_id; + -- Step 2: Delete existing sequences (for each table) + EXECUTE format('DROP SEQUENCE IF EXISTS %I CASCADE', seq_name); + -- Step 3: Recreate the sequence and set the maximum value (for each table) + EXECUTE format('CREATE SEQUENCE IF NOT EXISTS %I START WITH %s INCREMENT BY 1 ', + seq_name, + COALESCE(max_id, 0) + 1 -- The starting value is set to the maximum id value in the table plus 1, or starting with 1 if the table is empty + ); + -- Step 4: Set the default value of the id column in the table to the next value of the new series (for each table) + EXECUTE format('ALTER TABLE %I ALTER COLUMN id SET DEFAULT nextval(''%I'')', table_name, seq_name); + END; + i := i + 1; + END LOOP; + END $$; + + diff --git a/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml b/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml index d0b18cbd65..61ce9267cf 100644 --- a/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml +++ b/dinky-admin/src/main/resources/mapper/RoleMenuMapper.xml @@ -13,16 +13,4 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" - - - delete from dinky_sys_role_menu where role_id=#{roleId} - - - - insert into dinky_sys_role_menu(role_id, menu_id) values - - (#{item.roleId},#{item.menuId}) - - - \ No newline at end of file diff --git a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx index 70c959532c..4b3a29acbf 100644 --- a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx +++ b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx @@ -46,7 +46,7 @@ import { InitPushDolphinParams } from '@/types/Studio/init.d'; import { Button, Form, Tag } from 'antd'; import { DefaultOptionType } from 'antd/es/select'; import React, { useEffect } from 'react'; -import {TaskState} from "@/pages/DataStudio/type"; +import { TaskState } from '@/pages/DataStudio/type'; type PushDolphinProps = { onCancel: () => void; diff --git a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx index ab6e5d2953..50b3200eec 100644 --- a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx +++ b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx @@ -81,7 +81,12 @@ import { import { l } from '@/utils/intl'; import { editor } from 'monaco-editor'; import { DataStudioActionType } from '@/pages/DataStudio/data.d'; -import {getDataByParams, handleOption, handlePutDataJson, queryDataByParams} from '@/services/BusinessCrud'; +import { + getDataByParams, + handleOption, + handlePutDataJson, + queryDataByParams +} from '@/services/BusinessCrud'; import { API_CONSTANTS } from '@/services/endpoints'; import { Jobs, LineageDetailInfo } from '@/types/DevOps/data'; import { lockTask, matchLanguage } from '@/pages/DataStudio/function'; @@ -98,8 +103,12 @@ import { ResourceInfo } from '@/types/RegCenter/data'; import { buildResourceTreeDataAtTreeForm } from '@/pages/RegCenter/Resource/components/FileTree/function'; import { ProFormDependency } from '@ant-design/pro-form'; import { SavePoint } from '@/pages/DataStudio/CenterTabContent/SqlTask/SavePoint'; -import {DolphinTaskDefinition, DolphinTaskGroupInfo, DolphinTaskMinInfo} from "@/types/Studio/data"; -import PushDolphin from "@/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin"; +import { + DolphinTaskDefinition, + DolphinTaskGroupInfo, + DolphinTaskMinInfo +} from '@/types/Studio/data'; +import PushDolphin from '@/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin'; export type FlinkSqlProps = { showDesc: boolean; @@ -663,15 +672,15 @@ export const SqlTask = memo((props: FlinkSqlProps & any) => { ); let dolphinTaskGroup: DolphinTaskGroupInfo[] | undefined = undefined; - if (dolphinTaskDefinition?.projectCode){ - dolphinTaskGroup = await queryDataByParams< - DolphinTaskGroupInfo[] - >(API_CONSTANTS.SCHEDULER_QUERY_TASK_GROUP, { - projectCode: dolphinTaskDefinition?.projectCode || undefined - }); + if (dolphinTaskDefinition?.projectCode) { + dolphinTaskGroup = await queryDataByParams( + API_CONSTANTS.SCHEDULER_QUERY_TASK_GROUP, + { + projectCode: dolphinTaskDefinition?.projectCode || undefined + } + ); } - const formValuesInfo = dolphinTaskDefinition ? JSON.parse(JSON.stringify(dolphinTaskDefinition)) : {}; From 381396f45878bc68dabb70c0f4ba1e5444156eb9 Mon Sep 17 00:00:00 2001 From: Wink <809097465@qq.com> Date: Tue, 17 Dec 2024 21:45:50 +0800 Subject: [PATCH 3/3] Release 1.2.0 (LTS) (#4054) --- docs/download/dinky-1.2.0.md | 16 ++++++++-------- pom.xml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/download/dinky-1.2.0.md b/docs/download/dinky-1.2.0.md index 49878b6a59..0dd35b2d26 100644 --- a/docs/download/dinky-1.2.0.md +++ b/docs/download/dinky-1.2.0.md @@ -5,15 +5,15 @@ title: 1.2.0 release | Dinky 版本 | Flink 版本 | 二进制程序 | Source | |----------|----------|-----------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------| -| 1.2.0 | 1.14 | [dinky-release-1.14-1.2.0-rc5.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.14-1.2.0-rc5.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | -| 1.2.0 | 1.15 | [dinky-release-1.15-1.2.0-rc5.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.15-1.2.0-rc5.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | -| 1.2.0 | 1.16 | [dinky-release-1.16-1.2.0-rc5.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.16-1.2.0-rc5.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | -| 1.2.0 | 1.17 | [dinky-release-1.17-1.2.0-rc5.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.17-1.2.0-rc5.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | -| 1.2.0 | 1.18 | [dinky-release-1.18-1.2.0-rc5.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.18-1.2.0-rc5.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | -| 1.2.0 | 1.19 | [dinky-release-1.19-1.2.0-rc5.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.19-1.2.0-rc5.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | -| 1.2.0 | 1.20 | [dinky-release-1.20-1.2.0-rc5.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.20-1.2.0-rc5.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | +| 1.2.0 | 1.14 | [dinky-release-1.14-1.2.0.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.14-1.2.0.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | +| 1.2.0 | 1.15 | [dinky-release-1.15-1.2.0.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.15-1.2.0.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | +| 1.2.0 | 1.16 | [dinky-release-1.16-1.2.0.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.16-1.2.0.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | +| 1.2.0 | 1.17 | [dinky-release-1.17-1.2.0.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.17-1.2.0.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | +| 1.2.0 | 1.18 | [dinky-release-1.18-1.2.0.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.18-1.2.0.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | +| 1.2.0 | 1.19 | [dinky-release-1.19-1.2.0.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.19-1.2.0.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | +| 1.2.0 | 1.20 | [dinky-release-1.20-1.2.0.tar.gz](https://github.com/DataLinkDC/dinky/releases/download/v1.2.0/dinky-release-1.20-1.2.0.tar.gz) | [Source code (zip)](https://github.com/DataLinkDC/dinky/archive/refs/tags/v1.2.0.zip) | -## Dinky-1.2.0-rc5 发行说明 +## Dinky-1.2.0 发行说明 ### 升级说明 diff --git a/pom.xml b/pom.xml index 7cca543b4e..46d00688d6 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ UTF-8 2.5.0 0.10.2 - 1.2.0-rc5 + 1.2.0 1.37.0 2.12.10