From 46efd8d96e28dedc64bf85e09a50602cb6cd6545 Mon Sep 17 00:00:00 2001 From: liuxtq Date: Thu, 28 Nov 2024 10:10:48 +0800 Subject: [PATCH 1/8] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0=E6=A0=91=E7=8A=B6?= =?UTF-8?q?=E5=9B=BE=E5=B1=95=E7=A4=BAshow=20columns=E7=BB=93=E6=9E=9C=202?= =?UTF-8?q?.=E4=BF=AE=E5=A4=8D=E7=BD=91=E7=8A=B6=E5=9B=BE=E9=83=A8?= =?UTF-8?q?=E5=88=86bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/interpreter-setting.json | 7 + .../zeppelin/iginx/IginxInterpreter8.java | 77 ++++++++-- .../zeppelin/iginx/SimpleFileServer.java | 1 - .../iginx/util/HighchartsTreeNode.java | 51 +++++++ .../zeppelin/iginx/util/MultiwayTree.java | 19 ++- .../main/resources/interpreter-setting.json | 7 + .../resources/static/highcharts/tree.html | 134 ++++++++++++++++++ v8/src/main/resources/static/vis/main.html | 2 +- v8/src/main/resources/static/vis/network.html | 23 ++- 9 files changed, 292 insertions(+), 29 deletions(-) create mode 100644 v8/src/main/java/org/apache/zeppelin/iginx/util/HighchartsTreeNode.java create mode 100644 v8/src/main/resources/static/highcharts/tree.html diff --git a/v11/src/main/resources/interpreter-setting.json b/v11/src/main/resources/interpreter-setting.json index 3974000..c6f0d27 100644 --- a/v11/src/main/resources/interpreter-setting.json +++ b/v11/src/main/resources/interpreter-setting.json @@ -108,6 +108,13 @@ "defaultValue": "", "description": "The host of File HTTP server, Default = ''", "type": "string" + }, + "iginx.graph.tree.enable": { + "envName": null, + "propertyName": "iginx.graph.tree.enable", + "defaultValue": true, + "description": "If true, the result of show columns will be displayed as a tree; otherwise, it will be displayed as forest, Default = true", + "type": "checkbox" } }, "editor": { diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java index 5a22ef4..d4ad821 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java @@ -32,10 +32,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; -import org.apache.zeppelin.iginx.util.FileUtil; -import org.apache.zeppelin.iginx.util.HttpUtil; -import org.apache.zeppelin.iginx.util.MultiwayTree; -import org.apache.zeppelin.iginx.util.SqlCmdUtil; +import org.apache.zeppelin.iginx.util.*; import org.apache.zeppelin.interpreter.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +56,7 @@ public class IginxInterpreter8 extends Interpreter { private static final String IGINX_UPLOAD_DIR_MAX_SIZE = "iginx.zeppelin.upload.dir.max.size"; private static final String IGINX_NOTE_FONT_SIZE_ENABLE = "iginx.zeppelin.note.font.size.enable"; private static final String IGINX_NOTE_FONT_SIZE = "iginx.zeppelin.note.font.size"; + private static final String IGINX_GRAPH_TREE_ENABLE = "iginx.graph.tree.enable"; private static final String DEFAULT_HOST = "127.0.0.1"; private static final String DEFAULT_PORT = "6888"; @@ -76,6 +74,7 @@ public class IginxInterpreter8 extends Interpreter { private static final String DEFAULT_UPLOAD_DIR_MAX_SIZE = "200"; // GB private static final String DEFAULT_NOTE_FONT_SIZE_ENABLE = "false"; private static final String DEFAULT_NOTE_FONT_SIZE = "9.0"; + private static final String DEFAULT_IGINX_GRAPH_TREE_ENABLE = "true"; private static final String TAB = "\t"; private static final String NEWLINE = "\n"; @@ -101,6 +100,7 @@ public class IginxInterpreter8 extends Interpreter { private long uploadDirMaxSize = 0; private boolean noteFontSizeEnable = false; private double noteFontSize = 9.0; + private boolean graphTreeEnable = true; private Queue downloadFileQueue = new LinkedList<>(); private Queue downloadFileSizeQueue = new LinkedList<>(); @@ -165,7 +165,9 @@ public void open() throws InterpreterException { noteFontSize = Double.parseDouble( properties.getProperty(IGINX_NOTE_FONT_SIZE, DEFAULT_NOTE_FONT_SIZE).trim()); - + graphTreeEnable = + Boolean.parseBoolean( + properties.getProperty(IGINX_GRAPH_TREE_ENABLE, DEFAULT_IGINX_GRAPH_TREE_ENABLE)); localIpAddress = getLocalHostExactAddress(); if (localIpAddress == null) { localIpAddress = "127.0.0.1"; @@ -298,7 +300,8 @@ private InterpreterResult processSql(String sql, InterpreterContext context) { if (SqlType.ShowColumns == sqlResult.getSqlType()) { interpreterResult.add( new InterpreterResultMessage( - InterpreterResult.Type.HTML, buildNetworkForShowColumns(sqlResult))); + InterpreterResult.Type.HTML, + buildTreeForShowColumns(sqlResult))); // buildNetworkForShowColumns(sqlResult) } msg = buildSingleFormResult( @@ -345,8 +348,9 @@ public String buildNetworkForShowColumns(SessionExecuteSqlResult sqlResult) { row -> { MultiwayTree.addTreeNodeFromString(tree, row.get(0)); }); + String htmlTemplate = "static/vis/network.html"; try (InputStream inputStream = - IginxInterpreter8.class.getClassLoader().getResourceAsStream("static/vis/network.html")) { + IginxInterpreter8.class.getClassLoader().getResourceAsStream(htmlTemplate)) { BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); StringBuilder content = new StringBuilder(); String line; @@ -355,7 +359,8 @@ public String buildNetworkForShowColumns(SessionExecuteSqlResult sqlResult) { } Gson gson = new Gson(); String jsonString = gson.toJson(tree); - String html = content.toString().replace("ALL_NODE_STR", jsonString); + String html = + content.toString().replace("ALL_NODE_STR", jsonString).replace("TREE_ENABLED", "false"); // 写入vis等库文件，只在新环境执行一次 String targetPath = outfileDir + "/graphs/lib/"; if (!FileUtil.isDirectoryLoaded(targetPath)) { @@ -387,6 +392,60 @@ public String buildNetworkForShowColumns(SessionExecuteSqlResult sqlResult) { return ""; } + /** + * 为show columns 命令创建树状状图 + * + * @param sqlResult + */ + public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult) { + List> queryList = + sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision); + MultiwayTree tree = MultiwayTree.getMultiwayTree(); + queryList + .subList(1, queryList.size()) + .forEach( + row -> { + MultiwayTree.addTreeNodeFromString(tree, row.get(0)); + }); + String htmlTemplate = "static/highcharts/tree.html"; + try (InputStream inputStream = + IginxInterpreter8.class.getClassLoader().getResourceAsStream(htmlTemplate)) { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + StringBuilder content = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + content.append(line).append("\n"); + } + List nodeList = new ArrayList<>(); + tree.traverseToHighchartsTreeNodes(tree.getRoot(), nodeList, 1); + if (!graphTreeEnable) { + nodeList.remove(0); // 删掉根节点，展现森林 + } + Gson gson = new Gson(); + String jsonString = gson.toJson(nodeList); + String html = + content.toString().replace("NODE_LIST", jsonString).replace("TREE_ENABLED", "false"); + LOGGER.info("html={}", html); + // 写入Highcharts库文件，只在新环境执行一次 + // String targetPath = outfileDir + "/graphs/lib/"; + // if (!FileUtil.isDirectoryLoaded(targetPath)) { + // String sourcePath = "static/highcharts/lib/"; + // String jarUrl = + // + // Objects.requireNonNull(IginxInterpreter8.class.getClassLoader().getResource(sourcePath)) + // .toString(); + // String jarPath = jarUrl.substring(jarUrl.indexOf("file:") + 5, + // jarUrl.indexOf(".jar") + 4); + // FileUtil.extractDirectoryFromJar(jarPath, sourcePath, targetPath); + // } + + return html; + } catch (IOException e) { + LOGGER.warn("load show columns to tree error", e); + } + return ""; + } + private static boolean isLoadDataFromCsv(String sql) { return sql.startsWith("load data from infile ") && sql.contains("as csv"); } @@ -471,7 +530,7 @@ private InterpreterResult processLoadCsv(String sql, InterpreterContext context) + fileSizeGB + "GB."); } - + LOGGER.info("load data sql execute, sql={}", sql); List columns = null; long recordsNum = 0; byte[] bytes = FileUtils.readFileToByteArray(file); diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java b/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java index 30e8957..cd2bea0 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java @@ -88,7 +88,6 @@ public void handle(HttpExchange exchange) throws IOException { String requestPath = exchange.getRequestURI().getPath(); File file = new File(basePath + requestPath); if (file.exists() && !file.isDirectory()) { - // 设置响应头为文件下载 if (requestPath.endsWith("html")) { exchange.getResponseHeaders().set("Content-Type", "text/html"); } else { diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/util/HighchartsTreeNode.java b/v8/src/main/java/org/apache/zeppelin/iginx/util/HighchartsTreeNode.java new file mode 100644 index 0000000..ee3bb1b --- /dev/null +++ b/v8/src/main/java/org/apache/zeppelin/iginx/util/HighchartsTreeNode.java @@ -0,0 +1,51 @@ +package org.apache.zeppelin.iginx.util; + +public class HighchartsTreeNode { + private String id; + private String name; + private String parent; + private int level; + + public HighchartsTreeNode(String id, String name, String parent, int level) { + this.id = id; + this.name = name; + this.level = level; + if (level == 1) { + this.parent = "undefined"; + } else { + this.parent = parent; + } + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + + public int getLevel() { + return level; + } + + public void setLevel(int level) { + this.level = level; + } +} diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java b/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java index a4afffa..98e8ba4 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java @@ -1,10 +1,11 @@ package org.apache.zeppelin.iginx.util; +import java.util.List; import org.apache.commons.lang3.StringUtils; public class MultiwayTree { public static final String ROOT_NODE_NAME = "数据资产"; - public static final String ROOT_NODE_PATH = "root"; + public static final String ROOT_NODE_PATH = "rootId"; public TreeNode getRoot() { return root; @@ -40,11 +41,21 @@ private TreeNode findNode(TreeNode node, TreeNode nodeToFind) { return null; } - public void traversePreorder(TreeNode node) { + /** + * 广度优先遍历树，转换为Highcharts节点数组 + * + * @param node + * @param nodeList + * @param level 从0开始，0代表虚拟根节点 + */ + public void traverseToHighchartsTreeNodes( + TreeNode node, List nodeList, int level) { if (node != null) { - System.out.print(node.value + " "); + nodeList.add( + new HighchartsTreeNode( + node.path, node.value, StringUtils.substringBeforeLast(node.path, "."), level++)); for (TreeNode child : node.children) { - traversePreorder(child); + traverseToHighchartsTreeNodes(child, nodeList, level); } } } diff --git a/v8/src/main/resources/interpreter-setting.json b/v8/src/main/resources/interpreter-setting.json index 3974000..c6f0d27 100644 --- a/v8/src/main/resources/interpreter-setting.json +++ b/v8/src/main/resources/interpreter-setting.json @@ -108,6 +108,13 @@ "defaultValue": "", "description": "The host of File HTTP server, Default = ''", "type": "string" + }, + "iginx.graph.tree.enable": { + "envName": null, + "propertyName": "iginx.graph.tree.enable", + "defaultValue": true, + "description": "If true, the result of show columns will be displayed as a tree; otherwise, it will be displayed as forest, Default = true", + "type": "checkbox" } }, "editor": { diff --git a/v8/src/main/resources/static/highcharts/tree.html b/v8/src/main/resources/static/highcharts/tree.html new file mode 100644 index 0000000..e92c515 --- /dev/null +++ b/v8/src/main/resources/static/highcharts/tree.html @@ -0,0 +1,134 @@ + + + + + + +
+
+
+ + + diff --git a/v8/src/main/resources/static/vis/main.html b/v8/src/main/resources/static/vis/main.html index 7c83c4c..9d3265f 100644 --- a/v8/src/main/resources/static/vis/main.html +++ b/v8/src/main/resources/static/vis/main.html @@ -18,7 +18,7 @@
diff --git a/v8/src/main/resources/static/vis/network.html b/v8/src/main/resources/static/vis/network.html index fa1c84c..0d3edc0 100644 --- a/v8/src/main/resources/static/vis/network.html +++ b/v8/src/main/resources/static/vis/network.html @@ -7,17 +7,12 @@

- + crossorigin="anonymous"/> + crossorigin="anonymous">
@@ -65,8 +60,6 @@

From e290b2ddcf41b34b91ec486ef4a471a3a7eae63e Mon Sep 17 00:00:00 2001 From: liuxtq Date: Fri, 29 Nov 2024 17:13:04 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E5=9B=9E=E6=BB=9A=E5=AF=B9network=20graph?= =?UTF-8?q?=E7=9A=84=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- v8/src/main/resources/static/vis/network.html | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/v8/src/main/resources/static/vis/network.html b/v8/src/main/resources/static/vis/network.html index 0d3edc0..96713ab 100644 --- a/v8/src/main/resources/static/vis/network.html +++ b/v8/src/main/resources/static/vis/network.html @@ -7,14 +7,16 @@

- + - - + integrity="sha384-JEW9xMcG8R+pH31jmWH6WWP0WintQrMb4s7ZOdauHnUtxwoG2vI5DkLtS3qm9Ekf" + crossorigin="anonymous" + >

@@ -60,6 +62,8 @@

+ +

From 1fd90ba209404d0b0681c7b171417cfde0dcc5f7 Mon Sep 17 00:00:00 2001 From: liuxtq Date: Fri, 29 Nov 2024 18:20:55 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E4=BC=98=E5=8C=96tree=20graph=E7=BC=A9?= =?UTF-8?q?=E6=94=BE=E9=80=BB=E8=BE=91=EF=BC=8C=E6=A0=B9=E6=8D=AE=E6=A0=91?= =?UTF-8?q?=E7=9A=84=E5=B1=82=E6=95=B0=E8=87=AA=E9=80=82=E5=BA=94=E7=BC=A9?= =?UTF-8?q?=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/static/highcharts/tree.html | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/v8/src/main/resources/static/highcharts/tree.html b/v8/src/main/resources/static/highcharts/tree.html index 06c39be..dcd8baf 100644 --- a/v8/src/main/resources/static/highcharts/tree.html +++ b/v8/src/main/resources/static/highcharts/tree.html @@ -2,24 +2,28 @@