diff --git a/README.md b/README.md index 7eda735..ce39514 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,8 @@ docker run -v ~/code/zeppelin-interpreter/:/opt/zeppelin/interpreter/iginx --pri 12. **iginx.zeppelin.upload.file.max.size**：IGinX上传文件大小限制，单位GB。 13. **iginx.zeppelin.note.font.size.enable**:是否激活Note范围内统一字体尺寸，默认不开启。 14. **iginx.zeppelin.note.font.size**：Note范围内字体尺寸，默认16，可选值9-20。 +15. **iginx.file.http.host**：IGinX 中文件下载服务要占用的IP，默认为 127.0.0.1。 +16. **iginx.graph.tree.enable**: 如果设置成true，命令返回结果展现成树，否则展现成森林，默认true。 ### 新建IGinX笔记本 点击红框内的 Create new note diff --git a/v11/src/main/resources/interpreter-setting.json b/v11/src/main/resources/interpreter-setting.json index b7140dc..c6f0d27 100644 --- a/v11/src/main/resources/interpreter-setting.json +++ b/v11/src/main/resources/interpreter-setting.json @@ -73,6 +73,48 @@ "defaultValue": "18082", "description": "The port of File HTTP server, Default = '18082'", "type": "number" + }, + "iginx.zeppelin.upload.file.max.size": { + "envName": null, + "propertyName": "iginx.zeppelin.upload.file.max.size", + "defaultValue": "10", + "description": "The Max Size of upload file, Default = 10GB", + "type": "number" + }, + "iginx.zeppelin.upload.dir.max.size": { + "envName": null, + "propertyName": "iginx.zeppelin.upload.dir.max.size", + "defaultValue": "200", + "description": "The Max Size of upload directory, Default = 200GB", + "type": "number" + }, + "iginx.zeppelin.note.font.size.enable": { + "envName": null, + "propertyName": "iginx.zeppelin.note.font.size.enable", + "defaultValue": false, + "description": "If enable the same font size in the note scope, Default = false", + "type": "checkbox" + }, + "iginx.zeppelin.note.font.size": { + "envName": null, + "propertyName": "iginx.zeppelin.note.font.size", + "defaultValue": "16", + "description": "Font size in not scope, Default = 9", + "type": "number" + }, + "iginx.file.http.host": { + "envName": null, + "propertyName": "iginx.http.host", + "defaultValue": "", + "description": "The host of File HTTP server, Default = ''", + "type": "string" + }, + "iginx.graph.tree.enable": { + "envName": null, + "propertyName": "iginx.graph.tree.enable", + "defaultValue": true, + "description": "If true, the result of show columns will be displayed as a tree; otherwise, it will be displayed as forest, Default = true", + "type": "checkbox" } }, "editor": { diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java index 053e1ea..23bc7f2 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java @@ -12,6 +12,7 @@ import cn.edu.tsinghua.iginx.thrift.SqlType; import cn.edu.tsinghua.iginx.utils.FormatUtils; import cn.edu.tsinghua.iginx.utils.Pair; +import com.google.gson.Gson; import java.io.*; import java.math.BigDecimal; import java.math.RoundingMode; @@ -31,6 +32,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; +import org.apache.zeppelin.iginx.util.*; import org.apache.zeppelin.iginx.util.HttpUtil; import org.apache.zeppelin.iginx.util.SqlCmdUtil; import org.apache.zeppelin.interpreter.*; @@ -50,11 +52,13 @@ public class IginxInterpreter8 extends Interpreter { private static final String IGINX_OUTFILE_MAX_NUM = "iginx.outfile.max.num"; private static final String IGINX_OUTFILE_MAX_SIZE = "iginx.outfile.max.size"; private static final String IGINX_FILE_HTTP_PORT = "iginx.http.file.port"; + private static final String IGINX_FILE_HTTP_HOST = "iginx.file.http.host"; private static final String IGINX_ZEPPELIN_IP = "iginx.zeppelin.ip"; private static final String IGINX_UPLOAD_FILE_MAX_SIZE = "iginx.zeppelin.upload.file.max.size"; private static final String IGINX_UPLOAD_DIR_MAX_SIZE = "iginx.zeppelin.upload.dir.max.size"; private static final String IGINX_NOTE_FONT_SIZE_ENABLE = "iginx.zeppelin.note.font.size.enable"; private static final String IGINX_NOTE_FONT_SIZE = "iginx.zeppelin.note.font.size"; + private static final String IGINX_GRAPH_TREE_ENABLE = "iginx.graph.tree.enable"; private static final String DEFAULT_HOST = "127.0.0.1"; private static final String DEFAULT_PORT = "6888"; @@ -66,11 +70,13 @@ public class IginxInterpreter8 extends Interpreter { private static final String DEFAULT_OUTFILE_MAX_NUM = "100"; private static final String DEFAULT_OUTFILE_MAX_SIZE = "10240"; private static final String DEFAULT_FILE_HTTP_PORT = "18082"; + private static final String DEFAULT_FILE_HTTP_HOST = "127.0.0.1"; private static final String DEFAULT_UPLOAD_DIR = "uploads"; private static final String DEFAULT_UPLOAD_FILE_MAX_SIZE = "10"; // GB private static final String DEFAULT_UPLOAD_DIR_MAX_SIZE = "200"; // GB private static final String DEFAULT_NOTE_FONT_SIZE_ENABLE = "false"; private static final String DEFAULT_NOTE_FONT_SIZE = "9.0"; + private static final String DEFAULT_IGINX_GRAPH_TREE_ENABLE = "true"; private static final String TAB = "\t"; private static final String NEWLINE = "\n"; @@ -90,11 +96,13 @@ public class IginxInterpreter8 extends Interpreter { private int outfileMaxNum = 0; private int outfileMaxSize = 0; private int fileHttpPort = 0; + private String fileHttpHost = ""; private String localIpAddress = ""; private long uploadFileMaxSize = 0; private long uploadDirMaxSize = 0; private boolean noteFontSizeEnable = false; private double noteFontSize = 9.0; + private boolean graphTreeEnable = true; private Queue downloadFileQueue = new LinkedList<>(); private Queue downloadFileSizeQueue = new LinkedList<>(); @@ -144,6 +152,7 @@ public void open() throws InterpreterException { fileHttpPort = Integer.parseInt( properties.getProperty(IGINX_FILE_HTTP_PORT, DEFAULT_FILE_HTTP_PORT).trim()); + fileHttpHost = properties.getProperty(IGINX_FILE_HTTP_HOST, DEFAULT_FILE_HTTP_HOST).trim(); uploadFileMaxSize = Long.parseLong( properties @@ -158,7 +167,9 @@ public void open() throws InterpreterException { noteFontSize = Double.parseDouble( properties.getProperty(IGINX_NOTE_FONT_SIZE, DEFAULT_NOTE_FONT_SIZE).trim()); - + graphTreeEnable = + Boolean.parseBoolean( + properties.getProperty(IGINX_GRAPH_TREE_ENABLE, DEFAULT_IGINX_GRAPH_TREE_ENABLE)); localIpAddress = getLocalHostExactAddress(); if (localIpAddress == null) { localIpAddress = "127.0.0.1"; @@ -285,23 +296,24 @@ private InterpreterResult processSql(String sql, InterpreterContext context) { return new InterpreterResult(InterpreterResult.Code.ERROR, sqlResult.getParseErrorMsg()); } - InterpreterResult interpreterResult; + InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); String msg; - if (singleFormSqlType.contains(sqlResult.getSqlType()) && !sql.startsWith("explain")) { + if (SqlType.ShowColumns == sqlResult.getSqlType()) { + interpreterResult.add( + new InterpreterResultMessage( + InterpreterResult.Type.HTML, buildTreeForShowColumns(sqlResult))); + } msg = buildSingleFormResult( sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision)); - interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); interpreterResult.add(InterpreterResult.Type.TABLE, msg); } else if (sqlResult.getSqlType() == SqlType.Query && sql.startsWith("explain")) { msg = buildExplainResult( sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision)); - interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); interpreterResult.add(InterpreterResult.Type.TABLE, msg); } else if (sqlResult.getSqlType() == SqlType.ShowClusterInfo) { - interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); buildClusterInfoResult( interpreterResult, sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision)); @@ -321,6 +333,63 @@ private InterpreterResult processSql(String sql, InterpreterContext context) { } } + /** + * 为show columns 命令创建树状状图 + * + * @param sqlResult + */ + public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult) { + List> queryList = + sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision); + MultiwayTree tree = MultiwayTree.getMultiwayTree(); + queryList + .subList(1, queryList.size()) + .forEach( + row -> { + MultiwayTree.addTreeNodeFromString(tree, row.get(0)); + }); + String htmlTemplate = "static/highcharts/tree.html"; + try (InputStream inputStream = + IginxInterpreter8.class.getClassLoader().getResourceAsStream(htmlTemplate)) { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + StringBuilder content = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + content.append(line).append("\n"); + } + List nodeList = new ArrayList<>(); + int depth = tree.traverseToHighchartsTreeNodes(tree.getRoot(), nodeList); + if (!graphTreeEnable) { + nodeList.remove(0); // 删掉根节点，展现森林 + } + Gson gson = new Gson(); + String jsonString = gson.toJson(nodeList); + String html = + content + .toString() + .replace("NODE_LIST", jsonString) + .replace("TREE_DEPTH", String.valueOf(depth)); + // LOGGER.info("depth={},html={}", depth, html); + // 写入Highcharts库文件，只在新环境执行一次 + // String targetPath = outfileDir + "/graphs/lib/"; + // if (!FileUtil.isDirectoryLoaded(targetPath)) { + // String sourcePath = "static/highcharts/lib/"; + // String jarUrl = + // + // Objects.requireNonNull(IginxInterpreter8.class.getClassLoader().getResource(sourcePath)) + // .toString(); + // String jarPath = jarUrl.substring(jarUrl.indexOf("file:") + 5, + // jarUrl.indexOf(".jar") + 4); + // FileUtil.extractDirectoryFromJar(jarPath, sourcePath, targetPath); + // } + + return html; + } catch (IOException e) { + LOGGER.warn("load show columns to tree error", e); + } + return ""; + } + private static boolean isLoadDataFromCsv(String sql) { return sql.startsWith("load data from infile ") && sql.contains("as csv"); } diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/util/HighchartsTreeNode.java b/v8/src/main/java/org/apache/zeppelin/iginx/util/HighchartsTreeNode.java new file mode 100644 index 0000000..cf2c222 --- /dev/null +++ b/v8/src/main/java/org/apache/zeppelin/iginx/util/HighchartsTreeNode.java @@ -0,0 +1,51 @@ +package org.apache.zeppelin.iginx.util; + +public class HighchartsTreeNode { + private String id; + private String name; + private String parent; + private int depth; + + public HighchartsTreeNode(String id, String name, String parent, int depth) { + this.id = id; + this.name = name; + this.depth = depth; + if (depth == 0) { + this.parent = "undefined"; + } else { + this.parent = parent; + } + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + + public int getDepth() { + return depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } +} diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java b/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java new file mode 100644 index 0000000..4452525 --- /dev/null +++ b/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java @@ -0,0 +1,92 @@ +package org.apache.zeppelin.iginx.util; + +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import org.apache.commons.lang3.StringUtils; + +public class MultiwayTree { + public static final String ROOT_NODE_NAME = "数据资产"; + public static final String ROOT_NODE_PATH = "rootId"; + + public TreeNode getRoot() { + return root; + } + + public void setRoot(TreeNode root) { + this.root = root; + } + + TreeNode root; + + public TreeNode insert(TreeNode parenNode, TreeNode newNode) { + TreeNode childNode = findNode(parenNode, newNode); + if (childNode != null) { + System.out.println("node already exists"); + } else { + parenNode.children.add(newNode); + return newNode; + } + return childNode; + } + + // 查找节点操作 + private TreeNode findNode(TreeNode node, TreeNode nodeToFind) { + if (node == null) { + return null; + } + for (TreeNode child : node.children) { + if (child.value.equals(nodeToFind.value)) { + return child; + } + } + return null; + } + + /** + * 广度优先遍历树，转换为Highcharts节点数组 + * + * @param root + * @param nodeList + */ + public int traverseToHighchartsTreeNodes(TreeNode root, List nodeList) { + if (root == null) { + return 0; + } + + Queue queue = new LinkedList<>(); + queue.offer(root); + int depth = 0; + + while (!queue.isEmpty()) { + int levelSize = queue.size(); // 当前层的节点数 + for (int i = 0; i < levelSize; i++) { + TreeNode node = queue.poll(); + nodeList.add( + new HighchartsTreeNode( + node.path, node.value, StringUtils.substringBeforeLast(node.path, "."), depth)); + for (TreeNode child : node.children) { + queue.offer(child); + } + } + depth++; + } + return depth; + } + + public static MultiwayTree getMultiwayTree() { + MultiwayTree tree = new MultiwayTree(); + tree.root = new TreeNode(ROOT_NODE_PATH, ROOT_NODE_NAME); // 初始化 + return tree; + } + + public static void addTreeNodeFromString(MultiwayTree tree, String nodeString) { + String[] nodes = nodeString.split("\\."); + TreeNode newNode = tree.root; + for (int i = 0; i < nodes.length; i++) { + newNode = + tree.insert( + newNode, new TreeNode(StringUtils.join(newNode.path, ".", nodes[i]), nodes[i])); + } + } +} diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/util/TreeNode.java b/v8/src/main/java/org/apache/zeppelin/iginx/util/TreeNode.java new file mode 100644 index 0000000..630906e --- /dev/null +++ b/v8/src/main/java/org/apache/zeppelin/iginx/util/TreeNode.java @@ -0,0 +1,48 @@ +package org.apache.zeppelin.iginx.util; + +import java.util.ArrayList; +import java.util.List; + +public class TreeNode { + String path; + String value; + List children; + List columns; + + public TreeNode(String path, String value, List columns) { + this.path = path; + this.value = value; + this.children = new ArrayList<>(); + this.columns = columns; + } + + public TreeNode(String path, String value) { + this.path = path; + this.value = value; + this.children = new ArrayList<>(); + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } + + public List getColumns() { + return columns; + } + + public void setColumns(List columns) { + this.columns = columns; + } +} diff --git a/v8/src/main/resources/interpreter-setting.json b/v8/src/main/resources/interpreter-setting.json index d9c7b1b..c6f0d27 100644 --- a/v8/src/main/resources/interpreter-setting.json +++ b/v8/src/main/resources/interpreter-setting.json @@ -101,6 +101,20 @@ "defaultValue": "16", "description": "Font size in not scope, Default = 9", "type": "number" + }, + "iginx.file.http.host": { + "envName": null, + "propertyName": "iginx.http.host", + "defaultValue": "", + "description": "The host of File HTTP server, Default = ''", + "type": "string" + }, + "iginx.graph.tree.enable": { + "envName": null, + "propertyName": "iginx.graph.tree.enable", + "defaultValue": true, + "description": "If true, the result of show columns will be displayed as a tree; otherwise, it will be displayed as forest, Default = true", + "type": "checkbox" } }, "editor": { diff --git a/v8/src/main/resources/static/highcharts/tree.html b/v8/src/main/resources/static/highcharts/tree.html new file mode 100644 index 0000000..dcd8baf --- /dev/null +++ b/v8/src/main/resources/static/highcharts/tree.html @@ -0,0 +1,153 @@ + + + + + + +
+
+
+ + +