Skip to content

Commit

Permalink
Merge pull request #18 from liuxtq/dev-fix-network
Browse files Browse the repository at this point in the history
feat(interpreter): add tree graph to display the results of the Show Columns cmd
  • Loading branch information
zhuyuqing authored Nov 29, 2024
2 parents 9af67d7 + 4414fec commit 2bdcfa7
Show file tree
Hide file tree
Showing 8 changed files with 477 additions and 6 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ docker run -v ~/code/zeppelin-interpreter/:/opt/zeppelin/interpreter/iginx --pri
12. **iginx.zeppelin.upload.file.max.size**:IGinX上传文件大小限制,单位GB。
13. **iginx.zeppelin.note.font.size.enable**:是否激活Note范围内统一字体尺寸,默认不开启。
14. **iginx.zeppelin.note.font.size**:Note范围内字体尺寸,默认16,可选值9-20。
15. **iginx.file.http.host**:IGinX 中文件下载服务要占用的IP,默认为 127.0.0.1。
16. **iginx.graph.tree.enable**: 如果设置成true,命令返回结果展现成树,否则展现成森林,默认true。
### 新建IGinX笔记本

点击红框内的 Create new note
Expand Down
42 changes: 42 additions & 0 deletions v11/src/main/resources/interpreter-setting.json
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,48 @@
"defaultValue": "18082",
"description": "The port of File HTTP server, Default = '18082'",
"type": "number"
},
"iginx.zeppelin.upload.file.max.size": {
"envName": null,
"propertyName": "iginx.zeppelin.upload.file.max.size",
"defaultValue": "10",
"description": "The Max Size of upload file, Default = 10GB",
"type": "number"
},
"iginx.zeppelin.upload.dir.max.size": {
"envName": null,
"propertyName": "iginx.zeppelin.upload.dir.max.size",
"defaultValue": "200",
"description": "The Max Size of upload directory, Default = 200GB",
"type": "number"
},
"iginx.zeppelin.note.font.size.enable": {
"envName": null,
"propertyName": "iginx.zeppelin.note.font.size.enable",
"defaultValue": false,
"description": "If enable the same font size in the note scope, Default = false",
"type": "checkbox"
},
"iginx.zeppelin.note.font.size": {
"envName": null,
"propertyName": "iginx.zeppelin.note.font.size",
"defaultValue": "16",
"description": "Font size in not scope, Default = 9",
"type": "number"
},
"iginx.file.http.host": {
"envName": null,
"propertyName": "iginx.http.host",
"defaultValue": "",
"description": "The host of File HTTP server, Default = ''",
"type": "string"
},
"iginx.graph.tree.enable": {
"envName": null,
"propertyName": "iginx.graph.tree.enable",
"defaultValue": true,
"description": "If true, the result of show columns will be displayed as a tree; otherwise, it will be displayed as forest, Default = true",
"type": "checkbox"
}
},
"editor": {
Expand Down
81 changes: 75 additions & 6 deletions v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import cn.edu.tsinghua.iginx.thrift.SqlType;
import cn.edu.tsinghua.iginx.utils.FormatUtils;
import cn.edu.tsinghua.iginx.utils.Pair;
import com.google.gson.Gson;
import java.io.*;
import java.math.BigDecimal;
import java.math.RoundingMode;
Expand All @@ -31,6 +32,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.zeppelin.iginx.util.*;
import org.apache.zeppelin.iginx.util.HttpUtil;
import org.apache.zeppelin.iginx.util.SqlCmdUtil;
import org.apache.zeppelin.interpreter.*;
Expand All @@ -50,11 +52,13 @@ public class IginxInterpreter8 extends Interpreter {
private static final String IGINX_OUTFILE_MAX_NUM = "iginx.outfile.max.num";
private static final String IGINX_OUTFILE_MAX_SIZE = "iginx.outfile.max.size";
private static final String IGINX_FILE_HTTP_PORT = "iginx.http.file.port";
private static final String IGINX_FILE_HTTP_HOST = "iginx.file.http.host";
private static final String IGINX_ZEPPELIN_IP = "iginx.zeppelin.ip";
private static final String IGINX_UPLOAD_FILE_MAX_SIZE = "iginx.zeppelin.upload.file.max.size";
private static final String IGINX_UPLOAD_DIR_MAX_SIZE = "iginx.zeppelin.upload.dir.max.size";
private static final String IGINX_NOTE_FONT_SIZE_ENABLE = "iginx.zeppelin.note.font.size.enable";
private static final String IGINX_NOTE_FONT_SIZE = "iginx.zeppelin.note.font.size";
private static final String IGINX_GRAPH_TREE_ENABLE = "iginx.graph.tree.enable";

private static final String DEFAULT_HOST = "127.0.0.1";
private static final String DEFAULT_PORT = "6888";
Expand All @@ -66,11 +70,13 @@ public class IginxInterpreter8 extends Interpreter {
private static final String DEFAULT_OUTFILE_MAX_NUM = "100";
private static final String DEFAULT_OUTFILE_MAX_SIZE = "10240";
private static final String DEFAULT_FILE_HTTP_PORT = "18082";
private static final String DEFAULT_FILE_HTTP_HOST = "127.0.0.1";
private static final String DEFAULT_UPLOAD_DIR = "uploads";
private static final String DEFAULT_UPLOAD_FILE_MAX_SIZE = "10"; // GB
private static final String DEFAULT_UPLOAD_DIR_MAX_SIZE = "200"; // GB
private static final String DEFAULT_NOTE_FONT_SIZE_ENABLE = "false";
private static final String DEFAULT_NOTE_FONT_SIZE = "9.0";
private static final String DEFAULT_IGINX_GRAPH_TREE_ENABLE = "true";

private static final String TAB = "\t";
private static final String NEWLINE = "\n";
Expand All @@ -90,11 +96,13 @@ public class IginxInterpreter8 extends Interpreter {
private int outfileMaxNum = 0;
private int outfileMaxSize = 0;
private int fileHttpPort = 0;
private String fileHttpHost = "";
private String localIpAddress = "";
private long uploadFileMaxSize = 0;
private long uploadDirMaxSize = 0;
private boolean noteFontSizeEnable = false;
private double noteFontSize = 9.0;
private boolean graphTreeEnable = true;

private Queue<String> downloadFileQueue = new LinkedList<>();
private Queue<Double> downloadFileSizeQueue = new LinkedList<>();
Expand Down Expand Up @@ -144,6 +152,7 @@ public void open() throws InterpreterException {
fileHttpPort =
Integer.parseInt(
properties.getProperty(IGINX_FILE_HTTP_PORT, DEFAULT_FILE_HTTP_PORT).trim());
fileHttpHost = properties.getProperty(IGINX_FILE_HTTP_HOST, DEFAULT_FILE_HTTP_HOST).trim();
uploadFileMaxSize =
Long.parseLong(
properties
Expand All @@ -158,7 +167,9 @@ public void open() throws InterpreterException {
noteFontSize =
Double.parseDouble(
properties.getProperty(IGINX_NOTE_FONT_SIZE, DEFAULT_NOTE_FONT_SIZE).trim());

graphTreeEnable =
Boolean.parseBoolean(
properties.getProperty(IGINX_GRAPH_TREE_ENABLE, DEFAULT_IGINX_GRAPH_TREE_ENABLE));
localIpAddress = getLocalHostExactAddress();
if (localIpAddress == null) {
localIpAddress = "127.0.0.1";
Expand Down Expand Up @@ -285,23 +296,24 @@ private InterpreterResult processSql(String sql, InterpreterContext context) {
return new InterpreterResult(InterpreterResult.Code.ERROR, sqlResult.getParseErrorMsg());
}

InterpreterResult interpreterResult;
InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
String msg;

if (singleFormSqlType.contains(sqlResult.getSqlType()) && !sql.startsWith("explain")) {
if (SqlType.ShowColumns == sqlResult.getSqlType()) {
interpreterResult.add(
new InterpreterResultMessage(
InterpreterResult.Type.HTML, buildTreeForShowColumns(sqlResult)));
}
msg =
buildSingleFormResult(
sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision));
interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
interpreterResult.add(InterpreterResult.Type.TABLE, msg);
} else if (sqlResult.getSqlType() == SqlType.Query && sql.startsWith("explain")) {
msg =
buildExplainResult(
sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision));
interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
interpreterResult.add(InterpreterResult.Type.TABLE, msg);
} else if (sqlResult.getSqlType() == SqlType.ShowClusterInfo) {
interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
buildClusterInfoResult(
interpreterResult,
sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision));
Expand All @@ -321,6 +333,63 @@ private InterpreterResult processSql(String sql, InterpreterContext context) {
}
}

/**
* 为show columns 命令创建树状状图
*
* @param sqlResult
*/
public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult) {
List<List<String>> queryList =
sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision);
MultiwayTree tree = MultiwayTree.getMultiwayTree();
queryList
.subList(1, queryList.size())
.forEach(
row -> {
MultiwayTree.addTreeNodeFromString(tree, row.get(0));
});
String htmlTemplate = "static/highcharts/tree.html";
try (InputStream inputStream =
IginxInterpreter8.class.getClassLoader().getResourceAsStream(htmlTemplate)) {
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
StringBuilder content = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
content.append(line).append("\n");
}
List<HighchartsTreeNode> nodeList = new ArrayList<>();
int depth = tree.traverseToHighchartsTreeNodes(tree.getRoot(), nodeList);
if (!graphTreeEnable) {
nodeList.remove(0); // 删掉根节点,展现森林
}
Gson gson = new Gson();
String jsonString = gson.toJson(nodeList);
String html =
content
.toString()
.replace("NODE_LIST", jsonString)
.replace("TREE_DEPTH", String.valueOf(depth));
// LOGGER.info("depth={},html={}", depth, html);
// 写入Highcharts库文件,只在新环境执行一次
// String targetPath = outfileDir + "/graphs/lib/";
// if (!FileUtil.isDirectoryLoaded(targetPath)) {
// String sourcePath = "static/highcharts/lib/";
// String jarUrl =
//
// Objects.requireNonNull(IginxInterpreter8.class.getClassLoader().getResource(sourcePath))
// .toString();
// String jarPath = jarUrl.substring(jarUrl.indexOf("file:") + 5,
// jarUrl.indexOf(".jar") + 4);
// FileUtil.extractDirectoryFromJar(jarPath, sourcePath, targetPath);
// }

return html;
} catch (IOException e) {
LOGGER.warn("load show columns to tree error", e);
}
return "";
}

private static boolean isLoadDataFromCsv(String sql) {
return sql.startsWith("load data from infile ") && sql.contains("as csv");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package org.apache.zeppelin.iginx.util;

public class HighchartsTreeNode {
private String id;
private String name;
private String parent;
private int depth;

public HighchartsTreeNode(String id, String name, String parent, int depth) {
this.id = id;
this.name = name;
this.depth = depth;
if (depth == 0) {
this.parent = "undefined";
} else {
this.parent = parent;
}
}

public String getId() {
return id;
}

public void setId(String id) {
this.id = id;
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

public String getParent() {
return parent;
}

public void setParent(String parent) {
this.parent = parent;
}

public int getDepth() {
return depth;
}

public void setDepth(int depth) {
this.depth = depth;
}
}
92 changes: 92 additions & 0 deletions v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package org.apache.zeppelin.iginx.util;

import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import org.apache.commons.lang3.StringUtils;

public class MultiwayTree {
public static final String ROOT_NODE_NAME = "数据资产";
public static final String ROOT_NODE_PATH = "rootId";

public TreeNode getRoot() {
return root;
}

public void setRoot(TreeNode root) {
this.root = root;
}

TreeNode root;

public TreeNode insert(TreeNode parenNode, TreeNode newNode) {
TreeNode childNode = findNode(parenNode, newNode);
if (childNode != null) {
System.out.println("node already exists");
} else {
parenNode.children.add(newNode);
return newNode;
}
return childNode;
}

// 查找节点操作
private TreeNode findNode(TreeNode node, TreeNode nodeToFind) {
if (node == null) {
return null;
}
for (TreeNode child : node.children) {
if (child.value.equals(nodeToFind.value)) {
return child;
}
}
return null;
}

/**
* 广度优先遍历树,转换为Highcharts节点数组
*
* @param root
* @param nodeList
*/
public int traverseToHighchartsTreeNodes(TreeNode root, List<HighchartsTreeNode> nodeList) {
if (root == null) {
return 0;
}

Queue<TreeNode> queue = new LinkedList<>();
queue.offer(root);
int depth = 0;

while (!queue.isEmpty()) {
int levelSize = queue.size(); // 当前层的节点数
for (int i = 0; i < levelSize; i++) {
TreeNode node = queue.poll();
nodeList.add(
new HighchartsTreeNode(
node.path, node.value, StringUtils.substringBeforeLast(node.path, "."), depth));
for (TreeNode child : node.children) {
queue.offer(child);
}
}
depth++;
}
return depth;
}

public static MultiwayTree getMultiwayTree() {
MultiwayTree tree = new MultiwayTree();
tree.root = new TreeNode(ROOT_NODE_PATH, ROOT_NODE_NAME); // 初始化
return tree;
}

public static void addTreeNodeFromString(MultiwayTree tree, String nodeString) {
String[] nodes = nodeString.split("\\.");
TreeNode newNode = tree.root;
for (int i = 0; i < nodes.length; i++) {
newNode =
tree.insert(
newNode, new TreeNode(StringUtils.join(newNode.path, ".", nodes[i]), nodes[i]));
}
}
}
Loading

0 comments on commit 2bdcfa7

Please sign in to comment.