From 2c494583edcc2850c333cdc168d23c3610a8407d Mon Sep 17 00:00:00 2001 From: liuxtq Date: Thu, 5 Dec 2024 20:05:05 +0800 Subject: [PATCH 1/4] =?UTF-8?q?1.=E4=BC=98=E5=8C=96=E6=A0=91=E7=8A=B6?= =?UTF-8?q?=E5=9B=BE=202.=E4=BF=AE=E5=A4=8Dload=20data=E8=AF=AD=E5=8F=A5?= =?UTF-8?q?=E4=BB=A5\\t=E6=9C=80=E4=B8=BA=E5=88=86=E9=9A=94=E7=AC=A6?= =?UTF-8?q?=E6=97=B6=E7=B3=BB=E7=BB=9F=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../zeppelin/iginx/IginxInterpreter8.java | 74 ++++++---- .../zeppelin/iginx/SimpleFileServer.java | 44 ++++++ .../apache/zeppelin/iginx/util/FileUtil.java | 17 +++ .../zeppelin/iginx/util/MultiwayTree.java | 2 +- .../resources/static/highcharts/main.html | 49 +++++++ .../resources/static/highcharts/tree.html | 135 ++++++++++++++++-- 6 files changed, 277 insertions(+), 44 deletions(-) create mode 100644 v8/src/main/java/org/apache/zeppelin/iginx/util/FileUtil.java create mode 100644 v8/src/main/resources/static/highcharts/main.html diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java index 23bc7f2..5cfa0d5 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java @@ -30,7 +30,6 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.zeppelin.iginx.util.*; import org.apache.zeppelin.iginx.util.HttpUtil; @@ -302,7 +301,8 @@ private InterpreterResult processSql(String sql, InterpreterContext context) { if (SqlType.ShowColumns == sqlResult.getSqlType()) { interpreterResult.add( new InterpreterResultMessage( - InterpreterResult.Type.HTML, buildTreeForShowColumns(sqlResult))); + InterpreterResult.Type.HTML, + buildTreeForShowColumns(sqlResult, context.getParagraphId()))); } msg = buildSingleFormResult( @@ -338,7 +338,7 @@ private InterpreterResult processSql(String sql, InterpreterContext context) { * * @param sqlResult */ - public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult) { + public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult, String paragraphId) { List> queryList = sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision); MultiwayTree tree = MultiwayTree.getMultiwayTree(); @@ -359,30 +359,41 @@ public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult) { } List nodeList = new ArrayList<>(); int depth = tree.traverseToHighchartsTreeNodes(tree.getRoot(), nodeList); - if (!graphTreeEnable) { - nodeList.remove(0); // 删掉根节点，展现森林 - } Gson gson = new Gson(); String jsonString = gson.toJson(nodeList); String html = content .toString() .replace("NODE_LIST", jsonString) - .replace("TREE_DEPTH", String.valueOf(depth)); - // LOGGER.info("depth={},html={}", depth, html); - // 写入Highcharts库文件，只在新环境执行一次 - // String targetPath = outfileDir + "/graphs/lib/"; - // if (!FileUtil.isDirectoryLoaded(targetPath)) { - // String sourcePath = "static/highcharts/lib/"; - // String jarUrl = - // - // Objects.requireNonNull(IginxInterpreter8.class.getClassLoader().getResource(sourcePath)) - // .toString(); - // String jarPath = jarUrl.substring(jarUrl.indexOf("file:") + 5, - // jarUrl.indexOf(".jar") + 4); - // FileUtil.extractDirectoryFromJar(jarPath, sourcePath, targetPath); - // } - + .replace("TREE_DEPTH", String.valueOf(depth)) + .replace("TREE_ENABLE", String.valueOf(graphTreeEnable)); + String fileName = paragraphId + "_tree.html"; + // 写入文件服务器paragraphID_tree.html + String targetPath = outfileDir + "/graphs/tree/" + fileName; + FileUtil.writeToFile(html, targetPath); + + // 返回框架页面 + InputStream mainInputStream = + IginxInterpreter8.class + .getClassLoader() + .getResourceAsStream("static/highcharts/main.html"); + reader = new BufferedReader(new InputStreamReader(mainInputStream)); + content.setLength(0); + while ((line = reader.readLine()) != null) { + content.append(line).append("\n"); + } + html = + content + .toString() + .replace("PARAGRAPH_ID", paragraphId) + .replace( + "PARAGRAPH_TREE_URL", + String.format( + "http://%s:%d/graphs/tree/%s", fileHttpHost, fileHttpPort, fileName)); + LOGGER.info("depth={},html={}", depth, html); + + inputStream.close(); + mainInputStream.close(); return html; } catch (IOException e) { LOGGER.warn("load show columns to tree error", e); @@ -455,14 +466,17 @@ private InterpreterResult processLoadCsv(String sql, InterpreterContext context) if (!file.isFile()) { throw new InvalidParameterException(path + " is not a file!"); } - String[] paths = sql.split(" "); - for (int i = 0; i < paths.length; i++) { - if ("INFILE".equalsIgnoreCase(paths[i])) { - paths[i + 1] = "\"" + path + "\""; - break; - } + + String lowerCaseStr = sql.toLowerCase(); + int start = lowerCaseStr.indexOf("INFILE".toLowerCase()); + int end = lowerCaseStr.indexOf("AS CSV".toLowerCase()); + StringBuffer stringBuffer = new StringBuffer(sql); + sql = stringBuffer.replace(start + 7, end - 1, "\"" + path + "\"").toString(); + + if (sql.contains("\\")) { // 保持 + sql = sql.replace("\\\\", "\\"); } - sql = StringUtils.join(paths, " "); + LOGGER.info("load data sql execute, sql={}", sql); double fileSizeGB = new BigDecimal(file.length() / 1024 / 1024 / 1024) .setScale(2, RoundingMode.HALF_UP) @@ -1081,9 +1095,7 @@ public InterpreterResult tuneFontSize( InterpreterResult.Type.HTML, String.format("%s", hTagNumber, item.getData(), hTagNumber)); } else if (item.getType().equals(InterpreterResult.Type.HTML)) { - return new InterpreterResultMessage( - item.getType(), - String.format("%s", hTagNumber, item.getData(), hTagNumber)); + return new InterpreterResultMessage(item.getType(), item.getData()); } else { LOGGER.warn("unexpected result type {}", item.getType()); } diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java b/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java index a5d6102..80fe5a0 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java @@ -18,6 +18,7 @@ public class SimpleFileServer { public static String PREFIX = "/files"; public static String PREFIX_UPLOAD = "/files/upload"; + public static String PREFIX_GRAPH = "/graphs"; private int port; private String fileDir; private String uploadFileDir; @@ -58,6 +59,7 @@ public void start() throws IOException { httpServer = HttpServer.create(new InetSocketAddress(port), 0); httpServer.createContext(PREFIX, new FileHandler(fileDir)); httpServer.createContext(PREFIX_UPLOAD, new UploadHandler(uploadFileDir)); + httpServer.createContext(PREFIX_GRAPH, new GraphHandler(fileDir)); httpServer.start(); } catch (IOException e) { LOGGER.error("Error starting SimpleFileServer", e); @@ -215,6 +217,48 @@ public void handle(HttpExchange exchange) { } } + static class GraphHandler implements HttpHandler { + private String basePath; + + public GraphHandler(String basePath) { + this.basePath = basePath; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + exchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*"); + + // 获取请求的文件名，并构建文件路径 + String requestPath = exchange.getRequestURI().getPath(); + File file = new File(basePath + requestPath); + if (file.exists() && !file.isDirectory()) { + // 设置响应头为文件下载 + if (requestPath.endsWith("html")) { + exchange.getResponseHeaders().set("Content-Type", "text/html"); + } else { + exchange.getResponseHeaders().set("Content-Type", "text/plain"); + } + exchange.sendResponseHeaders(200, file.length()); + + // 读取文件并写入响应体 + OutputStream os = exchange.getResponseBody(); + FileInputStream fs = new FileInputStream(file); + final byte[] buffer = new byte[0x10000]; + int count = 0; + while ((count = fs.read(buffer)) >= 0) { + os.write(buffer, 0, count); + } + fs.close(); + os.close(); + } else { + String responseMeg = "404 (Not Found)，可能文件已被删除，请重新执行查询"; + exchange.sendResponseHeaders(404, responseMeg.length()); + Writer response = new OutputStreamWriter(exchange.getResponseBody()); + response.write(responseMeg); + response.close(); + } + } + } /** clean earliest files when upload file director exceeds 100GB */ public void cleanUpLoadDir() { new Thread( diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/util/FileUtil.java b/v8/src/main/java/org/apache/zeppelin/iginx/util/FileUtil.java new file mode 100644 index 0000000..33d4060 --- /dev/null +++ b/v8/src/main/java/org/apache/zeppelin/iginx/util/FileUtil.java @@ -0,0 +1,17 @@ +package org.apache.zeppelin.iginx.util; + +import java.io.*; +import java.nio.charset.StandardCharsets; + +public class FileUtil { + public static void writeToFile(String content, String filePath) { + try (FileOutputStream fos = new FileOutputStream(filePath); + OutputStreamWriter osw = new OutputStreamWriter(fos, StandardCharsets.UTF_8); + BufferedWriter writer = new BufferedWriter(osw)) { // 使用BufferedWriter提高效率 + + writer.write(content); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java b/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java index 4452525..703ffc2 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java @@ -6,7 +6,7 @@ import org.apache.commons.lang3.StringUtils; public class MultiwayTree { - public static final String ROOT_NODE_NAME = "数据资产"; + public static final String ROOT_NODE_NAME = ""; public static final String ROOT_NODE_PATH = "rootId"; public TreeNode getRoot() { diff --git a/v8/src/main/resources/static/highcharts/main.html b/v8/src/main/resources/static/highcharts/main.html new file mode 100644 index 0000000..94c454c --- /dev/null +++ b/v8/src/main/resources/static/highcharts/main.html @@ -0,0 +1,49 @@ + + + + + Title + + + +
+ +
+ + + \ No newline at end of file diff --git a/v8/src/main/resources/static/highcharts/tree.html b/v8/src/main/resources/static/highcharts/tree.html index 641e043..a4dd237 100644 --- a/v8/src/main/resources/static/highcharts/tree.html +++ b/v8/src/main/resources/static/highcharts/tree.html @@ -1,5 +1,6 @@ + - - -
- -
- - - \ No newline at end of file diff --git a/v8/src/main/resources/static/highcharts/tree.html b/v8/src/main/resources/static/highcharts/tree.html index a4dd237..615e4e7 100644 --- a/v8/src/main/resources/static/highcharts/tree.html +++ b/v8/src/main/resources/static/highcharts/tree.html @@ -71,7 +71,7 @@ marginRight: 200, style: { fontFamily: "Helvetica, Arial, sans-serif", - fontSize: '1.5rem' + fontSize: '2rem' } }, credits: { From 664b4ef2e7e618c8c38ac68bb5553d396b353393 Mon Sep 17 00:00:00 2001 From: liuxtq Date: Fri, 6 Dec 2024 09:43:31 +0800 Subject: [PATCH 4/4] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E6=8C=89=E9=92=AE=EF=BC=8C=E5=88=87=E6=8D=A2=E5=9B=BE=E5=BD=A2?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=B1=95=E7=8E=B0=E5=92=8C=E9=9A=90=E8=97=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../zeppelin/iginx/IginxInterpreter8.java | 1 + .../resources/static/highcharts/tree.html | 43 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java index 40f06c3..356074f 100644 --- a/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java +++ b/v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java @@ -364,6 +364,7 @@ public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult, String String html = content .toString() + .replace("PARAGRAPH_ID", paragraphId) .replace("NODE_LIST", jsonString) .replace("TREE_DEPTH", String.valueOf(depth)) .replace("TREE_ENABLE", String.valueOf(graphTreeEnable)); diff --git a/v8/src/main/resources/static/highcharts/tree.html b/v8/src/main/resources/static/highcharts/tree.html index 615e4e7..fdae7a0 100644 --- a/v8/src/main/resources/static/highcharts/tree.html +++ b/v8/src/main/resources/static/highcharts/tree.html @@ -9,7 +9,7 @@ max-width: 100vw; min-width: 60vw; } - #container { + #PARAGRAPH_ID_single_container { height: 35rem; } .large-tree{ @@ -24,11 +24,30 @@ width: 55vw; margin: 0 calc((100vw - 60vw) / 2); } + .toggle-button { + padding: 0 5px; + border: none; + background-color: transparent; + color: inherit; + cursor: pointer; + font-size: 14px; + } + .toggle-button::before { + content: ">> "; + color: #333; + } + .toggle-button:hover { + color: #2e70a8; + } + .toggle-button:hover::before { + color: #2e70a8; + } +
-
+