Skip to content

Commit

Permalink
Merge pull request #20 from liuxtq/dev-delimiter
Browse files Browse the repository at this point in the history 
fix(interpreter): Optimize the tree graph and fix the "load data err with \\t" issue.
  • Loading branch information
@zhuyuqing
zhuyuqing authored Dec 6, 2024
2 parents 08302c8 + 664b4ef commit 6ad71c0
Show file tree
Hide file tree
Showing 5 changed files with 238 additions and 45 deletions.
52 changes: 21 additions & 31 deletions v8/src/main/java/org/apache/zeppelin/iginx/IginxInterpreter8.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.zeppelin.iginx.util.*;
import org.apache.zeppelin.iginx.util.HttpUtil;
Expand Down Expand Up @@ -302,7 +301,8 @@ private InterpreterResult processSql(String sql, InterpreterContext context) {
if (SqlType.ShowColumns == sqlResult.getSqlType()) {
interpreterResult.add(
new InterpreterResultMessage(
InterpreterResult.Type.HTML, buildTreeForShowColumns(sqlResult)));
InterpreterResult.Type.HTML,
buildTreeForShowColumns(sqlResult, context.getParagraphId())));
}
msg =
buildSingleFormResult(
Expand Down Expand Up @@ -338,7 +338,7 @@ private InterpreterResult processSql(String sql, InterpreterContext context) {
*
* @param sqlResult
*/
public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult) {
public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult, String paragraphId) {
List<List<String>> queryList =
sqlResult.getResultInList(true, FormatUtils.DEFAULT_TIME_FORMAT, timePrecision);
MultiwayTree tree = MultiwayTree.getMultiwayTree();
Expand All @@ -359,30 +359,19 @@ public String buildTreeForShowColumns(SessionExecuteSqlResult sqlResult) {
}
List<HighchartsTreeNode> nodeList = new ArrayList<>();
int depth = tree.traverseToHighchartsTreeNodes(tree.getRoot(), nodeList);
if (!graphTreeEnable) {
nodeList.remove(0); // 删掉根节点，展现森林
}
Gson gson = new Gson();
String jsonString = gson.toJson(nodeList);
String html =
content
.toString()
.replace("PARAGRAPH_ID", paragraphId)
.replace("NODE_LIST", jsonString)
.replace("TREE_DEPTH", String.valueOf(depth));
// LOGGER.info("depth={},html={}", depth, html);
// 写入Highcharts库文件，只在新环境执行一次
// String targetPath = outfileDir + "/graphs/lib/";
// if (!FileUtil.isDirectoryLoaded(targetPath)) {
// String sourcePath = "static/highcharts/lib/";
// String jarUrl =
//
// Objects.requireNonNull(IginxInterpreter8.class.getClassLoader().getResource(sourcePath))
// .toString();
// String jarPath = jarUrl.substring(jarUrl.indexOf("file:") + 5,
// jarUrl.indexOf(".jar") + 4);
// FileUtil.extractDirectoryFromJar(jarPath, sourcePath, targetPath);
// }

.replace("TREE_DEPTH", String.valueOf(depth))
.replace("TREE_ENABLE", String.valueOf(graphTreeEnable));
String fileName = paragraphId + "_tree.html";
// 写入文件服务器paragraphID_tree.html
String targetPath = outfileDir + "/graphs/tree/" + fileName;
FileUtil.writeToFile(html, targetPath);
return html;
} catch (IOException e) {
LOGGER.warn("load show columns to tree error", e);
Expand Down Expand Up @@ -455,14 +444,17 @@ private InterpreterResult processLoadCsv(String sql, InterpreterContext context)
if (!file.isFile()) {
throw new InvalidParameterException(path + " is not a file!");
}
String[] paths = sql.split(" ");
for (int i = 0; i < paths.length; i++) {
if ("INFILE".equalsIgnoreCase(paths[i])) {
paths[i + 1] = "\"" + path + "\"";
break;
}

String lowerCaseStr = sql.toLowerCase();
int start = lowerCaseStr.indexOf("INFILE".toLowerCase());
int end = lowerCaseStr.indexOf("AS CSV".toLowerCase());
StringBuffer stringBuffer = new StringBuffer(sql);
sql = stringBuffer.replace(start + 7, end - 1, "\"" + path + "\"").toString();

if (sql.contains("\\")) {
sql = sql.replace("\\\\", "\\");
}
sql = StringUtils.join(paths, " ");
LOGGER.info("load data sql execute, sql={}", sql);
double fileSizeGB =
new BigDecimal(file.length() / 1024 / 1024 / 1024)
.setScale(2, RoundingMode.HALF_UP)
Expand Down Expand Up @@ -1081,9 +1073,7 @@ public InterpreterResult tuneFontSize(
InterpreterResult.Type.HTML,
String.format("<h%d>%s</h%d>", hTagNumber, item.getData(), hTagNumber));
} else if (item.getType().equals(InterpreterResult.Type.HTML)) {
return new InterpreterResultMessage(
item.getType(),
String.format("<h%d>%s</h%d>", hTagNumber, item.getData(), hTagNumber));
return new InterpreterResultMessage(item.getType(), item.getData());
} else {
LOGGER.warn("unexpected result type {}", item.getType());
}
Expand Down
44 changes: 44 additions & 0 deletions v8/src/main/java/org/apache/zeppelin/iginx/SimpleFileServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class SimpleFileServer {

public static String PREFIX = "/files";
public static String PREFIX_UPLOAD = "/files/upload";
public static String PREFIX_GRAPH = "/graphs";
private int port;
private String fileDir;
private String uploadFileDir;
Expand Down Expand Up @@ -58,6 +59,7 @@ public void start() throws IOException {
httpServer = HttpServer.create(new InetSocketAddress(port), 0);
httpServer.createContext(PREFIX, new FileHandler(fileDir));
httpServer.createContext(PREFIX_UPLOAD, new UploadHandler(uploadFileDir));
httpServer.createContext(PREFIX_GRAPH, new GraphHandler(fileDir));
httpServer.start();
} catch (IOException e) {
LOGGER.error("Error starting SimpleFileServer", e);
Expand Down Expand Up @@ -215,6 +217,48 @@ public void handle(HttpExchange exchange) {
}
}

static class GraphHandler implements HttpHandler {
private String basePath;

public GraphHandler(String basePath) {
this.basePath = basePath;
}

@Override
public void handle(HttpExchange exchange) throws IOException {
exchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*");

// 获取请求的文件名，并构建文件路径
String requestPath = exchange.getRequestURI().getPath();
File file = new File(basePath + requestPath);
if (file.exists() && !file.isDirectory()) {
// 设置响应头为文件下载
if (requestPath.endsWith("html")) {
exchange.getResponseHeaders().set("Content-Type", "text/html");
} else {
exchange.getResponseHeaders().set("Content-Type", "text/plain");
}
exchange.sendResponseHeaders(200, file.length());

// 读取文件并写入响应体
OutputStream os = exchange.getResponseBody();
FileInputStream fs = new FileInputStream(file);
final byte[] buffer = new byte[0x10000];
int count = 0;
while ((count = fs.read(buffer)) >= 0) {
os.write(buffer, 0, count);
}
fs.close();
os.close();
} else {
String responseMeg = "404 (Not Found)，可能文件已被删除，请重新执行查询";
exchange.sendResponseHeaders(404, responseMeg.length());
Writer response = new OutputStreamWriter(exchange.getResponseBody());
response.write(responseMeg);
response.close();
}
}
}
/** clean earliest files when upload file director exceeds 100GB */
public void cleanUpLoadDir() {
new Thread(
Expand Down
17 changes: 17 additions & 0 deletions v8/src/main/java/org/apache/zeppelin/iginx/util/FileUtil.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package org.apache.zeppelin.iginx.util;

import java.io.*;
import java.nio.charset.StandardCharsets;

public class FileUtil {
public static void writeToFile(String content, String filePath) {
try (FileOutputStream fos = new FileOutputStream(filePath);
OutputStreamWriter osw = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
BufferedWriter writer = new BufferedWriter(osw)) { // 使用BufferedWriter提高效率

writer.write(content);
} catch (IOException e) {
e.printStackTrace();
}
}
}
2 changes: 1 addition & 1 deletion v8/src/main/java/org/apache/zeppelin/iginx/util/MultiwayTree.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import org.apache.commons.lang3.StringUtils;

public class MultiwayTree {
public static final String ROOT_NODE_NAME = "数据资产";
public static final String ROOT_NODE_NAME = "";
public static final String ROOT_NODE_PATH = "rootId";

public TreeNode getRoot() {
Expand Down
Loading

0 comments on commit 6ad71c0

Please sign in to comment.